ci: build through the internal package cache; pin almalinux base by digest #1

Merged
jknapp merged 11 commits from ci/pkg-cache into main 2026-10-22 14:52:20 +00:00
20 changed files with 1721 additions and 49 deletions

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
## resolve-base-image.sh IMAGE:TAG — CI only (not copied into images).
## Prints IMAGE@sha256:<digest> for a Docker Hub tag, looked up DIRECTLY at
## registry-1.docker.io over HTTPS (never via the pkg-cache mirror), so the
## build can say `FROM IMAGE@sha256:...` and the mirror can only serve
## content-addressed blobs that BuildKit verifies. Exits non-zero (prints
## nothing on stdout) if the lookup fails; the workflow then skips the mirror
## and pulls the tag directly from Docker Hub. HEAD requests do not count
## against Docker Hub pull rate limits.
## Canonical copy: cloud-apache-container/.gitea/scripts/resolve-base-image.sh
## (docs/pkg-cache.md there); other repos carry identical copies.
set -euo pipefail
ref="${1:?usage: resolve-base-image.sh IMAGE:TAG}"
name_re='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)?$'
tag_re='^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$'
name="${ref%:*}"; tag="${ref##*:}"
[ "$name" != "$ref" ] || { echo "resolve-base-image: '$ref' has no :tag" >&2; exit 2; }
[[ "$name" =~ $name_re ]] || { echo "resolve-base-image: bad image name '$name'" >&2; exit 2; }
[[ "$tag" =~ $tag_re ]] || { echo "resolve-base-image: bad tag '$tag'" >&2; exit 2; }
repo="$name"; [[ "$repo" == */* ]] || repo="library/$repo" # official images
curl_opts=(-fsS --proto '=https' --retry 3 --retry-all-errors --max-time 20)
token=$(curl "${curl_opts[@]}" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" |
python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')
[ -n "$token" ] || { echo "resolve-base-image: no Docker Hub token for $repo" >&2; exit 1; }
hdrs=$(curl "${curl_opts[@]}" -I \
-H "Authorization: Bearer ${token}" \
-H 'Accept: application/vnd.oci.image.index.v1+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
"https://registry-1.docker.io/v2/${repo}/manifests/${tag}")
digest=$(printf '%s\n' "$hdrs" | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:" {print $2}' | tail -n1)
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "resolve-base-image: no valid digest for $ref (got '${digest}')" >&2; exit 1; }
printf '%s@%s\n' "$name" "$digest"
+60
View File
@@ -14,10 +14,64 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
# Base image pinned by DIGEST, looked up directly at Docker Hub (never via
# the pkg-cache mirror), so the mirror can only serve content-addressed
# blobs that BuildKit verifies. If the lookup fails this job does NOT use
# the mirror and pulls the tag straight from Docker Hub.
# Canonical docs: cloud-apache-container docs/pkg-cache.md.
- name: Resolve base image digest (Docker Hub, direct)
id: base
env:
BASE_TAG: almalinux:10-minimal
run: |
set -uo pipefail
if ref=$(.gitea/scripts/resolve-base-image.sh "$BASE_TAG") && [[ "$ref" =~ ^[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}$ ]]; then
echo "base-image: ${BASE_TAG} -> ${ref} (digest from registry-1.docker.io)"
echo "image=${ref}" >> "$GITHUB_OUTPUT"
echo "pinned=true" >> "$GITHUB_OUTPUT"
else
echo "::warning::base image ${BASE_TAG}: Docker Hub digest lookup failed - pulling the tag directly from Docker Hub, pkg-cache registry mirror NOT used"
echo "image=${BASE_TAG}" >> "$GITHUB_OUTPUT"
echo "pinned=false" >> "$GITHUB_OUTPUT"
fi
# Internal package cache (CT130 pkg-cache). Used ONLY if it answers over
# TLS with our pinned CA; otherwise (down, unreachable, bad/expired cert)
# this job builds from the upstream repos and Docker Hub exactly as before.
# The repo cache and the registry mirror are probed separately so either
# can fail back on its own.
- name: Probe package cache
id: pkgcache
# The registry mirror is only used for a digest-pinned base image.
env:
BASE_PINNED: ${{ steps.base.outputs.pinned }}
run: |
url=""; mirror=""
if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
if [ "$BASE_PINNED" = true ] && curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi
echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=${mirror:-<not used: Docker Hub direct>}"
[ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
echo "url=${url}" >> "$GITHUB_OUTPUT"
echo "mirror=${mirror}" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
# BuildKit (docker-container driver) ignores dockerd's registry-mirrors,
# so the mirror is configured on the builder; BuildKit itself still falls
# back to Docker Hub if the mirror stops answering mid-run.
- name: Set up Docker Buildx (Docker Hub via pkg-cache mirror)
if: steps.pkgcache.outputs.mirror != ''
uses: docker/setup-buildx-action@v3
with:
buildkitd-config-inline: |
[registry."docker.io"]
mirrors = ["192.168.1.40:5000"]
[registry."192.168.1.40:5000"]
ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"]
- name: Set up Docker Buildx
if: steps.pkgcache.outputs.mirror == ''
uses: docker/setup-buildx-action@v3
- name: Login to Gitea
@@ -58,6 +112,9 @@ jobs:
platforms: linux/amd64
push: false
load: true
build-args: |
BASE_IMAGE=${{ steps.base.outputs.image }}
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
tags: cpanel-importer:smoke
no-cache: true
@@ -101,6 +158,9 @@ jobs:
context: .
platforms: linux/amd64
push: true
build-args: |
BASE_IMAGE=${{ steps.base.outputs.image }}
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
tags: ${{ steps.tags.outputs.tags }}
cache-from: type=registry,ref=repo.anhonesthost.net/cloud-hosting-platform/cpanel-importer:latest
cache-to: type=inline
+15 -1
View File
@@ -121,9 +121,23 @@ ClamAV signature DBs).
bash tests/build-fixtures.sh
```
Two tarballs land under `tests/fixtures/`:
Three tarballs land under `tests/fixtures/`:
- `cpmove-clean.tar.gz` — a benign cpmove with a WordPress MyISAM dump.
- `cpmove-alfa.tar.gz` — same shape PLUS an ALFA-style symlink to /etc.
- `cpmove-mail.tar.gz` — same shape PLUS a mail tree (two Maildirs, one
empty) with an ALFA-style symlink inside it.
### Host-side regression tests
```bash
bash tests/run-tests.sh
```
Runs `extract.sh` and `scan-files.php` straight from the checkout (bash,
php-cli, python3, GNU tar; `clamscan` is stubbed). Covers the mail-tree
opt-in (`IMPORT_INCLUDE_MAIL=true`): skipped by default, extracted when
opted in, dangerous symlinks inside it still stripped, and never scanned
or quarantined by `scan-files.php --skip-mail`.
### Run against the clean fixture
+27 -9
View File
@@ -5,7 +5,12 @@
# Build: docker build -t cpanel-importer:dev .
# Run: see README.md for the docker run invocation the WHP panel uses.
FROM almalinux:10-minimal
# Base image. CI passes BASE_IMAGE=almalinux@sha256:<digest>, resolved
# DIRECTLY from Docker Hub in the same run (.gitea/scripts/resolve-base-image.sh),
# so the pkg-cache registry mirror can only serve content-addressed blobs.
# Local builds use the tag. Canonical docs: cloud-apache-container docs/pkg-cache.md.
ARG BASE_IMAGE=almalinux:10-minimal
FROM ${BASE_IMAGE}
LABEL org.opencontainers.image.title="cpanel-importer"
LABEL org.opencontainers.image.description="cPanel cpmove sanitization sandbox (ClamAV + SaneSecurity + WP content scan)"
@@ -28,9 +33,9 @@ ENV LANG=C.UTF-8 \
# the SaneSecurity rsync temp files at the end of the layer.
#
# Pinning strategy:
# - PHP 8.4: AlmaLinux 10 stock ships PHP 8.3 only; the spec asks for
# 8.4 specifically. We add Remi's modular repo and enable the
# `php:remi-8.4` stream. We DO NOT pin to a specific 8.4.X because
# - PHP 8.5: AlmaLinux 10 stock ships PHP 8.3 only; the spec asks for
# a current Remi stream (8.5; the 8.4 stream had a known exploit). We
# add Remi's modular repo and enable the `php:remi-8.5` stream. We DO NOT pin to a specific 8.5.X because
# Remi rolls security patches into the same minor and an exact pin
# would block updates.
# - clamav / clamav-update: track the AL10 EPEL stream. CI builds
@@ -43,18 +48,30 @@ ENV LANG=C.UTF-8 \
# clamav first, that lands at GID 999 — which then collides with the
# UID/GID we want for whp-import. We pre-create our user FIRST so
# virusgroup ends up at 998.
#
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
# (CT130 pkg-cache, https only). Unset = upstream repositories. epel-release
# and remi-release are GPG-verified against keys/pkg-cache/ either way
# (pkg-cache-repos.sh release-install). Repo files are restored at the end of
# the RUN, so no layer keeps the cache address. Same script as
# cloud-apache-container (scripts/pkg-cache-repos.sh, docs/pkg-cache.md there).
ARG PKG_CACHE=
COPY scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
COPY keys/pkg-cache/ /usr/local/share/pkg-cache/
RUN set -eux; \
/usr/local/sbin/pkg-cache-repos.sh rpm ; \
# microdnf is what almalinux:10-minimal ships with by default.
microdnf -y install --setopt=install_weak_deps=0 \
epel-release \
dnf \
shadow-utils \
; \
# Add Remi's repo for PHP 8.4 (AL10 stock has 8.3 only).
dnf -y --setopt=install_weak_deps=0 install \
https://rpms.remirepo.net/enterprise/remi-release-10.rpm ; \
# EPEL + Remi's repo for PHP 8.5 (AL10 stock has 8.3 only); release
# RPMs signature-checked against the committed keys, then the new repo
# files are pointed at the cache too (no-op without PKG_CACHE).
/usr/local/sbin/pkg-cache-repos.sh release-install epel remi ; \
/usr/local/sbin/pkg-cache-repos.sh rpm ; \
dnf -y --setopt=install_weak_deps=0 module reset php ; \
dnf -y --setopt=install_weak_deps=0 module enable php:remi-8.4 ; \
dnf -y --setopt=install_weak_deps=0 module enable php:remi-8.5 ; \
# Pre-create the worker BEFORE installing clamav so virusgroup
# doesn't claim our GID.
groupadd --system --gid ${WHP_GID} whp-import ; \
@@ -98,6 +115,7 @@ RUN set -eux; \
# an EXISTING host dir keep host ownership and are independent of
# this — the panel sets up its own dirs with mode 750 owner 999.)
chown whp-import:whp-import /host/quarantine /host/sanitized ; \
/usr/local/sbin/pkg-cache-repos.sh restore ; \
# Strip dnf cache.
dnf -y clean all ; \
rm -rf /var/cache/dnf /var/cache/yum /var/cache/ldconfig/* \
+11
View File
@@ -50,6 +50,7 @@ docker run \
--env IMPORT_USERNAME=${USERNAME} \
--env IMPORT_BACKUP_FILE=/host/backup/${BACKUP_NAME} \
--env CLAMAV_REFRESH=true \
--env IMPORT_INCLUDE_MAIL=true \
--memory=4g \
--memory-swap=4g \
--cpus=2 \
@@ -57,6 +58,15 @@ docker run \
repo.anhonesthost.net/cloud-hosting-platform/cpanel-importer:2026.05.NNN
```
`IMPORT_INCLUDE_MAIL=true` is optional: WHP passes it only when the import
copies mailboxes. Without it `cpmove-*/homedir/mail` is not extracted at all.
With it the mail tree is extracted (dangerous symlinks inside it are still
stripped) but **not** ClamAV-scanned: mail is copied verbatim into the
customer's own mailbox and verified message-by-message, so quarantining a
message would silently drop mail, and multi-GB maildirs would dominate the
run. `report.json` then carries `"mail_included": true`; images older than
this option never write the key, which is how WHP tells them apart.
Container exits with status `0` on success, non-zero on any failure
(missing/unreadable backup, dangerous symlink found, scanner error).
Even on failure, `/host/sanitized/<importid>/report.json` is written
@@ -94,6 +104,7 @@ run, success or failure.
"files_clean": 28432,
"files_cleaned": 0,
"files_quarantined": 39,
"mail_included": false,
"actions": [
{
"path": "cpmove-testuser/homedir/public_html/example.com/ALFA_DATA/index.php",
+32
View File
@@ -0,0 +1,32 @@
Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh,
docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing
here is added to the system trust store.
ca.crt "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18.
NAME-CONSTRAINED to IP 192.168.1.40 and DNS
pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local.
SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3
(private key lives only on CT130: /etc/pkgcache-ca/ca.key)
RPM-GPG-KEY-EPEL-10 Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158
fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and
mirror.math.princeton.edu (identical); fingerprint matches
https://fedoraproject.org/security/
RPM-GPG-KEY-remi2024 Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4
fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical
to the key shipped in remi-release-10.2-1.el10.remi. Not found on
keys.openpgp.org / keyserver.ubuntu.com - confirm out of band.
Remi rotates keys (remi2025, ...): when remi-release starts being
signed by a new key, `release-install remi` fails loudly - add the
new key here (and to pkg-cache-repos.sh).
RPM-GPG-KEY-EPEL-9 Fedora (epel9) FF8A D134 4597 106E CE81 3B91 8A38 72BF 3228 467C
fetched 2026-10-22 over HTTPS from dl.fedoraproject.org and
mirror.math.princeton.edu (identical); used by `release-install epel`
on EL9 images (cloud-memcached-container, cloud-haproxy-container).
RPM-GPG-KEY-nodesource Nodesource Operations <operations@nodesource.com>
242B 8138 31AF 0956 2B6C 46F7 6B88 DA4E 3AF2 8A14 (rsa4096, 2023-12-22)
fetched 2026-10-22 over HTTPS from
https://rpm.nodesource.com/gpgkey/ns-operations-public.key (the key
NodeSource's own setup_N.x scripts reference); verified by dnf
gpgcheck against the nodejs RPMs. Not on keys.openpgp.org /
keyserver.ubuntu.com - confirm out of band. Used by
`pkg-cache-repos.sh nodesource MAJOR` (cloud-node-container).
+29
View File
@@ -0,0 +1,29 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA
Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK
Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L
URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf
mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2
XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI
uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60
GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix
fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z
BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77
HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB
tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB
CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC
BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p
l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp
yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg
Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u
scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs
7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK
F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg
sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB
0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY
1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt
TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k
p9Dfd2pKuxK9Mys=
=mhQZ
-----END PGP PUBLIC KEY BLOCK-----
+29
View File
@@ -0,0 +1,29 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGE3mOsBEACsU+XwJWDJVkItBaugXhXIIkb9oe+7aadELuVo0kBmc3HXt/Yp
CJW9hHEiGZ6z2jwgPqyJjZhCvcAWvgzKcvqE+9i0NItV1rzfxrBe2BtUtZmVcuE6
2b+SPfxQ2Hr8llaawRjt8BCFX/ZzM4/1Qk+EzlfTcEcpkMf6wdO7kD6ulBk/tbsW
DHX2lNcxszTf+XP9HXHWJlA2xBfP+Dk4gl4DnO2Y1xR0OSywE/QtvEbN5cY94ieu
n7CBy29AleMhmbnx9pw3NyxcFIAsEZHJoU4ZW9ulAJ/ogttSyAWeacW7eJGW31/Z
39cS+I4KXJgeGRI20RmpqfH0tuT+X5Da59YpjYxkbhSK3HYBVnNPhoJFUc2j5iKy
XLgkapu1xRnEJhw05kr4LCbud0NTvfecqSqa+59kuVc+zWmfTnGTYc0PXZ6Oa3rK
44UOmE6eAT5zd/ToleDO0VesN+EO7CXfRsm7HWGpABF5wNK3vIEF2uRr2VJMvgqS
9eNwhJyOzoca4xFSwCkc6dACGGkV+CqhufdFBhmcAsUotSxe3zmrBjqA0B/nxIvH
DVgOAMnVCe+Lmv8T0mFgqZSJdIUdKjnOLu/GRFhjDKIak4jeMBMTYpVnU+HhMHLq
uDiZkNEvEEGhBQmZuI8J55F/a6UURnxUwT3piyi3Pmr2IFD7ahBxPzOBCQARAQAB
tCdGZWRvcmEgKGVwZWw5KSA8ZXBlbEBmZWRvcmFwcm9qZWN0Lm9yZz6JAk4EEwEI
ADgWIQT/itE0RZcQbs6BO5GKOHK/MihGfAUCYTeY6wIbDwULCQgHAgYVCgkICwIE
FgIDAQIeAQIXgAAKCRCKOHK/MihGfFX/EACBPWv20+ttYu1A5WvtHJPzwbj0U4yF
3zTQpBglQ2UfkRpYdipTlT3Ih6j5h2VmgRPtINCc/ZE28adrWpBoeFIS2YAKOCLC
nZYtHl2nCoLq1U7FSttUGsZ/t8uGCBgnugTfnIYcmlP1jKKA6RJAclK89evDQX5n
R9ZD+Cq3CBMlttvSTCht0qQVlwycedH8iWyYgP/mF0W35BIn7NuuZwWhgR00n/VG
4nbKPOzTWbsP45awcmivdrS74P6mL84WfkghipdmcoyVb1B8ZP4Y/Ke0RXOnLhNe
CfrXXvuW+Pvg2RTfwRDtehGQPAgXbmLmz2ZkV69RGIr54HJv84NDbqZovRTMr7gL
9k3ciCzXCiYQgM8yAyGHV0KEhFSQ1HV7gMnt9UmxbxBE2pGU7vu3CwjYga5DpwU7
w5wu1TmM5KgZtZvuWOTDnqDLf0cKoIbW8FeeCOn24elcj32bnQDuF9DPey1mqcvT
/yEo/Ushyz6CVYxN8DGgcy2M9JOsnmjDx02h6qgWGWDuKgb9jZrvRedpAQCeemEd
fhEs6ihqVxRFl16HxC4EVijybhAL76SsM2nbtIqW1apBQJQpXWtQwwdvgTVpdEtE
r4ArVJYX5LrswnWEQMOelugUG6S3ZjMfcyOa/O0364iY73vyVgaYK+2XtT2usMux
VL469Kj5m13T6w==
=Mjs/
-----END PGP PUBLIC KEY BLOCK-----
+52
View File
@@ -0,0 +1,52 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGWFsqMBEAC4YlSuNpHjOiQVTC8StolVx3J8E9SC6G4wmVWbytMD1Zivfanj
3LKDqdy7vpbViXA9whpyGQgQYdOlPKaNHWw4GSeGTnP1FbvaK1NJ9MnAzUysh49R
cqoktYeIQbqgOgZHw5/gVH5bYObf1KJQ2LY/je9WTlW3QMRe1TaW0F0AZnJMBIOG
JllE33QIpxDnOQHnvdUGq+HscUOTrW4Q3AdzwsQxU4GX82d9qFFr8HjAmUjlUKyh
PgvMpb8VDvR7NE6et+wWh6BMYILo3VJFP7/gTXGiy3WWs5GW/PNXIXw5/f2w1B+e
Dh5tlfGG4q/4P8mbv7C6RPHoF2Lq+W69HLqSq/l42WUL41tHlFTsluCIKpeeMxAu
olxx3Z5momHMMyh7rbZtQDqFMTAqxNjT1NinKkWyj7KP5WE9hVosyd3UMR4UYDMm
Zb6LI4N7Hht887+CY7grxTJ9cGQhMFfvLyvlf/FWQY4S6G7sEaUV71N9StvXBNG/
iNoIMzhwjBmrmvT/UGPkLSi0SIBOxeGw2rwngnYikIJxuTQ4V40lm8o6eMFca7rW
BQvAEncV8pvi/3FqrUBZ/U53s7NO24hBDoaQPzFbVCw0GHHf9GD7KoQb5c9ONoCZ
Qu2YWo7ajPLzl/h2pJ2Kn29Ywev9MpuuPTGvifiKilcBVLYP3ZJg4ZfRBwARAQAB
tDFOb2Rlc291cmNlIE9wZXJhdGlvbnMgPG9wZXJhdGlvbnNAbm9kZXNvdXJjZS5j
b20+iQJOBBMBCgA4FiEEJCuBODGvCVYrbEb3a4jaTjryihQFAmWFsqMCGwMFCwkI
BwIGFQoJCAsCBBYCAwECHgECF4AACgkQa4jaTjryihTD+hAAtTU155ke7xmnT0ph
odbGjm3mOGdkdaDH+xmWtPtufiPncgHMCYh4aIDRloog8rFE+pP3qfwJmUKxthwD
drrLAzfUwOrGxPnYH3MWwg66RVVoiiU+Hh42gJTez+P/K0VMQZXH0XTZD96TiwT/
S0+WvQPuJ9pda2L4cjV+YP2+m0ERUG9QPj0Shgw+/PWCGWKD6J9DRToiwfwCTOvc
DNolF1EcRlfPMTsqI2Bdc+TQywqU1Py1MUDSAPXxdK6nW2oM7eC5l3XXefcQ6JOS
AuNdEDIsaf+H76j6/wSGsuIcJjUo9wmKjEV1iJYwT6jLbjaVQxXgEPHBSb+kzanx
FWyrYP6TZpeY2gT4bBMsOKd7ay1I3/nIHwe6x8Yzkqzz+7YVhNpMbybH72NTxDWQ
UmiboJs9r7j1lkHlBjxU9RLF5laYSeCIXD62KQVoP5ao1LSPKsCXFqcdpUgLYtRv
CXxdriiHp4m9eGzbS7RW1TLm6tMOKbI4LzOtMm4DoZzGD2FTfsUtn6ALx9FSgvl/
VMlzqZes+rclcukACtutvtB8e0yRhuzPUN56ctLEZfI1XzQFoN8Vdj6HWtiXWRiJ
EohuufE2cEebmX90NRsbh4KAFXNEQROlMeMf7ufuSsk7QXLTutxmWSQvWwyS+P5/
ZqCapIDlRCJFZX4N9a1IXo0OgCC5Ag0EZYWyowEQAKz7sA6WBKFVcmzcyr6y9TI5
RemnzotRUdqXj8tD70xvV1nBEf+BfTw3hsiV019ZboPBae+cRaD4plQbNb0du/+Q
UFq4ja055ZEld01RqE49L562pWvA5cwSmjs9+ZMI1ja0HbTb2K4razpzX3be+Zul
TbH20X5FHTz+Jyfmw+G3Y9okw3OIzuYo16Lv/Qs8VoBmj01PItzq8dX5wJiIlVxU
8wKHKy1kiKGCgyPnvqcXCGen0EoUS9wsWK1QHfryscYGVF9y1TBQvJa9HIpWPrIZ
slSiVS4tQhGHfLDVlaDsYFrH9SIUkvW7peSEiPE1TKQjvbMS3CcUZRaMcGfgyDPe
gXwnNwZbJ2/cisXbdAJaGT49V1Y+Sc1oKSGSG4FaGUNhS3p3cB7qlX2zS/sR2ael
90fXR6nhjvhMAS98i7663pCUv9q0Z/NXNZwdXjbSF4W8gq3PVt7hCnJrngsGSkGu
Qz0dc227/sAjh9CxYm8kQqUuQtiPVfKwkcUUT+O7jr2/6eAXZbTYVZbvaLEGMrKA
bdWKpZMMOKAI45f+ZAcEmJG4D68N0jzVrLCXMuuLq/yqqyEmxDjSOH0qVutl8jxg
vfxPxKYX8nnAzJZJHGHmV/ihjfClSdYePKExHuFVVhvc4BxE1XfPUeCZG4JpB9/B
gNCiDPsg4JIw6GO5LS2xABEBAAGJAjYEGAEKACAWIQQkK4E4Ma8JVitsRvdriNpO
OvKKFAUCZYWyowIbDAAKCRBriNpOOvKKFOTKD/40WN6ZRfP6hVLAX26DdyN9Bqzg
D+0hJilmHL1YFeLt9U/YPhOZJxPaDGrMfldC0FSXKKr7H5C7FQErl936108wGohz
53KGbNid0crNeaWrQuWk3QjaMF62Yc+NVB8b5LoEk0TEt9KPO6kveYDtW6Ygkw9i
5aaS/AC+8GChOH6lWhLHcO91T7+Hi5qRez7qyme/AlHUTV9/jE6JgFJ7thCCAqBQ
AfX3mGFlTwSTMHDeYq81MMl9UhYgoFJtx8PuqmSV7Eb29ccJofz3J+b5OJvpmLSZ
Hu22Csa8k2V/1ojeL0DjQHD06V75F7BerxLR/rbJ64GkIiOdm4ntXC4a335gjQfM
5tI6LtGsIpr+Le4C068eDj7YpzS0ql3g6o4A1TbtapLccgEj4iw95H4KKh0Ro5r0
0s6LoSLu9uva3LiLVn6xXvpw3iv8FnhTr5PEiPnorXvxcCnUrB3Rkk6Wh1SKjpxa
u6KiFzH0CXR1AgUdWSC0rAmvEdFjXgaoQd+Z2eJFHEAsg91URd9NtdeDsyMwdbHh
iBZPGX5ddPjJA7nwKbkibCv7xiodd9ph8NmGLqD/2SDU1rqvPuCDMqLuxc89Gvhs
3CeLTSJK9Uzb3oNdZLxL8CECip1ZkowVfTL+jY23phS9+Z0Db/yZqot3ZZJAAG8g
oSVkzT/Mwp9zFVQu5w==
=drMP
-----END PGP PUBLIC KEY BLOCK-----
+52
View File
@@ -0,0 +1,52 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W
Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe
UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa
qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN
ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB
jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO
/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL
yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ
w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN
0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l
ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB
tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly
ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd
/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX
gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx
Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W
4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/
/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz
/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc
3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof
ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS
dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy
sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n
i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd
DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P
7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI
T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ
OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3
cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W
RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x
5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m
8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD
pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r
N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm
Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG
Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA
AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9
OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI
wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA
qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c
WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV
u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI
qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S
4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4
MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II
Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl
bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s
r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1
=meaU
-----END PGP PUBLIC KEY BLOCK-----
+15
View File
@@ -0,0 +1,15 @@
-----BEGIN CERTIFICATE-----
MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw
TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV
BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN
MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg
KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG
ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV
OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD
VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/
v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw
a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu
Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg
cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM
YAIl
-----END CERTIFICATE-----
+30 -2
View File
@@ -8,6 +8,11 @@
# IMPORT_BACKUP_FILE absolute path inside the container, typically
# /host/backup/cpmove-<user>.tar.gz
# CLAMAV_REFRESH "true" to run freshclam at start (default: true)
# IMPORT_INCLUDE_MAIL "true" to extract cpmove-*/homedir/mail for WHP's
# mailbox import (default: false = mail tree skipped).
# The mail tree is never ClamAV-scanned. report.json
# records "mail_included" so the panel can tell this
# image (which honours the opt-in) from an older one.
#
# Flow (spec §0):
# 1. validate env
@@ -71,8 +76,20 @@ log "cpanel-importer starting (container UID=$(id -u) GID=$(id -g))"
: "${IMPORT_USERNAME:?IMPORT_USERNAME env var is required}"
: "${IMPORT_BACKUP_FILE:?IMPORT_BACKUP_FILE env var is required}"
CLAMAV_REFRESH="${CLAMAV_REFRESH:-true}"
# Exactly "true" opts in; anything else is the default (no mail tree).
if [[ "${IMPORT_INCLUDE_MAIL:-false}" == "true" ]]; then
IMPORT_INCLUDE_MAIL=true
else
IMPORT_INCLUDE_MAIL=false
fi
export IMPORT_INCLUDE_MAIL
# The mail-tree verify's readability check (os.access) is meaningless for
# root; the image runs as uid 999 and the opt-in path insists on it.
if [[ "$IMPORT_INCLUDE_MAIL" == "true" && "$(id -u)" == "0" ]]; then
die "IMPORT_INCLUDE_MAIL=true must not run as root (container --user 999:999 expected)"
fi
log "import_id=$IMPORT_ID username=$IMPORT_USERNAME backup=$IMPORT_BACKUP_FILE"
log "import_id=$IMPORT_ID username=$IMPORT_USERNAME backup=$IMPORT_BACKUP_FILE include_mail=$IMPORT_INCLUDE_MAIL"
if [[ ! -f "$IMPORT_BACKUP_FILE" ]]; then
die "backup file does not exist or is not a regular file: $IMPORT_BACKUP_FILE"
@@ -140,11 +157,17 @@ fi
STAGE="scan_files"
log "stage: scan_files"
SCAN_FILES_EXTRA=()
if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then
# The extracted mail tree is not scanned (see scan-files.php --skip-mail).
SCAN_FILES_EXTRA+=(--skip-mail)
fi
php /scripts/scan-files.php \
--extract "$EXTRACT_DIR" \
--quarantine "$QUARANTINE_DIR" \
--report /tmp/scan-files-report.json \
--import-id "$IMPORT_ID" \
"${SCAN_FILES_EXTRA[@]}" \
|| die "scan-files.php failed; see stderr above"
# --- DB engine swap + WP content scan -------------------------------------
@@ -192,6 +215,7 @@ $filesPath = $argv[3];
$dbsPath = $argv[4];
$strippedPath = $argv[5];
$outPath = $argv[6];
$mailIncluded = $argv[7] === "true";
$files = is_file($filesPath) ? json_decode(file_get_contents($filesPath), true) : null;
$dbs = is_file($dbsPath) ? json_decode(file_get_contents($dbsPath), true) : null;
@@ -252,6 +276,10 @@ $report = [
"files_clean" => $filesClean,
"files_cleaned" => $filesCleaned,
"files_quarantined" => $filesQuarantined,
// true only when IMPORT_INCLUDE_MAIL=true AND this image extracted the
// mail tree. Older images never write the key; the panel treats a
// missing/false value as "mail was not extracted".
"mail_included" => $mailIncluded,
"actions" => $actions,
"databases" => $databases,
"summary_for_panel" => [
@@ -263,7 +291,7 @@ $report = [
file_put_contents($outPath, json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n");
fprintf(STDERR, "report written: %s\n", $outPath);
' "$IMPORT_ID" "$DURATION" /tmp/scan-files-report.json /tmp/scan-dbs-report.json "$STRIPPED_SYMLINKS_FILE" "$SANITIZED_DIR/report.json" \
' "$IMPORT_ID" "$DURATION" /tmp/scan-files-report.json /tmp/scan-dbs-report.json "$STRIPPED_SYMLINKS_FILE" "$SANITIZED_DIR/report.json" "$IMPORT_INCLUDE_MAIL" \
|| die "report merge failed"
log "done — exited cleanly after ${DURATION}s"
+94 -17
View File
@@ -4,6 +4,10 @@
#
# Usage: extract.sh <tarball> <dest> <username> [<actions_out>]
#
# Env: IMPORT_INCLUDE_MAIL "true" = extract cpmove-*/homedir/mail too
# (WHP's mailbox import needs the Maildirs).
# Anything else (default) = skip the mail tree.
#
# Calls scripts/lib/scan-symlinks.php first, then untars the cpmove with
# every DANGEROUS-classified symlink entry stripped via tar --exclude.
# The stripped-symlinks list is written as JSON to <actions_out> (default
@@ -21,23 +25,45 @@ DEST="${2:?usage: extract.sh <tarball> <dest> <username> [<actions_out>]}"
USERNAME="${3:?usage: extract.sh <tarball> <dest> <username> [<actions_out>]}"
ACTIONS_OUT="${4:-${DEST}/.cpanel-importer-stripped-symlinks.json}"
IMPORT_INCLUDE_MAIL="${IMPORT_INCLUDE_MAIL:-false}"
# Resolve sibling scripts relative to this file (/scripts in the image) so
# tests/run-tests.sh can run the same script from a checkout.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ts() { date -u +'%Y-%m-%dT%H:%M:%SZ'; }
log() { printf '[%s] extract: %s\n' "$(ts)" "$*"; }
[[ -f "$TARBALL" ]] || { log "tarball not found: $TARBALL"; exit 2; }
mkdir -p "$DEST"
# Any failure after this point removes the (partial) extract before exiting:
# it is the container's own scratch dir inside the bind-mounted output, and a
# half-extracted tree, or one the link check refused, must not be left
# behind for anything to read. Directories the archive made unreadable are
# opened up first (find -P never follows links; -type d skips them) so rm
# can descend.
fail_extract() {
local code="$1"; shift
log "$*"
cd /
if [[ -n "$DEST" && "$DEST" != "/" && -d "$DEST" ]]; then
find -P "$DEST" -type d ! -perm -u=rwx -exec chmod u+rwx {} \; 2>/dev/null || true
rm -rf -- "$DEST" || log "WARN: could not remove the failed extract $DEST"
log "removed the failed extract $DEST"
fi
exit "$code"
}
# --- pre-extract symlink scan ---------------------------------------------
log "scanning tarball for dangerous symlinks (cpmove vector check)"
SYMLINK_REPORT=$(mktemp -p /tmp scan-symlinks.XXXXXX.json)
if ! php /scripts/lib/scan-symlinks.php \
if ! php "$SCRIPT_DIR/lib/scan-symlinks.php" \
--tarball "$TARBALL" \
--username "$USERNAME" \
--report "$SYMLINK_REPORT"; then
log "scan-symlinks.php exited with usage/IO error; aborting (this is not a sanitize-able state)"
cat "$SYMLINK_REPORT" >&2 || true
exit 3
fail_extract 3 "scan-symlinks.php exited with usage/IO error; aborting (this is not a sanitize-able state)"
fi
# --- compute exclude list from dangerous findings -------------------------
@@ -86,19 +112,35 @@ if [[ "$DANGEROUS_COUNT" -gt 0 ]]; then
done < "$EXCLUDES_FILE"
fi
# Also skip the cpanel user's mail spool entirely. WHP doesn't import
# email (mail-import is a panel-side roadmap item, not a sandbox-mode
# step) so extracting + scanning the mailbox tree wastes time and disk:
# on real customer accounts the mail dir often dwarfs everything else
# (10+ GB of historical mbox/maildir). The exclude pattern uses tar's
# fnmatch globs and covers both maildir + spool layouts at any depth
# under homedir/. Logged as an info line so it shows up in the panel
# import log alongside the symlink strips.
{
echo "cpmove-*/homedir/mail"
echo "cpmove-*/homedir/mail/*"
} >> "$EXCLUDES_FILE"
log "skipping mail tree (cpmove-*/homedir/mail) — WHP does not import cPanel mail"
# Mail tree (cpmove-*/homedir/mail). Skipped by default: a plain site
# import doesn't need it, and on real customer accounts the mail dir often
# dwarfs everything else (10+ GB of historical maildir). The exclude
# pattern uses tar's fnmatch globs and covers the whole tree at any depth.
#
# IMPORT_INCLUDE_MAIL=true (WHP passes it only when the import copies
# mailboxes) extracts it. The link policy and verify below apply to
# <any top dir>/homedir/mail, not only cpmove-*: WHP accepts any top-level
# dir name. (The default-mode exclude keeps its historical cpmove-* scope.) The DANGEROUS-symlink strips computed above apply
# to the mail tree exactly as to the rest of the archive: scan-symlinks.php
# lists every entry in the tarball, mail included.
if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then
log "including mail tree (cpmove-*/homedir/mail) — IMPORT_INCLUDE_MAIL=true"
# Link policy for the mail tree (scripts/lib/mail-tree-links.py): every
# symlink in it, and every hard link crossing its boundary, is excluded
# here; Dovecot's in-tree hard-link copies are kept. The archive is
# refused outright if any member name could be read differently by the
# classifier and by tar (control characters, non-canonical paths).
# scan-symlinks.php alone only strips DANGEROUS absolute targets.
MAIL_LINKS=$(python3 -I "$SCRIPT_DIR/lib/mail-tree-links.py" scan "$TARBALL" "$EXCLUDES_FILE" "$ACTIONS_OUT") \
|| fail_extract 3 "mail-tree link scan refused the archive (see above); aborting — the same backup may import with 'Import without email'"
log "stripped $MAIL_LINKS symlink / boundary-crossing hard-link entr(y/ies) in the mail tree"
else
{
echo "cpmove-*/homedir/mail"
echo "cpmove-*/homedir/mail/*"
} >> "$EXCLUDES_FILE"
log "skipping mail tree (cpmove-*/homedir/mail) — IMPORT_INCLUDE_MAIL not set"
fi
# --- extract --------------------------------------------------------------
@@ -125,9 +167,44 @@ log "extracting with hardened tar flags into $DEST"
# match is effectively a literal-path skip.
# --absolute-names is NOT used — leading / in a member name is stripped.
cd "$DEST"
# tar's exit code is captured, not fatal on the spot: the mail-tree verify
# below must run on whatever tar did write, and a failed tar must still get
# its partial tree removed.
TAR_RC=0
tar --no-same-owner --no-same-permissions --no-overwrite-dir \
--exclude-from="$EXCLUDES_FILE" \
$TAR_FLAGS "$TARBALL"
$TAR_FLAGS "$TARBALL" || TAR_RC=$?
# N2-1 authority for the WHOLE tree: the classifier above resolves relative
# targets lexically, so a chain through another link (a -> b/.., b -> x/y/z)
# can pass it. strip-escaping-links.py asks realpath() where every extracted
# symlink really lands and removes (records) any that leave $DEST, point at
# $DEST itself, dangle or loop. A walk error fails the extract closed.
# Only the TRUSTED username (<username>, the WHP account WHP passes) is
# exempted, and only re-rooted inside the extract's homedir; names read from
# the archive (top-dir names) are never trusted.
STRIP_RC=0
ESCAPING=$(python3 -I "$SCRIPT_DIR/lib/strip-escaping-links.py" "$DEST" "$ACTIONS_OUT" "$USERNAME") || STRIP_RC=$?
if [[ $STRIP_RC -eq 0 && "${ESCAPING:-0}" -gt 0 ]]; then
log "stripped $ESCAPING symlink(s) that resolve outside the extract, dangle or loop (post-extract check)"
fi
VERIFY_RC=0
if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then
# The authority: whatever the classifier above decided, no symlink, no
# hard link leaving the mail tree, and no uninspectable directory may
# exist in the extract. Always runs, even after a failed tar.
python3 -I "$SCRIPT_DIR/lib/mail-tree-links.py" verify "$DEST" || VERIFY_RC=$?
fi
if [[ $TAR_RC -ne 0 ]]; then
fail_extract "$TAR_RC" "tar failed (rc=$TAR_RC)$( [[ $VERIFY_RC -ne 0 ]] && echo "; mail-tree verify also failed (rc=$VERIFY_RC)" ); aborting"
fi
if [[ $STRIP_RC -ne 0 ]]; then
fail_extract 4 "post-extract symlink check failed (rc=$STRIP_RC); aborting"
fi
if [[ $VERIFY_RC -ne 0 ]]; then
fail_extract 4 "extracted mail tree failed the link/readability check (see above); aborting — the same backup may import with 'Import without email'"
fi
log "extracted OK ($(find "$DEST" -type f | wc -l) files; $DANGEROUS_COUNT symlinks stripped)"
exit 0
+215
View File
@@ -0,0 +1,215 @@
#!/usr/bin/env python3
"""
mail-tree-links.py — link policy for cpmove-*/homedir/mail when the mail
tree is extracted (IMPORT_INCLUDE_MAIL=true). Called by extract.sh.
A Maildir has no legitimate symlinks. Hard links are different: Dovecot
copies a message between folders as a hard link (maildir_copy_with_hardlinks,
default yes), so a link whose name AND target are both inside the same
account's mail tree is real mail and is kept. Every other link touching a
mail tree is stripped (pre-extract) or fails the extract (post-extract).
scan <tarball> <excludes_file> <actions_json>
Pre-extract classifier (an optimisation; `verify` is the authority).
Reads the archive's own member list with tarfile (exact names and
types, no `tar -tv` text parsing) and REFUSES the archive (exit 3)
when any member name, or hard-link target, could be read differently
by this classifier and by tar:
- a control character (\\x00-\\x1f, \\x7f) in any name or link
target (a newline would split one --exclude-from pattern in two);
- a non-canonical name: absolute, a '..' component, or
posixpath.normpath() differing from the name after stripping one
leading './' (e.g. 'x/./homedir', 'x/homedir//mail').
Symlink TARGETS are not canonical-checked: cPanel archives carry
ordinary relative links (homedir/access-logs, about -> ../public_html)
outside the mail tree, and every symlink inside it is stripped anyway.
Then appends a glob-escaped --exclude-from line for every symlink under
a mail tree and every hard link that crosses a mail-tree boundary, and
records them as 'stripped_mail_link' actions. Prints the count.
verify <dest>
Post-extract authority. lstat-walks every cpmove-*/homedir/mail under
<dest> (never following links) and exits 4 when it finds a symlink, or
a regular file whose st_nlink is larger than the number of its names
inside that same mail tree (some other name exists outside it), or any
directory it cannot read+search (an uninspectable subtree fails).
scan also cross-checks its member count against `tar -tf` and refuses on
any difference or stream error, so nothing GNU tar extracts goes
unclassified (verify remains the authority either way).
"""
import json
import os
import posixpath
import re
import stat
import subprocess
import sys
import tarfile
# ANY top-level directory: WHP's CpanelArchivePaths::selectTopDir() takes the
# first non-hidden dir of any name (cpmove-<u>, cPanel's backup-<date>_<u>,
# acct, ...). Covering every top dir is a superset of whichever it picks.
MAIL = re.compile(r'^([^/]+)/homedir/mail(/|$)')
CTRL = re.compile(r'[\x00-\x1f\x7f]')
def refuse(code, msg):
sys.stderr.write(f'mail-tree-links: REFUSING archive: {msg}\n')
sys.exit(code)
def canonical(raw):
"""The name tar will use, or None when it is not canonical."""
n = raw[2:] if raw.startswith('./') else raw
if n == '' or n.startswith('/') or '..' in n.split('/'):
return None
return n if posixpath.normpath(n) == n else None
def owner(n):
m = MAIL.match(n)
return m.group(1) if m else None
def scan(tar_path, excl_path, actions_path):
try:
with open(actions_path) as ah:
doc = json.load(ah)
except Exception:
doc = {'actions': [], 'count': 0}
already = {a.get('path') for a in doc.get('actions', [])}
found = []
seen = 0
try:
tf = tarfile.open(tar_path, 'r|*')
except Exception as e:
refuse(3, f'archive unreadable by the classifier: {e}')
try:
with tf:
for m in tf:
seen += 1
for label, value in (('name', m.name), ('link target', m.linkname)):
if CTRL.search(value or ''):
refuse(3, f'member {label} contains a control character: {value!r}')
name = canonical(m.name)
if name is None:
refuse(3, f'non-canonical member name: {m.name!r}')
if m.issym():
if owner(name):
found.append((m.name, 'symlink', m.linkname))
elif m.islnk():
target = canonical(m.linkname)
if target is None:
refuse(3, f'non-canonical hard-link target: {m.name!r} -> {m.linkname!r}')
a, b = owner(name), owner(target)
if (a or b) and a != b:
found.append((m.name, 'hardlink', m.linkname))
except Exception as e: # TarError, EOFError, zlib.error ... (SystemExit passes through)
refuse(3, f'archive stream error after {seen} member(s): {e}')
# The classifier must have seen every member GNU tar will extract: a
# stream that stops early (zero block, truncation, a header tarfile
# skips) would leave members unclassified. Count GNU tar's own listing
# (escape quoting = one line per member, even with a newline in a name).
try:
lst = subprocess.run(['tar', '--quoting-style=escape', '-tf', tar_path],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
except OSError as e:
refuse(3, f'cannot list the archive with tar: {e}')
if lst.returncode != 0:
refuse(3, f'tar -tf failed (rc={lst.returncode}): {lst.stderr.decode(errors="replace").strip()[:300]}')
tar_count = lst.stdout.count(b'\n')
if tar_count != seen:
refuse(3, f'member count differs: classifier saw {seen}, tar lists {tar_count}')
found = [f for f in found if f[0] not in already]
with open(excl_path, 'a') as eh:
for name, _, _ in found:
eh.write(re.sub(r'([*?\[\\])', r'\\\1', name) + '\n')
doc['actions'] = doc.get('actions', []) + [
{'action': 'stripped_mail_link', 'path': name, 'target': target,
'reason': f'{kind} inside the mail tree (Maildirs have no legitimate links outside it)'}
for name, kind, target in found
]
doc['count'] = len(doc['actions'])
with open(actions_path, 'w') as ah:
json.dump(doc, ah, indent=2)
for name, kind, target in found:
sys.stderr.write(f' STRIP {kind}: {name} -> {target}\n')
print(len(found))
def verify(dest):
bad = []
def walk_error(err):
bad.append(f'cannot inspect: {os.path.relpath(getattr(err, "filename", "") or dest, dest)} ({err.strerror})')
try:
tops = sorted(os.listdir(dest))
except OSError as e:
refuse(4, f'cannot list the extract: {e}')
for top in tops:
try:
tst = os.lstat(os.path.join(dest, top))
except OSError as err:
walk_error(err)
continue
if stat.S_ISLNK(tst.st_mode):
# Never walk through it (WHP refuses a top-level symlink anyway).
bad.append(f'top-level symlink: {top}')
continue
if not stat.S_ISDIR(tst.st_mode):
continue
mail = os.path.join(dest, top, 'homedir', 'mail')
try:
hst = os.lstat(os.path.join(dest, top, 'homedir'))
if stat.S_ISLNK(hst.st_mode):
bad.append(f'symlink: {top}/homedir')
continue
st = os.lstat(mail)
except FileNotFoundError:
continue
except OSError as err:
walk_error(err)
continue
if stat.S_ISLNK(st.st_mode):
bad.append(f'symlink: {top}/homedir/mail')
continue
if stat.S_ISDIR(st.st_mode) and not os.access(mail, os.R_OK | os.X_OK):
bad.append(f'directory not readable+searchable: {top}/homedir/mail')
continue
names = {} # (dev, ino) -> [count of names in this tree, st_nlink, a path]
# Fail closed: os.walk silently skips what it can't read unless told.
for root, dirs, files in os.walk(mail, followlinks=False, onerror=walk_error):
for e in dirs + files:
p = os.path.join(root, e)
try:
s = os.lstat(p)
except OSError as err:
walk_error(err)
continue
if stat.S_ISLNK(s.st_mode):
bad.append(f'symlink: {os.path.relpath(p, dest)}')
elif stat.S_ISDIR(s.st_mode) and not os.access(p, os.R_OK | os.X_OK):
bad.append(f'directory not readable+searchable: {os.path.relpath(p, dest)}')
elif stat.S_ISREG(s.st_mode) and s.st_nlink > 1:
rec = names.setdefault((s.st_dev, s.st_ino), [0, s.st_nlink, p])
rec[0] += 1
for count, nlink, p in names.values():
if nlink > count:
bad.append(f'hard link leaving the mail tree: {os.path.relpath(p, dest)} '
f'(st_nlink {nlink}, {count} name(s) inside the mail tree)')
for b in bad:
sys.stderr.write(f'mail-tree-links: {b}\n')
if bad:
refuse(4, f'{len(bad)} problem(s) in the extracted mail tree (links or uninspectable directories)')
if __name__ == '__main__':
if len(sys.argv) == 5 and sys.argv[1] == 'scan':
scan(sys.argv[2], sys.argv[3], sys.argv[4])
elif len(sys.argv) == 3 and sys.argv[1] == 'verify':
verify(sys.argv[2])
else:
sys.stderr.write('usage: mail-tree-links.py scan <tarball> <excludes> <actions_json> | verify <dest>\n')
sys.exit(2)
+78 -11
View File
@@ -90,8 +90,40 @@ $dangerousPrefixes = [
'/dev',
];
/**
* Lexical resolution of a relative link from its own directory in the
* archive: a reason when it reaches/climbs above the archive root, when a
* link under <top>/homedir/ resolves outside that homedir, or when any
* component on the way is itself a symlink entry ($linkSet: normalized
* archive path => true); NULL otherwise. extract.sh's post-extract
* strip-escaping-links.py (realpath of every link) is the authority.
*/
function relative_symlink_escape(string $archivePath, string $target, array $linkSet = []): ?string {
$ap = preg_replace('#^(\./)+#', '', $archivePath);
$dir = array_values(array_filter(explode('/', (string) $ap), fn($x) => $x !== '' && $x !== '.'));
array_pop($dir);
$home = (count($dir) >= 2 && $dir[1] === 'homedir') ? [$dir[0], 'homedir'] : null;
$stack = $dir;
foreach (explode('/', $target) as $seg) {
if ($seg === '' || $seg === '.') continue;
if ($seg === '..') {
if (count($stack) <= 1) return 'escapes the archive root';
array_pop($stack);
continue;
}
$stack[] = $seg;
// No chaining: a component that is itself a symlink entry in this
// archive would be expanded by the kernel, which this lexical walk
// cannot model (`a -> b/..` with `b -> x/y/z`).
if (isset($linkSet[implode('/', $stack)])) return 'passes through another symlink in the archive';
}
if ($home !== null && (count($stack) < 2 || $stack[0] !== $home[0] || $stack[1] !== 'homedir')) {
return 'escapes the account homedir';
}
return null;
}
$findings = [];
$cpanelUsername = null;
$cmd = 'tar -tvf ' . escapeshellarg($tarPath) . ' 2>/dev/null';
$fh = @popen($cmd, 'r');
@@ -100,6 +132,9 @@ if (!$fh) {
exit(2);
}
// Pass 1: every symlink entry (the chain rule needs the whole set).
$links = [];
$linkSet = [];
while (($line = fgets($fh)) !== false) {
if ($line === '' || $line[0] !== 'l') continue;
$arrow = strpos($line, ' -> ');
@@ -108,24 +143,44 @@ while (($line = fgets($fh)) !== false) {
$right = rtrim(substr($line, $arrow + 4), "\r\n");
$parts = preg_split('/\s+/', $left, 6);
if (count($parts) < 6) continue;
$archivePath = $parts[5];
$target = $right;
$links[] = [$parts[5], $right];
$linkSet[trim((string) preg_replace('#^(\./)+#', '', $parts[5]), '/')] = true;
}
pclose($fh);
if ($target === '' || $target[0] !== '/') continue;
// Pass 2: classify.
foreach ($links as [$archivePath, $target]) {
if ($cpanelUsername === null) {
if (preg_match('#^cpmove-([^/]+)/#', $archivePath, $m)) {
$cpanelUsername = $m[1];
// N2-1: a RELATIVE target that climbs out of the archive (or out of
// the account's homedir/) is DANGEROUS (stripped), not safe: GNU tar
// extracts `cpmove-u/sds -> ../../../../root/.my.cnf` as-is. Same rule
// as WHP's CpanelBackupImporter::relativeSymlinkEscape().
if ($target !== '' && $target[0] !== '/') {
$escape = relative_symlink_escape($archivePath, $target, $linkSet);
if ($escape !== null) {
$findings[] = [
'type' => 'DANGEROUS',
'archive_path' => $archivePath,
'target' => $target,
'reason' => "relative target $escape",
];
}
continue;
}
if ($target === '') continue;
// (1) user-internal — accept symlinks pointing into the customer's
// own /home/<user>/ tree. The panel rewrites these on extract.
$userInternal = false;
$usernames = [];
if ($cpanelUsername !== null && $cpanelUsername !== '') $usernames[] = $cpanelUsername;
if ($username !== '') $usernames[] = $username;
foreach ($usernames as $u) {
// N2-1 review: only the TRUSTED --username (the WHP account WHP passes);
// a name read from the archive's top dir (cpmove-<name>) is never trusted.
if (preg_match('/\A[a-z_][a-z0-9_-]{0,31}\z/', $username)) $usernames[] = $username;
// N2-1 review: only a NORMALIZED target can be user-internal
// (`/home/<u>/../../etc/shadow` is not).
$normalized = !preg_match('#(^|/)\.{1,2}(/|$)|//#', $target);
foreach ($normalized ? $usernames : [] as $u) {
$prefix = '/home/' . $u . '/';
if (strpos($target, $prefix) === 0 || $target === rtrim($prefix, '/')) {
$userInternal = true;
@@ -138,6 +193,19 @@ while (($line = fgets($fh)) !== false) {
}
if ($userInternal) continue;
// N2-1 review: a non-normalized absolute target ('.', '..', '//') is
// never compared by prefix (`/home/<u>/../../etc` would dodge /etc):
// DANGEROUS outright.
if (!$normalized) {
$findings[] = [
'type' => 'DANGEROUS',
'archive_path' => $archivePath,
'target' => $target,
'reason' => "absolute target is not normalized ('.', '..' or empty component)",
];
continue;
}
// (2) exact root.
$type = null;
$reason = '';
@@ -170,7 +238,6 @@ while (($line = fgets($fh)) !== false) {
'reason' => $reason,
];
}
pclose($fh);
$dangerousCount = count(array_filter($findings, fn($f) => $f['type'] === 'DANGEROUS'));
$uncertainCount = count(array_filter($findings, fn($f) => $f['type'] === 'UNCERTAIN'));
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/python3 -I
# strip-escaping-links.py — post-extract symlink authority (N2-1).
#
# python3 -I strip-escaping-links.py <dest> <actions.json> [username ...]
#
# The pre-extract classifier (scan-symlinks.php) resolves a relative target
# LEXICALLY, so a chain through another link (`a -> b/..` with `b -> x/y/z`,
# or `l1 -> l2/../../..`) can slip past it. This pass is the authority: it
# lstat-walks the WHOLE extracted tree (never following a link) and asks the
# kernel (realpath strict) where every symlink really resolves. A link is
# removed and recorded as a `stripped_escaping_symlink` action when:
# - it resolves outside <dest>, or to <dest> itself;
# - it is dangling (strict resolution fails), or loops.
# Own-account absolute links (cPanel's wp-content -> /home/<u>/shared and
# the like) are judged RE-ROOTED, never against the container's own /home:
# a target that is already normalized (no '.', '..' or empty component)
# and lies under /home[N]/<user>/ for the TRUSTED username only (the WHP
# account name WHP passes as extract.sh's <username>; names read from the
# archive are never trusted) is mapped to <dest>/<top>/homedir/<rest>, where
# <top> is the link's own top-level dir, and kept only if THAT resolves
# (realpath strict) inside <top>/homedir. Every other absolute target either
# dangles here or resolves into the container's own /etc, /usr, ... — both
# outside <dest> — so it is stripped. A link to the extract itself is
# stripped too.
#
# Repeats until a pass strips nothing (removing one link can leave another
# that resolved through it dangling). Any walk error fails closed (exit 1).
# stdout: number of links stripped. Exit 0 ok, 1 error, 2 usage.
import json
import os
import posixpath
import re
import stat
import sys
def main(argv):
if len(argv) < 3:
sys.stderr.write("usage: strip-escaping-links.py <dest> <actions.json> [trusted-username]\n")
return 2
dest = os.path.realpath(argv[1])
actions_path = argv[2]
trusted = argv[3] if len(argv) > 3 else ''
if not re.fullmatch(r'[a-z_][a-z0-9_-]{0,31}', trusted):
trusted = '' # no (valid) trusted name: exempt nothing
own_re = re.compile(r'\A/home\d*/' + re.escape(trusted) + r'(?:/(.*))?\Z') if trusted else None
def own_account_ok(link_path, target):
"""True when an absolute own-account target, re-rooted at the link's
<top>/homedir, resolves inside that homedir."""
if own_re is None or posixpath.normpath(target) != target.rstrip('/') or '/../' in target + '/' or '/./' in target + '/':
return False
m = own_re.match(target.rstrip('/'))
if not m:
return False
rel = os.path.relpath(link_path, dest).split(os.sep)
if len(rel) < 2:
return False
home = os.path.join(dest, rel[0], 'homedir')
try:
hst = os.lstat(home)
if not stat.S_ISDIR(hst.st_mode):
return False
home_real = os.path.realpath(home, strict=True)
cand = os.path.join(home_real, m.group(1)) if m.group(1) else home_real
real = os.path.realpath(cand, strict=True)
except OSError:
return False
return real == home_real or real.startswith(home_real + '/')
if not os.path.isdir(dest):
sys.stderr.write(f"strip-escaping-links: not a directory: {dest}\n")
return 1
def walk_error(err):
raise err
stripped = []
while True:
this_pass = 0
for dirpath, dirnames, filenames in os.walk(dest, topdown=True, onerror=walk_error, followlinks=False):
for name in list(dirnames) + list(filenames):
p = os.path.join(dirpath, name)
st = os.lstat(p)
if not stat.S_ISLNK(st.st_mode):
continue
target = os.readlink(p)
reason = None
if target.startswith('/') and own_re is not None and own_re.match(target.rstrip('/') or '/'):
if own_account_ok(p, target):
continue
reason = 'own-account target does not resolve inside the extracted homedir'
try:
if reason is None:
real = os.path.realpath(p, strict=True)
if not real.startswith(dest + '/'):
reason = 'resolves outside the extract'
except OSError:
reason = 'dangling or looping'
if reason is None:
continue
os.unlink(p)
if name in dirnames:
dirnames.remove(name)
stripped.append({
'action': 'stripped_escaping_symlink',
'archive_path': os.path.relpath(p, dest),
'target': target,
'reason': reason,
})
sys.stderr.write(f" STRIP {os.path.relpath(p, dest)} -> {target!r} ({reason})\n")
this_pass += 1
if this_pass == 0:
break
if stripped:
try:
with open(actions_path) as fh:
doc = json.load(fh)
except (OSError, ValueError):
doc = {'actions': []}
doc.setdefault('actions', []).extend(stripped)
doc['count'] = len(doc['actions'])
with open(actions_path, 'w') as fh:
json.dump(doc, fh, indent=2)
print(len(stripped))
return 0
if __name__ == '__main__':
try:
sys.exit(main(sys.argv))
except OSError as e:
sys.stderr.write(f"strip-escaping-links: walk failed: {e}\n")
sys.exit(1)
+296
View File
@@ -0,0 +1,296 @@
#!/bin/bash
## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal
## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster)
## when PKG_CACHE is set; without it the repo files are left alone and builds
## use the upstream repositories as before.
##
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi/NodeSource
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu, Debian, LiteSpeed
## pkg-cache-repos.sh release-install epel [remi] # GPG-verified release RPMs (EL9/EL10)
## pkg-cache-repos.sh nodesource MAJOR # NodeSource repo file + pinned key
## eval "$(pkg-cache-repos.sh go-env)" # GOPROXY via the cache (go.sum/sumdb verified)
## pkg-cache-repos.sh restore # undo everything above
##
## CANONICAL copy: cloud-apache-container/scripts/pkg-cache-repos.sh (docs/pkg-cache.md
## there). Other repos carry byte-identical copies - change it there first.
##
## Every RUN that rewrites must `restore` before it ends: it puts rewritten files
## back, deletes files this script created, deletes any NEW repo/apt file that
## still references the cache, then FAILS if any reference is left — so no shipped
## layer carries the cache's LAN address (images run on hosts that can't reach it).
##
## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging
## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror
## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with
## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and
## immutable .rpm/.deb files long-term.
##
## Integrity (security review 2026-10-21):
## * PKG_CACHE must be https://. The cache's certificate is issued by the
## name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only
## for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten
## repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN —
## never added to the system trust store.
## * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and
## AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL
## publishes no repomd signature (404), so its metadata relies on HTTPS end
## to end (client->cache->dl.fedoraproject.org/mirrors, all verified).
## * release-install downloads epel-release / remi-release (from the cache, or
## from the HTTPS origin without PKG_CACHE), checks the package name, imports
## the signing keys committed in keys/pkg-cache/ and requires
## `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1.
## * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail
## a build that gets old PHP (replay of old-but-signed metadata).
##
## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if
## the cache dies mid-build dnf fails over to the origin by itself.
## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the
## cache (with the CA), and a build without PKG_CACHE uses upstream sources.
set -euo pipefail
MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|nodesource MAJOR|go-env|restore}"
shift || true
STATE=/var/lib/pkg-cache-repos # saved originals + manifest + created files
SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}" # ca.crt + RPM-GPG-KEY-*
C="${PKG_CACHE:-}"
C="${C%/}"
die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; }
if [ -n "$C" ]; then
case "$C" in
https://*) ;;
*) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;;
esac
[[ "$C" =~ ^https://[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]*)?$ ]] || die "PKG_CACHE has unexpected characters: '$C'"
HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}"
[ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing"
fi
# System CA bundle (EL vs Debian/Ubuntu path); the pkg-cache CA is appended to a
# copy in the state dir, never to the system store.
sys_bundle() {
local b
for b in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do
[ -s "$b" ] && { echo "$b"; return 0; }
done
die "no system CA bundle found"
}
make_bundle() { cat "$(sys_bundle)" "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"; }
# Files that may hold repo configuration; restore re-checks all of them.
repo_files() {
find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true
}
start_session() {
[ -d "$STATE" ] && return 0
mkdir -p "$STATE/orig"
repo_files > "$STATE/manifest" # what existed before this RUN touched anything
: > "$STATE/created"
printf '%s\n' "$HOSTPORT" > "$STATE/host"
}
save() { # keep the first-seen (pre-rewrite) copy
[ -e "$STATE/orig$1" ] && return 0
mkdir -p "$STATE/orig$(dirname "$1")"
cp -p "$1" "$STATE/orig$1"
}
created() { printf '%s\n' "$1" >> "$STATE/created"; }
if [ "$MODE" = restore ]; then
[ -d "$STATE" ] || exit 0
host=$(cat "$STATE/host")
# mapfile from a process substitution, not `cmd | while read`: no pipeline,
# so nothing for pipefail to misreport (scripts/tests/lsphp-info-probe.test.sh).
mapfile -t saved < <(cd "$STATE/orig" && find . -type f)
for rel in "${saved[@]}"; do
cat "$STATE/orig/$rel" > "/${rel#./}"
done
while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created"
# A repo/apt file that appeared during this RUN and still names the cache
# (e.g. written by a package from the rewritten repos) is removed too.
mapfile -t current < <(repo_files)
for f in "${current[@]}"; do
if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then
echo "pkg-cache-repos: removing new file that references the cache: $f"
rm -f -- "$f"
fi
done
left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true)
rm -rf "$STATE"
[ -z "$left" ] || die "cache address $host still present after restore in: $left"
echo "pkg-cache-repos: original repository files restored (no reference to $host left)"
exit 0
fi
if [ "$MODE" = release-install ]; then
[ $# -gt 0 ] || die "release-install needs: epel and/or remi"
tmp=$(mktemp -d)
curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120)
if [ -n "$C" ]; then
start_session
make_bundle
curl_opts+=(--cacert "$STATE/ca-bundle.pem")
fi
files=()
el=$(rpm -E '%{?rhel}' 2>/dev/null || true)
case "$el" in 9|10) ;; *) die "release-install supports EL9/EL10 only (rpm -E %rhel = '$el')" ;; esac
for name in "$@"; do
case "$name" in
epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-$el
path=/epel/epel-release-latest-$el.noarch.rpm
origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-$el.noarch.rpm ;;
remi) pkg=remi-release key=RPM-GPG-KEY-remi2024
path=/remi/enterprise/remi-release-$el.rpm
origin=https://rpms.remirepo.net/enterprise/remi-release-$el.rpm ;;
*) die "unknown release package '$name'" ;;
esac
url="$origin"; [ -n "$C" ] && url="$C$path"
f="$tmp/$pkg.rpm"
curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url"
rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key"
got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true)
[ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg"
chk=$(rpm -K "$f" 2>&1 || true)
[[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk"
echo "pkg-cache-repos: $pkg verified ($chk) from $url"
files+=("$f")
done
dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}"
rm -rf "$tmp"
exit 0
fi
if [ -z "$C" ] && [ "$MODE" != nodesource ] && [ "$MODE" != go-env ]; then
echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories"
exit 0
fi
rewrite_rpm() {
local f="$1"
grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net|rpm\.nodesource\.com)/' "$f" || return 0
save "$f"
# Per [section]: a (commented or live) baseurl on a known origin becomes
# "baseurl=<cache>/<prefix>/<path> <origin>/<path>"; that section's
# mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it
# gets sslcacert=<system bundle + pkg-cache CA>, gpgcheck=1 and, for repos
# that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched.
awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" '
function flush( i) {
for (i = 1; i <= n; i++) {
if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i]
if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue
print buf[i]
}
if (hit) {
print "sslcacert=" BUNDLE
print "gpgcheck=1"
if (signed) print "repo_gpgcheck=1"
}
n = 0; hit = 0; signed = 0
}
function rw(line, re, prefix, origin, sig, p) {
if (match(line, re)) {
p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p)
hit = 1; if (sig) signed = 1
return "baseurl=" C "/" prefix "/" p " " origin "/" p
}
return line
}
/^\[/ { flush() }
{
l = $0
if (l ~ /^#? ?baseurl=/) {
l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1)
l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0)
l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1)
# NodeSource signs its RPMs but not repomd.xml (404): gpgcheck only.
l = rw(l, "^#? ?baseurl=https?://rpm\\.nodesource\\.com/", "nodesource", "https://rpm.nodesource.com", 0)
}
buf[++n] = l
}
END { flush() }
' "$f" > "$f.pkgcache" && cat "$f.pkgcache" > "$f" && rm -f "$f.pkgcache"
}
rewrite_apt() {
local f="$1"
grep -q -E 'https?://([a-z]{2}\.)?(archive|security)\.ubuntu\.com/|https?://rpms\.litespeedtech\.com/|https?://(deb|security)\.debian\.org/' "$f" || return 0
save "$f"
# Debian: debian-security first, and "debian" only as a whole path segment,
# so .../debian-security is never rewritten as /debian/.
sed -i -E \
-e "s%https?://(archive|[a-z]{2}\.archive)\.ubuntu\.com/ubuntu/?%$C/ubuntu/%g" \
-e "s%https?://security\.ubuntu\.com/ubuntu/?%$C/ubuntu-security/%g" \
-e "s%https?://rpms\.litespeedtech\.com/%$C/litespeed/%g" \
-e "s%https?://(deb|security)\.debian\.org/debian-security/?([[:space:]]|\$)%$C/debian-security/\\2%g" \
-e "s%https?://security\.debian\.org/?([[:space:]]|\$)%$C/debian-security/\\1%g" \
-e "s%https?://deb\.debian\.org/debian/?([[:space:]]|\$)%$C/debian/\\1%g" \
"$f"
}
if [ "$MODE" = go-env ]; then
# Prints shell exports for `eval`. Without PKG_CACHE: nothing (Go defaults).
# GOPROXY "a|b" falls through on ANY error, so a dead cache costs one
# failed request, not the build. GOSUMDB stays the default sum.golang.org,
# reached directly: modules are still verified against go.sum + the sumdb.
[ -n "$C" ] || { echo "pkg-cache-repos(go-env): PKG_CACHE not set - default GOPROXY" >&2; exit 0; }
start_session
make_bundle
printf "export GOPROXY='%s/goproxy|https://proxy.golang.org|direct' SSL_CERT_FILE='%s'\n" "$C" "$STATE/ca-bundle.pem"
echo "pkg-cache-repos(go-env): GOPROXY via $C/goproxy (fallback proxy.golang.org)" >&2
exit 0
fi
if [ "$MODE" = nodesource ]; then
# Replaces `curl https://rpm.nodesource.com/setup_N.x | bash` (unverified
# remote script): writes the same nodesource-nodejs repo (origin URL, so the
# shipped image keeps a normal repo file) with gpgcheck=1 and the signing key
# committed in keys/pkg-cache/; with PKG_CACHE it is then rewritten like any
# other repo and `restore` puts the origin version back.
major="${1:?usage: pkg-cache-repos.sh nodesource MAJOR}"
[[ "$major" =~ ^[0-9]{1,3}$ ]] || die "bad Node major '$major'"
key="$SHARE/RPM-GPG-KEY-nodesource"
[ -s "$key" ] || die "missing $key"
rpm --import "$key" || die "cannot import $key"
rm -f /etc/yum.repos.d/nodesource*.repo
f=/etc/yum.repos.d/nodesource-nodejs.repo
printf '%s\n' "[nodesource-nodejs]" \
"name=Node.js Packages for Linux RPM based distros - \$basearch" \
"baseurl=https://rpm.nodesource.com/pub_${major}.x/nodistro/nodejs/\$basearch" \
"priority=9" "enabled=1" "gpgcheck=1" \
"gpgkey=file://$key" "module_hotfixes=1" > "$f"
echo "pkg-cache-repos: wrote $f (Node ${major}.x, key $(basename "$key"))"
if [ -n "$C" ]; then
start_session
make_bundle
rewrite_rpm "$f"
grep -h '^baseurl=' "$f" | sed 's/^/ /'
fi
exit 0
fi
case "$MODE" in
rpm)
start_session
make_bundle
for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done
echo "pkg-cache-repos(rpm): repos now via $C:"
grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /'
;;
apt)
start_session
conf=/etc/apt/apt.conf.d/99pkg-cache
# Trust the pkg-cache CA for the cache host ONLY (apt per-host option).
printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf"
created "$conf"
for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
[ -f "$f" ] && rewrite_apt "$f"
done
echo "pkg-cache-repos(apt): sources now via $C:"
# (|| true: Debian 13 images have no /etc/apt/sources.list; grep's exit 2
# for the missing file must not fail the build under pipefail)
{ grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null || true; } | grep -F "$C/" | sort -u | sed 's/^/ /' || true
;;
*) die "unknown mode '$MODE'" ;;
esac
+34 -5
View File
@@ -7,7 +7,11 @@
* for v1.1 expansion; see CONTRIBUTING.md for how to wire one in.
*
* Usage:
* scan-files.php --extract <dir> --quarantine <dir> --report <out.json> --import-id <id>
* scan-files.php --extract <dir> --quarantine <dir> --report <out.json> --import-id <id> [--skip-mail]
*
* --skip-mail: do not scan (or count, or quarantine anything in) the
* <top dir>/homedir/mail tree (any top-level dir). entrypoint.sh passes it when the mail tree
* was extracted for WHP's mailbox import (IMPORT_INCLUDE_MAIL=true).
*
* Exit codes:
* 0 — scan completed (regardless of how many hits)
@@ -28,10 +32,10 @@ require __DIR__ . '/lib/safety-net.php';
const SCANNER_VERSION = '1.0.0';
$opts = getopt('', ['extract:', 'quarantine:', 'report:', 'import-id:']);
$opts = getopt('', ['extract:', 'quarantine:', 'report:', 'import-id:', 'skip-mail']);
foreach (['extract', 'quarantine', 'report', 'import-id'] as $k) {
if (!isset($opts[$k])) {
fwrite(STDERR, "usage: scan-files.php --extract <dir> --quarantine <dir> --report <out.json> --import-id <id>\n");
fwrite(STDERR, "usage: scan-files.php --extract <dir> --quarantine <dir> --report <out.json> --import-id <id> [--skip-mail]\n");
exit(2);
}
}
@@ -39,6 +43,17 @@ $extractDir = rtrim($opts['extract'], '/');
$quarantineDir = rtrim($opts['quarantine'], '/');
$reportPath = $opts['report'];
$importId = $opts['import-id'];
$skipMail = array_key_exists('skip-mail', $opts);
// <any top dir>/homedir/mail: WHP accepts any top-level dir name
// (cpmove-<u>, backup-<date>_<u>, ...), so the skip covers all of them.
// The mail tree is copied verbatim into the customer's own mailbox, and
// WHP verifies the copy message-by-message against what it packaged. So it
// is NOT scanned: quarantining a message would silently drop mail, and
// ClamAV over a multi-GB maildir would dominate the run. PCRE form for the
// PHP walk, POSIX ERE form for clamscan --exclude-dir.
$mailDirPcre = '#\A' . preg_quote($extractDir, '#') . '/[^/]+/homedir/mail(?:/|\z)#';
$mailDirEre = '^' . preg_replace('/[.\[\]()*+?{}|^$\\\\]/', '\\\\$0', $extractDir) . '/[^/]+/homedir/mail(/.*)?$';
if (!is_dir($extractDir)) {
fwrite(STDERR, "scan-files: extract dir does not exist: $extractDir\n");
@@ -48,6 +63,9 @@ if (!is_dir($extractDir)) {
@mkdir($quarantineDir, 0750, true);
fwrite(STDERR, "scan-files: starting (extract=$extractDir, quarantine=$quarantineDir)\n");
if ($skipMail) {
fwrite(STDERR, "scan-files: NOT scanning the mail tree (<top dir>/homedir/mail): it is copied verbatim into the customer's own mailbox and verified against the source; scanning would quarantine messages out of it and multi-GB maildirs would dominate the run\n");
}
// -- v1.0 cleaner registry (intentionally empty) ----------------------------
//
@@ -75,7 +93,8 @@ $cleaners = [];
// Output format per line on a hit:
// /tmp/extract/foo/bar.php: Some.Signature.Name FOUND
$cmd = sprintf(
'clamscan --infected --recursive --no-summary --stdout %s 2>/dev/null',
'clamscan --infected --recursive --no-summary --stdout %s%s 2>/dev/null',
$skipMail ? '--exclude-dir=' . escapeshellarg($mailDirEre) . ' ' : '',
escapeshellarg($extractDir)
);
@@ -99,6 +118,12 @@ while (($line = fgets($fh)) !== false) {
// Defensive: shouldn't happen with our invocation.
continue;
}
if ($skipMail && preg_match($mailDirPcre, $path)) {
// Belt: --exclude-dir should have kept clamscan out of the mail
// tree; never move a message out of a Maildir regardless.
fwrite(STDERR, "scan-files: WARN ignoring a hit inside the mail tree: $path\n");
continue;
}
$hits[] = ['path' => $path, 'signature' => $sig];
}
pclose($fh);
@@ -156,12 +181,15 @@ $rdi = new RecursiveDirectoryIterator(
$extractDir,
FilesystemIterator::SKIP_DOTS | FilesystemIterator::CURRENT_AS_PATHNAME
);
$filter = new RecursiveCallbackFilterIterator($rdi, function ($pathname, $key, $iterator) use (&$skippedLinks) {
$filter = new RecursiveCallbackFilterIterator($rdi, function ($pathname, $key, $iterator) use (&$skippedLinks, $skipMail, $mailDirPcre) {
// $pathname is a string when CURRENT_AS_PATHNAME is set.
if (is_link($pathname)) {
$skippedLinks++;
return false;
}
if ($skipMail && preg_match($mailDirPcre, $pathname)) {
return false; // not scanned, so not counted as scanned
}
return true;
});
$it = new RecursiveIteratorIterator($filter, RecursiveIteratorIterator::LEAVES_ONLY);
@@ -276,6 +304,7 @@ $report = [
'files_clean' => max(0, $filesScanned - count($hits)),
'files_cleaned' => $cleaned,
'files_quarantined' => $quarantined,
'mail_scan_skipped' => $skipMail,
'actions' => $actions,
];
+166 -4
View File
@@ -13,8 +13,17 @@
# `cpmove-testuser/homedir/.../alfasymlink -> /etc`
# — the pre-extract scan MUST refuse this.
#
# cpmove-mail.tar.gz — same shape PLUS a cPanel mail tree:
# homedir/etc/<domain>/{passwd,shadow} and
# homedir/mail/<domain>/{alice,empty}/ Maildirs
# (alice has 2 messages incl. one in .Archive;
# empty has cur/new/tmp and no messages), plus a
# DANGEROUS symlink INSIDE the mail tree
# (homedir/mail/<domain>/alice/.evil -> /etc) that
# must be stripped even when mail is included.
#
# Run: bash tests/build-fixtures.sh
# Output: tests/fixtures/cpmove-clean.tar.gz, tests/fixtures/cpmove-alfa.tar.gz
# Output: tests/fixtures/cpmove-{clean,alfa,mail,mail-xlink}.tar.gz
set -euo pipefail
@@ -44,8 +53,10 @@ EOF
echo "<?php phpinfo();" > "$root/cpmove-$USER/homedir/public_html/index.php"
echo "Hello world." > "$root/cpmove-$USER/homedir/public_html/about.txt"
# benign user-internal relative symlink — must NOT trigger the scan
ln -sf "../public_html/about.txt" "$root/cpmove-$USER/homedir/about-shortcut"
# benign user-internal relative symlink — must NOT trigger the scan.
# (It used to be ../public_html/about.txt, which resolves OUTSIDE
# homedir/ — cpmove-$USER/public_html — and is now DANGEROUS, N2-1.)
ln -sf "public_html/about.txt" "$root/cpmove-$USER/homedir/about-shortcut"
# one synthetic WordPress mysql dump with ENGINE=MyISAM + a clean siteurl
cat > "$root/cpmove-$USER/mysql/${USER}_wp.sql" <<EOF
@@ -82,7 +93,7 @@ EOF
# ---- cpmove-clean.tar.gz --------------------------------------------------
CLEAN_TMP="$(mktemp -d)"
trap 'rm -rf "$CLEAN_TMP" "$ALFA_TMP" 2>/dev/null || true' EXIT
trap 'rm -rf "$CLEAN_TMP" "$ALFA_TMP" "$MAIL_TMP" 2>/dev/null || true' EXIT
build_common_tree "$CLEAN_TMP"
tar -C "$CLEAN_TMP" -czf "$FIXTURES_DIR/cpmove-clean.tar.gz" "cpmove-$USER"
@@ -108,6 +119,157 @@ ln -sf "/etc" "$ALFA_TMP/cpmove-$USER/homedir/public_html/$DOMAIN/ALFA_DATA/root
tar -C "$ALFA_TMP" -czf "$FIXTURES_DIR/cpmove-alfa.tar.gz" "cpmove-$USER"
echo "wrote $FIXTURES_DIR/cpmove-alfa.tar.gz ($(stat -c%s "$FIXTURES_DIR/cpmove-alfa.tar.gz") bytes)"
# ---- cpmove-relesc.tar.gz (N2-1) --------------------------------------------
#
# RELATIVE symlinks that climb out: `sds` (a metadata file WHP parses) out of
# the archive to /etc/passwd, and a docroot file out of homedir/ into the
# cpmove's own mysql/ dump. Both must be DANGEROUS (stripped), while
# homedir/www -> public_html (inside homedir) stays.
RELESC_TMP="$(mktemp -d)"
build_common_tree "$RELESC_TMP"
rm -rf "$RELESC_TMP/cpmove-$USER/sds"
ln -sf "../../../../../../../../etc/passwd" "$RELESC_TMP/cpmove-$USER/sds"
ln -sf "../../mysql/${USER}_wp.sql" "$RELESC_TMP/cpmove-$USER/homedir/public_html/dump.sql"
ln -sf "public_html" "$RELESC_TMP/cpmove-$USER/homedir/www"
tar -C "$RELESC_TMP" -czf "$FIXTURES_DIR/cpmove-relesc.tar.gz" "cpmove-$USER"
rm -rf "$RELESC_TMP"
echo "wrote $FIXTURES_DIR/cpmove-relesc.tar.gz"
# ---- cpmove-chain.tar.gz (N2-1 review: symlink chains) ----------------------
# Each link alone resolves lexically inside the archive; through the other
# link the kernel resolves it ABOVE the extract:
# d1/d2/b -> ../.. (= cpmove-$USER, fine on its own)
# d1/d2/a -> b/../.. (b/.. = extract root, /.. = its parent)
# d1/d2/l2 -> ../..
# d1/d2/l1 -> l2/../../..
CHAIN_TMP="$(mktemp -d)"
build_common_tree "$CHAIN_TMP"
C="$CHAIN_TMP/cpmove-$USER/d1/d2"
mkdir -p "$C"
ln -s "../.." "$C/b"; ln -s "b/../.." "$C/a"
ln -s "../.." "$C/l2"; ln -s "l2/../../.." "$C/l1"
tar -C "$CHAIN_TMP" -czf "$FIXTURES_DIR/cpmove-chain.tar.gz" "cpmove-$USER"
rm -rf "$CHAIN_TMP"
echo "wrote $FIXTURES_DIR/cpmove-chain.tar.gz"
# ---- cpmove-mail.tar.gz ---------------------------------------------------
MAIL_TMP="$(mktemp -d)"
build_common_tree "$MAIL_TMP"
MROOT="$MAIL_TMP/cpmove-$USER/homedir"
mkdir -p "$MROOT/etc/$DOMAIN"
printf 'alice:x:1:1::/home/%s/mail/%s/alice:/bin/false\nempty:x:1:1::/home/%s/mail/%s/empty:/bin/false\n' \
"$USER" "$DOMAIN" "$USER" "$DOMAIN" > "$MROOT/etc/$DOMAIN/passwd"
# Not real hashes: the importer image never reads these, it only extracts.
printf 'alice:$6$fixturesaltfixtu$notarealhash:19000::::::\nempty:$6$fixturesaltfixtu$notarealhash:19000::::::\n' \
> "$MROOT/etc/$DOMAIN/shadow"
for box in alice empty; do
mkdir -p "$MROOT/mail/$DOMAIN/$box"/{cur,new,tmp}
done
mkdir -p "$MROOT/mail/$DOMAIN/alice/.Archive"/{cur,new,tmp}
printf 'From: a@example.net\nSubject: one\n\nhello\n' \
> "$MROOT/mail/$DOMAIN/alice/cur/1700000000.M1P1.host,S=40:2,S"
printf 'From: b@example.net\nSubject: two\n\narchived\n' \
> "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000001.M2P1.host,S=43:2,"
printf 'Archive\n' > "$MROOT/mail/$DOMAIN/alice/subscriptions"
# DANGEROUS symlink inside the mail tree: stripped whether or not mail is included.
ln -sf "/etc" "$MROOT/mail/$DOMAIN/alice/.evil"
# Relative symlinks inside the mail tree (not DANGEROUS-classified): a file
# link climbing out to /etc/shadow and a folder link to another domain's box.
ln -sf "../../../../../../../../etc/shadow" "$MROOT/mail/$DOMAIN/alice/.evil-rel"
mkdir -p "$MROOT/mail/other.example/bob"/{cur,new,tmp}
ln -sf "../../other.example/bob" "$MROOT/mail/$DOMAIN/alice/.Other"
# Hard links. With --sort=name below, the first name of an inode is stored
# as the file and later names as hard-link entries:
# .Archive/cur/1700000002... (file) <- cur/1700000002... Dovecot-style
# copy inside the mail tree: legitimate, must be KEPT;
# public_html/stolen.txt -> that message: crosses OUT of the mail tree
# (cpmove-mail-xlink.tar.gz only, below);
# etc/<domain>/shadow (file) <- mail/<domain>/alice/cur/leak: crosses IN.
printf 'From: c@example.net\nSubject: three\n\ncopied\n' \
> "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S"
ln "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S" \
"$MROOT/mail/$DOMAIN/alice/cur/1700000002.M3P1.host,S=41:2,S"
ln "$MROOT/etc/$DOMAIN/shadow" "$MROOT/mail/$DOMAIN/alice/cur/leak"
tar -C "$MAIL_TMP" --sort=name -czf "$FIXTURES_DIR/cpmove-mail.tar.gz" "cpmove-$USER"
# cpmove-mail-xlink.tar.gz: the same PLUS public_html/stolen.txt as a hard
# link to a message (crosses OUT of the mail tree). Separate fixture: in
# the default (no-mail) mode tar cannot create a link to an excluded
# member and the extract fails (fail-closed, as before this change).
ln "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S" \
"$MROOT/public_html/stolen.txt"
tar -C "$MAIL_TMP" --sort=name -czf "$FIXTURES_DIR/cpmove-mail-xlink.tar.gz" "cpmove-$USER"
echo "wrote $FIXTURES_DIR/cpmove-mail.tar.gz + cpmove-mail-xlink.tar.gz"
# ---- parser-differential fixtures (must be REFUSED with IMPORT_INCLUDE_MAIL=true)
# Written with python tarfile because GNU tar normalises such names on create.
# cpmove-bad-dot ./ component: cpmove-testuser/./homedir/mail/.../.x -> ../../../../etc
# cpmove-bad-dslash empty component: cpmove-testuser/homedir//mail/.../.x -> ../../../../etc
# cpmove-bad-dotdot '..' component: cpmove-testuser/homedir/../homedir/mail/.../.x -> ../../../../etc
# cpmove-bad-abs absolute name: /cpmove-testuser/homedir/mail/.../.x -> ../../../../etc
# cpmove-bad-newline '\n' in a name (would split one --exclude-from pattern in two)
python3 - "$FIXTURES_DIR" <<'PY'
import io, sys, tarfile
out = sys.argv[1]
def build(fname, members):
with tarfile.open(f'{out}/{fname}', 'w:gz', format=tarfile.PAX_FORMAT) as tf:
for d in ('cpmove-testuser', 'cpmove-testuser/homedir', 'cpmove-testuser/homedir/mail',
'cpmove-testuser/homedir/mail/example.com', 'cpmove-testuser/homedir/mail/example.com/alice',
'cpmove-testuser/homedir/mail/example.com/alice/cur'):
ti = tarfile.TarInfo(d); ti.type = tarfile.DIRTYPE; ti.mode = 0o755; tf.addfile(ti)
for name, kind, arg in members:
ti = tarfile.TarInfo(name); ti.mode = 0o644
if kind == 'sym':
ti.type = tarfile.SYMTYPE; ti.linkname = arg; tf.addfile(ti)
elif kind == 'hard':
ti.type = tarfile.LNKTYPE; ti.linkname = arg; tf.addfile(ti)
elif kind in ('dir', 'dir0'):
ti.type = tarfile.DIRTYPE; ti.mode = 0o755 if kind == 'dir' else 0; tf.addfile(ti)
else:
ti.size = len(arg); tf.addfile(ti, io.BytesIO(arg))
mb = 'homedir/mail/example.com/alice'
build('cpmove-bad-dot.tar.gz', [(f'cpmove-testuser/./{mb}/.x', 'sym', '../../../../etc')])
build('cpmove-bad-dslash.tar.gz', [(f'cpmove-testuser/homedir//mail/example.com/alice/.x', 'sym', '../../../../etc')])
build('cpmove-bad-dotdot.tar.gz', [(f'cpmove-testuser/homedir/../{mb}/.x', 'sym', '../../../../etc')])
build('cpmove-bad-abs.tar.gz', [(f'/cpmove-testuser/{mb}/.x', 'sym', '../../../../etc')])
build('cpmove-bad-newline.tar.gz', [(f'cpmove-testuser/{mb}/cur/a\ncpmove-testuser/homedir/public_html', 'file', b'x')])
# A mail-tree directory the archive makes unreadable (mode 0000): verify
# cannot inspect it, so the extract must fail (and be removed).
build('cpmove-bad-unreadable.tar.gz', [(f'cpmove-testuser/{mb}/cur/m1', 'file', b'hello'),
(f'cpmove-testuser/{mb}/.Locked', 'dir0', None),
(f'cpmove-testuser/{mb}/.Locked/cur', 'dir0', None)])
# Passes the classifier but GNU tar fails mid-way (hard link to a member that
# does not exist), after it already wrote the mail files: the partial tree
# must be removed.
build('cpmove-bad-tarfail.tar.gz', [(f'cpmove-testuser/{mb}/cur/m1', 'file', b'hello'),
('cpmove-testuser/homedir/public_html', 'dir', None),
('cpmove-testuser/homedir/public_html/x', 'hard', 'cpmove-testuser/homedir/nonexistent'),
(f'cpmove-testuser/{mb}/cur/m2', 'file', b'after the failure')])
PY
# The same mail fixture under other top-level dir names WHP accepts
# (CpanelArchivePaths::selectTopDir takes the first non-hidden dir of ANY
# name): `acct/` and cPanel's user-download layout `backup-<date>_<user>/`.
python3 - "$FIXTURES_DIR" <<'PY'
import io, sys, tarfile
out = sys.argv[1]
src = f'{out}/cpmove-mail.tar.gz'
for top, fname in (('acct', 'cpmove-acct.tar.gz'), ('backup-10.20.2026_10-00-00_testuser', 'cpmove-backupdir.tar.gz')):
with tarfile.open(src) as tin, tarfile.open(f'{out}/{fname}', 'w:gz', format=tarfile.PAX_FORMAT) as tout:
for m in tin.getmembers():
data = tin.extractfile(m).read() if m.isfile() else None
m.name = m.name.replace('cpmove-testuser', top, 1)
if m.islnk():
m.linkname = m.linkname.replace('cpmove-testuser', top, 1)
tout.addfile(m, io.BytesIO(data) if data is not None else None)
PY
# Truncated archive: the first half of a valid one.
head -c "$(( $(stat -c%s "$FIXTURES_DIR/cpmove-mail.tar.gz") / 2 ))" "$FIXTURES_DIR/cpmove-mail.tar.gz" \
> "$FIXTURES_DIR/cpmove-bad-truncated.tar.gz"
echo "wrote $FIXTURES_DIR/cpmove-bad-{dot,dslash,dotdot,abs,newline,unreadable,tarfail,truncated}.tar.gz"
echo ""
echo "fixtures built:"
ls -la "$FIXTURES_DIR"
+316
View File
@@ -0,0 +1,316 @@
#!/usr/bin/env bash
#
# run-tests.sh — host-side regression tests for the sandbox scripts.
#
# Runs extract.sh and scan-files.php straight from the checkout (no image
# build): needs bash, php-cli, python3 and GNU tar. clamscan is replaced by
# a stub on PATH that records its argv and reports fixed "FOUND" hits, so
# the scan-files.php plumbing (exclude flags, quarantine, counts) is tested
# without ClamAV or signatures. The full image is covered by the docker
# commands in CONTRIBUTING.md.
#
# Run: bash tests/run-tests.sh (exit 0 = all pass)
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
FIX="$REPO/tests/fixtures"
# Under /tmp: scan-files.php sets a container open_basedir that allows /tmp.
WORK="$(mktemp -d -p /tmp cpi-tests.XXXXXX)"
trap 'rm -rf "$WORK"' EXIT
FAILS=0
pass() { printf ' PASS %s\n' "$1"; }
fail() { printf ' FAIL %s\n' "$1"; FAILS=$((FAILS + 1)); }
check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; }
bash "$REPO/tests/build-fixtures.sh" >/dev/null
# Copied under /tmp: scan-symlinks.php opens the tarball under the same open_basedir.
cp "$FIX/cpmove-mail.tar.gz" "$WORK/"
MAILTAR="$WORK/cpmove-mail.tar.gz"
U=cpmove-testuser
D=example.com
# ---- extract.sh: default (no opt-in) skips the mail tree -------------------
echo "extract.sh without IMPORT_INCLUDE_MAIL"
OUT="$WORK/ex-default"
LOG="$WORK/ex-default.log"
env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$MAILTAR" "$OUT" testuser "$WORK/ex-default.actions.json" >"$LOG" 2>&1
check "default: homedir/mail not extracted" "[[ ! -e '$OUT/$U/homedir/mail' ]]"
check "default: homedir/etc/<domain>/passwd still extracted" "[[ -f '$OUT/$U/homedir/etc/$D/passwd' ]]"
check "default: log says the mail tree was skipped" "grep -q 'skipping mail tree' '$LOG'"
# ---- extract.sh: opt-in includes it -----------------------------------------
echo "extract.sh with IMPORT_INCLUDE_MAIL=true"
OUT="$WORK/ex-mail"
LOG="$WORK/ex-mail.log"
IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$MAILTAR" "$OUT" testuser "$WORK/ex-mail.actions.json" >"$LOG" 2>&1
check "opt-in: INBOX message extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/cur/1700000000.M1P1.host,S=40:2,S' ]]"
check "opt-in: subfolder message extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/.Archive/cur/1700000001.M2P1.host,S=43:2,' ]]"
check "opt-in: subscriptions extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/subscriptions' ]]"
check "opt-in: empty Maildir kept with cur/new/tmp" "[[ -d '$OUT/$U/homedir/mail/$D/empty/cur' && -d '$OUT/$U/homedir/mail/$D/empty/new' && -d '$OUT/$U/homedir/mail/$D/empty/tmp' ]]"
check "opt-in: DANGEROUS symlink inside the mail tree stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/.evil' && ! -L '$OUT/$U/homedir/mail/$D/alice/.evil' ]]"
check "opt-in: the strip is recorded as an action" "grep -q 'homedir/mail/$D/alice/.evil' '$WORK/ex-mail.actions.json'"
check "opt-in: relative file symlink (../..../etc/shadow) inside the mail tree stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/.evil-rel' && ! -L '$OUT/$U/homedir/mail/$D/alice/.evil-rel' ]]"
check "opt-in: relative folder symlink to another domain's box stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/.Other' && ! -L '$OUT/$U/homedir/mail/$D/alice/.Other' ]]"
check "opt-in: no symlink at all left under the mail tree" "[[ -z \$(find '$OUT/$U/homedir/mail' -type l) ]]"
check "opt-in: the DANGEROUS in-mail symlink is listed once, not twice" "[[ \$(grep -c 'alice/.evil\"' '$WORK/ex-mail.actions.json') == 1 ]]"
check "opt-in: relative symlinks recorded as stripped_mail_link actions" "grep -q 'alice/.evil-rel' '$WORK/ex-mail.actions.json' && grep -q 'alice/.Other' '$WORK/ex-mail.actions.json'"
check "opt-in: hard link INSIDE the mail tree (Dovecot copy) kept" "[[ -f '$OUT/$U/homedir/mail/$D/alice/cur/1700000002.M3P1.host,S=41:2,S' && -f '$OUT/$U/homedir/mail/$D/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S' ]]"
check "opt-in: hard link crossing INTO the mail tree (message -> etc/shadow) stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/cur/leak' ]]"
check "opt-in: log says the mail tree was included" "grep -q 'including mail tree' '$LOG'"
check "opt-in: no 'skipping mail tree' line" "! grep -q 'skipping mail tree' '$LOG'"
echo "N2-1: relative symlinks that escape the archive / the homedir (cpmove-relesc)"
cp "$FIX/cpmove-relesc.tar.gz" "$WORK/"
php "$REPO/scripts/lib/scan-symlinks.php" --tarball "$WORK/cpmove-relesc.tar.gz" --report "$WORK/relesc-scan.json" --username testuser 2>/dev/null
relesc_type() { php -r '$r = json_decode(file_get_contents($argv[1]), true); foreach ($r["findings"] as $f) if ($f["archive_path"] === $argv[2]) { echo $f["type"]; exit; } echo "NONE";' "$WORK/relesc-scan.json" "$1"; }
check "relesc: sds -> ../../../../etc/passwd is DANGEROUS" "[[ \$(relesc_type '$U/sds') == DANGEROUS ]]"
check "relesc: homedir/public_html/dump.sql -> ../../mysql/... (escapes homedir) is DANGEROUS" "[[ \$(relesc_type '$U/homedir/public_html/dump.sql') == DANGEROUS ]]"
check "relesc: homedir/www -> public_html (inside homedir) is not a finding" "[[ \$(relesc_type '$U/homedir/www') == NONE ]]"
check "relesc: homedir/about-shortcut -> public_html/about.txt is not a finding" "[[ \$(relesc_type '$U/homedir/about-shortcut') == NONE ]]"
OUT="$WORK/ex-relesc"
env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/cpmove-relesc.tar.gz" "$OUT" testuser "$WORK/ex-relesc.actions.json" >"$WORK/ex-relesc.log" 2>&1
check "relesc: sds stripped from the extract" "[[ ! -e '$OUT/$U/sds' && ! -L '$OUT/$U/sds' ]]"
check "relesc: dump.sql link stripped from the extract" "[[ ! -L '$OUT/$U/homedir/public_html/dump.sql' ]]"
check "relesc: homedir/www link kept" "[[ -L '$OUT/$U/homedir/www' ]]"
check "relesc: both strips recorded" "grep -q '$U/sds' '$WORK/ex-relesc.actions.json' && grep -q 'public_html/dump.sql' '$WORK/ex-relesc.actions.json'"
echo "N2-1 review: symlink chains (cpmove-chain)"
cp "$FIX/cpmove-chain.tar.gz" "$WORK/"
php "$REPO/scripts/lib/scan-symlinks.php" --tarball "$WORK/cpmove-chain.tar.gz" --report "$WORK/relesc-scan.json" --username testuser 2>/dev/null
check "chain: d1/d2/a -> b/../.. (through link b) is DANGEROUS" "[[ \$(relesc_type '$U/d1/d2/a') == DANGEROUS ]]"
check "chain: d1/d2/l1 -> l2/../../.. (through link l2) is DANGEROUS" "[[ \$(relesc_type '$U/d1/d2/l1') == DANGEROUS ]]"
check "chain: d1/d2/b -> ../.. alone is not a finding" "[[ \$(relesc_type '$U/d1/d2/b') == NONE ]]"
OUT="$WORK/ex-chain"
env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/cpmove-chain.tar.gz" "$OUT" testuser "$WORK/ex-chain.actions.json" >"$WORK/ex-chain.log" 2>&1
check "chain: a and l1 absent from the extract" "[[ ! -L '$OUT/$U/d1/d2/a' && ! -L '$OUT/$U/d1/d2/l1' ]]"
check "chain: b and l2 kept" "[[ -L '$OUT/$U/d1/d2/b' && -L '$OUT/$U/d1/d2/l2' ]]"
echo "N2-1 review: post-extract authority strip-escaping-links.py (links the classifier never saw)"
P="$WORK/postx"
rm -rf "$P"; mkdir -p "$P/$U/d1/d2" "$P/$U/homedir/public_html/inner"
echo x > "$P/$U/homedir/public_html/inner/f"
ln -s "../.." "$P/$U/d1/d2/b"; ln -s "b/../.." "$P/$U/d1/d2/a"
ln -s "../.." "$P/$U/d1/d2/l2"; ln -s "l2/../../.." "$P/$U/d1/d2/l1"
ln -s "/etc/passwd" "$P/$U/homedir/abs-etc"
ln -s "nonexistent" "$P/$U/homedir/public_html/x.php.bak.cpanel-import"
ln -s "../../../../../../victim/shell.php" "$P/$U/homedir/public_html/y.bak"
ln -s "loop2" "$P/$U/homedir/loop1"; ln -s "loop1" "$P/$U/homedir/loop2"
ln -s "../.." "$P/$U/top-link"
ln -s "inner/f" "$P/$U/homedir/public_html/ok-file"
ln -s "public_html" "$P/$U/homedir/www"
ln -s "/home/testuser/public_html/inner" "$P/$U/homedir/own-abs"
# Review round 2: own-account absolute links are judged re-rooted in the
# extract, normalized, and only for the TRUSTED username argument.
ln -s "/home/testuser/../../etc/shadow" "$P/$U/homedir/own-dotdot"
ln -s "/home/testuser/./public_html/inner" "$P/$U/homedir/own-dot"
ln -s "/home/testuser/public_html/missing" "$P/$U/homedir/own-dangling"
ln -s "../../.." "$P/$U/homedir/esc"; ln -s "/home/testuser/esc" "$P/$U/homedir/own-via-esc"
ln -s "/home/root/x" "$P/$U/homedir/other-user"
mkdir -p "$P/cpmove-root/homedir/x"; ln -s "/home/root/x" "$P/cpmove-root/homedir/archive-named-user"
echo '{"actions":[{"action":"pre"}]}' > "$WORK/postx.json"
n=$(python3 -I "$REPO/scripts/lib/strip-escaping-links.py" "$P" "$WORK/postx.json" testuser 2>/dev/null)
for gone in d1/d2/a d1/d2/l1 homedir/abs-etc homedir/public_html/x.php.bak.cpanel-import homedir/public_html/y.bak homedir/loop1 homedir/loop2 top-link \
homedir/own-dotdot homedir/own-dot homedir/own-dangling homedir/esc homedir/own-via-esc homedir/other-user; do
check "post-extract: $gone stripped" "[[ ! -L '$P/$U/$gone' ]]"
done
for kept in d1/d2/b d1/d2/l2 homedir/public_html/ok-file homedir/www homedir/own-abs; do
check "post-extract: $kept kept" "[[ -L '$P/$U/$kept' ]]"
done
check "post-extract: cpmove-root/…/archive-named-user (/home/root/x; 'root' only named by the archive) stripped" "[[ ! -L '$P/cpmove-root/homedir/archive-named-user' ]]"
check "post-extract: 15 links stripped (got $n)" "[[ '$n' == 15 ]]"
check "post-extract: strips appended as stripped_escaping_symlink actions" "grep -q '\"pre\"' '$WORK/postx.json' && [[ \$(grep -c stripped_escaping_symlink '$WORK/postx.json') == 15 ]]"
rm -rf "$P"; mkdir -p "$P/$U/homedir/public_html/inner"
ln -s "/home/testuser/public_html/inner" "$P/$U/homedir/own-abs"
python3 -I "$REPO/scripts/lib/strip-escaping-links.py" "$P" "$WORK/postx2.json" >/dev/null 2>&1
check "post-extract: with no trusted username, an own-account link is NOT exempt" "[[ ! -L '$P/$U/homedir/own-abs' ]]"
rm -rf "$P"; mkdir -p "$P/$U/homedir/public_html/inner"
ln -s "/home/testuser/public_html/inner" "$P/$U/homedir/own-abs"
python3 -I "$REPO/scripts/lib/strip-escaping-links.py" "$P" "$WORK/postx3.json" 'testuser/../x' >/dev/null 2>&1
check "post-extract: an invalid 'trusted' username exempts nothing" "[[ ! -L '$P/$U/homedir/own-abs' ]]"
echo "extract.sh: an archive cannot name its own exempt user (top dir cpmove-root)"
AR="$WORK/archuser-src"; rm -rf "$AR"; mkdir -p "$AR/cpmove-root/homedir/x"
ln -s "/home/root/x" "$AR/cpmove-root/homedir/self-exempt"
tar -C "$AR" -czf "$WORK/archuser.tar.gz" cpmove-root
env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/archuser.tar.gz" "$WORK/ex-archuser" testuser "$WORK/archuser.json" >/dev/null 2>&1
check "archive-named user: /home/root/x link stripped (only the WHP username is trusted)" "[[ -d '$WORK/ex-archuser/cpmove-root/homedir/x' && ! -L '$WORK/ex-archuser/cpmove-root/homedir/self-exempt' ]]"
env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/archuser.tar.gz" "$WORK/ex-archuser2" root "$WORK/archuser2.json" >/dev/null 2>&1
check "control: the same link is kept when WHP itself passes that username" "[[ -L '$WORK/ex-archuser2/cpmove-root/homedir/self-exempt' ]]"
echo "scan-symlinks: /home/<user>/../../etc is not user-internal"
DT="$WORK/dotdot-src"; rm -rf "$DT"; mkdir -p "$DT/$U/homedir"
ln -s "/home/testuser/../../etc/shadow" "$DT/$U/homedir/sneaky"
tar -C "$DT" -czf "$WORK/dotdot.tar.gz" "$U"
php "$REPO/scripts/lib/scan-symlinks.php" --tarball "$WORK/dotdot.tar.gz" --report "$WORK/relesc-scan.json" --username testuser 2>/dev/null
check "scan: /home/testuser/../../etc/shadow is DANGEROUS (not BENIGN_USER_INTERNAL)" "[[ \$(relesc_type '$U/homedir/sneaky') == DANGEROUS ]]"
echo "hard link crossing OUT of the mail tree (cpmove-mail-xlink)"
cp "$FIX/cpmove-mail-xlink.tar.gz" "$WORK/"
OUT="$WORK/ex-xlink"
IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-mail-xlink.tar.gz" "$OUT" testuser "$WORK/ex-xlink.actions.json" >"$WORK/ex-xlink.log" 2>&1
check "opt-in: public_html/stolen.txt (hard link to a message) stripped" "[[ ! -e '$OUT/$U/homedir/public_html/stolen.txt' ]]"
check "opt-in: the linked message itself still extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S' ]]"
check "default mode: same archive fails closed (tar cannot link to the excluded mail member)" \
"! env -u IMPORT_INCLUDE_MAIL bash '$REPO/scripts/extract.sh' '$WORK/cpmove-mail-xlink.tar.gz' '$WORK/ex-xlink-default' testuser '$WORK/x.json' >/dev/null 2>&1"
echo "parser-differential archives are refused (IMPORT_INCLUDE_MAIL=true)"
for bad in dot dslash dotdot abs newline; do
cp "$FIX/cpmove-bad-$bad.tar.gz" "$WORK/"
OUT="$WORK/ex-bad-$bad"
rc=0
IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-bad-$bad.tar.gz" "$OUT" testuser "$WORK/bad-$bad.json" >"$WORK/bad-$bad.log" 2>&1 || rc=$?
check "bad-$bad: extract exits non-zero (rc=$rc)" "[[ $rc -ne 0 ]]"
check "bad-$bad: refusal logged" "grep -q 'REFUSING archive' '$WORK/bad-$bad.log'"
check "bad-$bad: extract dir removed" "[[ ! -e '$OUT' ]]"
done
echo "failures after extraction starts remove the extract (IMPORT_INCLUDE_MAIL=true)"
for bad in unreadable tarfail truncated; do
cp "$FIX/cpmove-bad-$bad.tar.gz" "$WORK/"
OUT="$WORK/ex-bad-$bad"
rc=0
IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-bad-$bad.tar.gz" "$OUT" testuser "$WORK/bad-$bad.json" >"$WORK/bad-$bad.log" 2>&1 || rc=$?
check "bad-$bad: extract exits non-zero (rc=$rc)" "[[ $rc -ne 0 ]]"
check "bad-$bad: extract dir removed" "[[ ! -e '$OUT' ]]"
done
check "bad-unreadable: failed in verify (cannot inspect the 0000 dir)" "grep -q 'not readable+searchable\|cannot inspect' '$WORK/bad-unreadable.log'"
check "bad-tarfail: tar itself failed (rc logged), verify still ran" "grep -q 'tar failed (rc=' '$WORK/bad-tarfail.log'"
check "bad-truncated: refused by the classifier" "grep -q 'REFUSING archive' '$WORK/bad-truncated.log'"
echo "classifier refuses when GNU tar lists a different member count"
FAKEBIN="$WORK/faketar"
mkdir -p "$FAKEBIN"
REALTAR="$(command -v tar)"
cat > "$FAKEBIN/tar" <<FAKE
#!/usr/bin/env bash
# Real tar, plus one extra member in the escape-quoted listing only.
"$REALTAR" "\$@"; rc=\$?
[[ " \$* " == *" --quoting-style=escape "* ]] && echo "cpmove-testuser/homedir/mail/hidden-from-the-classifier"
exit \$rc
FAKE
chmod +x "$FAKEBIN/tar"
: > "$WORK/cm.excl"; echo '{"actions":[]}' > "$WORK/cm.json"
rc=0
PATH="$FAKEBIN:$PATH" python3 -I "$REPO/scripts/lib/mail-tree-links.py" scan "$MAILTAR" "$WORK/cm.excl" "$WORK/cm.json" >/dev/null 2>"$WORK/cm.log" || rc=$?
check "count mismatch: scan exits 3 (rc=$rc)" "[[ $rc -eq 3 ]]"
check "count mismatch: reason logged" "grep -q 'member count differs' '$WORK/cm.log'"
rc=0
python3 -I "$REPO/scripts/lib/mail-tree-links.py" scan "$MAILTAR" "$WORK/cm.excl" "$WORK/cm.json" >/dev/null 2>&1 || rc=$?
check "control: real tar, same archive => scan exits 0" "[[ $rc -eq 0 ]]"
echo "post-extract authority: mail-tree-links.py verify"
V="$WORK/verify"
mkvt() { rm -rf "$V"; mkdir -p "$V/$U/homedir/mail/$D/a/cur" "$V/$U/homedir/mail/$D/a/.F/cur" "$V/$U/homedir/public_html" "$V/cpmove-other/homedir/mail/$D/b/cur"; echo m > "$V/$U/homedir/mail/$D/a/cur/1"; }
vrc() { local rc=0; python3 -I "$REPO/scripts/lib/mail-tree-links.py" verify "$V" >/dev/null 2>&1 || rc=$?; echo $rc; }
mkvt; check "verify: clean tree => 0" "[[ \$(vrc) == 0 ]]"
mkvt; ln "$V/$U/homedir/mail/$D/a/cur/1" "$V/$U/homedir/mail/$D/a/.F/cur/1"
check "verify: Dovecot hard link inside one mail tree => 0 (kept)" "[[ \$(vrc) == 0 ]]"
mkvt; ln -s ../../../../../etc "$V/$U/homedir/mail/$D/a/.evil"
check "verify: symlink in the mail tree (survived the pre-filter) => 4" "[[ \$(vrc) == 4 ]]"
mkvt; ln "$V/$U/homedir/mail/$D/a/cur/1" "$V/$U/homedir/public_html/out"
check "verify: hard link OUT of the mail tree (survived the pre-filter) => 4" "[[ \$(vrc) == 4 ]]"
mkvt; echo s > "$V/$U/homedir/shadow"; ln "$V/$U/homedir/shadow" "$V/$U/homedir/mail/$D/a/cur/in"
check "verify: hard link INTO the mail tree (survived the pre-filter) => 4" "[[ \$(vrc) == 4 ]]"
mkvt; ln "$V/$U/homedir/mail/$D/a/cur/1" "$V/cpmove-other/homedir/mail/$D/b/cur/1"
check "verify: hard link between two accounts' mail trees => 4" "[[ \$(vrc) == 4 ]]"
mkvt; rm -rf "$V/$U/homedir/mail"; ln -s /etc "$V/$U/homedir/mail"
check "verify: homedir/mail itself a symlink => 4" "[[ \$(vrc) == 4 ]]"
echo "extract.sh with IMPORT_INCLUDE_MAIL=1 (only \"true\" opts in)"
OUT="$WORK/ex-one"
IMPORT_INCLUDE_MAIL=1 bash "$REPO/scripts/extract.sh" "$MAILTAR" "$OUT" testuser "$WORK/ex-one.actions.json" >/dev/null 2>&1
check "IMPORT_INCLUDE_MAIL=1: mail tree skipped" "[[ ! -e '$OUT/$U/homedir/mail' ]]"
# ---- scan-files.php with a stub clamscan ------------------------------------
# The stub reports one hit inside the mail tree and one in public_html.
STUBBIN="$WORK/bin"
mkdir -p "$STUBBIN"
cat > "$STUBBIN/clamscan" <<'STUB'
#!/usr/bin/env bash
printf '%s\n' "$@" > "$CLAMSCAN_ARGS_OUT"
dir="${!#}"
top="${CLAM_TOP:-cpmove-testuser}"
echo "$dir/$top/homedir/mail/example.com/alice/cur/1700000000.M1P1.host,S=40:2,S: Test.Mail.Sig FOUND"
echo "$dir/$top/homedir/public_html/index.php: Test.Php.Sig FOUND"
STUB
chmod +x "$STUBBIN/clamscan"
run_scan() { # <extract dir> <tag> [extra args]
local ex="$1" tag="$2"; shift 2
mkdir -p "$WORK/q-$tag"
CLAMSCAN_ARGS_OUT="$WORK/clam-$tag.args" PATH="$STUBBIN:$PATH" \
php "$REPO/scripts/scan-files.php" --extract "$ex" --quarantine "$WORK/q-$tag" \
--report "$WORK/scan-$tag.json" --import-id "t-$tag" "$@" 2>"$WORK/scan-$tag.log"
}
jq_php() { php -r '$r = json_decode(file_get_contents($argv[1]), true); echo json_encode($r[$argv[2]] ?? null);' "$1" "$2"; }
echo "scan-files.php --skip-mail (mail tree extracted)"
cp -a "$WORK/ex-mail" "$WORK/scan-skip"
run_scan "$WORK/scan-skip" skip --skip-mail
# Expected argv entry: the extract dir with ERE metacharacters escaped
# (the only one mktemp's path can hold is "."), then the mail-tree pattern.
SKIPDIR="$WORK/scan-skip"
EXCL="--exclude-dir=^${SKIPDIR//./\\.}/[^/]+/homedir/mail(/.*)?\$"
if grep -qxF -- "$EXCL" "$WORK/clam-skip.args"; then
pass "--skip-mail: clamscan gets --exclude-dir for the mail tree"
else
fail "--skip-mail: clamscan gets --exclude-dir for the mail tree (want $EXCL; got: $(tr '\n' ' ' < "$WORK/clam-skip.args"))"
fi
check "--skip-mail: a hit inside the mail tree is NOT quarantined (message stays in the Maildir)" \
"[[ -f '$WORK/scan-skip/$U/homedir/mail/$D/alice/cur/1700000000.M1P1.host,S=40:2,S' ]]"
check "--skip-mail: a hit outside the mail tree is still quarantined" \
"[[ ! -e '$WORK/scan-skip/$U/homedir/public_html/index.php' && -f '$WORK/q-skip/$U/homedir/public_html/index.php' ]]"
check "--skip-mail: report files_quarantined = 1" "[[ \$(jq_php '$WORK/scan-skip.json' files_quarantined) == 1 ]]"
check "--skip-mail: report mail_scan_skipped = true" "[[ \$(jq_php '$WORK/scan-skip.json' mail_scan_skipped) == true ]]"
NONMAIL=$(cd "$WORK/scan-skip" && find . -type f ! -path "./$U/homedir/mail/*" | wc -l)
check "--skip-mail: files_scanned counts only non-mail files ($NONMAIL, after the quarantine move)" \
"[[ \$(jq_php '$WORK/scan-skip.json' files_scanned) == $((NONMAIL + 1)) ]]"
check "--skip-mail: the skip is logged with its reason" "grep -q 'NOT scanning the mail tree' '$WORK/scan-skip.log'"
echo "I-1: top-level dirs other than cpmove-* (acct/, backup-<date>_<user>/)"
for fx in acct:acct backupdir:backup-10.20.2026_10-00-00_testuser; do
name="${fx%%:*}"; top="${fx#*:}"
cp "$FIX/cpmove-$name.tar.gz" "$WORK/"
OUT="$WORK/ex-$name"
rc=0
IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-$name.tar.gz" "$OUT" testuser "$WORK/ex-$name.actions.json" >"$WORK/ex-$name.log" 2>&1 || rc=$?
M="$OUT/$top/homedir/mail/$D/alice"
check "$name: extract exits 0 (rc=$rc)" "[[ $rc -eq 0 ]]"
check "$name: relative symlinks in the Maildir stripped (.evil-rel, .Other)" "[[ ! -L '$M/.evil-rel' && ! -L '$M/.Other' ]]"
check "$name: no symlink at all under $top/homedir/mail" "[[ -z \$(find '$OUT/$top/homedir/mail' -type l) ]]"
check "$name: hard link INTO the mail tree stripped" "[[ ! -e '$M/cur/leak' ]]"
check "$name: in-tree Dovecot hard link kept" "[[ -f '$M/cur/1700000002.M3P1.host,S=41:2,S' ]]"
check "$name: strips recorded as stripped_mail_link" "grep -q '$top/homedir/mail/$D/alice/.evil-rel' '$WORK/ex-$name.actions.json'"
rm -rf "$WORK/q-$name"
CLAM_TOP="$top" CLAMSCAN_ARGS_OUT="$WORK/clam-$name.args" PATH="$STUBBIN:$PATH" \
php "$REPO/scripts/scan-files.php" --extract "$OUT" --quarantine "$WORK/q-$name" \
--report "$WORK/scan-$name.json" --import-id "t-$name" --skip-mail 2>/dev/null
check "$name: --skip-mail keeps a hit inside $top/homedir/mail in the Maildir" "[[ -f '$M/cur/1700000000.M1P1.host,S=40:2,S' ]]"
check "$name: --skip-mail still quarantines public_html" "[[ ! -e '$OUT/$top/homedir/public_html/index.php' ]]"
done
mkvt; mkdir -p "$V/acct/homedir/mail/$D/a/cur"; ln -s ../../../../../etc "$V/acct/homedir/mail/$D/a/x"
check "verify: symlink under acct/homedir/mail => 4" "[[ \$(vrc) == 4 ]]"
mkvt; ln -s "$U" "$V/zz-link"
check "verify: top-level symlink => 4 (never walked through)" "[[ \$(vrc) == 4 ]]"
if [[ -n "${WHP_REPO:-}" ]]; then
echo "M-4: contract with the WHP panel at \$WHP_REPO"
SAN="$WHP_REPO/web-files/libs/CpanelImportSanitizationContainer.php"
check "WHP sends the env name this image reads (INCLUDE_MAIL_ENV = 'IMPORT_INCLUDE_MAIL')" "grep -q \"INCLUDE_MAIL_ENV = 'IMPORT_INCLUDE_MAIL'\" '$SAN'"
check "WHP sends exactly '=true'" "grep -q \"INCLUDE_MAIL_ENV . '=true'\" '$SAN'"
check "WHP reads the report key this image writes (mail_included === true)" "grep -q \"\\['mail_included'\\] ?? null) === true\" '$SAN'"
check "this image writes mail_included" "grep -q '\"mail_included\"' '$REPO/scripts/entrypoint.sh'"
fi
echo "scan-files.php without --skip-mail (unchanged behaviour)"
cp -a "$WORK/ex-mail" "$WORK/scan-noskip"
run_scan "$WORK/scan-noskip" noskip
check "no flag: no --exclude-dir passed" "! grep -q -- '--exclude-dir' '$WORK/clam-noskip.args'"
check "no flag: report mail_scan_skipped = false" "[[ \$(jq_php '$WORK/scan-noskip.json' mail_scan_skipped) == false ]]"
check "no flag: both hits quarantined" "[[ \$(jq_php '$WORK/scan-noskip.json' files_quarantined) == 2 ]]"
echo
if [[ $FAILS -eq 0 ]]; then echo "ALL PASS"; else echo "$FAILS FAILURE(S)"; fi
exit $(( FAILS == 0 ? 0 : 1 ))