diff --git a/.gitea/scripts/resolve-base-image.sh b/.gitea/scripts/resolve-base-image.sh new file mode 100755 index 0000000..1c5b074 --- /dev/null +++ b/.gitea/scripts/resolve-base-image.sh @@ -0,0 +1,35 @@ +#!/bin/bash +## resolve-base-image.sh IMAGE:TAG — CI only (not copied into images). +## Prints IMAGE@sha256: for a Docker Hub tag, looked up DIRECTLY at +## registry-1.docker.io over HTTPS (never via the pkg-cache mirror), so the +## build can say `FROM IMAGE@sha256:...` and the mirror can only serve +## content-addressed blobs that BuildKit verifies. Exits non-zero (prints +## nothing on stdout) if the lookup fails; the workflow then skips the mirror +## and pulls the tag directly from Docker Hub. HEAD requests do not count +## against Docker Hub pull rate limits. +## Canonical copy: cloud-apache-container/.gitea/scripts/resolve-base-image.sh +## (docs/pkg-cache.md there); other repos carry identical copies. +set -euo pipefail +ref="${1:?usage: resolve-base-image.sh IMAGE:TAG}" +name_re='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)?$' +tag_re='^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$' +name="${ref%:*}"; tag="${ref##*:}" +[ "$name" != "$ref" ] || { echo "resolve-base-image: '$ref' has no :tag" >&2; exit 2; } +[[ "$name" =~ $name_re ]] || { echo "resolve-base-image: bad image name '$name'" >&2; exit 2; } +[[ "$tag" =~ $tag_re ]] || { echo "resolve-base-image: bad tag '$tag'" >&2; exit 2; } +repo="$name"; [[ "$repo" == */* ]] || repo="library/$repo" # official images + +curl_opts=(-fsS --proto '=https' --retry 3 --retry-all-errors --max-time 20) +token=$(curl "${curl_opts[@]}" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" | + python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])') +[ -n "$token" ] || { echo "resolve-base-image: no Docker Hub token for $repo" >&2; exit 1; } +hdrs=$(curl "${curl_opts[@]}" -I \ + -H "Authorization: Bearer ${token}" \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + "https://registry-1.docker.io/v2/${repo}/manifests/${tag}") +digest=$(printf '%s\n' "$hdrs" | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:" {print $2}' | tail -n1) +[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "resolve-base-image: no valid digest for $ref (got '${digest}')" >&2; exit 1; } +printf '%s@%s\n' "$name" "$digest" diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index 8d3a9d5..897439c 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -14,10 +14,64 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Base image pinned by DIGEST, looked up directly at Docker Hub (never via + # the pkg-cache mirror), so the mirror can only serve content-addressed + # blobs that BuildKit verifies. If the lookup fails this job does NOT use + # the mirror and pulls the tag straight from Docker Hub. + # Canonical docs: cloud-apache-container docs/pkg-cache.md. + - name: Resolve base image digest (Docker Hub, direct) + id: base + env: + BASE_TAG: almalinux:10-minimal + run: | + set -uo pipefail + if ref=$(.gitea/scripts/resolve-base-image.sh "$BASE_TAG") && [[ "$ref" =~ ^[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}$ ]]; then + echo "base-image: ${BASE_TAG} -> ${ref} (digest from registry-1.docker.io)" + echo "image=${ref}" >> "$GITHUB_OUTPUT" + echo "pinned=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::base image ${BASE_TAG}: Docker Hub digest lookup failed - pulling the tag directly from Docker Hub, pkg-cache registry mirror NOT used" + echo "image=${BASE_TAG}" >> "$GITHUB_OUTPUT" + echo "pinned=false" >> "$GITHUB_OUTPUT" + fi + + # Internal package cache (CT130 pkg-cache). Used ONLY if it answers over + # TLS with our pinned CA; otherwise (down, unreachable, bad/expired cert) + # this job builds from the upstream repos and Docker Hub exactly as before. + # The repo cache and the registry mirror are probed separately so either + # can fail back on its own. + - name: Probe package cache + id: pkgcache + # The registry mirror is only used for a digest-pinned base image. + env: + BASE_PINNED: ${{ steps.base.outputs.pinned }} + run: | + url=""; mirror="" + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi + if [ "$BASE_PINNED" = true ] && curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi + echo "pkg-cache: repos=${url:-} registry-mirror=${mirror:-}" + [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories" + echo "url=${url}" >> "$GITHUB_OUTPUT" + echo "mirror=${mirror}" >> "$GITHUB_OUTPUT" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + # BuildKit (docker-container driver) ignores dockerd's registry-mirrors, + # so the mirror is configured on the builder; BuildKit itself still falls + # back to Docker Hub if the mirror stops answering mid-run. + - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror) + if: steps.pkgcache.outputs.mirror != '' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["192.168.1.40:5000"] + [registry."192.168.1.40:5000"] + ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"] + - name: Set up Docker Buildx + if: steps.pkgcache.outputs.mirror == '' uses: docker/setup-buildx-action@v3 - name: Login to Gitea @@ -58,6 +112,9 @@ jobs: platforms: linux/amd64 push: false load: true + build-args: | + BASE_IMAGE=${{ steps.base.outputs.image }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} tags: cpanel-importer:smoke no-cache: true @@ -101,6 +158,9 @@ jobs: context: . platforms: linux/amd64 push: true + build-args: | + BASE_IMAGE=${{ steps.base.outputs.image }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} tags: ${{ steps.tags.outputs.tags }} cache-from: type=registry,ref=repo.anhonesthost.net/cloud-hosting-platform/cpanel-importer:latest cache-to: type=inline diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 67bf69d..f394afc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,9 +121,23 @@ ClamAV signature DBs). bash tests/build-fixtures.sh ``` -Two tarballs land under `tests/fixtures/`: +Three tarballs land under `tests/fixtures/`: - `cpmove-clean.tar.gz` — a benign cpmove with a WordPress MyISAM dump. - `cpmove-alfa.tar.gz` — same shape PLUS an ALFA-style symlink to /etc. +- `cpmove-mail.tar.gz` — same shape PLUS a mail tree (two Maildirs, one + empty) with an ALFA-style symlink inside it. + +### Host-side regression tests + +```bash +bash tests/run-tests.sh +``` + +Runs `extract.sh` and `scan-files.php` straight from the checkout (bash, +php-cli, python3, GNU tar; `clamscan` is stubbed). Covers the mail-tree +opt-in (`IMPORT_INCLUDE_MAIL=true`): skipped by default, extracted when +opted in, dangerous symlinks inside it still stripped, and never scanned +or quarantined by `scan-files.php --skip-mail`. ### Run against the clean fixture diff --git a/Dockerfile b/Dockerfile index e50d8ad..6f567c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,12 @@ # Build: docker build -t cpanel-importer:dev . # Run: see README.md for the docker run invocation the WHP panel uses. -FROM almalinux:10-minimal +# Base image. CI passes BASE_IMAGE=almalinux@sha256:, resolved +# DIRECTLY from Docker Hub in the same run (.gitea/scripts/resolve-base-image.sh), +# so the pkg-cache registry mirror can only serve content-addressed blobs. +# Local builds use the tag. Canonical docs: cloud-apache-container docs/pkg-cache.md. +ARG BASE_IMAGE=almalinux:10-minimal +FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title="cpanel-importer" LABEL org.opencontainers.image.description="cPanel cpmove sanitization sandbox (ClamAV + SaneSecurity + WP content scan)" @@ -28,9 +33,9 @@ ENV LANG=C.UTF-8 \ # the SaneSecurity rsync temp files at the end of the layer. # # Pinning strategy: -# - PHP 8.4: AlmaLinux 10 stock ships PHP 8.3 only; the spec asks for -# 8.4 specifically. We add Remi's modular repo and enable the -# `php:remi-8.4` stream. We DO NOT pin to a specific 8.4.X because +# - PHP 8.5: AlmaLinux 10 stock ships PHP 8.3 only; the spec asks for +# a current Remi stream (8.5; the 8.4 stream had a known exploit). We +# add Remi's modular repo and enable the `php:remi-8.5` stream. We DO NOT pin to a specific 8.5.X because # Remi rolls security patches into the same minor and an exact pin # would block updates. # - clamav / clamav-update: track the AL10 EPEL stream. CI builds @@ -43,18 +48,30 @@ ENV LANG=C.UTF-8 \ # clamav first, that lands at GID 999 — which then collides with the # UID/GID we want for whp-import. We pre-create our user FIRST so # virusgroup ends up at 998. +# +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 +# (CT130 pkg-cache, https only). Unset = upstream repositories. epel-release +# and remi-release are GPG-verified against keys/pkg-cache/ either way +# (pkg-cache-repos.sh release-install). Repo files are restored at the end of +# the RUN, so no layer keeps the cache address. Same script as +# cloud-apache-container (scripts/pkg-cache-repos.sh, docs/pkg-cache.md there). +ARG PKG_CACHE= +COPY scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh +COPY keys/pkg-cache/ /usr/local/share/pkg-cache/ RUN set -eux; \ + /usr/local/sbin/pkg-cache-repos.sh rpm ; \ # microdnf is what almalinux:10-minimal ships with by default. microdnf -y install --setopt=install_weak_deps=0 \ - epel-release \ dnf \ shadow-utils \ ; \ - # Add Remi's repo for PHP 8.4 (AL10 stock has 8.3 only). - dnf -y --setopt=install_weak_deps=0 install \ - https://rpms.remirepo.net/enterprise/remi-release-10.rpm ; \ + # EPEL + Remi's repo for PHP 8.5 (AL10 stock has 8.3 only); release + # RPMs signature-checked against the committed keys, then the new repo + # files are pointed at the cache too (no-op without PKG_CACHE). + /usr/local/sbin/pkg-cache-repos.sh release-install epel remi ; \ + /usr/local/sbin/pkg-cache-repos.sh rpm ; \ dnf -y --setopt=install_weak_deps=0 module reset php ; \ - dnf -y --setopt=install_weak_deps=0 module enable php:remi-8.4 ; \ + dnf -y --setopt=install_weak_deps=0 module enable php:remi-8.5 ; \ # Pre-create the worker BEFORE installing clamav so virusgroup # doesn't claim our GID. groupadd --system --gid ${WHP_GID} whp-import ; \ @@ -98,6 +115,7 @@ RUN set -eux; \ # an EXISTING host dir keep host ownership and are independent of # this — the panel sets up its own dirs with mode 750 owner 999.) chown whp-import:whp-import /host/quarantine /host/sanitized ; \ + /usr/local/sbin/pkg-cache-repos.sh restore ; \ # Strip dnf cache. dnf -y clean all ; \ rm -rf /var/cache/dnf /var/cache/yum /var/cache/ldconfig/* \ diff --git a/README.md b/README.md index cee07e8..7aeb393 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ docker run \ --env IMPORT_USERNAME=${USERNAME} \ --env IMPORT_BACKUP_FILE=/host/backup/${BACKUP_NAME} \ --env CLAMAV_REFRESH=true \ + --env IMPORT_INCLUDE_MAIL=true \ --memory=4g \ --memory-swap=4g \ --cpus=2 \ @@ -57,6 +58,15 @@ docker run \ repo.anhonesthost.net/cloud-hosting-platform/cpanel-importer:2026.05.NNN ``` +`IMPORT_INCLUDE_MAIL=true` is optional: WHP passes it only when the import +copies mailboxes. Without it `cpmove-*/homedir/mail` is not extracted at all. +With it the mail tree is extracted (dangerous symlinks inside it are still +stripped) but **not** ClamAV-scanned: mail is copied verbatim into the +customer's own mailbox and verified message-by-message, so quarantining a +message would silently drop mail, and multi-GB maildirs would dominate the +run. `report.json` then carries `"mail_included": true`; images older than +this option never write the key, which is how WHP tells them apart. + Container exits with status `0` on success, non-zero on any failure (missing/unreadable backup, dangerous symlink found, scanner error). Even on failure, `/host/sanitized//report.json` is written @@ -94,6 +104,7 @@ run, success or failure. "files_clean": 28432, "files_cleaned": 0, "files_quarantined": 39, + "mail_included": false, "actions": [ { "path": "cpmove-testuser/homedir/public_html/example.com/ALFA_DATA/index.php", diff --git a/keys/pkg-cache/README b/keys/pkg-cache/README new file mode 100644 index 0000000..c8edeec --- /dev/null +++ b/keys/pkg-cache/README @@ -0,0 +1,32 @@ +Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh, +docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing +here is added to the system trust store. + +ca.crt "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18. + NAME-CONSTRAINED to IP 192.168.1.40 and DNS + pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local. + SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3 + (private key lives only on CT130: /etc/pkgcache-ca/ca.key) +RPM-GPG-KEY-EPEL-10 Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158 + fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and + mirror.math.princeton.edu (identical); fingerprint matches + https://fedoraproject.org/security/ +RPM-GPG-KEY-remi2024 Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4 + fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical + to the key shipped in remi-release-10.2-1.el10.remi. Not found on + keys.openpgp.org / keyserver.ubuntu.com - confirm out of band. + Remi rotates keys (remi2025, ...): when remi-release starts being + signed by a new key, `release-install remi` fails loudly - add the + new key here (and to pkg-cache-repos.sh). +RPM-GPG-KEY-EPEL-9 Fedora (epel9) FF8A D134 4597 106E CE81 3B91 8A38 72BF 3228 467C + fetched 2026-10-22 over HTTPS from dl.fedoraproject.org and + mirror.math.princeton.edu (identical); used by `release-install epel` + on EL9 images (cloud-memcached-container, cloud-haproxy-container). +RPM-GPG-KEY-nodesource Nodesource Operations + 242B 8138 31AF 0956 2B6C 46F7 6B88 DA4E 3AF2 8A14 (rsa4096, 2023-12-22) + fetched 2026-10-22 over HTTPS from + https://rpm.nodesource.com/gpgkey/ns-operations-public.key (the key + NodeSource's own setup_N.x scripts reference); verified by dnf + gpgcheck against the nodejs RPMs. Not on keys.openpgp.org / + keyserver.ubuntu.com - confirm out of band. Used by + `pkg-cache-repos.sh nodesource MAJOR` (cloud-node-container). diff --git a/keys/pkg-cache/RPM-GPG-KEY-EPEL-10 b/keys/pkg-cache/RPM-GPG-KEY-EPEL-10 new file mode 100644 index 0000000..adf4f88 --- /dev/null +++ b/keys/pkg-cache/RPM-GPG-KEY-EPEL-10 @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA +Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK +Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L +URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf +mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2 +XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI +uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60 +GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix +fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z +BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77 +HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB +tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB +CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC +BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p +l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp +yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg +Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u +scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs +7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK +F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg +sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB +0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY +1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt +TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k +p9Dfd2pKuxK9Mys= +=mhQZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/keys/pkg-cache/RPM-GPG-KEY-EPEL-9 b/keys/pkg-cache/RPM-GPG-KEY-EPEL-9 new file mode 100644 index 0000000..234c12f --- /dev/null +++ b/keys/pkg-cache/RPM-GPG-KEY-EPEL-9 @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGE3mOsBEACsU+XwJWDJVkItBaugXhXIIkb9oe+7aadELuVo0kBmc3HXt/Yp +CJW9hHEiGZ6z2jwgPqyJjZhCvcAWvgzKcvqE+9i0NItV1rzfxrBe2BtUtZmVcuE6 +2b+SPfxQ2Hr8llaawRjt8BCFX/ZzM4/1Qk+EzlfTcEcpkMf6wdO7kD6ulBk/tbsW +DHX2lNcxszTf+XP9HXHWJlA2xBfP+Dk4gl4DnO2Y1xR0OSywE/QtvEbN5cY94ieu +n7CBy29AleMhmbnx9pw3NyxcFIAsEZHJoU4ZW9ulAJ/ogttSyAWeacW7eJGW31/Z +39cS+I4KXJgeGRI20RmpqfH0tuT+X5Da59YpjYxkbhSK3HYBVnNPhoJFUc2j5iKy +XLgkapu1xRnEJhw05kr4LCbud0NTvfecqSqa+59kuVc+zWmfTnGTYc0PXZ6Oa3rK +44UOmE6eAT5zd/ToleDO0VesN+EO7CXfRsm7HWGpABF5wNK3vIEF2uRr2VJMvgqS +9eNwhJyOzoca4xFSwCkc6dACGGkV+CqhufdFBhmcAsUotSxe3zmrBjqA0B/nxIvH +DVgOAMnVCe+Lmv8T0mFgqZSJdIUdKjnOLu/GRFhjDKIak4jeMBMTYpVnU+HhMHLq +uDiZkNEvEEGhBQmZuI8J55F/a6UURnxUwT3piyi3Pmr2IFD7ahBxPzOBCQARAQAB +tCdGZWRvcmEgKGVwZWw5KSA8ZXBlbEBmZWRvcmFwcm9qZWN0Lm9yZz6JAk4EEwEI +ADgWIQT/itE0RZcQbs6BO5GKOHK/MihGfAUCYTeY6wIbDwULCQgHAgYVCgkICwIE +FgIDAQIeAQIXgAAKCRCKOHK/MihGfFX/EACBPWv20+ttYu1A5WvtHJPzwbj0U4yF +3zTQpBglQ2UfkRpYdipTlT3Ih6j5h2VmgRPtINCc/ZE28adrWpBoeFIS2YAKOCLC +nZYtHl2nCoLq1U7FSttUGsZ/t8uGCBgnugTfnIYcmlP1jKKA6RJAclK89evDQX5n +R9ZD+Cq3CBMlttvSTCht0qQVlwycedH8iWyYgP/mF0W35BIn7NuuZwWhgR00n/VG +4nbKPOzTWbsP45awcmivdrS74P6mL84WfkghipdmcoyVb1B8ZP4Y/Ke0RXOnLhNe +CfrXXvuW+Pvg2RTfwRDtehGQPAgXbmLmz2ZkV69RGIr54HJv84NDbqZovRTMr7gL +9k3ciCzXCiYQgM8yAyGHV0KEhFSQ1HV7gMnt9UmxbxBE2pGU7vu3CwjYga5DpwU7 +w5wu1TmM5KgZtZvuWOTDnqDLf0cKoIbW8FeeCOn24elcj32bnQDuF9DPey1mqcvT +/yEo/Ushyz6CVYxN8DGgcy2M9JOsnmjDx02h6qgWGWDuKgb9jZrvRedpAQCeemEd +fhEs6ihqVxRFl16HxC4EVijybhAL76SsM2nbtIqW1apBQJQpXWtQwwdvgTVpdEtE +r4ArVJYX5LrswnWEQMOelugUG6S3ZjMfcyOa/O0364iY73vyVgaYK+2XtT2usMux +VL469Kj5m13T6w== +=Mjs/ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/keys/pkg-cache/RPM-GPG-KEY-nodesource b/keys/pkg-cache/RPM-GPG-KEY-nodesource new file mode 100644 index 0000000..0618885 --- /dev/null +++ b/keys/pkg-cache/RPM-GPG-KEY-nodesource @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGWFsqMBEAC4YlSuNpHjOiQVTC8StolVx3J8E9SC6G4wmVWbytMD1Zivfanj +3LKDqdy7vpbViXA9whpyGQgQYdOlPKaNHWw4GSeGTnP1FbvaK1NJ9MnAzUysh49R +cqoktYeIQbqgOgZHw5/gVH5bYObf1KJQ2LY/je9WTlW3QMRe1TaW0F0AZnJMBIOG +JllE33QIpxDnOQHnvdUGq+HscUOTrW4Q3AdzwsQxU4GX82d9qFFr8HjAmUjlUKyh +PgvMpb8VDvR7NE6et+wWh6BMYILo3VJFP7/gTXGiy3WWs5GW/PNXIXw5/f2w1B+e +Dh5tlfGG4q/4P8mbv7C6RPHoF2Lq+W69HLqSq/l42WUL41tHlFTsluCIKpeeMxAu +olxx3Z5momHMMyh7rbZtQDqFMTAqxNjT1NinKkWyj7KP5WE9hVosyd3UMR4UYDMm +Zb6LI4N7Hht887+CY7grxTJ9cGQhMFfvLyvlf/FWQY4S6G7sEaUV71N9StvXBNG/ +iNoIMzhwjBmrmvT/UGPkLSi0SIBOxeGw2rwngnYikIJxuTQ4V40lm8o6eMFca7rW +BQvAEncV8pvi/3FqrUBZ/U53s7NO24hBDoaQPzFbVCw0GHHf9GD7KoQb5c9ONoCZ +Qu2YWo7ajPLzl/h2pJ2Kn29Ywev9MpuuPTGvifiKilcBVLYP3ZJg4ZfRBwARAQAB +tDFOb2Rlc291cmNlIE9wZXJhdGlvbnMgPG9wZXJhdGlvbnNAbm9kZXNvdXJjZS5j +b20+iQJOBBMBCgA4FiEEJCuBODGvCVYrbEb3a4jaTjryihQFAmWFsqMCGwMFCwkI +BwIGFQoJCAsCBBYCAwECHgECF4AACgkQa4jaTjryihTD+hAAtTU155ke7xmnT0ph +odbGjm3mOGdkdaDH+xmWtPtufiPncgHMCYh4aIDRloog8rFE+pP3qfwJmUKxthwD +drrLAzfUwOrGxPnYH3MWwg66RVVoiiU+Hh42gJTez+P/K0VMQZXH0XTZD96TiwT/ +S0+WvQPuJ9pda2L4cjV+YP2+m0ERUG9QPj0Shgw+/PWCGWKD6J9DRToiwfwCTOvc +DNolF1EcRlfPMTsqI2Bdc+TQywqU1Py1MUDSAPXxdK6nW2oM7eC5l3XXefcQ6JOS +AuNdEDIsaf+H76j6/wSGsuIcJjUo9wmKjEV1iJYwT6jLbjaVQxXgEPHBSb+kzanx +FWyrYP6TZpeY2gT4bBMsOKd7ay1I3/nIHwe6x8Yzkqzz+7YVhNpMbybH72NTxDWQ +UmiboJs9r7j1lkHlBjxU9RLF5laYSeCIXD62KQVoP5ao1LSPKsCXFqcdpUgLYtRv +CXxdriiHp4m9eGzbS7RW1TLm6tMOKbI4LzOtMm4DoZzGD2FTfsUtn6ALx9FSgvl/ +VMlzqZes+rclcukACtutvtB8e0yRhuzPUN56ctLEZfI1XzQFoN8Vdj6HWtiXWRiJ +EohuufE2cEebmX90NRsbh4KAFXNEQROlMeMf7ufuSsk7QXLTutxmWSQvWwyS+P5/ +ZqCapIDlRCJFZX4N9a1IXo0OgCC5Ag0EZYWyowEQAKz7sA6WBKFVcmzcyr6y9TI5 +RemnzotRUdqXj8tD70xvV1nBEf+BfTw3hsiV019ZboPBae+cRaD4plQbNb0du/+Q +UFq4ja055ZEld01RqE49L562pWvA5cwSmjs9+ZMI1ja0HbTb2K4razpzX3be+Zul +TbH20X5FHTz+Jyfmw+G3Y9okw3OIzuYo16Lv/Qs8VoBmj01PItzq8dX5wJiIlVxU +8wKHKy1kiKGCgyPnvqcXCGen0EoUS9wsWK1QHfryscYGVF9y1TBQvJa9HIpWPrIZ +slSiVS4tQhGHfLDVlaDsYFrH9SIUkvW7peSEiPE1TKQjvbMS3CcUZRaMcGfgyDPe +gXwnNwZbJ2/cisXbdAJaGT49V1Y+Sc1oKSGSG4FaGUNhS3p3cB7qlX2zS/sR2ael +90fXR6nhjvhMAS98i7663pCUv9q0Z/NXNZwdXjbSF4W8gq3PVt7hCnJrngsGSkGu +Qz0dc227/sAjh9CxYm8kQqUuQtiPVfKwkcUUT+O7jr2/6eAXZbTYVZbvaLEGMrKA +bdWKpZMMOKAI45f+ZAcEmJG4D68N0jzVrLCXMuuLq/yqqyEmxDjSOH0qVutl8jxg +vfxPxKYX8nnAzJZJHGHmV/ihjfClSdYePKExHuFVVhvc4BxE1XfPUeCZG4JpB9/B +gNCiDPsg4JIw6GO5LS2xABEBAAGJAjYEGAEKACAWIQQkK4E4Ma8JVitsRvdriNpO +OvKKFAUCZYWyowIbDAAKCRBriNpOOvKKFOTKD/40WN6ZRfP6hVLAX26DdyN9Bqzg +D+0hJilmHL1YFeLt9U/YPhOZJxPaDGrMfldC0FSXKKr7H5C7FQErl936108wGohz +53KGbNid0crNeaWrQuWk3QjaMF62Yc+NVB8b5LoEk0TEt9KPO6kveYDtW6Ygkw9i +5aaS/AC+8GChOH6lWhLHcO91T7+Hi5qRez7qyme/AlHUTV9/jE6JgFJ7thCCAqBQ +AfX3mGFlTwSTMHDeYq81MMl9UhYgoFJtx8PuqmSV7Eb29ccJofz3J+b5OJvpmLSZ +Hu22Csa8k2V/1ojeL0DjQHD06V75F7BerxLR/rbJ64GkIiOdm4ntXC4a335gjQfM +5tI6LtGsIpr+Le4C068eDj7YpzS0ql3g6o4A1TbtapLccgEj4iw95H4KKh0Ro5r0 +0s6LoSLu9uva3LiLVn6xXvpw3iv8FnhTr5PEiPnorXvxcCnUrB3Rkk6Wh1SKjpxa +u6KiFzH0CXR1AgUdWSC0rAmvEdFjXgaoQd+Z2eJFHEAsg91URd9NtdeDsyMwdbHh +iBZPGX5ddPjJA7nwKbkibCv7xiodd9ph8NmGLqD/2SDU1rqvPuCDMqLuxc89Gvhs +3CeLTSJK9Uzb3oNdZLxL8CECip1ZkowVfTL+jY23phS9+Z0Db/yZqot3ZZJAAG8g +oSVkzT/Mwp9zFVQu5w== +=drMP +-----END PGP PUBLIC KEY BLOCK----- diff --git a/keys/pkg-cache/RPM-GPG-KEY-remi2024 b/keys/pkg-cache/RPM-GPG-KEY-remi2024 new file mode 100644 index 0000000..fe82d62 --- /dev/null +++ b/keys/pkg-cache/RPM-GPG-KEY-remi2024 @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W +Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe +UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa +qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN +ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB +jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO +/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL +yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ +w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN +0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l +ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB +tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly +ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd +/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX +gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx +Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W +4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/ +/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz +/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc +3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof +ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS +dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy +sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n +i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd +DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P +7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI +T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ +OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3 +cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W +RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x +5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m +8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD +pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r +N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm +Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG +Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA +AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9 +OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI +wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA +qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c +WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV +u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI +qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S +4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4 +MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II +Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl +bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s +r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1 +=meaU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/keys/pkg-cache/ca.crt b/keys/pkg-cache/ca.crt new file mode 100644 index 0000000..4413e94 --- /dev/null +++ b/keys/pkg-cache/ca.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw +TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV +BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN +MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg +KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV +OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/ +v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw +a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu +Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg +cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM +YAIl +-----END CERTIFICATE----- diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index d78564c..ed2ca83 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -8,6 +8,11 @@ # IMPORT_BACKUP_FILE absolute path inside the container, typically # /host/backup/cpmove-.tar.gz # CLAMAV_REFRESH "true" to run freshclam at start (default: true) +# IMPORT_INCLUDE_MAIL "true" to extract cpmove-*/homedir/mail for WHP's +# mailbox import (default: false = mail tree skipped). +# The mail tree is never ClamAV-scanned. report.json +# records "mail_included" so the panel can tell this +# image (which honours the opt-in) from an older one. # # Flow (spec §0): # 1. validate env @@ -71,8 +76,20 @@ log "cpanel-importer starting (container UID=$(id -u) GID=$(id -g))" : "${IMPORT_USERNAME:?IMPORT_USERNAME env var is required}" : "${IMPORT_BACKUP_FILE:?IMPORT_BACKUP_FILE env var is required}" CLAMAV_REFRESH="${CLAMAV_REFRESH:-true}" +# Exactly "true" opts in; anything else is the default (no mail tree). +if [[ "${IMPORT_INCLUDE_MAIL:-false}" == "true" ]]; then + IMPORT_INCLUDE_MAIL=true +else + IMPORT_INCLUDE_MAIL=false +fi +export IMPORT_INCLUDE_MAIL +# The mail-tree verify's readability check (os.access) is meaningless for +# root; the image runs as uid 999 and the opt-in path insists on it. +if [[ "$IMPORT_INCLUDE_MAIL" == "true" && "$(id -u)" == "0" ]]; then + die "IMPORT_INCLUDE_MAIL=true must not run as root (container --user 999:999 expected)" +fi -log "import_id=$IMPORT_ID username=$IMPORT_USERNAME backup=$IMPORT_BACKUP_FILE" +log "import_id=$IMPORT_ID username=$IMPORT_USERNAME backup=$IMPORT_BACKUP_FILE include_mail=$IMPORT_INCLUDE_MAIL" if [[ ! -f "$IMPORT_BACKUP_FILE" ]]; then die "backup file does not exist or is not a regular file: $IMPORT_BACKUP_FILE" @@ -140,11 +157,17 @@ fi STAGE="scan_files" log "stage: scan_files" +SCAN_FILES_EXTRA=() +if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then + # The extracted mail tree is not scanned (see scan-files.php --skip-mail). + SCAN_FILES_EXTRA+=(--skip-mail) +fi php /scripts/scan-files.php \ --extract "$EXTRACT_DIR" \ --quarantine "$QUARANTINE_DIR" \ --report /tmp/scan-files-report.json \ --import-id "$IMPORT_ID" \ + "${SCAN_FILES_EXTRA[@]}" \ || die "scan-files.php failed; see stderr above" # --- DB engine swap + WP content scan ------------------------------------- @@ -192,6 +215,7 @@ $filesPath = $argv[3]; $dbsPath = $argv[4]; $strippedPath = $argv[5]; $outPath = $argv[6]; +$mailIncluded = $argv[7] === "true"; $files = is_file($filesPath) ? json_decode(file_get_contents($filesPath), true) : null; $dbs = is_file($dbsPath) ? json_decode(file_get_contents($dbsPath), true) : null; @@ -252,6 +276,10 @@ $report = [ "files_clean" => $filesClean, "files_cleaned" => $filesCleaned, "files_quarantined" => $filesQuarantined, + // true only when IMPORT_INCLUDE_MAIL=true AND this image extracted the + // mail tree. Older images never write the key; the panel treats a + // missing/false value as "mail was not extracted". + "mail_included" => $mailIncluded, "actions" => $actions, "databases" => $databases, "summary_for_panel" => [ @@ -263,7 +291,7 @@ $report = [ file_put_contents($outPath, json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); fprintf(STDERR, "report written: %s\n", $outPath); -' "$IMPORT_ID" "$DURATION" /tmp/scan-files-report.json /tmp/scan-dbs-report.json "$STRIPPED_SYMLINKS_FILE" "$SANITIZED_DIR/report.json" \ +' "$IMPORT_ID" "$DURATION" /tmp/scan-files-report.json /tmp/scan-dbs-report.json "$STRIPPED_SYMLINKS_FILE" "$SANITIZED_DIR/report.json" "$IMPORT_INCLUDE_MAIL" \ || die "report merge failed" log "done — exited cleanly after ${DURATION}s" diff --git a/scripts/extract.sh b/scripts/extract.sh index e42c447..07e6927 100755 --- a/scripts/extract.sh +++ b/scripts/extract.sh @@ -4,6 +4,10 @@ # # Usage: extract.sh [] # +# Env: IMPORT_INCLUDE_MAIL "true" = extract cpmove-*/homedir/mail too +# (WHP's mailbox import needs the Maildirs). +# Anything else (default) = skip the mail tree. +# # Calls scripts/lib/scan-symlinks.php first, then untars the cpmove with # every DANGEROUS-classified symlink entry stripped via tar --exclude. # The stripped-symlinks list is written as JSON to (default @@ -21,23 +25,45 @@ DEST="${2:?usage: extract.sh []}" USERNAME="${3:?usage: extract.sh []}" ACTIONS_OUT="${4:-${DEST}/.cpanel-importer-stripped-symlinks.json}" +IMPORT_INCLUDE_MAIL="${IMPORT_INCLUDE_MAIL:-false}" +# Resolve sibling scripts relative to this file (/scripts in the image) so +# tests/run-tests.sh can run the same script from a checkout. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ts() { date -u +'%Y-%m-%dT%H:%M:%SZ'; } log() { printf '[%s] extract: %s\n' "$(ts)" "$*"; } [[ -f "$TARBALL" ]] || { log "tarball not found: $TARBALL"; exit 2; } mkdir -p "$DEST" +# Any failure after this point removes the (partial) extract before exiting: +# it is the container's own scratch dir inside the bind-mounted output, and a +# half-extracted tree, or one the link check refused, must not be left +# behind for anything to read. Directories the archive made unreadable are +# opened up first (find -P never follows links; -type d skips them) so rm +# can descend. +fail_extract() { + local code="$1"; shift + log "$*" + cd / + if [[ -n "$DEST" && "$DEST" != "/" && -d "$DEST" ]]; then + find -P "$DEST" -type d ! -perm -u=rwx -exec chmod u+rwx {} \; 2>/dev/null || true + rm -rf -- "$DEST" || log "WARN: could not remove the failed extract $DEST" + log "removed the failed extract $DEST" + fi + exit "$code" +} + # --- pre-extract symlink scan --------------------------------------------- log "scanning tarball for dangerous symlinks (cpmove vector check)" SYMLINK_REPORT=$(mktemp -p /tmp scan-symlinks.XXXXXX.json) -if ! php /scripts/lib/scan-symlinks.php \ +if ! php "$SCRIPT_DIR/lib/scan-symlinks.php" \ --tarball "$TARBALL" \ --username "$USERNAME" \ --report "$SYMLINK_REPORT"; then - log "scan-symlinks.php exited with usage/IO error; aborting (this is not a sanitize-able state)" cat "$SYMLINK_REPORT" >&2 || true - exit 3 + fail_extract 3 "scan-symlinks.php exited with usage/IO error; aborting (this is not a sanitize-able state)" fi # --- compute exclude list from dangerous findings ------------------------- @@ -86,19 +112,35 @@ if [[ "$DANGEROUS_COUNT" -gt 0 ]]; then done < "$EXCLUDES_FILE" fi -# Also skip the cpanel user's mail spool entirely. WHP doesn't import -# email (mail-import is a panel-side roadmap item, not a sandbox-mode -# step) so extracting + scanning the mailbox tree wastes time and disk: -# on real customer accounts the mail dir often dwarfs everything else -# (10+ GB of historical mbox/maildir). The exclude pattern uses tar's -# fnmatch globs and covers both maildir + spool layouts at any depth -# under homedir/. Logged as an info line so it shows up in the panel -# import log alongside the symlink strips. -{ - echo "cpmove-*/homedir/mail" - echo "cpmove-*/homedir/mail/*" -} >> "$EXCLUDES_FILE" -log "skipping mail tree (cpmove-*/homedir/mail) — WHP does not import cPanel mail" +# Mail tree (cpmove-*/homedir/mail). Skipped by default: a plain site +# import doesn't need it, and on real customer accounts the mail dir often +# dwarfs everything else (10+ GB of historical maildir). The exclude +# pattern uses tar's fnmatch globs and covers the whole tree at any depth. +# +# IMPORT_INCLUDE_MAIL=true (WHP passes it only when the import copies +# mailboxes) extracts it. The link policy and verify below apply to +# /homedir/mail, not only cpmove-*: WHP accepts any top-level +# dir name. (The default-mode exclude keeps its historical cpmove-* scope.) The DANGEROUS-symlink strips computed above apply +# to the mail tree exactly as to the rest of the archive: scan-symlinks.php +# lists every entry in the tarball, mail included. +if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then + log "including mail tree (cpmove-*/homedir/mail) — IMPORT_INCLUDE_MAIL=true" + # Link policy for the mail tree (scripts/lib/mail-tree-links.py): every + # symlink in it, and every hard link crossing its boundary, is excluded + # here; Dovecot's in-tree hard-link copies are kept. The archive is + # refused outright if any member name could be read differently by the + # classifier and by tar (control characters, non-canonical paths). + # scan-symlinks.php alone only strips DANGEROUS absolute targets. + MAIL_LINKS=$(python3 -I "$SCRIPT_DIR/lib/mail-tree-links.py" scan "$TARBALL" "$EXCLUDES_FILE" "$ACTIONS_OUT") \ + || fail_extract 3 "mail-tree link scan refused the archive (see above); aborting — the same backup may import with 'Import without email'" + log "stripped $MAIL_LINKS symlink / boundary-crossing hard-link entr(y/ies) in the mail tree" +else + { + echo "cpmove-*/homedir/mail" + echo "cpmove-*/homedir/mail/*" + } >> "$EXCLUDES_FILE" + log "skipping mail tree (cpmove-*/homedir/mail) — IMPORT_INCLUDE_MAIL not set" +fi # --- extract -------------------------------------------------------------- @@ -125,9 +167,44 @@ log "extracting with hardened tar flags into $DEST" # match is effectively a literal-path skip. # --absolute-names is NOT used — leading / in a member name is stripped. cd "$DEST" +# tar's exit code is captured, not fatal on the spot: the mail-tree verify +# below must run on whatever tar did write, and a failed tar must still get +# its partial tree removed. +TAR_RC=0 tar --no-same-owner --no-same-permissions --no-overwrite-dir \ --exclude-from="$EXCLUDES_FILE" \ - $TAR_FLAGS "$TARBALL" + $TAR_FLAGS "$TARBALL" || TAR_RC=$? + +# N2-1 authority for the WHOLE tree: the classifier above resolves relative +# targets lexically, so a chain through another link (a -> b/.., b -> x/y/z) +# can pass it. strip-escaping-links.py asks realpath() where every extracted +# symlink really lands and removes (records) any that leave $DEST, point at +# $DEST itself, dangle or loop. A walk error fails the extract closed. +# Only the TRUSTED username (, the WHP account WHP passes) is +# exempted, and only re-rooted inside the extract's homedir; names read from +# the archive (top-dir names) are never trusted. +STRIP_RC=0 +ESCAPING=$(python3 -I "$SCRIPT_DIR/lib/strip-escaping-links.py" "$DEST" "$ACTIONS_OUT" "$USERNAME") || STRIP_RC=$? +if [[ $STRIP_RC -eq 0 && "${ESCAPING:-0}" -gt 0 ]]; then + log "stripped $ESCAPING symlink(s) that resolve outside the extract, dangle or loop (post-extract check)" +fi + +VERIFY_RC=0 +if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then + # The authority: whatever the classifier above decided, no symlink, no + # hard link leaving the mail tree, and no uninspectable directory may + # exist in the extract. Always runs, even after a failed tar. + python3 -I "$SCRIPT_DIR/lib/mail-tree-links.py" verify "$DEST" || VERIFY_RC=$? +fi +if [[ $TAR_RC -ne 0 ]]; then + fail_extract "$TAR_RC" "tar failed (rc=$TAR_RC)$( [[ $VERIFY_RC -ne 0 ]] && echo "; mail-tree verify also failed (rc=$VERIFY_RC)" ); aborting" +fi +if [[ $STRIP_RC -ne 0 ]]; then + fail_extract 4 "post-extract symlink check failed (rc=$STRIP_RC); aborting" +fi +if [[ $VERIFY_RC -ne 0 ]]; then + fail_extract 4 "extracted mail tree failed the link/readability check (see above); aborting — the same backup may import with 'Import without email'" +fi log "extracted OK ($(find "$DEST" -type f | wc -l) files; $DANGEROUS_COUNT symlinks stripped)" exit 0 diff --git a/scripts/lib/mail-tree-links.py b/scripts/lib/mail-tree-links.py new file mode 100644 index 0000000..17b2a98 --- /dev/null +++ b/scripts/lib/mail-tree-links.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +mail-tree-links.py — link policy for cpmove-*/homedir/mail when the mail +tree is extracted (IMPORT_INCLUDE_MAIL=true). Called by extract.sh. + +A Maildir has no legitimate symlinks. Hard links are different: Dovecot +copies a message between folders as a hard link (maildir_copy_with_hardlinks, +default yes), so a link whose name AND target are both inside the same +account's mail tree is real mail and is kept. Every other link touching a +mail tree is stripped (pre-extract) or fails the extract (post-extract). + + scan + Pre-extract classifier (an optimisation; `verify` is the authority). + Reads the archive's own member list with tarfile (exact names and + types, no `tar -tv` text parsing) and REFUSES the archive (exit 3) + when any member name, or hard-link target, could be read differently + by this classifier and by tar: + - a control character (\\x00-\\x1f, \\x7f) in any name or link + target (a newline would split one --exclude-from pattern in two); + - a non-canonical name: absolute, a '..' component, or + posixpath.normpath() differing from the name after stripping one + leading './' (e.g. 'x/./homedir', 'x/homedir//mail'). + Symlink TARGETS are not canonical-checked: cPanel archives carry + ordinary relative links (homedir/access-logs, about -> ../public_html) + outside the mail tree, and every symlink inside it is stripped anyway. + Then appends a glob-escaped --exclude-from line for every symlink under + a mail tree and every hard link that crosses a mail-tree boundary, and + records them as 'stripped_mail_link' actions. Prints the count. + + verify + Post-extract authority. lstat-walks every cpmove-*/homedir/mail under + (never following links) and exits 4 when it finds a symlink, or + a regular file whose st_nlink is larger than the number of its names + inside that same mail tree (some other name exists outside it), or any + directory it cannot read+search (an uninspectable subtree fails). + + scan also cross-checks its member count against `tar -tf` and refuses on + any difference or stream error, so nothing GNU tar extracts goes + unclassified (verify remains the authority either way). +""" +import json +import os +import posixpath +import re +import stat +import subprocess +import sys +import tarfile + +# ANY top-level directory: WHP's CpanelArchivePaths::selectTopDir() takes the +# first non-hidden dir of any name (cpmove-, cPanel's backup-_, +# acct, ...). Covering every top dir is a superset of whichever it picks. +MAIL = re.compile(r'^([^/]+)/homedir/mail(/|$)') +CTRL = re.compile(r'[\x00-\x1f\x7f]') + + +def refuse(code, msg): + sys.stderr.write(f'mail-tree-links: REFUSING archive: {msg}\n') + sys.exit(code) + + +def canonical(raw): + """The name tar will use, or None when it is not canonical.""" + n = raw[2:] if raw.startswith('./') else raw + if n == '' or n.startswith('/') or '..' in n.split('/'): + return None + return n if posixpath.normpath(n) == n else None + + +def owner(n): + m = MAIL.match(n) + return m.group(1) if m else None + + +def scan(tar_path, excl_path, actions_path): + try: + with open(actions_path) as ah: + doc = json.load(ah) + except Exception: + doc = {'actions': [], 'count': 0} + already = {a.get('path') for a in doc.get('actions', [])} + found = [] + seen = 0 + try: + tf = tarfile.open(tar_path, 'r|*') + except Exception as e: + refuse(3, f'archive unreadable by the classifier: {e}') + try: + with tf: + for m in tf: + seen += 1 + for label, value in (('name', m.name), ('link target', m.linkname)): + if CTRL.search(value or ''): + refuse(3, f'member {label} contains a control character: {value!r}') + name = canonical(m.name) + if name is None: + refuse(3, f'non-canonical member name: {m.name!r}') + if m.issym(): + if owner(name): + found.append((m.name, 'symlink', m.linkname)) + elif m.islnk(): + target = canonical(m.linkname) + if target is None: + refuse(3, f'non-canonical hard-link target: {m.name!r} -> {m.linkname!r}') + a, b = owner(name), owner(target) + if (a or b) and a != b: + found.append((m.name, 'hardlink', m.linkname)) + except Exception as e: # TarError, EOFError, zlib.error ... (SystemExit passes through) + refuse(3, f'archive stream error after {seen} member(s): {e}') + # The classifier must have seen every member GNU tar will extract: a + # stream that stops early (zero block, truncation, a header tarfile + # skips) would leave members unclassified. Count GNU tar's own listing + # (escape quoting = one line per member, even with a newline in a name). + try: + lst = subprocess.run(['tar', '--quoting-style=escape', '-tf', tar_path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + except OSError as e: + refuse(3, f'cannot list the archive with tar: {e}') + if lst.returncode != 0: + refuse(3, f'tar -tf failed (rc={lst.returncode}): {lst.stderr.decode(errors="replace").strip()[:300]}') + tar_count = lst.stdout.count(b'\n') + if tar_count != seen: + refuse(3, f'member count differs: classifier saw {seen}, tar lists {tar_count}') + found = [f for f in found if f[0] not in already] + with open(excl_path, 'a') as eh: + for name, _, _ in found: + eh.write(re.sub(r'([*?\[\\])', r'\\\1', name) + '\n') + doc['actions'] = doc.get('actions', []) + [ + {'action': 'stripped_mail_link', 'path': name, 'target': target, + 'reason': f'{kind} inside the mail tree (Maildirs have no legitimate links outside it)'} + for name, kind, target in found + ] + doc['count'] = len(doc['actions']) + with open(actions_path, 'w') as ah: + json.dump(doc, ah, indent=2) + for name, kind, target in found: + sys.stderr.write(f' STRIP {kind}: {name} -> {target}\n') + print(len(found)) + + +def verify(dest): + bad = [] + + def walk_error(err): + bad.append(f'cannot inspect: {os.path.relpath(getattr(err, "filename", "") or dest, dest)} ({err.strerror})') + + try: + tops = sorted(os.listdir(dest)) + except OSError as e: + refuse(4, f'cannot list the extract: {e}') + for top in tops: + try: + tst = os.lstat(os.path.join(dest, top)) + except OSError as err: + walk_error(err) + continue + if stat.S_ISLNK(tst.st_mode): + # Never walk through it (WHP refuses a top-level symlink anyway). + bad.append(f'top-level symlink: {top}') + continue + if not stat.S_ISDIR(tst.st_mode): + continue + mail = os.path.join(dest, top, 'homedir', 'mail') + try: + hst = os.lstat(os.path.join(dest, top, 'homedir')) + if stat.S_ISLNK(hst.st_mode): + bad.append(f'symlink: {top}/homedir') + continue + st = os.lstat(mail) + except FileNotFoundError: + continue + except OSError as err: + walk_error(err) + continue + if stat.S_ISLNK(st.st_mode): + bad.append(f'symlink: {top}/homedir/mail') + continue + if stat.S_ISDIR(st.st_mode) and not os.access(mail, os.R_OK | os.X_OK): + bad.append(f'directory not readable+searchable: {top}/homedir/mail') + continue + names = {} # (dev, ino) -> [count of names in this tree, st_nlink, a path] + # Fail closed: os.walk silently skips what it can't read unless told. + for root, dirs, files in os.walk(mail, followlinks=False, onerror=walk_error): + for e in dirs + files: + p = os.path.join(root, e) + try: + s = os.lstat(p) + except OSError as err: + walk_error(err) + continue + if stat.S_ISLNK(s.st_mode): + bad.append(f'symlink: {os.path.relpath(p, dest)}') + elif stat.S_ISDIR(s.st_mode) and not os.access(p, os.R_OK | os.X_OK): + bad.append(f'directory not readable+searchable: {os.path.relpath(p, dest)}') + elif stat.S_ISREG(s.st_mode) and s.st_nlink > 1: + rec = names.setdefault((s.st_dev, s.st_ino), [0, s.st_nlink, p]) + rec[0] += 1 + for count, nlink, p in names.values(): + if nlink > count: + bad.append(f'hard link leaving the mail tree: {os.path.relpath(p, dest)} ' + f'(st_nlink {nlink}, {count} name(s) inside the mail tree)') + for b in bad: + sys.stderr.write(f'mail-tree-links: {b}\n') + if bad: + refuse(4, f'{len(bad)} problem(s) in the extracted mail tree (links or uninspectable directories)') + + +if __name__ == '__main__': + if len(sys.argv) == 5 and sys.argv[1] == 'scan': + scan(sys.argv[2], sys.argv[3], sys.argv[4]) + elif len(sys.argv) == 3 and sys.argv[1] == 'verify': + verify(sys.argv[2]) + else: + sys.stderr.write('usage: mail-tree-links.py scan | verify \n') + sys.exit(2) diff --git a/scripts/lib/scan-symlinks.php b/scripts/lib/scan-symlinks.php index 43da433..aae5c2b 100644 --- a/scripts/lib/scan-symlinks.php +++ b/scripts/lib/scan-symlinks.php @@ -90,8 +90,40 @@ $dangerousPrefixes = [ '/dev', ]; +/** + * Lexical resolution of a relative link from its own directory in the + * archive: a reason when it reaches/climbs above the archive root, when a + * link under /homedir/ resolves outside that homedir, or when any + * component on the way is itself a symlink entry ($linkSet: normalized + * archive path => true); NULL otherwise. extract.sh's post-extract + * strip-escaping-links.py (realpath of every link) is the authority. + */ +function relative_symlink_escape(string $archivePath, string $target, array $linkSet = []): ?string { + $ap = preg_replace('#^(\./)+#', '', $archivePath); + $dir = array_values(array_filter(explode('/', (string) $ap), fn($x) => $x !== '' && $x !== '.')); + array_pop($dir); + $home = (count($dir) >= 2 && $dir[1] === 'homedir') ? [$dir[0], 'homedir'] : null; + $stack = $dir; + foreach (explode('/', $target) as $seg) { + if ($seg === '' || $seg === '.') continue; + if ($seg === '..') { + if (count($stack) <= 1) return 'escapes the archive root'; + array_pop($stack); + continue; + } + $stack[] = $seg; + // No chaining: a component that is itself a symlink entry in this + // archive would be expanded by the kernel, which this lexical walk + // cannot model (`a -> b/..` with `b -> x/y/z`). + if (isset($linkSet[implode('/', $stack)])) return 'passes through another symlink in the archive'; + } + if ($home !== null && (count($stack) < 2 || $stack[0] !== $home[0] || $stack[1] !== 'homedir')) { + return 'escapes the account homedir'; + } + return null; +} + $findings = []; -$cpanelUsername = null; $cmd = 'tar -tvf ' . escapeshellarg($tarPath) . ' 2>/dev/null'; $fh = @popen($cmd, 'r'); @@ -100,6 +132,9 @@ if (!$fh) { exit(2); } +// Pass 1: every symlink entry (the chain rule needs the whole set). +$links = []; +$linkSet = []; while (($line = fgets($fh)) !== false) { if ($line === '' || $line[0] !== 'l') continue; $arrow = strpos($line, ' -> '); @@ -108,24 +143,44 @@ while (($line = fgets($fh)) !== false) { $right = rtrim(substr($line, $arrow + 4), "\r\n"); $parts = preg_split('/\s+/', $left, 6); if (count($parts) < 6) continue; - $archivePath = $parts[5]; - $target = $right; + $links[] = [$parts[5], $right]; + $linkSet[trim((string) preg_replace('#^(\./)+#', '', $parts[5]), '/')] = true; +} +pclose($fh); - if ($target === '' || $target[0] !== '/') continue; +// Pass 2: classify. +foreach ($links as [$archivePath, $target]) { - if ($cpanelUsername === null) { - if (preg_match('#^cpmove-([^/]+)/#', $archivePath, $m)) { - $cpanelUsername = $m[1]; + // N2-1: a RELATIVE target that climbs out of the archive (or out of + // the account's homedir/) is DANGEROUS (stripped), not safe: GNU tar + // extracts `cpmove-u/sds -> ../../../../root/.my.cnf` as-is. Same rule + // as WHP's CpanelBackupImporter::relativeSymlinkEscape(). + if ($target !== '' && $target[0] !== '/') { + $escape = relative_symlink_escape($archivePath, $target, $linkSet); + if ($escape !== null) { + $findings[] = [ + 'type' => 'DANGEROUS', + 'archive_path' => $archivePath, + 'target' => $target, + 'reason' => "relative target $escape", + ]; } + continue; } + if ($target === '') continue; + // (1) user-internal — accept symlinks pointing into the customer's // own /home// tree. The panel rewrites these on extract. $userInternal = false; $usernames = []; - if ($cpanelUsername !== null && $cpanelUsername !== '') $usernames[] = $cpanelUsername; - if ($username !== '') $usernames[] = $username; - foreach ($usernames as $u) { + // N2-1 review: only the TRUSTED --username (the WHP account WHP passes); + // a name read from the archive's top dir (cpmove-) is never trusted. + if (preg_match('/\A[a-z_][a-z0-9_-]{0,31}\z/', $username)) $usernames[] = $username; + // N2-1 review: only a NORMALIZED target can be user-internal + // (`/home//../../etc/shadow` is not). + $normalized = !preg_match('#(^|/)\.{1,2}(/|$)|//#', $target); + foreach ($normalized ? $usernames : [] as $u) { $prefix = '/home/' . $u . '/'; if (strpos($target, $prefix) === 0 || $target === rtrim($prefix, '/')) { $userInternal = true; @@ -138,6 +193,19 @@ while (($line = fgets($fh)) !== false) { } if ($userInternal) continue; + // N2-1 review: a non-normalized absolute target ('.', '..', '//') is + // never compared by prefix (`/home//../../etc` would dodge /etc): + // DANGEROUS outright. + if (!$normalized) { + $findings[] = [ + 'type' => 'DANGEROUS', + 'archive_path' => $archivePath, + 'target' => $target, + 'reason' => "absolute target is not normalized ('.', '..' or empty component)", + ]; + continue; + } + // (2) exact root. $type = null; $reason = ''; @@ -170,7 +238,6 @@ while (($line = fgets($fh)) !== false) { 'reason' => $reason, ]; } -pclose($fh); $dangerousCount = count(array_filter($findings, fn($f) => $f['type'] === 'DANGEROUS')); $uncertainCount = count(array_filter($findings, fn($f) => $f['type'] === 'UNCERTAIN')); diff --git a/scripts/lib/strip-escaping-links.py b/scripts/lib/strip-escaping-links.py new file mode 100644 index 0000000..29cc768 --- /dev/null +++ b/scripts/lib/strip-escaping-links.py @@ -0,0 +1,135 @@ +#!/usr/bin/python3 -I +# strip-escaping-links.py — post-extract symlink authority (N2-1). +# +# python3 -I strip-escaping-links.py [username ...] +# +# The pre-extract classifier (scan-symlinks.php) resolves a relative target +# LEXICALLY, so a chain through another link (`a -> b/..` with `b -> x/y/z`, +# or `l1 -> l2/../../..`) can slip past it. This pass is the authority: it +# lstat-walks the WHOLE extracted tree (never following a link) and asks the +# kernel (realpath strict) where every symlink really resolves. A link is +# removed and recorded as a `stripped_escaping_symlink` action when: +# - it resolves outside , or to itself; +# - it is dangling (strict resolution fails), or loops. +# Own-account absolute links (cPanel's wp-content -> /home//shared and +# the like) are judged RE-ROOTED, never against the container's own /home: +# a target that is already normalized (no '.', '..' or empty component) +# and lies under /home[N]// for the TRUSTED username only (the WHP +# account name WHP passes as extract.sh's ; names read from the +# archive are never trusted) is mapped to //homedir/, where +# is the link's own top-level dir, and kept only if THAT resolves +# (realpath strict) inside /homedir. Every other absolute target either +# dangles here or resolves into the container's own /etc, /usr, ... — both +# outside — so it is stripped. A link to the extract itself is +# stripped too. +# +# Repeats until a pass strips nothing (removing one link can leave another +# that resolved through it dangling). Any walk error fails closed (exit 1). +# stdout: number of links stripped. Exit 0 ok, 1 error, 2 usage. + +import json +import os +import posixpath +import re +import stat +import sys + + +def main(argv): + if len(argv) < 3: + sys.stderr.write("usage: strip-escaping-links.py [trusted-username]\n") + return 2 + dest = os.path.realpath(argv[1]) + actions_path = argv[2] + trusted = argv[3] if len(argv) > 3 else '' + if not re.fullmatch(r'[a-z_][a-z0-9_-]{0,31}', trusted): + trusted = '' # no (valid) trusted name: exempt nothing + own_re = re.compile(r'\A/home\d*/' + re.escape(trusted) + r'(?:/(.*))?\Z') if trusted else None + + def own_account_ok(link_path, target): + """True when an absolute own-account target, re-rooted at the link's + /homedir, resolves inside that homedir.""" + if own_re is None or posixpath.normpath(target) != target.rstrip('/') or '/../' in target + '/' or '/./' in target + '/': + return False + m = own_re.match(target.rstrip('/')) + if not m: + return False + rel = os.path.relpath(link_path, dest).split(os.sep) + if len(rel) < 2: + return False + home = os.path.join(dest, rel[0], 'homedir') + try: + hst = os.lstat(home) + if not stat.S_ISDIR(hst.st_mode): + return False + home_real = os.path.realpath(home, strict=True) + cand = os.path.join(home_real, m.group(1)) if m.group(1) else home_real + real = os.path.realpath(cand, strict=True) + except OSError: + return False + return real == home_real or real.startswith(home_real + '/') + if not os.path.isdir(dest): + sys.stderr.write(f"strip-escaping-links: not a directory: {dest}\n") + return 1 + + def walk_error(err): + raise err + + stripped = [] + while True: + this_pass = 0 + for dirpath, dirnames, filenames in os.walk(dest, topdown=True, onerror=walk_error, followlinks=False): + for name in list(dirnames) + list(filenames): + p = os.path.join(dirpath, name) + st = os.lstat(p) + if not stat.S_ISLNK(st.st_mode): + continue + target = os.readlink(p) + reason = None + if target.startswith('/') and own_re is not None and own_re.match(target.rstrip('/') or '/'): + if own_account_ok(p, target): + continue + reason = 'own-account target does not resolve inside the extracted homedir' + try: + if reason is None: + real = os.path.realpath(p, strict=True) + if not real.startswith(dest + '/'): + reason = 'resolves outside the extract' + except OSError: + reason = 'dangling or looping' + if reason is None: + continue + os.unlink(p) + if name in dirnames: + dirnames.remove(name) + stripped.append({ + 'action': 'stripped_escaping_symlink', + 'archive_path': os.path.relpath(p, dest), + 'target': target, + 'reason': reason, + }) + sys.stderr.write(f" STRIP {os.path.relpath(p, dest)} -> {target!r} ({reason})\n") + this_pass += 1 + if this_pass == 0: + break + + if stripped: + try: + with open(actions_path) as fh: + doc = json.load(fh) + except (OSError, ValueError): + doc = {'actions': []} + doc.setdefault('actions', []).extend(stripped) + doc['count'] = len(doc['actions']) + with open(actions_path, 'w') as fh: + json.dump(doc, fh, indent=2) + print(len(stripped)) + return 0 + + +if __name__ == '__main__': + try: + sys.exit(main(sys.argv)) + except OSError as e: + sys.stderr.write(f"strip-escaping-links: walk failed: {e}\n") + sys.exit(1) diff --git a/scripts/pkg-cache-repos.sh b/scripts/pkg-cache-repos.sh new file mode 100755 index 0000000..ee2bd7c --- /dev/null +++ b/scripts/pkg-cache-repos.sh @@ -0,0 +1,296 @@ +#!/bin/bash +## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal +## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster) +## when PKG_CACHE is set; without it the repo files are left alone and builds +## use the upstream repositories as before. +## +## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi/NodeSource +## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu, Debian, LiteSpeed +## pkg-cache-repos.sh release-install epel [remi] # GPG-verified release RPMs (EL9/EL10) +## pkg-cache-repos.sh nodesource MAJOR # NodeSource repo file + pinned key +## eval "$(pkg-cache-repos.sh go-env)" # GOPROXY via the cache (go.sum/sumdb verified) +## pkg-cache-repos.sh restore # undo everything above +## +## CANONICAL copy: cloud-apache-container/scripts/pkg-cache-repos.sh (docs/pkg-cache.md +## there). Other repos carry byte-identical copies - change it there first. +## +## Every RUN that rewrites must `restore` before it ends: it puts rewritten files +## back, deletes files this script created, deletes any NEW repo/apt file that +## still references the cache, then FAILS if any reference is left — so no shipped +## layer carries the cache's LAN address (images run on hosts that can't reach it). +## +## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging +## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror +## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with +## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and +## immutable .rpm/.deb files long-term. +## +## Integrity (security review 2026-10-21): +## * PKG_CACHE must be https://. The cache's certificate is issued by the +## name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only +## for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten +## repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN — +## never added to the system trust store. +## * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and +## AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL +## publishes no repomd signature (404), so its metadata relies on HTTPS end +## to end (client->cache->dl.fedoraproject.org/mirrors, all verified). +## * release-install downloads epel-release / remi-release (from the cache, or +## from the HTTPS origin without PKG_CACHE), checks the package name, imports +## the signing keys committed in keys/pkg-cache/ and requires +## `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1. +## * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail +## a build that gets old PHP (replay of old-but-signed metadata). +## +## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if +## the cache dies mid-build dnf fails over to the origin by itself. +## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the +## cache (with the CA), and a build without PKG_CACHE uses upstream sources. +set -euo pipefail + +MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|nodesource MAJOR|go-env|restore}" +shift || true +STATE=/var/lib/pkg-cache-repos # saved originals + manifest + created files +SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}" # ca.crt + RPM-GPG-KEY-* +C="${PKG_CACHE:-}" +C="${C%/}" + +die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; } + +if [ -n "$C" ]; then + case "$C" in + https://*) ;; + *) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;; + esac + [[ "$C" =~ ^https://[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]*)?$ ]] || die "PKG_CACHE has unexpected characters: '$C'" + HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}" + [ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing" +fi + +# System CA bundle (EL vs Debian/Ubuntu path); the pkg-cache CA is appended to a +# copy in the state dir, never to the system store. +sys_bundle() { + local b + for b in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do + [ -s "$b" ] && { echo "$b"; return 0; } + done + die "no system CA bundle found" +} +make_bundle() { cat "$(sys_bundle)" "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"; } + +# Files that may hold repo configuration; restore re-checks all of them. +repo_files() { + find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true +} + +start_session() { + [ -d "$STATE" ] && return 0 + mkdir -p "$STATE/orig" + repo_files > "$STATE/manifest" # what existed before this RUN touched anything + : > "$STATE/created" + printf '%s\n' "$HOSTPORT" > "$STATE/host" +} +save() { # keep the first-seen (pre-rewrite) copy + [ -e "$STATE/orig$1" ] && return 0 + mkdir -p "$STATE/orig$(dirname "$1")" + cp -p "$1" "$STATE/orig$1" +} +created() { printf '%s\n' "$1" >> "$STATE/created"; } + +if [ "$MODE" = restore ]; then + [ -d "$STATE" ] || exit 0 + host=$(cat "$STATE/host") + # mapfile from a process substitution, not `cmd | while read`: no pipeline, + # so nothing for pipefail to misreport (scripts/tests/lsphp-info-probe.test.sh). + mapfile -t saved < <(cd "$STATE/orig" && find . -type f) + for rel in "${saved[@]}"; do + cat "$STATE/orig/$rel" > "/${rel#./}" + done + while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created" + # A repo/apt file that appeared during this RUN and still names the cache + # (e.g. written by a package from the rewritten repos) is removed too. + mapfile -t current < <(repo_files) + for f in "${current[@]}"; do + if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then + echo "pkg-cache-repos: removing new file that references the cache: $f" + rm -f -- "$f" + fi + done + left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true) + rm -rf "$STATE" + [ -z "$left" ] || die "cache address $host still present after restore in: $left" + echo "pkg-cache-repos: original repository files restored (no reference to $host left)" + exit 0 +fi + +if [ "$MODE" = release-install ]; then + [ $# -gt 0 ] || die "release-install needs: epel and/or remi" + tmp=$(mktemp -d) + curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120) + if [ -n "$C" ]; then + start_session + make_bundle + curl_opts+=(--cacert "$STATE/ca-bundle.pem") + fi + files=() + el=$(rpm -E '%{?rhel}' 2>/dev/null || true) + case "$el" in 9|10) ;; *) die "release-install supports EL9/EL10 only (rpm -E %rhel = '$el')" ;; esac + for name in "$@"; do + case "$name" in + epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-$el + path=/epel/epel-release-latest-$el.noarch.rpm + origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-$el.noarch.rpm ;; + remi) pkg=remi-release key=RPM-GPG-KEY-remi2024 + path=/remi/enterprise/remi-release-$el.rpm + origin=https://rpms.remirepo.net/enterprise/remi-release-$el.rpm ;; + *) die "unknown release package '$name'" ;; + esac + url="$origin"; [ -n "$C" ] && url="$C$path" + f="$tmp/$pkg.rpm" + curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url" + rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key" + got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true) + [ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg" + chk=$(rpm -K "$f" 2>&1 || true) + [[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk" + echo "pkg-cache-repos: $pkg verified ($chk) from $url" + files+=("$f") + done + dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}" + rm -rf "$tmp" + exit 0 +fi + +if [ -z "$C" ] && [ "$MODE" != nodesource ] && [ "$MODE" != go-env ]; then + echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories" + exit 0 +fi + +rewrite_rpm() { + local f="$1" + grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net|rpm\.nodesource\.com)/' "$f" || return 0 + save "$f" + # Per [section]: a (commented or live) baseurl on a known origin becomes + # "baseurl=// /"; that section's + # mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it + # gets sslcacert=, gpgcheck=1 and, for repos + # that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched. + awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" ' + function flush( i) { + for (i = 1; i <= n; i++) { + if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i] + if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue + print buf[i] + } + if (hit) { + print "sslcacert=" BUNDLE + print "gpgcheck=1" + if (signed) print "repo_gpgcheck=1" + } + n = 0; hit = 0; signed = 0 + } + function rw(line, re, prefix, origin, sig, p) { + if (match(line, re)) { + p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p) + hit = 1; if (sig) signed = 1 + return "baseurl=" C "/" prefix "/" p " " origin "/" p + } + return line + } + /^\[/ { flush() } + { + l = $0 + if (l ~ /^#? ?baseurl=/) { + l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1) + l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0) + l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1) + # NodeSource signs its RPMs but not repomd.xml (404): gpgcheck only. + l = rw(l, "^#? ?baseurl=https?://rpm\\.nodesource\\.com/", "nodesource", "https://rpm.nodesource.com", 0) + } + buf[++n] = l + } + END { flush() } + ' "$f" > "$f.pkgcache" && cat "$f.pkgcache" > "$f" && rm -f "$f.pkgcache" +} + +rewrite_apt() { + local f="$1" + grep -q -E 'https?://([a-z]{2}\.)?(archive|security)\.ubuntu\.com/|https?://rpms\.litespeedtech\.com/|https?://(deb|security)\.debian\.org/' "$f" || return 0 + save "$f" + # Debian: debian-security first, and "debian" only as a whole path segment, + # so .../debian-security is never rewritten as /debian/. + sed -i -E \ + -e "s%https?://(archive|[a-z]{2}\.archive)\.ubuntu\.com/ubuntu/?%$C/ubuntu/%g" \ + -e "s%https?://security\.ubuntu\.com/ubuntu/?%$C/ubuntu-security/%g" \ + -e "s%https?://rpms\.litespeedtech\.com/%$C/litespeed/%g" \ + -e "s%https?://(deb|security)\.debian\.org/debian-security/?([[:space:]]|\$)%$C/debian-security/\\2%g" \ + -e "s%https?://security\.debian\.org/?([[:space:]]|\$)%$C/debian-security/\\1%g" \ + -e "s%https?://deb\.debian\.org/debian/?([[:space:]]|\$)%$C/debian/\\1%g" \ + "$f" +} + +if [ "$MODE" = go-env ]; then + # Prints shell exports for `eval`. Without PKG_CACHE: nothing (Go defaults). + # GOPROXY "a|b" falls through on ANY error, so a dead cache costs one + # failed request, not the build. GOSUMDB stays the default sum.golang.org, + # reached directly: modules are still verified against go.sum + the sumdb. + [ -n "$C" ] || { echo "pkg-cache-repos(go-env): PKG_CACHE not set - default GOPROXY" >&2; exit 0; } + start_session + make_bundle + printf "export GOPROXY='%s/goproxy|https://proxy.golang.org|direct' SSL_CERT_FILE='%s'\n" "$C" "$STATE/ca-bundle.pem" + echo "pkg-cache-repos(go-env): GOPROXY via $C/goproxy (fallback proxy.golang.org)" >&2 + exit 0 +fi + +if [ "$MODE" = nodesource ]; then + # Replaces `curl https://rpm.nodesource.com/setup_N.x | bash` (unverified + # remote script): writes the same nodesource-nodejs repo (origin URL, so the + # shipped image keeps a normal repo file) with gpgcheck=1 and the signing key + # committed in keys/pkg-cache/; with PKG_CACHE it is then rewritten like any + # other repo and `restore` puts the origin version back. + major="${1:?usage: pkg-cache-repos.sh nodesource MAJOR}" + [[ "$major" =~ ^[0-9]{1,3}$ ]] || die "bad Node major '$major'" + key="$SHARE/RPM-GPG-KEY-nodesource" + [ -s "$key" ] || die "missing $key" + rpm --import "$key" || die "cannot import $key" + rm -f /etc/yum.repos.d/nodesource*.repo + f=/etc/yum.repos.d/nodesource-nodejs.repo + printf '%s\n' "[nodesource-nodejs]" \ + "name=Node.js Packages for Linux RPM based distros - \$basearch" \ + "baseurl=https://rpm.nodesource.com/pub_${major}.x/nodistro/nodejs/\$basearch" \ + "priority=9" "enabled=1" "gpgcheck=1" \ + "gpgkey=file://$key" "module_hotfixes=1" > "$f" + echo "pkg-cache-repos: wrote $f (Node ${major}.x, key $(basename "$key"))" + if [ -n "$C" ]; then + start_session + make_bundle + rewrite_rpm "$f" + grep -h '^baseurl=' "$f" | sed 's/^/ /' + fi + exit 0 +fi + +case "$MODE" in + rpm) + start_session + make_bundle + for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done + echo "pkg-cache-repos(rpm): repos now via $C:" + grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /' + ;; + apt) + start_session + conf=/etc/apt/apt.conf.d/99pkg-cache + # Trust the pkg-cache CA for the cache host ONLY (apt per-host option). + printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf" + created "$conf" + for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + [ -f "$f" ] && rewrite_apt "$f" + done + echo "pkg-cache-repos(apt): sources now via $C:" + # (|| true: Debian 13 images have no /etc/apt/sources.list; grep's exit 2 + # for the missing file must not fail the build under pipefail) + { grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null || true; } | grep -F "$C/" | sort -u | sed 's/^/ /' || true + ;; + *) die "unknown mode '$MODE'" ;; +esac diff --git a/scripts/scan-files.php b/scripts/scan-files.php index 39de3ac..1e5f6c9 100755 --- a/scripts/scan-files.php +++ b/scripts/scan-files.php @@ -7,7 +7,11 @@ * for v1.1 expansion; see CONTRIBUTING.md for how to wire one in. * * Usage: - * scan-files.php --extract --quarantine --report --import-id + * scan-files.php --extract --quarantine --report --import-id [--skip-mail] + * + * --skip-mail: do not scan (or count, or quarantine anything in) the + * /homedir/mail tree (any top-level dir). entrypoint.sh passes it when the mail tree + * was extracted for WHP's mailbox import (IMPORT_INCLUDE_MAIL=true). * * Exit codes: * 0 — scan completed (regardless of how many hits) @@ -28,10 +32,10 @@ require __DIR__ . '/lib/safety-net.php'; const SCANNER_VERSION = '1.0.0'; -$opts = getopt('', ['extract:', 'quarantine:', 'report:', 'import-id:']); +$opts = getopt('', ['extract:', 'quarantine:', 'report:', 'import-id:', 'skip-mail']); foreach (['extract', 'quarantine', 'report', 'import-id'] as $k) { if (!isset($opts[$k])) { - fwrite(STDERR, "usage: scan-files.php --extract --quarantine --report --import-id \n"); + fwrite(STDERR, "usage: scan-files.php --extract --quarantine --report --import-id [--skip-mail]\n"); exit(2); } } @@ -39,6 +43,17 @@ $extractDir = rtrim($opts['extract'], '/'); $quarantineDir = rtrim($opts['quarantine'], '/'); $reportPath = $opts['report']; $importId = $opts['import-id']; +$skipMail = array_key_exists('skip-mail', $opts); + +// /homedir/mail: WHP accepts any top-level dir name +// (cpmove-, backup-_, ...), so the skip covers all of them. +// The mail tree is copied verbatim into the customer's own mailbox, and +// WHP verifies the copy message-by-message against what it packaged. So it +// is NOT scanned: quarantining a message would silently drop mail, and +// ClamAV over a multi-GB maildir would dominate the run. PCRE form for the +// PHP walk, POSIX ERE form for clamscan --exclude-dir. +$mailDirPcre = '#\A' . preg_quote($extractDir, '#') . '/[^/]+/homedir/mail(?:/|\z)#'; +$mailDirEre = '^' . preg_replace('/[.\[\]()*+?{}|^$\\\\]/', '\\\\$0', $extractDir) . '/[^/]+/homedir/mail(/.*)?$'; if (!is_dir($extractDir)) { fwrite(STDERR, "scan-files: extract dir does not exist: $extractDir\n"); @@ -48,6 +63,9 @@ if (!is_dir($extractDir)) { @mkdir($quarantineDir, 0750, true); fwrite(STDERR, "scan-files: starting (extract=$extractDir, quarantine=$quarantineDir)\n"); +if ($skipMail) { + fwrite(STDERR, "scan-files: NOT scanning the mail tree (/homedir/mail): it is copied verbatim into the customer's own mailbox and verified against the source; scanning would quarantine messages out of it and multi-GB maildirs would dominate the run\n"); +} // -- v1.0 cleaner registry (intentionally empty) ---------------------------- // @@ -75,7 +93,8 @@ $cleaners = []; // Output format per line on a hit: // /tmp/extract/foo/bar.php: Some.Signature.Name FOUND $cmd = sprintf( - 'clamscan --infected --recursive --no-summary --stdout %s 2>/dev/null', + 'clamscan --infected --recursive --no-summary --stdout %s%s 2>/dev/null', + $skipMail ? '--exclude-dir=' . escapeshellarg($mailDirEre) . ' ' : '', escapeshellarg($extractDir) ); @@ -99,6 +118,12 @@ while (($line = fgets($fh)) !== false) { // Defensive: shouldn't happen with our invocation. continue; } + if ($skipMail && preg_match($mailDirPcre, $path)) { + // Belt: --exclude-dir should have kept clamscan out of the mail + // tree; never move a message out of a Maildir regardless. + fwrite(STDERR, "scan-files: WARN ignoring a hit inside the mail tree: $path\n"); + continue; + } $hits[] = ['path' => $path, 'signature' => $sig]; } pclose($fh); @@ -156,12 +181,15 @@ $rdi = new RecursiveDirectoryIterator( $extractDir, FilesystemIterator::SKIP_DOTS | FilesystemIterator::CURRENT_AS_PATHNAME ); -$filter = new RecursiveCallbackFilterIterator($rdi, function ($pathname, $key, $iterator) use (&$skippedLinks) { +$filter = new RecursiveCallbackFilterIterator($rdi, function ($pathname, $key, $iterator) use (&$skippedLinks, $skipMail, $mailDirPcre) { // $pathname is a string when CURRENT_AS_PATHNAME is set. if (is_link($pathname)) { $skippedLinks++; return false; } + if ($skipMail && preg_match($mailDirPcre, $pathname)) { + return false; // not scanned, so not counted as scanned + } return true; }); $it = new RecursiveIteratorIterator($filter, RecursiveIteratorIterator::LEAVES_ONLY); @@ -276,6 +304,7 @@ $report = [ 'files_clean' => max(0, $filesScanned - count($hits)), 'files_cleaned' => $cleaned, 'files_quarantined' => $quarantined, + 'mail_scan_skipped' => $skipMail, 'actions' => $actions, ]; diff --git a/tests/build-fixtures.sh b/tests/build-fixtures.sh index 7914886..ac91f86 100755 --- a/tests/build-fixtures.sh +++ b/tests/build-fixtures.sh @@ -13,8 +13,17 @@ # `cpmove-testuser/homedir/.../alfasymlink -> /etc` # — the pre-extract scan MUST refuse this. # +# cpmove-mail.tar.gz — same shape PLUS a cPanel mail tree: +# homedir/etc//{passwd,shadow} and +# homedir/mail//{alice,empty}/ Maildirs +# (alice has 2 messages incl. one in .Archive; +# empty has cur/new/tmp and no messages), plus a +# DANGEROUS symlink INSIDE the mail tree +# (homedir/mail//alice/.evil -> /etc) that +# must be stripped even when mail is included. +# # Run: bash tests/build-fixtures.sh -# Output: tests/fixtures/cpmove-clean.tar.gz, tests/fixtures/cpmove-alfa.tar.gz +# Output: tests/fixtures/cpmove-{clean,alfa,mail,mail-xlink}.tar.gz set -euo pipefail @@ -44,8 +53,10 @@ EOF echo " "$root/cpmove-$USER/homedir/public_html/index.php" echo "Hello world." > "$root/cpmove-$USER/homedir/public_html/about.txt" - # benign user-internal relative symlink — must NOT trigger the scan - ln -sf "../public_html/about.txt" "$root/cpmove-$USER/homedir/about-shortcut" + # benign user-internal relative symlink — must NOT trigger the scan. + # (It used to be ../public_html/about.txt, which resolves OUTSIDE + # homedir/ — cpmove-$USER/public_html — and is now DANGEROUS, N2-1.) + ln -sf "public_html/about.txt" "$root/cpmove-$USER/homedir/about-shortcut" # one synthetic WordPress mysql dump with ENGINE=MyISAM + a clean siteurl cat > "$root/cpmove-$USER/mysql/${USER}_wp.sql" </dev/null || true' EXIT +trap 'rm -rf "$CLEAN_TMP" "$ALFA_TMP" "$MAIL_TMP" 2>/dev/null || true' EXIT build_common_tree "$CLEAN_TMP" tar -C "$CLEAN_TMP" -czf "$FIXTURES_DIR/cpmove-clean.tar.gz" "cpmove-$USER" @@ -108,6 +119,157 @@ ln -sf "/etc" "$ALFA_TMP/cpmove-$USER/homedir/public_html/$DOMAIN/ALFA_DATA/root tar -C "$ALFA_TMP" -czf "$FIXTURES_DIR/cpmove-alfa.tar.gz" "cpmove-$USER" echo "wrote $FIXTURES_DIR/cpmove-alfa.tar.gz ($(stat -c%s "$FIXTURES_DIR/cpmove-alfa.tar.gz") bytes)" +# ---- cpmove-relesc.tar.gz (N2-1) -------------------------------------------- +# +# RELATIVE symlinks that climb out: `sds` (a metadata file WHP parses) out of +# the archive to /etc/passwd, and a docroot file out of homedir/ into the +# cpmove's own mysql/ dump. Both must be DANGEROUS (stripped), while +# homedir/www -> public_html (inside homedir) stays. + +RELESC_TMP="$(mktemp -d)" +build_common_tree "$RELESC_TMP" +rm -rf "$RELESC_TMP/cpmove-$USER/sds" +ln -sf "../../../../../../../../etc/passwd" "$RELESC_TMP/cpmove-$USER/sds" +ln -sf "../../mysql/${USER}_wp.sql" "$RELESC_TMP/cpmove-$USER/homedir/public_html/dump.sql" +ln -sf "public_html" "$RELESC_TMP/cpmove-$USER/homedir/www" +tar -C "$RELESC_TMP" -czf "$FIXTURES_DIR/cpmove-relesc.tar.gz" "cpmove-$USER" +rm -rf "$RELESC_TMP" +echo "wrote $FIXTURES_DIR/cpmove-relesc.tar.gz" + +# ---- cpmove-chain.tar.gz (N2-1 review: symlink chains) ---------------------- +# Each link alone resolves lexically inside the archive; through the other +# link the kernel resolves it ABOVE the extract: +# d1/d2/b -> ../.. (= cpmove-$USER, fine on its own) +# d1/d2/a -> b/../.. (b/.. = extract root, /.. = its parent) +# d1/d2/l2 -> ../.. +# d1/d2/l1 -> l2/../../.. +CHAIN_TMP="$(mktemp -d)" +build_common_tree "$CHAIN_TMP" +C="$CHAIN_TMP/cpmove-$USER/d1/d2" +mkdir -p "$C" +ln -s "../.." "$C/b"; ln -s "b/../.." "$C/a" +ln -s "../.." "$C/l2"; ln -s "l2/../../.." "$C/l1" +tar -C "$CHAIN_TMP" -czf "$FIXTURES_DIR/cpmove-chain.tar.gz" "cpmove-$USER" +rm -rf "$CHAIN_TMP" +echo "wrote $FIXTURES_DIR/cpmove-chain.tar.gz" + +# ---- cpmove-mail.tar.gz --------------------------------------------------- + +MAIL_TMP="$(mktemp -d)" +build_common_tree "$MAIL_TMP" +MROOT="$MAIL_TMP/cpmove-$USER/homedir" +mkdir -p "$MROOT/etc/$DOMAIN" +printf 'alice:x:1:1::/home/%s/mail/%s/alice:/bin/false\nempty:x:1:1::/home/%s/mail/%s/empty:/bin/false\n' \ + "$USER" "$DOMAIN" "$USER" "$DOMAIN" > "$MROOT/etc/$DOMAIN/passwd" +# Not real hashes: the importer image never reads these, it only extracts. +printf 'alice:$6$fixturesaltfixtu$notarealhash:19000::::::\nempty:$6$fixturesaltfixtu$notarealhash:19000::::::\n' \ + > "$MROOT/etc/$DOMAIN/shadow" +for box in alice empty; do + mkdir -p "$MROOT/mail/$DOMAIN/$box"/{cur,new,tmp} +done +mkdir -p "$MROOT/mail/$DOMAIN/alice/.Archive"/{cur,new,tmp} +printf 'From: a@example.net\nSubject: one\n\nhello\n' \ + > "$MROOT/mail/$DOMAIN/alice/cur/1700000000.M1P1.host,S=40:2,S" +printf 'From: b@example.net\nSubject: two\n\narchived\n' \ + > "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000001.M2P1.host,S=43:2," +printf 'Archive\n' > "$MROOT/mail/$DOMAIN/alice/subscriptions" +# DANGEROUS symlink inside the mail tree: stripped whether or not mail is included. +ln -sf "/etc" "$MROOT/mail/$DOMAIN/alice/.evil" +# Relative symlinks inside the mail tree (not DANGEROUS-classified): a file +# link climbing out to /etc/shadow and a folder link to another domain's box. +ln -sf "../../../../../../../../etc/shadow" "$MROOT/mail/$DOMAIN/alice/.evil-rel" +mkdir -p "$MROOT/mail/other.example/bob"/{cur,new,tmp} +ln -sf "../../other.example/bob" "$MROOT/mail/$DOMAIN/alice/.Other" +# Hard links. With --sort=name below, the first name of an inode is stored +# as the file and later names as hard-link entries: +# .Archive/cur/1700000002... (file) <- cur/1700000002... Dovecot-style +# copy inside the mail tree: legitimate, must be KEPT; +# public_html/stolen.txt -> that message: crosses OUT of the mail tree +# (cpmove-mail-xlink.tar.gz only, below); +# etc//shadow (file) <- mail//alice/cur/leak: crosses IN. +printf 'From: c@example.net\nSubject: three\n\ncopied\n' \ + > "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S" +ln "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S" \ + "$MROOT/mail/$DOMAIN/alice/cur/1700000002.M3P1.host,S=41:2,S" +ln "$MROOT/etc/$DOMAIN/shadow" "$MROOT/mail/$DOMAIN/alice/cur/leak" + +tar -C "$MAIL_TMP" --sort=name -czf "$FIXTURES_DIR/cpmove-mail.tar.gz" "cpmove-$USER" + +# cpmove-mail-xlink.tar.gz: the same PLUS public_html/stolen.txt as a hard +# link to a message (crosses OUT of the mail tree). Separate fixture: in +# the default (no-mail) mode tar cannot create a link to an excluded +# member and the extract fails (fail-closed, as before this change). +ln "$MROOT/mail/$DOMAIN/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S" \ + "$MROOT/public_html/stolen.txt" +tar -C "$MAIL_TMP" --sort=name -czf "$FIXTURES_DIR/cpmove-mail-xlink.tar.gz" "cpmove-$USER" +echo "wrote $FIXTURES_DIR/cpmove-mail.tar.gz + cpmove-mail-xlink.tar.gz" + +# ---- parser-differential fixtures (must be REFUSED with IMPORT_INCLUDE_MAIL=true) +# Written with python tarfile because GNU tar normalises such names on create. +# cpmove-bad-dot ./ component: cpmove-testuser/./homedir/mail/.../.x -> ../../../../etc +# cpmove-bad-dslash empty component: cpmove-testuser/homedir//mail/.../.x -> ../../../../etc +# cpmove-bad-dotdot '..' component: cpmove-testuser/homedir/../homedir/mail/.../.x -> ../../../../etc +# cpmove-bad-abs absolute name: /cpmove-testuser/homedir/mail/.../.x -> ../../../../etc +# cpmove-bad-newline '\n' in a name (would split one --exclude-from pattern in two) +python3 - "$FIXTURES_DIR" <<'PY' +import io, sys, tarfile +out = sys.argv[1] +def build(fname, members): + with tarfile.open(f'{out}/{fname}', 'w:gz', format=tarfile.PAX_FORMAT) as tf: + for d in ('cpmove-testuser', 'cpmove-testuser/homedir', 'cpmove-testuser/homedir/mail', + 'cpmove-testuser/homedir/mail/example.com', 'cpmove-testuser/homedir/mail/example.com/alice', + 'cpmove-testuser/homedir/mail/example.com/alice/cur'): + ti = tarfile.TarInfo(d); ti.type = tarfile.DIRTYPE; ti.mode = 0o755; tf.addfile(ti) + for name, kind, arg in members: + ti = tarfile.TarInfo(name); ti.mode = 0o644 + if kind == 'sym': + ti.type = tarfile.SYMTYPE; ti.linkname = arg; tf.addfile(ti) + elif kind == 'hard': + ti.type = tarfile.LNKTYPE; ti.linkname = arg; tf.addfile(ti) + elif kind in ('dir', 'dir0'): + ti.type = tarfile.DIRTYPE; ti.mode = 0o755 if kind == 'dir' else 0; tf.addfile(ti) + else: + ti.size = len(arg); tf.addfile(ti, io.BytesIO(arg)) +mb = 'homedir/mail/example.com/alice' +build('cpmove-bad-dot.tar.gz', [(f'cpmove-testuser/./{mb}/.x', 'sym', '../../../../etc')]) +build('cpmove-bad-dslash.tar.gz', [(f'cpmove-testuser/homedir//mail/example.com/alice/.x', 'sym', '../../../../etc')]) +build('cpmove-bad-dotdot.tar.gz', [(f'cpmove-testuser/homedir/../{mb}/.x', 'sym', '../../../../etc')]) +build('cpmove-bad-abs.tar.gz', [(f'/cpmove-testuser/{mb}/.x', 'sym', '../../../../etc')]) +build('cpmove-bad-newline.tar.gz', [(f'cpmove-testuser/{mb}/cur/a\ncpmove-testuser/homedir/public_html', 'file', b'x')]) +# A mail-tree directory the archive makes unreadable (mode 0000): verify +# cannot inspect it, so the extract must fail (and be removed). +build('cpmove-bad-unreadable.tar.gz', [(f'cpmove-testuser/{mb}/cur/m1', 'file', b'hello'), + (f'cpmove-testuser/{mb}/.Locked', 'dir0', None), + (f'cpmove-testuser/{mb}/.Locked/cur', 'dir0', None)]) +# Passes the classifier but GNU tar fails mid-way (hard link to a member that +# does not exist), after it already wrote the mail files: the partial tree +# must be removed. +build('cpmove-bad-tarfail.tar.gz', [(f'cpmove-testuser/{mb}/cur/m1', 'file', b'hello'), + ('cpmove-testuser/homedir/public_html', 'dir', None), + ('cpmove-testuser/homedir/public_html/x', 'hard', 'cpmove-testuser/homedir/nonexistent'), + (f'cpmove-testuser/{mb}/cur/m2', 'file', b'after the failure')]) +PY +# The same mail fixture under other top-level dir names WHP accepts +# (CpanelArchivePaths::selectTopDir takes the first non-hidden dir of ANY +# name): `acct/` and cPanel's user-download layout `backup-_/`. +python3 - "$FIXTURES_DIR" <<'PY' +import io, sys, tarfile +out = sys.argv[1] +src = f'{out}/cpmove-mail.tar.gz' +for top, fname in (('acct', 'cpmove-acct.tar.gz'), ('backup-10.20.2026_10-00-00_testuser', 'cpmove-backupdir.tar.gz')): + with tarfile.open(src) as tin, tarfile.open(f'{out}/{fname}', 'w:gz', format=tarfile.PAX_FORMAT) as tout: + for m in tin.getmembers(): + data = tin.extractfile(m).read() if m.isfile() else None + m.name = m.name.replace('cpmove-testuser', top, 1) + if m.islnk(): + m.linkname = m.linkname.replace('cpmove-testuser', top, 1) + tout.addfile(m, io.BytesIO(data) if data is not None else None) +PY +# Truncated archive: the first half of a valid one. +head -c "$(( $(stat -c%s "$FIXTURES_DIR/cpmove-mail.tar.gz") / 2 ))" "$FIXTURES_DIR/cpmove-mail.tar.gz" \ + > "$FIXTURES_DIR/cpmove-bad-truncated.tar.gz" +echo "wrote $FIXTURES_DIR/cpmove-bad-{dot,dslash,dotdot,abs,newline,unreadable,tarfail,truncated}.tar.gz" + echo "" echo "fixtures built:" ls -la "$FIXTURES_DIR" diff --git a/tests/run-tests.sh b/tests/run-tests.sh new file mode 100755 index 0000000..ab8577e --- /dev/null +++ b/tests/run-tests.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# +# run-tests.sh — host-side regression tests for the sandbox scripts. +# +# Runs extract.sh and scan-files.php straight from the checkout (no image +# build): needs bash, php-cli, python3 and GNU tar. clamscan is replaced by +# a stub on PATH that records its argv and reports fixed "FOUND" hits, so +# the scan-files.php plumbing (exclude flags, quarantine, counts) is tested +# without ClamAV or signatures. The full image is covered by the docker +# commands in CONTRIBUTING.md. +# +# Run: bash tests/run-tests.sh (exit 0 = all pass) + +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIX="$REPO/tests/fixtures" +# Under /tmp: scan-files.php sets a container open_basedir that allows /tmp. +WORK="$(mktemp -d -p /tmp cpi-tests.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +FAILS=0 +pass() { printf ' PASS %s\n' "$1"; } +fail() { printf ' FAIL %s\n' "$1"; FAILS=$((FAILS + 1)); } +check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; } + +bash "$REPO/tests/build-fixtures.sh" >/dev/null + +# Copied under /tmp: scan-symlinks.php opens the tarball under the same open_basedir. +cp "$FIX/cpmove-mail.tar.gz" "$WORK/" +MAILTAR="$WORK/cpmove-mail.tar.gz" +U=cpmove-testuser +D=example.com + +# ---- extract.sh: default (no opt-in) skips the mail tree ------------------- +echo "extract.sh without IMPORT_INCLUDE_MAIL" +OUT="$WORK/ex-default" +LOG="$WORK/ex-default.log" +env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$MAILTAR" "$OUT" testuser "$WORK/ex-default.actions.json" >"$LOG" 2>&1 +check "default: homedir/mail not extracted" "[[ ! -e '$OUT/$U/homedir/mail' ]]" +check "default: homedir/etc//passwd still extracted" "[[ -f '$OUT/$U/homedir/etc/$D/passwd' ]]" +check "default: log says the mail tree was skipped" "grep -q 'skipping mail tree' '$LOG'" + +# ---- extract.sh: opt-in includes it ----------------------------------------- +echo "extract.sh with IMPORT_INCLUDE_MAIL=true" +OUT="$WORK/ex-mail" +LOG="$WORK/ex-mail.log" +IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$MAILTAR" "$OUT" testuser "$WORK/ex-mail.actions.json" >"$LOG" 2>&1 +check "opt-in: INBOX message extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/cur/1700000000.M1P1.host,S=40:2,S' ]]" +check "opt-in: subfolder message extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/.Archive/cur/1700000001.M2P1.host,S=43:2,' ]]" +check "opt-in: subscriptions extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/subscriptions' ]]" +check "opt-in: empty Maildir kept with cur/new/tmp" "[[ -d '$OUT/$U/homedir/mail/$D/empty/cur' && -d '$OUT/$U/homedir/mail/$D/empty/new' && -d '$OUT/$U/homedir/mail/$D/empty/tmp' ]]" +check "opt-in: DANGEROUS symlink inside the mail tree stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/.evil' && ! -L '$OUT/$U/homedir/mail/$D/alice/.evil' ]]" +check "opt-in: the strip is recorded as an action" "grep -q 'homedir/mail/$D/alice/.evil' '$WORK/ex-mail.actions.json'" +check "opt-in: relative file symlink (../..../etc/shadow) inside the mail tree stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/.evil-rel' && ! -L '$OUT/$U/homedir/mail/$D/alice/.evil-rel' ]]" +check "opt-in: relative folder symlink to another domain's box stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/.Other' && ! -L '$OUT/$U/homedir/mail/$D/alice/.Other' ]]" +check "opt-in: no symlink at all left under the mail tree" "[[ -z \$(find '$OUT/$U/homedir/mail' -type l) ]]" +check "opt-in: the DANGEROUS in-mail symlink is listed once, not twice" "[[ \$(grep -c 'alice/.evil\"' '$WORK/ex-mail.actions.json') == 1 ]]" +check "opt-in: relative symlinks recorded as stripped_mail_link actions" "grep -q 'alice/.evil-rel' '$WORK/ex-mail.actions.json' && grep -q 'alice/.Other' '$WORK/ex-mail.actions.json'" +check "opt-in: hard link INSIDE the mail tree (Dovecot copy) kept" "[[ -f '$OUT/$U/homedir/mail/$D/alice/cur/1700000002.M3P1.host,S=41:2,S' && -f '$OUT/$U/homedir/mail/$D/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S' ]]" +check "opt-in: hard link crossing INTO the mail tree (message -> etc/shadow) stripped" "[[ ! -e '$OUT/$U/homedir/mail/$D/alice/cur/leak' ]]" +check "opt-in: log says the mail tree was included" "grep -q 'including mail tree' '$LOG'" +check "opt-in: no 'skipping mail tree' line" "! grep -q 'skipping mail tree' '$LOG'" + +echo "N2-1: relative symlinks that escape the archive / the homedir (cpmove-relesc)" +cp "$FIX/cpmove-relesc.tar.gz" "$WORK/" +php "$REPO/scripts/lib/scan-symlinks.php" --tarball "$WORK/cpmove-relesc.tar.gz" --report "$WORK/relesc-scan.json" --username testuser 2>/dev/null +relesc_type() { php -r '$r = json_decode(file_get_contents($argv[1]), true); foreach ($r["findings"] as $f) if ($f["archive_path"] === $argv[2]) { echo $f["type"]; exit; } echo "NONE";' "$WORK/relesc-scan.json" "$1"; } +check "relesc: sds -> ../../../../etc/passwd is DANGEROUS" "[[ \$(relesc_type '$U/sds') == DANGEROUS ]]" +check "relesc: homedir/public_html/dump.sql -> ../../mysql/... (escapes homedir) is DANGEROUS" "[[ \$(relesc_type '$U/homedir/public_html/dump.sql') == DANGEROUS ]]" +check "relesc: homedir/www -> public_html (inside homedir) is not a finding" "[[ \$(relesc_type '$U/homedir/www') == NONE ]]" +check "relesc: homedir/about-shortcut -> public_html/about.txt is not a finding" "[[ \$(relesc_type '$U/homedir/about-shortcut') == NONE ]]" +OUT="$WORK/ex-relesc" +env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/cpmove-relesc.tar.gz" "$OUT" testuser "$WORK/ex-relesc.actions.json" >"$WORK/ex-relesc.log" 2>&1 +check "relesc: sds stripped from the extract" "[[ ! -e '$OUT/$U/sds' && ! -L '$OUT/$U/sds' ]]" +check "relesc: dump.sql link stripped from the extract" "[[ ! -L '$OUT/$U/homedir/public_html/dump.sql' ]]" +check "relesc: homedir/www link kept" "[[ -L '$OUT/$U/homedir/www' ]]" +check "relesc: both strips recorded" "grep -q '$U/sds' '$WORK/ex-relesc.actions.json' && grep -q 'public_html/dump.sql' '$WORK/ex-relesc.actions.json'" + +echo "N2-1 review: symlink chains (cpmove-chain)" +cp "$FIX/cpmove-chain.tar.gz" "$WORK/" +php "$REPO/scripts/lib/scan-symlinks.php" --tarball "$WORK/cpmove-chain.tar.gz" --report "$WORK/relesc-scan.json" --username testuser 2>/dev/null +check "chain: d1/d2/a -> b/../.. (through link b) is DANGEROUS" "[[ \$(relesc_type '$U/d1/d2/a') == DANGEROUS ]]" +check "chain: d1/d2/l1 -> l2/../../.. (through link l2) is DANGEROUS" "[[ \$(relesc_type '$U/d1/d2/l1') == DANGEROUS ]]" +check "chain: d1/d2/b -> ../.. alone is not a finding" "[[ \$(relesc_type '$U/d1/d2/b') == NONE ]]" +OUT="$WORK/ex-chain" +env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/cpmove-chain.tar.gz" "$OUT" testuser "$WORK/ex-chain.actions.json" >"$WORK/ex-chain.log" 2>&1 +check "chain: a and l1 absent from the extract" "[[ ! -L '$OUT/$U/d1/d2/a' && ! -L '$OUT/$U/d1/d2/l1' ]]" +check "chain: b and l2 kept" "[[ -L '$OUT/$U/d1/d2/b' && -L '$OUT/$U/d1/d2/l2' ]]" + +echo "N2-1 review: post-extract authority strip-escaping-links.py (links the classifier never saw)" +P="$WORK/postx" +rm -rf "$P"; mkdir -p "$P/$U/d1/d2" "$P/$U/homedir/public_html/inner" +echo x > "$P/$U/homedir/public_html/inner/f" +ln -s "../.." "$P/$U/d1/d2/b"; ln -s "b/../.." "$P/$U/d1/d2/a" +ln -s "../.." "$P/$U/d1/d2/l2"; ln -s "l2/../../.." "$P/$U/d1/d2/l1" +ln -s "/etc/passwd" "$P/$U/homedir/abs-etc" +ln -s "nonexistent" "$P/$U/homedir/public_html/x.php.bak.cpanel-import" +ln -s "../../../../../../victim/shell.php" "$P/$U/homedir/public_html/y.bak" +ln -s "loop2" "$P/$U/homedir/loop1"; ln -s "loop1" "$P/$U/homedir/loop2" +ln -s "../.." "$P/$U/top-link" +ln -s "inner/f" "$P/$U/homedir/public_html/ok-file" +ln -s "public_html" "$P/$U/homedir/www" +ln -s "/home/testuser/public_html/inner" "$P/$U/homedir/own-abs" +# Review round 2: own-account absolute links are judged re-rooted in the +# extract, normalized, and only for the TRUSTED username argument. +ln -s "/home/testuser/../../etc/shadow" "$P/$U/homedir/own-dotdot" +ln -s "/home/testuser/./public_html/inner" "$P/$U/homedir/own-dot" +ln -s "/home/testuser/public_html/missing" "$P/$U/homedir/own-dangling" +ln -s "../../.." "$P/$U/homedir/esc"; ln -s "/home/testuser/esc" "$P/$U/homedir/own-via-esc" +ln -s "/home/root/x" "$P/$U/homedir/other-user" +mkdir -p "$P/cpmove-root/homedir/x"; ln -s "/home/root/x" "$P/cpmove-root/homedir/archive-named-user" +echo '{"actions":[{"action":"pre"}]}' > "$WORK/postx.json" +n=$(python3 -I "$REPO/scripts/lib/strip-escaping-links.py" "$P" "$WORK/postx.json" testuser 2>/dev/null) +for gone in d1/d2/a d1/d2/l1 homedir/abs-etc homedir/public_html/x.php.bak.cpanel-import homedir/public_html/y.bak homedir/loop1 homedir/loop2 top-link \ + homedir/own-dotdot homedir/own-dot homedir/own-dangling homedir/esc homedir/own-via-esc homedir/other-user; do + check "post-extract: $gone stripped" "[[ ! -L '$P/$U/$gone' ]]" +done +for kept in d1/d2/b d1/d2/l2 homedir/public_html/ok-file homedir/www homedir/own-abs; do + check "post-extract: $kept kept" "[[ -L '$P/$U/$kept' ]]" +done +check "post-extract: cpmove-root/…/archive-named-user (/home/root/x; 'root' only named by the archive) stripped" "[[ ! -L '$P/cpmove-root/homedir/archive-named-user' ]]" +check "post-extract: 15 links stripped (got $n)" "[[ '$n' == 15 ]]" +check "post-extract: strips appended as stripped_escaping_symlink actions" "grep -q '\"pre\"' '$WORK/postx.json' && [[ \$(grep -c stripped_escaping_symlink '$WORK/postx.json') == 15 ]]" +rm -rf "$P"; mkdir -p "$P/$U/homedir/public_html/inner" +ln -s "/home/testuser/public_html/inner" "$P/$U/homedir/own-abs" +python3 -I "$REPO/scripts/lib/strip-escaping-links.py" "$P" "$WORK/postx2.json" >/dev/null 2>&1 +check "post-extract: with no trusted username, an own-account link is NOT exempt" "[[ ! -L '$P/$U/homedir/own-abs' ]]" +rm -rf "$P"; mkdir -p "$P/$U/homedir/public_html/inner" +ln -s "/home/testuser/public_html/inner" "$P/$U/homedir/own-abs" +python3 -I "$REPO/scripts/lib/strip-escaping-links.py" "$P" "$WORK/postx3.json" 'testuser/../x' >/dev/null 2>&1 +check "post-extract: an invalid 'trusted' username exempts nothing" "[[ ! -L '$P/$U/homedir/own-abs' ]]" +echo "extract.sh: an archive cannot name its own exempt user (top dir cpmove-root)" +AR="$WORK/archuser-src"; rm -rf "$AR"; mkdir -p "$AR/cpmove-root/homedir/x" +ln -s "/home/root/x" "$AR/cpmove-root/homedir/self-exempt" +tar -C "$AR" -czf "$WORK/archuser.tar.gz" cpmove-root +env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/archuser.tar.gz" "$WORK/ex-archuser" testuser "$WORK/archuser.json" >/dev/null 2>&1 +check "archive-named user: /home/root/x link stripped (only the WHP username is trusted)" "[[ -d '$WORK/ex-archuser/cpmove-root/homedir/x' && ! -L '$WORK/ex-archuser/cpmove-root/homedir/self-exempt' ]]" +env -u IMPORT_INCLUDE_MAIL bash "$REPO/scripts/extract.sh" "$WORK/archuser.tar.gz" "$WORK/ex-archuser2" root "$WORK/archuser2.json" >/dev/null 2>&1 +check "control: the same link is kept when WHP itself passes that username" "[[ -L '$WORK/ex-archuser2/cpmove-root/homedir/self-exempt' ]]" + +echo "scan-symlinks: /home//../../etc is not user-internal" +DT="$WORK/dotdot-src"; rm -rf "$DT"; mkdir -p "$DT/$U/homedir" +ln -s "/home/testuser/../../etc/shadow" "$DT/$U/homedir/sneaky" +tar -C "$DT" -czf "$WORK/dotdot.tar.gz" "$U" +php "$REPO/scripts/lib/scan-symlinks.php" --tarball "$WORK/dotdot.tar.gz" --report "$WORK/relesc-scan.json" --username testuser 2>/dev/null +check "scan: /home/testuser/../../etc/shadow is DANGEROUS (not BENIGN_USER_INTERNAL)" "[[ \$(relesc_type '$U/homedir/sneaky') == DANGEROUS ]]" + +echo "hard link crossing OUT of the mail tree (cpmove-mail-xlink)" +cp "$FIX/cpmove-mail-xlink.tar.gz" "$WORK/" +OUT="$WORK/ex-xlink" +IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-mail-xlink.tar.gz" "$OUT" testuser "$WORK/ex-xlink.actions.json" >"$WORK/ex-xlink.log" 2>&1 +check "opt-in: public_html/stolen.txt (hard link to a message) stripped" "[[ ! -e '$OUT/$U/homedir/public_html/stolen.txt' ]]" +check "opt-in: the linked message itself still extracted" "[[ -f '$OUT/$U/homedir/mail/$D/alice/.Archive/cur/1700000002.M3P1.host,S=41:2,S' ]]" +check "default mode: same archive fails closed (tar cannot link to the excluded mail member)" \ + "! env -u IMPORT_INCLUDE_MAIL bash '$REPO/scripts/extract.sh' '$WORK/cpmove-mail-xlink.tar.gz' '$WORK/ex-xlink-default' testuser '$WORK/x.json' >/dev/null 2>&1" + +echo "parser-differential archives are refused (IMPORT_INCLUDE_MAIL=true)" +for bad in dot dslash dotdot abs newline; do + cp "$FIX/cpmove-bad-$bad.tar.gz" "$WORK/" + OUT="$WORK/ex-bad-$bad" + rc=0 + IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-bad-$bad.tar.gz" "$OUT" testuser "$WORK/bad-$bad.json" >"$WORK/bad-$bad.log" 2>&1 || rc=$? + check "bad-$bad: extract exits non-zero (rc=$rc)" "[[ $rc -ne 0 ]]" + check "bad-$bad: refusal logged" "grep -q 'REFUSING archive' '$WORK/bad-$bad.log'" + check "bad-$bad: extract dir removed" "[[ ! -e '$OUT' ]]" +done + +echo "failures after extraction starts remove the extract (IMPORT_INCLUDE_MAIL=true)" +for bad in unreadable tarfail truncated; do + cp "$FIX/cpmove-bad-$bad.tar.gz" "$WORK/" + OUT="$WORK/ex-bad-$bad" + rc=0 + IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-bad-$bad.tar.gz" "$OUT" testuser "$WORK/bad-$bad.json" >"$WORK/bad-$bad.log" 2>&1 || rc=$? + check "bad-$bad: extract exits non-zero (rc=$rc)" "[[ $rc -ne 0 ]]" + check "bad-$bad: extract dir removed" "[[ ! -e '$OUT' ]]" +done +check "bad-unreadable: failed in verify (cannot inspect the 0000 dir)" "grep -q 'not readable+searchable\|cannot inspect' '$WORK/bad-unreadable.log'" +check "bad-tarfail: tar itself failed (rc logged), verify still ran" "grep -q 'tar failed (rc=' '$WORK/bad-tarfail.log'" +check "bad-truncated: refused by the classifier" "grep -q 'REFUSING archive' '$WORK/bad-truncated.log'" + +echo "classifier refuses when GNU tar lists a different member count" +FAKEBIN="$WORK/faketar" +mkdir -p "$FAKEBIN" +REALTAR="$(command -v tar)" +cat > "$FAKEBIN/tar" < "$WORK/cm.excl"; echo '{"actions":[]}' > "$WORK/cm.json" +rc=0 +PATH="$FAKEBIN:$PATH" python3 -I "$REPO/scripts/lib/mail-tree-links.py" scan "$MAILTAR" "$WORK/cm.excl" "$WORK/cm.json" >/dev/null 2>"$WORK/cm.log" || rc=$? +check "count mismatch: scan exits 3 (rc=$rc)" "[[ $rc -eq 3 ]]" +check "count mismatch: reason logged" "grep -q 'member count differs' '$WORK/cm.log'" +rc=0 +python3 -I "$REPO/scripts/lib/mail-tree-links.py" scan "$MAILTAR" "$WORK/cm.excl" "$WORK/cm.json" >/dev/null 2>&1 || rc=$? +check "control: real tar, same archive => scan exits 0" "[[ $rc -eq 0 ]]" + +echo "post-extract authority: mail-tree-links.py verify" +V="$WORK/verify" +mkvt() { rm -rf "$V"; mkdir -p "$V/$U/homedir/mail/$D/a/cur" "$V/$U/homedir/mail/$D/a/.F/cur" "$V/$U/homedir/public_html" "$V/cpmove-other/homedir/mail/$D/b/cur"; echo m > "$V/$U/homedir/mail/$D/a/cur/1"; } +vrc() { local rc=0; python3 -I "$REPO/scripts/lib/mail-tree-links.py" verify "$V" >/dev/null 2>&1 || rc=$?; echo $rc; } +mkvt; check "verify: clean tree => 0" "[[ \$(vrc) == 0 ]]" +mkvt; ln "$V/$U/homedir/mail/$D/a/cur/1" "$V/$U/homedir/mail/$D/a/.F/cur/1" +check "verify: Dovecot hard link inside one mail tree => 0 (kept)" "[[ \$(vrc) == 0 ]]" +mkvt; ln -s ../../../../../etc "$V/$U/homedir/mail/$D/a/.evil" +check "verify: symlink in the mail tree (survived the pre-filter) => 4" "[[ \$(vrc) == 4 ]]" +mkvt; ln "$V/$U/homedir/mail/$D/a/cur/1" "$V/$U/homedir/public_html/out" +check "verify: hard link OUT of the mail tree (survived the pre-filter) => 4" "[[ \$(vrc) == 4 ]]" +mkvt; echo s > "$V/$U/homedir/shadow"; ln "$V/$U/homedir/shadow" "$V/$U/homedir/mail/$D/a/cur/in" +check "verify: hard link INTO the mail tree (survived the pre-filter) => 4" "[[ \$(vrc) == 4 ]]" +mkvt; ln "$V/$U/homedir/mail/$D/a/cur/1" "$V/cpmove-other/homedir/mail/$D/b/cur/1" +check "verify: hard link between two accounts' mail trees => 4" "[[ \$(vrc) == 4 ]]" +mkvt; rm -rf "$V/$U/homedir/mail"; ln -s /etc "$V/$U/homedir/mail" +check "verify: homedir/mail itself a symlink => 4" "[[ \$(vrc) == 4 ]]" + +echo "extract.sh with IMPORT_INCLUDE_MAIL=1 (only \"true\" opts in)" +OUT="$WORK/ex-one" +IMPORT_INCLUDE_MAIL=1 bash "$REPO/scripts/extract.sh" "$MAILTAR" "$OUT" testuser "$WORK/ex-one.actions.json" >/dev/null 2>&1 +check "IMPORT_INCLUDE_MAIL=1: mail tree skipped" "[[ ! -e '$OUT/$U/homedir/mail' ]]" + +# ---- scan-files.php with a stub clamscan ------------------------------------ +# The stub reports one hit inside the mail tree and one in public_html. +STUBBIN="$WORK/bin" +mkdir -p "$STUBBIN" +cat > "$STUBBIN/clamscan" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$@" > "$CLAMSCAN_ARGS_OUT" +dir="${!#}" +top="${CLAM_TOP:-cpmove-testuser}" +echo "$dir/$top/homedir/mail/example.com/alice/cur/1700000000.M1P1.host,S=40:2,S: Test.Mail.Sig FOUND" +echo "$dir/$top/homedir/public_html/index.php: Test.Php.Sig FOUND" +STUB +chmod +x "$STUBBIN/clamscan" + +run_scan() { # [extra args] + local ex="$1" tag="$2"; shift 2 + mkdir -p "$WORK/q-$tag" + CLAMSCAN_ARGS_OUT="$WORK/clam-$tag.args" PATH="$STUBBIN:$PATH" \ + php "$REPO/scripts/scan-files.php" --extract "$ex" --quarantine "$WORK/q-$tag" \ + --report "$WORK/scan-$tag.json" --import-id "t-$tag" "$@" 2>"$WORK/scan-$tag.log" +} +jq_php() { php -r '$r = json_decode(file_get_contents($argv[1]), true); echo json_encode($r[$argv[2]] ?? null);' "$1" "$2"; } + +echo "scan-files.php --skip-mail (mail tree extracted)" +cp -a "$WORK/ex-mail" "$WORK/scan-skip" +run_scan "$WORK/scan-skip" skip --skip-mail +# Expected argv entry: the extract dir with ERE metacharacters escaped +# (the only one mktemp's path can hold is "."), then the mail-tree pattern. +SKIPDIR="$WORK/scan-skip" +EXCL="--exclude-dir=^${SKIPDIR//./\\.}/[^/]+/homedir/mail(/.*)?\$" +if grep -qxF -- "$EXCL" "$WORK/clam-skip.args"; then + pass "--skip-mail: clamscan gets --exclude-dir for the mail tree" +else + fail "--skip-mail: clamscan gets --exclude-dir for the mail tree (want $EXCL; got: $(tr '\n' ' ' < "$WORK/clam-skip.args"))" +fi +check "--skip-mail: a hit inside the mail tree is NOT quarantined (message stays in the Maildir)" \ + "[[ -f '$WORK/scan-skip/$U/homedir/mail/$D/alice/cur/1700000000.M1P1.host,S=40:2,S' ]]" +check "--skip-mail: a hit outside the mail tree is still quarantined" \ + "[[ ! -e '$WORK/scan-skip/$U/homedir/public_html/index.php' && -f '$WORK/q-skip/$U/homedir/public_html/index.php' ]]" +check "--skip-mail: report files_quarantined = 1" "[[ \$(jq_php '$WORK/scan-skip.json' files_quarantined) == 1 ]]" +check "--skip-mail: report mail_scan_skipped = true" "[[ \$(jq_php '$WORK/scan-skip.json' mail_scan_skipped) == true ]]" +NONMAIL=$(cd "$WORK/scan-skip" && find . -type f ! -path "./$U/homedir/mail/*" | wc -l) +check "--skip-mail: files_scanned counts only non-mail files ($NONMAIL, after the quarantine move)" \ + "[[ \$(jq_php '$WORK/scan-skip.json' files_scanned) == $((NONMAIL + 1)) ]]" +check "--skip-mail: the skip is logged with its reason" "grep -q 'NOT scanning the mail tree' '$WORK/scan-skip.log'" + +echo "I-1: top-level dirs other than cpmove-* (acct/, backup-_/)" +for fx in acct:acct backupdir:backup-10.20.2026_10-00-00_testuser; do + name="${fx%%:*}"; top="${fx#*:}" + cp "$FIX/cpmove-$name.tar.gz" "$WORK/" + OUT="$WORK/ex-$name" + rc=0 + IMPORT_INCLUDE_MAIL=true bash "$REPO/scripts/extract.sh" "$WORK/cpmove-$name.tar.gz" "$OUT" testuser "$WORK/ex-$name.actions.json" >"$WORK/ex-$name.log" 2>&1 || rc=$? + M="$OUT/$top/homedir/mail/$D/alice" + check "$name: extract exits 0 (rc=$rc)" "[[ $rc -eq 0 ]]" + check "$name: relative symlinks in the Maildir stripped (.evil-rel, .Other)" "[[ ! -L '$M/.evil-rel' && ! -L '$M/.Other' ]]" + check "$name: no symlink at all under $top/homedir/mail" "[[ -z \$(find '$OUT/$top/homedir/mail' -type l) ]]" + check "$name: hard link INTO the mail tree stripped" "[[ ! -e '$M/cur/leak' ]]" + check "$name: in-tree Dovecot hard link kept" "[[ -f '$M/cur/1700000002.M3P1.host,S=41:2,S' ]]" + check "$name: strips recorded as stripped_mail_link" "grep -q '$top/homedir/mail/$D/alice/.evil-rel' '$WORK/ex-$name.actions.json'" + rm -rf "$WORK/q-$name" + CLAM_TOP="$top" CLAMSCAN_ARGS_OUT="$WORK/clam-$name.args" PATH="$STUBBIN:$PATH" \ + php "$REPO/scripts/scan-files.php" --extract "$OUT" --quarantine "$WORK/q-$name" \ + --report "$WORK/scan-$name.json" --import-id "t-$name" --skip-mail 2>/dev/null + check "$name: --skip-mail keeps a hit inside $top/homedir/mail in the Maildir" "[[ -f '$M/cur/1700000000.M1P1.host,S=40:2,S' ]]" + check "$name: --skip-mail still quarantines public_html" "[[ ! -e '$OUT/$top/homedir/public_html/index.php' ]]" +done +mkvt; mkdir -p "$V/acct/homedir/mail/$D/a/cur"; ln -s ../../../../../etc "$V/acct/homedir/mail/$D/a/x" +check "verify: symlink under acct/homedir/mail => 4" "[[ \$(vrc) == 4 ]]" +mkvt; ln -s "$U" "$V/zz-link" +check "verify: top-level symlink => 4 (never walked through)" "[[ \$(vrc) == 4 ]]" + +if [[ -n "${WHP_REPO:-}" ]]; then + echo "M-4: contract with the WHP panel at \$WHP_REPO" + SAN="$WHP_REPO/web-files/libs/CpanelImportSanitizationContainer.php" + check "WHP sends the env name this image reads (INCLUDE_MAIL_ENV = 'IMPORT_INCLUDE_MAIL')" "grep -q \"INCLUDE_MAIL_ENV = 'IMPORT_INCLUDE_MAIL'\" '$SAN'" + check "WHP sends exactly '=true'" "grep -q \"INCLUDE_MAIL_ENV . '=true'\" '$SAN'" + check "WHP reads the report key this image writes (mail_included === true)" "grep -q \"\\['mail_included'\\] ?? null) === true\" '$SAN'" + check "this image writes mail_included" "grep -q '\"mail_included\"' '$REPO/scripts/entrypoint.sh'" +fi + +echo "scan-files.php without --skip-mail (unchanged behaviour)" +cp -a "$WORK/ex-mail" "$WORK/scan-noskip" +run_scan "$WORK/scan-noskip" noskip +check "no flag: no --exclude-dir passed" "! grep -q -- '--exclude-dir' '$WORK/clam-noskip.args'" +check "no flag: report mail_scan_skipped = false" "[[ \$(jq_php '$WORK/scan-noskip.json' mail_scan_skipped) == false ]]" +check "no flag: both hits quarantined" "[[ \$(jq_php '$WORK/scan-noskip.json' files_quarantined) == 2 ]]" + +echo +if [[ $FAILS -eq 0 ]]; then echo "ALL PASS"; else echo "$FAILS FAILURE(S)"; fi +exit $(( FAILS == 0 ? 0 : 1 ))