diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 67bf69d..f394afc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,9 +121,23 @@ ClamAV signature DBs). bash tests/build-fixtures.sh ``` -Two tarballs land under `tests/fixtures/`: +Three tarballs land under `tests/fixtures/`: - `cpmove-clean.tar.gz` — a benign cpmove with a WordPress MyISAM dump. - `cpmove-alfa.tar.gz` — same shape PLUS an ALFA-style symlink to /etc. +- `cpmove-mail.tar.gz` — same shape PLUS a mail tree (two Maildirs, one + empty) with an ALFA-style symlink inside it. + +### Host-side regression tests + +```bash +bash tests/run-tests.sh +``` + +Runs `extract.sh` and `scan-files.php` straight from the checkout (bash, +php-cli, python3, GNU tar; `clamscan` is stubbed). Covers the mail-tree +opt-in (`IMPORT_INCLUDE_MAIL=true`): skipped by default, extracted when +opted in, dangerous symlinks inside it still stripped, and never scanned +or quarantined by `scan-files.php --skip-mail`. ### Run against the clean fixture diff --git a/README.md b/README.md index cee07e8..7aeb393 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ docker run \ --env IMPORT_USERNAME=${USERNAME} \ --env IMPORT_BACKUP_FILE=/host/backup/${BACKUP_NAME} \ --env CLAMAV_REFRESH=true \ + --env IMPORT_INCLUDE_MAIL=true \ --memory=4g \ --memory-swap=4g \ --cpus=2 \ @@ -57,6 +58,15 @@ docker run \ repo.anhonesthost.net/cloud-hosting-platform/cpanel-importer:2026.05.NNN ``` +`IMPORT_INCLUDE_MAIL=true` is optional: WHP passes it only when the import +copies mailboxes. Without it `cpmove-*/homedir/mail` is not extracted at all. +With it the mail tree is extracted (dangerous symlinks inside it are still +stripped) but **not** ClamAV-scanned: mail is copied verbatim into the +customer's own mailbox and verified message-by-message, so quarantining a +message would silently drop mail, and multi-GB maildirs would dominate the +run. `report.json` then carries `"mail_included": true`; images older than +this option never write the key, which is how WHP tells them apart. + Container exits with status `0` on success, non-zero on any failure (missing/unreadable backup, dangerous symlink found, scanner error). Even on failure, `/host/sanitized//report.json` is written @@ -94,6 +104,7 @@ run, success or failure. "files_clean": 28432, "files_cleaned": 0, "files_quarantined": 39, + "mail_included": false, "actions": [ { "path": "cpmove-testuser/homedir/public_html/example.com/ALFA_DATA/index.php", diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index d78564c..0f19e89 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -8,6 +8,11 @@ # IMPORT_BACKUP_FILE absolute path inside the container, typically # /host/backup/cpmove-.tar.gz # CLAMAV_REFRESH "true" to run freshclam at start (default: true) +# IMPORT_INCLUDE_MAIL "true" to extract cpmove-*/homedir/mail for WHP's +# mailbox import (default: false = mail tree skipped). +# The mail tree is never ClamAV-scanned. report.json +# records "mail_included" so the panel can tell this +# image (which honours the opt-in) from an older one. # # Flow (spec §0): # 1. validate env @@ -71,8 +76,15 @@ log "cpanel-importer starting (container UID=$(id -u) GID=$(id -g))" : "${IMPORT_USERNAME:?IMPORT_USERNAME env var is required}" : "${IMPORT_BACKUP_FILE:?IMPORT_BACKUP_FILE env var is required}" CLAMAV_REFRESH="${CLAMAV_REFRESH:-true}" +# Exactly "true" opts in; anything else is the default (no mail tree). +if [[ "${IMPORT_INCLUDE_MAIL:-false}" == "true" ]]; then + IMPORT_INCLUDE_MAIL=true +else + IMPORT_INCLUDE_MAIL=false +fi +export IMPORT_INCLUDE_MAIL -log "import_id=$IMPORT_ID username=$IMPORT_USERNAME backup=$IMPORT_BACKUP_FILE" +log "import_id=$IMPORT_ID username=$IMPORT_USERNAME backup=$IMPORT_BACKUP_FILE include_mail=$IMPORT_INCLUDE_MAIL" if [[ ! -f "$IMPORT_BACKUP_FILE" ]]; then die "backup file does not exist or is not a regular file: $IMPORT_BACKUP_FILE" @@ -140,11 +152,17 @@ fi STAGE="scan_files" log "stage: scan_files" +SCAN_FILES_EXTRA=() +if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then + # The extracted mail tree is not scanned (see scan-files.php --skip-mail). + SCAN_FILES_EXTRA+=(--skip-mail) +fi php /scripts/scan-files.php \ --extract "$EXTRACT_DIR" \ --quarantine "$QUARANTINE_DIR" \ --report /tmp/scan-files-report.json \ --import-id "$IMPORT_ID" \ + "${SCAN_FILES_EXTRA[@]}" \ || die "scan-files.php failed; see stderr above" # --- DB engine swap + WP content scan ------------------------------------- @@ -192,6 +210,7 @@ $filesPath = $argv[3]; $dbsPath = $argv[4]; $strippedPath = $argv[5]; $outPath = $argv[6]; +$mailIncluded = $argv[7] === "true"; $files = is_file($filesPath) ? json_decode(file_get_contents($filesPath), true) : null; $dbs = is_file($dbsPath) ? json_decode(file_get_contents($dbsPath), true) : null; @@ -252,6 +271,10 @@ $report = [ "files_clean" => $filesClean, "files_cleaned" => $filesCleaned, "files_quarantined" => $filesQuarantined, + // true only when IMPORT_INCLUDE_MAIL=true AND this image extracted the + // mail tree. Older images never write the key; the panel treats a + // missing/false value as "mail was not extracted". + "mail_included" => $mailIncluded, "actions" => $actions, "databases" => $databases, "summary_for_panel" => [ @@ -263,7 +286,7 @@ $report = [ file_put_contents($outPath, json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); fprintf(STDERR, "report written: %s\n", $outPath); -' "$IMPORT_ID" "$DURATION" /tmp/scan-files-report.json /tmp/scan-dbs-report.json "$STRIPPED_SYMLINKS_FILE" "$SANITIZED_DIR/report.json" \ +' "$IMPORT_ID" "$DURATION" /tmp/scan-files-report.json /tmp/scan-dbs-report.json "$STRIPPED_SYMLINKS_FILE" "$SANITIZED_DIR/report.json" "$IMPORT_INCLUDE_MAIL" \ || die "report merge failed" log "done — exited cleanly after ${DURATION}s" diff --git a/scripts/extract.sh b/scripts/extract.sh index e42c447..3673878 100755 --- a/scripts/extract.sh +++ b/scripts/extract.sh @@ -4,6 +4,10 @@ # # Usage: extract.sh [] # +# Env: IMPORT_INCLUDE_MAIL "true" = extract cpmove-*/homedir/mail too +# (WHP's mailbox import needs the Maildirs). +# Anything else (default) = skip the mail tree. +# # Calls scripts/lib/scan-symlinks.php first, then untars the cpmove with # every DANGEROUS-classified symlink entry stripped via tar --exclude. # The stripped-symlinks list is written as JSON to (default @@ -21,6 +25,11 @@ DEST="${2:?usage: extract.sh []}" USERNAME="${3:?usage: extract.sh []}" ACTIONS_OUT="${4:-${DEST}/.cpanel-importer-stripped-symlinks.json}" +IMPORT_INCLUDE_MAIL="${IMPORT_INCLUDE_MAIL:-false}" +# Resolve sibling scripts relative to this file (/scripts in the image) so +# tests/run-tests.sh can run the same script from a checkout. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ts() { date -u +'%Y-%m-%dT%H:%M:%SZ'; } log() { printf '[%s] extract: %s\n' "$(ts)" "$*"; } @@ -31,7 +40,7 @@ mkdir -p "$DEST" log "scanning tarball for dangerous symlinks (cpmove vector check)" SYMLINK_REPORT=$(mktemp -p /tmp scan-symlinks.XXXXXX.json) -if ! php /scripts/lib/scan-symlinks.php \ +if ! php "$SCRIPT_DIR/lib/scan-symlinks.php" \ --tarball "$TARBALL" \ --username "$USERNAME" \ --report "$SYMLINK_REPORT"; then @@ -86,19 +95,24 @@ if [[ "$DANGEROUS_COUNT" -gt 0 ]]; then done < "$EXCLUDES_FILE" fi -# Also skip the cpanel user's mail spool entirely. WHP doesn't import -# email (mail-import is a panel-side roadmap item, not a sandbox-mode -# step) so extracting + scanning the mailbox tree wastes time and disk: -# on real customer accounts the mail dir often dwarfs everything else -# (10+ GB of historical mbox/maildir). The exclude pattern uses tar's -# fnmatch globs and covers both maildir + spool layouts at any depth -# under homedir/. Logged as an info line so it shows up in the panel -# import log alongside the symlink strips. -{ - echo "cpmove-*/homedir/mail" - echo "cpmove-*/homedir/mail/*" -} >> "$EXCLUDES_FILE" -log "skipping mail tree (cpmove-*/homedir/mail) — WHP does not import cPanel mail" +# Mail tree (cpmove-*/homedir/mail). Skipped by default: a plain site +# import doesn't need it, and on real customer accounts the mail dir often +# dwarfs everything else (10+ GB of historical maildir). The exclude +# pattern uses tar's fnmatch globs and covers the whole tree at any depth. +# +# IMPORT_INCLUDE_MAIL=true (WHP passes it only when the import copies +# mailboxes) extracts it. The DANGEROUS-symlink strips computed above apply +# to the mail tree exactly as to the rest of the archive: scan-symlinks.php +# lists every entry in the tarball, mail included. +if [[ "$IMPORT_INCLUDE_MAIL" == "true" ]]; then + log "including mail tree (cpmove-*/homedir/mail) — IMPORT_INCLUDE_MAIL=true" +else + { + echo "cpmove-*/homedir/mail" + echo "cpmove-*/homedir/mail/*" + } >> "$EXCLUDES_FILE" + log "skipping mail tree (cpmove-*/homedir/mail) — IMPORT_INCLUDE_MAIL not set" +fi # --- extract -------------------------------------------------------------- diff --git a/scripts/scan-files.php b/scripts/scan-files.php index 39de3ac..ebc126d 100755 --- a/scripts/scan-files.php +++ b/scripts/scan-files.php @@ -7,7 +7,11 @@ * for v1.1 expansion; see CONTRIBUTING.md for how to wire one in. * * Usage: - * scan-files.php --extract --quarantine --report --import-id + * scan-files.php --extract --quarantine --report --import-id [--skip-mail] + * + * --skip-mail: do not scan (or count, or quarantine anything in) the + * cpmove-/homedir/mail tree. entrypoint.sh passes it when the mail tree + * was extracted for WHP's mailbox import (IMPORT_INCLUDE_MAIL=true). * * Exit codes: * 0 — scan completed (regardless of how many hits) @@ -28,10 +32,10 @@ require __DIR__ . '/lib/safety-net.php'; const SCANNER_VERSION = '1.0.0'; -$opts = getopt('', ['extract:', 'quarantine:', 'report:', 'import-id:']); +$opts = getopt('', ['extract:', 'quarantine:', 'report:', 'import-id:', 'skip-mail']); foreach (['extract', 'quarantine', 'report', 'import-id'] as $k) { if (!isset($opts[$k])) { - fwrite(STDERR, "usage: scan-files.php --extract --quarantine --report --import-id \n"); + fwrite(STDERR, "usage: scan-files.php --extract --quarantine --report --import-id [--skip-mail]\n"); exit(2); } } @@ -39,6 +43,15 @@ $extractDir = rtrim($opts['extract'], '/'); $quarantineDir = rtrim($opts['quarantine'], '/'); $reportPath = $opts['report']; $importId = $opts['import-id']; +$skipMail = array_key_exists('skip-mail', $opts); + +// The mail tree is copied verbatim into the customer's own mailbox, and +// WHP verifies the copy message-by-message against what it packaged. So it +// is NOT scanned: quarantining a message would silently drop mail, and +// ClamAV over a multi-GB maildir would dominate the run. PCRE form for the +// PHP walk, POSIX ERE form for clamscan --exclude-dir. +$mailDirPcre = '#\A' . preg_quote($extractDir, '#') . '/cpmove-[^/]+/homedir/mail(?:/|\z)#'; +$mailDirEre = '^' . preg_replace('/[.\[\]()*+?{}|^$\\\\]/', '\\\\$0', $extractDir) . '/cpmove-[^/]+/homedir/mail(/.*)?$'; if (!is_dir($extractDir)) { fwrite(STDERR, "scan-files: extract dir does not exist: $extractDir\n"); @@ -48,6 +61,9 @@ if (!is_dir($extractDir)) { @mkdir($quarantineDir, 0750, true); fwrite(STDERR, "scan-files: starting (extract=$extractDir, quarantine=$quarantineDir)\n"); +if ($skipMail) { + fwrite(STDERR, "scan-files: NOT scanning the mail tree (cpmove-*/homedir/mail): it is copied verbatim into the customer's own mailbox and verified against the source; scanning would quarantine messages out of it and multi-GB maildirs would dominate the run\n"); +} // -- v1.0 cleaner registry (intentionally empty) ---------------------------- // @@ -75,7 +91,8 @@ $cleaners = []; // Output format per line on a hit: // /tmp/extract/foo/bar.php: Some.Signature.Name FOUND $cmd = sprintf( - 'clamscan --infected --recursive --no-summary --stdout %s 2>/dev/null', + 'clamscan --infected --recursive --no-summary --stdout %s%s 2>/dev/null', + $skipMail ? '--exclude-dir=' . escapeshellarg($mailDirEre) . ' ' : '', escapeshellarg($extractDir) ); @@ -99,6 +116,12 @@ while (($line = fgets($fh)) !== false) { // Defensive: shouldn't happen with our invocation. continue; } + if ($skipMail && preg_match($mailDirPcre, $path)) { + // Belt: --exclude-dir should have kept clamscan out of the mail + // tree; never move a message out of a Maildir regardless. + fwrite(STDERR, "scan-files: WARN ignoring a hit inside the mail tree: $path\n"); + continue; + } $hits[] = ['path' => $path, 'signature' => $sig]; } pclose($fh); @@ -156,12 +179,15 @@ $rdi = new RecursiveDirectoryIterator( $extractDir, FilesystemIterator::SKIP_DOTS | FilesystemIterator::CURRENT_AS_PATHNAME ); -$filter = new RecursiveCallbackFilterIterator($rdi, function ($pathname, $key, $iterator) use (&$skippedLinks) { +$filter = new RecursiveCallbackFilterIterator($rdi, function ($pathname, $key, $iterator) use (&$skippedLinks, $skipMail, $mailDirPcre) { // $pathname is a string when CURRENT_AS_PATHNAME is set. if (is_link($pathname)) { $skippedLinks++; return false; } + if ($skipMail && preg_match($mailDirPcre, $pathname)) { + return false; // not scanned, so not counted as scanned + } return true; }); $it = new RecursiveIteratorIterator($filter, RecursiveIteratorIterator::LEAVES_ONLY); @@ -276,6 +302,7 @@ $report = [ 'files_clean' => max(0, $filesScanned - count($hits)), 'files_cleaned' => $cleaned, 'files_quarantined' => $quarantined, + 'mail_scan_skipped' => $skipMail, 'actions' => $actions, ];