Triple-C/app/src-tauri/src/docker/migration.rs

//! Container **base-image migration** — the Docker-level machinery.
//!
//! The orchestration (which Tauri command does what, in which order) lives in
//! [`crate::commands::migration_commands`]. This module holds the two things
//! that benefit from being separate: the *pure* delta computation, which is
//! fully unit-tested below, and the small set of Docker operations migration
//! needs that nothing else in the app does.
//!
//! # Why this is a diff of two image manifests and not `docker diff`
//!
//! `docker diff` reports changes since the container's **last commit**. Every
//! Triple-C project container is created from its own snapshot image and
//! re-committed on each recreation, so `docker diff` on one reports only what
//! happened since the most recent commit — measured on a real project: 2,533
//! entries, almost all of them `/tmp` churn, and none of the actual
//! divergence from the base. It is the wrong tool here and is not used.
//!
//! # Why the diff is filtered through dpkg ownership
//!
//! Raw path diffing lies. On a real project, 11,088 paths differed between the
//! snapshot and the current base and approximately **zero** were user-authored:
//! the rest were the base's *own* AWS CLI and pnpm trees at different versions.
//! Two filters make the set honest:
//!
//! 1. **dpkg ownership** — anything listed in `/var/lib/dpkg/info/*.list` in
//!    either image belongs to a package, not to the user.
//! 2. **presence in the new base** — if the current base already ships a path,
//!    the base's copy wins by definition (that is the point of migrating), so
//!    it is never carried across. This is also what makes the extraction's
//!    never-clobber guarantee cheap: the payload does not even contain the
//!    conflicting files.
//!
//! `pip3 list` is likewise a liar on Ubuntu — its apparent extras are
//! `dist-packages` installed by apt — so Python packages are covered by the apt
//! delta rather than by a pip diff.

use std::collections::{BTreeMap, BTreeSet, HashMap};

use bollard::container::{
    Config, CreateContainerOptions, LogOutput, LogsOptions, RemoveContainerOptions,
    StartContainerOptions, WaitContainerOptions,
};
use bollard::image::TagImageOptions;
use bollard::models::HostConfig;
use futures_util::StreamExt;

use super::client::get_docker;
use crate::models::{ProjectPath, UnpreservedData};

// ─────────────────────────────────────────────────────────────────────────────
// Policy constants
// ─────────────────────────────────────────────────────────────────────────────

/// Roots whose *non-package, not-in-the-base* contents are carried across
/// verbatim.
///
/// `/usr/local` is narrowed to the four directories that hold executables and
/// data rather than configuration, per the migration design. `/workspace` is
/// here because loose files at the workspace root live in the container's
/// writable layer — they are not on any bind mount and are genuinely lost today
/// when a container is recreated from a different image.
pub const COPY_ROOTS: &[&str] = &[
    "/usr/local/bin",
    "/usr/local/sbin",
    "/usr/local/lib",
    "/usr/local/share",
    "/opt",
    "/srv",
    "/workspace",
];

/// Subtrees never copied even though they sit under a [`COPY_ROOTS`] entry.
///
/// Both are the *base image's own* content, shipped by the Dockerfile. Copying
/// them forward would pin the new base to the old base's version of them, which
/// is the exact failure migration exists to fix. (The presence-in-base filter
/// would already catch them; naming them is cheap insurance against a base that
/// relocates one.)
pub const COPY_EXCLUSIONS: &[&str] = &["/usr/local/aws-cli", "/opt/mission-control"];

/// Roots the filesystem manifest walks. Wider than [`COPY_ROOTS`] so the
/// manifest stays useful for debugging; [`compute_verbatim_paths`] applies the
/// narrower policy.
///
/// [`DATA_ROOTS`] are in here for a different reason: they are never copied,
/// but they *are* destroyed by the container swap, so the walk has to see them
/// in order to warn about them.
pub const MANIFEST_ROOTS: &[&str] = &[
    "/usr/local",
    "/opt",
    "/srv",
    "/workspace",
    "/var/lib",
    "/var/www",
];

/// Roots holding **state a base-image swap destroys and no replay can put
/// back**. Reported by [`unpreserved_data`], never copied.
///
/// A container running Postgres, MySQL, Redis or nginx keeps its actual data in
/// `/var/lib/<service>` or `/var/www`. Replaying the apt delta reinstalls the
/// *package* onto the new base and gets an empty data directory back — the
/// database is gone. That is worse than the ordinary recreate path, which
/// creates from the project's snapshot and therefore keeps `/var` intact.
///
/// These are deliberately **not** in [`COPY_ROOTS`]. A live database's on-disk
/// files cannot be tarred out from under a running server and restored into a
/// different base's version of the same package with any confidence — a copy
/// that half-works is worse than a warning that lets the user take a proper
/// dump first. So migration's answer is disclosure, loudly, before anything is
/// touched.
pub const DATA_ROOTS: &[&str] = &["/var/lib", "/var/www"];

/// Base-image capabilities worth telling the user they are missing, as
/// `(path, human label)`.
///
/// A feature is only ever reported as missing when the **current base actually
/// ships it** and the container does not, so this table needs no maintenance
/// when a capability is dropped from the image — it simply stops appearing.
pub const FEATURE_PROBES: &[(&str, &str)] = &[
    ("/usr/bin/socat", "Auth bridge tunnel (socat)"),
    ("/usr/bin/bwrap", "Sandbox mode (bubblewrap)"),
    ("/usr/bin/cron", "Cron daemon (scheduled tasks)"),
    ("/usr/bin/jq", "JSON tooling (jq)"),
    ("/usr/bin/rg", "Fast search (ripgrep)"),
    ("/usr/bin/gh", "GitHub CLI"),
    ("/usr/bin/git", "git"),
    ("/usr/bin/docker", "Docker CLI"),
    ("/usr/bin/node", "Node.js"),
    ("/usr/bin/python3", "Python 3"),
    ("/usr/local/bin/triple-c-open", "Host browser URL relay"),
    ("/usr/local/bin/osc52-clipboard", "Clipboard bridge (OSC 52)"),
    ("/usr/local/bin/audio-shim", "Voice mode audio capture"),
    ("/usr/local/bin/triple-c-scheduler", "Scheduled tasks"),
    ("/usr/local/bin/triple-c-task-runner", "Scheduled task runner"),
    ("/usr/local/bin/triple-c-sso-refresh", "AWS SSO auto-refresh"),
    ("/opt/mission-control", "Mission Control (Flight Control)"),
    ("/usr/bin/wg", "VPN tooling for the VPN Support toggle (WireGuard)"),
    ("/opt/triple-c-skills", "Bundled skills for the VPN Support toggle (PIA VPN)"),
];

/// Headroom demanded on Docker's storage backend on top of the measured
/// payload, so a migration cannot be the thing that fills the disk. The new
/// snapshot commit is a delta layer over the base (the base itself is already
/// on disk), and a 524 MB commit was measured at 25.6 s — 2 GiB is a generous
/// ceiling for that plus the replayed packages.
pub const DISK_HEADROOM_BYTES: u64 = 2 * 1024 * 1024 * 1024;

/// Label carrying the image ID of the base a container's lineage descends from.
pub const LABEL_BASE_IMAGE_ID: &str = "triple-c.base-image-id";
/// Label carrying the image this container was actually created from.
pub const LABEL_CREATE_IMAGE: &str = "triple-c.create-image";
/// Label stamped on a container created *by* a migration, so a crash between
/// the container swap and the final commit is recognisable on restart.
pub const LABEL_MIGRATION_STATE: &str = "triple-c.migration-state";
/// Value of [`LABEL_MIGRATION_STATE`] while a migration is unfinished.
pub const MIGRATION_LABEL_IN_PROGRESS: &str = "in-progress";
/// Label stamped on the short-lived probe containers [`run_throwaway`] creates.
///
/// They are removed on every path including failure, but a hard crash of the
/// app (or of Docker) between create and remove would otherwise leave a
/// container that carries no `triple-c.*` marking at all — invisible to every
/// cleanup this app has, and unattributable by hand. The label makes
/// `docker ps -a --filter label=triple-c.probe=migration` find them.
pub const LABEL_PROBE: &str = "triple-c.probe";
/// Value of [`LABEL_PROBE`] on a migration manifest/pre-flight probe container.
pub const PROBE_LABEL_MIGRATION: &str = "migration";

// ─────────────────────────────────────────────────────────────────────────────
// Manifests
// ─────────────────────────────────────────────────────────────────────────────

/// One entry from the filesystem walk.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ManifestEntry {
    /// `find`'s `%y`: `f` regular, `d` directory, `l` symlink, …
    pub kind: char,
    pub size: u64,
    pub path: String,
}

impl ManifestEntry {
    pub fn is_dir(&self) -> bool {
        self.kind == 'd'
    }
}

/// Everything one probe run learned about an image or a running container.
#[derive(Debug, Clone, Default)]
pub struct Manifest {
    /// Filesystem walk of [`MANIFEST_ROOTS`].
    pub paths: Vec<ManifestEntry>,
    /// Paths under those roots that `dpkg` owns.
    pub dpkg_owned: BTreeSet<String>,
    /// `apt-mark showmanual`.
    pub apt_manual: BTreeSet<String>,
    /// Globally installed npm package names (scoped names kept intact).
    pub npm_global: BTreeSet<String>,
    /// Which [`FEATURE_PROBES`] paths exist.
    pub features: BTreeSet<String>,
    /// Filesystem walk of `/etc`.
    pub etc_paths: BTreeSet<String>,
    /// `package -> version` for every installed dpkg package.
    pub dpkg_versions: BTreeMap<String, String>,
}

impl Manifest {
    /// Index of the filesystem walk, for O(log n) presence tests.
    fn path_set(&self) -> BTreeSet<&str> {
        self.paths.iter().map(|e| e.path.as_str()).collect()
    }
}

/// The shell program run inside a throwaway container (or, when the project is
/// running, inside the container itself) to produce a [`Manifest`].
///
/// Sections are separated by sentinel lines so one exec answers every question;
/// on a 5.49 GB image the whole thing takes about three seconds. Every command
/// is failure-tolerant (`2>/dev/null`, no `set -e`) because a missing `npm` or
/// an unreadable directory must degrade one section, not the run.
pub fn manifest_script() -> String {
    let feature_paths = FEATURE_PROBES
        .iter()
        .map(|(p, _)| shell_single_quote(p))
        .collect::<Vec<_>>()
        .join(" ");
    let roots = MANIFEST_ROOTS
        .iter()
        .map(|p| shell_single_quote(p))
        .collect::<Vec<_>>()
        .join(" ");
    // The dpkg grep is anchored to the manifest roots so the section stays a
    // few hundred kB instead of the ~40 MB a full ownership dump would be.
    let dpkg_filter = MANIFEST_ROOTS
        .iter()
        .map(|r| r.trim_start_matches('/'))
        .collect::<Vec<_>>()
        .join("|");
    format!(
        r#"
echo '###PATHS'
find {roots} -xdev -printf '%y\t%s\t%p\n' 2>/dev/null
echo '###DPKG'
cat /var/lib/dpkg/info/*.list 2>/dev/null | grep -E '^/({dpkg_filter})(/|$)'
echo '###APT'
apt-mark showmanual 2>/dev/null
echo '###NPM'
npm ls -g --depth=0 --parseable 2>/dev/null
echo '###FEATURES'
for p in {feature_paths}; do
  if [ -e "$p" ]; then echo "$p"; fi
done
echo '###ETC'
find /etc -xdev -printf '%y\t%s\t%p\n' 2>/dev/null
echo '###PKGVER'
dpkg-query -W -f='${{Package}}\t${{Version}}\n' 2>/dev/null
echo '###END'
exit 0
"#
    )
}

/// Parse the output of [`manifest_script`].
///
/// Unknown sections and malformed lines are skipped rather than failing: the
/// probe runs against images this build has never seen, and one odd line must
/// not cost the whole manifest.
pub fn parse_manifest(raw: &str) -> Manifest {
    let mut m = Manifest::default();
    let mut section = "";
    for line in raw.lines() {
        let line = line.strip_suffix('\r').unwrap_or(line);
        if let Some(name) = line.strip_prefix("###") {
            section = match name {
                "PATHS" | "DPKG" | "APT" | "NPM" | "FEATURES" | "ETC" | "PKGVER" | "END" => name,
                _ => "",
            };
            continue;
        }
        if line.is_empty() {
            continue;
        }
        match section {
            "PATHS" => {
                if let Some(entry) = parse_find_line(line) {
                    m.paths.push(entry);
                }
            }
            "DPKG" => {
                m.dpkg_owned.insert(line.to_string());
            }
            "APT" => {
                m.apt_manual.insert(line.trim().to_string());
            }
            "NPM" => {
                if let Some(name) = npm_package_from_path(line) {
                    m.npm_global.insert(name);
                }
            }
            "FEATURES" => {
                m.features.insert(line.to_string());
            }
            "ETC" => {
                if let Some(entry) = parse_find_line(line) {
                    m.etc_paths.insert(entry.path);
                }
            }
            "PKGVER" => {
                if let Some((pkg, ver)) = line.split_once('\t') {
                    m.dpkg_versions.insert(pkg.to_string(), ver.to_string());
                }
            }
            _ => {}
        }
    }
    m
}

fn parse_find_line(line: &str) -> Option<ManifestEntry> {
    let mut parts = line.splitn(3, '\t');
    let kind = parts.next()?.chars().next()?;
    let size = parts.next()?.parse::<u64>().ok()?;
    let path = parts.next()?;
    if !path.starts_with('/') {
        return None;
    }
    Some(ManifestEntry {
        kind,
        size,
        path: path.to_string(),
    })
}

/// `/usr/lib/node_modules/@scope/pkg` → `@scope/pkg`.
///
/// `npm ls -g --parseable` prints the prefix directory on its first line and
/// one path per installed package after it; splitting on the *last*
/// `/node_modules/` is what keeps scoped names intact.
fn npm_package_from_path(line: &str) -> Option<String> {
    let idx = line.rfind("/node_modules/")?;
    let name = line[idx + "/node_modules/".len()..].trim();
    if name.is_empty() {
        return None;
    }
    Some(name.to_string())
}

/// Quote a string for safe interpolation into a single-quoted shell word.
fn shell_single_quote(s: &str) -> String {
    format!("'{}'", s.replace('\'', r#"'\''"#))
}

// ─────────────────────────────────────────────────────────────────────────────
// Pure delta computation
// ─────────────────────────────────────────────────────────────────────────────

/// Set difference, sorted. Used for both the apt and the `npm -g` delta.
pub fn set_delta(from: &BTreeSet<String>, base: &BTreeSet<String>) -> Vec<String> {
    from.difference(base).cloned().collect()
}

/// The `/workspace/<mount_name>` targets a project's bind mounts occupy.
///
/// Everything under one of these belongs to the host filesystem and must never
/// be staged: it is not lost by a container swap, and copying a whole mounted
/// repository into a tar would be both pointless and enormous. Computed from
/// `project.paths` rather than hardcoded, because the mount names are
/// user-chosen.
pub fn bind_mount_exclusions(paths: &[ProjectPath]) -> Vec<String> {
    let mut out: Vec<String> = paths
        .iter()
        // **The same filter `project_path_mounts` applies, and it has to be.**
        // That function skips a row with an empty `host_path` or `mount_name`
        // so a legacy row cannot brick the create. The consequence is that
        // `/workspace/<name>` for such a row is *not* a bind mount — it is
        // ordinary writable-layer content. Excluding it here would tell
        // `compute_verbatim_paths` to skip staging it, and the container swap
        // would then destroy whatever the user has put there. The two
        // predicates must agree or a migration silently eats a directory.
        .filter(|p| !p.mount_name.trim().is_empty() && !p.host_path.trim().is_empty())
        .map(|p| format!("/workspace/{}", p.mount_name))
        .collect();
    out.sort();
    out.dedup();
    out
}

/// Whether `path` is `root` itself or lives beneath it.
pub fn is_under(path: &str, root: &str) -> bool {
    path == root || path.starts_with(&format!("{}/", root))
}

/// Drop every path that already has an ancestor in the set.
///
/// Turns "one entry per file" into "one entry per newly-added subtree", which
/// is what makes both the reported list and the `tar -T` include list small
/// when someone has installed something large into `/usr/local/lib`.
pub fn prune_to_roots(paths: &BTreeSet<String>) -> Vec<String> {
    let mut kept: Vec<String> = Vec::new();
    // BTreeSet iterates lexicographically, so a parent is always visited before
    // any of its children ("/a" < "/a/b"), and checking only the last kept
    // entry is not enough — a sibling can intervene. Check all kept roots, but
    // short-circuit on the common case.
    for p in paths {
        if kept.iter().any(|k| is_under(p, k) && k != p) {
            continue;
        }
        kept.push(p.clone());
    }
    kept
}

/// The set of paths a migration would carry across verbatim.
///
/// A path qualifies when **all** of:
/// * it lives under a [`COPY_ROOTS`] entry,
/// * it is not under a [`COPY_EXCLUSIONS`] entry or a bind-mount target,
/// * neither image's dpkg database owns it,
/// * the current base image does not already have it.
///
/// The result is then pruned to subtree roots. An empty result means the copy
/// step is skipped entirely.
pub fn compute_verbatim_paths(
    from: &Manifest,
    base: &Manifest,
    bind_targets: &[String],
) -> Vec<String> {
    let base_paths = base.path_set();
    let mut candidates: BTreeSet<String> = BTreeSet::new();

    for entry in &from.paths {
        let p = entry.path.as_str();
        if !COPY_ROOTS.iter().any(|r| is_under(p, r)) {
            continue;
        }
        // A copy root itself is a container for new content, never new content.
        if COPY_ROOTS.contains(&p) {
            continue;
        }
        if COPY_EXCLUSIONS.iter().any(|x| is_under(p, x)) {
            continue;
        }
        if bind_targets.iter().any(|t| is_under(p, t)) {
            continue;
        }
        if from.dpkg_owned.contains(p) || base.dpkg_owned.contains(p) {
            continue;
        }
        if base_paths.contains(p) {
            continue;
        }
        candidates.insert(entry.path.clone());
    }

    let dirs: BTreeSet<&str> = from
        .paths
        .iter()
        .filter(|e| e.is_dir())
        .map(|e| e.path.as_str())
        .collect();

    prune_to_roots(&candidates)
        .into_iter()
        // Drop empty directory trees. Measured on a real project, these were
        // three of the five hits: `/usr/local/share/{fonts,sgml,xml}`, which a
        // package's postinst creates and dpkg does not own, so no other filter
        // catches them. They carry nothing, and replaying the packages that
        // made them recreates them anyway.
        .filter(|p| {
            !dirs.contains(p.as_str())
                || from
                    .paths
                    .iter()
                    .any(|e| !e.is_dir() && is_under(&e.path, p))
        })
        .collect()
}

/// Total on-disk size of a verbatim set, for the pre-flight disk estimate.
pub fn verbatim_payload_bytes(from: &Manifest, verbatim: &[String]) -> u64 {
    from.paths
        .iter()
        .filter(|e| !e.is_dir())
        .filter(|e| verbatim.iter().any(|root| is_under(&e.path, root)))
        .map(|e| e.size)
        .sum()
}

/// The reporting unit under a [`DATA_ROOTS`] entry: the first path component
/// below the root, e.g. `/var/lib/postgresql`. Directory-level, because that is
/// the granularity a user can actually act on ("dump this database"), and
/// because a per-file list of a Postgres cluster would be thousands of lines.
fn data_unit(path: &str) -> Option<String> {
    for root in DATA_ROOTS {
        let prefix = format!("{}/", root);
        if let Some(rest) = path.strip_prefix(&prefix) {
            let first = rest.split('/').next()?;
            if first.is_empty() {
                return None;
            }
            return Some(format!("{}/{}", root, first));
        }
    }
    None
}

/// Data-bearing subtrees under [`DATA_ROOTS`] that the migration will destroy
/// and cannot restore, with the size and file count of what is at risk.
///
/// A subtree qualifies when **all** of:
/// * it is a first-level directory under a [`DATA_ROOTS`] entry,
/// * the current base image does not have that directory **at all** — if the
///   base ships it, it is the base's own machinery (`/var/lib/apt`,
///   `/var/lib/dpkg`, `/var/lib/systemd`, …) and the base's copy is the right
///   one, exactly as for `/etc`,
/// * it contains at least one regular file that neither image's dpkg database
///   owns — a package's own scaffolding is recreated by the apt replay, the
///   data written into it is not.
///
/// That pair of filters is what keeps this quiet on an ordinary container and
/// loud on one running a database: `/var/lib/postgresql` is absent from the
/// base and full of unowned files, while `/var/lib/apt/lists` is present in the
/// base and never reported.
pub fn unpreserved_data(from: &Manifest, base: &Manifest) -> Vec<UnpreservedData> {
    let base_paths = base.path_set();
    let mut acc: BTreeMap<String, (u64, u32)> = BTreeMap::new();

    for entry in &from.paths {
        let Some(unit) = data_unit(&entry.path) else {
            continue;
        };
        if base_paths.contains(unit.as_str()) {
            continue;
        }
        if entry.is_dir() {
            continue;
        }
        if from.dpkg_owned.contains(&entry.path) || base.dpkg_owned.contains(&entry.path) {
            continue;
        }
        let slot = acc.entry(unit).or_insert((0, 0));
        slot.0 = slot.0.saturating_add(entry.size);
        slot.1 += 1;
    }

    acc.into_iter()
        .map(|(path, (bytes, file_count))| UnpreservedData {
            path,
            bytes,
            file_count,
        })
        .collect()
}

/// Base-image capabilities the container does not have, as
/// `(concrete paths, human labels)`.
///
/// Only paths the base actually ships are considered, so this can never
/// recommend migrating to gain something the new base does not have either.
pub fn missing_features(from: &Manifest, base: &Manifest) -> (Vec<String>, Vec<String>) {
    let mut paths = Vec::new();
    let mut labels = Vec::new();
    for (path, label) in FEATURE_PROBES {
        if base.features.contains(*path) && !from.features.contains(*path) {
            paths.push((*path).to_string());
            labels.push((*label).to_string());
        }
    }
    (paths, labels)
}

/// How many dpkg packages the current base carries at a version the container
/// does not have — either a different version, or a package the container is
/// missing entirely.
///
/// A rough drift measure, deliberately not a claim that every one is *newer*:
/// comparing Debian version strings properly needs `dpkg --compare-versions`,
/// and the number exists to answer "is this container far behind?", which
/// inequality answers just as well.
pub fn outdated_package_count(from: &Manifest, base: &Manifest) -> u32 {
    base.dpkg_versions
        .iter()
        .filter(|(pkg, base_ver)| from.dpkg_versions.get(*pkg) != Some(*base_ver))
        .count() as u32
}

/// `/etc` paths the base has that the container does not, and vice versa.
///
/// **Reported, never copied.** The snapshot lineage carries
/// `/etc/apt/sources.list.d/nodesource.sources` where the current base has
/// `nodesource.list`; copying `/etc` wholesale would leave both in place and
/// every `apt-get update` would fail on a duplicate-source conflict. Since
/// `/etc` is also where the base's own configuration lives, the base's copy is
/// always the right one.
pub fn etc_deltas(from: &Manifest, base: &Manifest) -> (Vec<String>, Vec<String>) {
    let only_in_container: Vec<String> = from
        .etc_paths
        .difference(&base.etc_paths)
        .cloned()
        .collect();
    let only_in_base: Vec<String> = base
        .etc_paths
        .difference(&from.etc_paths)
        .cloned()
        .collect();
    (only_in_container, only_in_base)
}

/// The `tar` member names for a verbatim set: absolute paths made relative to
/// `/`, so the archive extracts with `-C /`.
pub fn tar_member_names(verbatim: &[String]) -> Vec<String> {
    verbatim
        .iter()
        .map(|p| p.trim_start_matches('/').to_string())
        .filter(|p| !p.is_empty())
        .collect()
}

// ─────────────────────────────────────────────────────────────────────────────
// Crash-recovery state machine
// ─────────────────────────────────────────────────────────────────────────────

/// What to do about a migration state found on startup.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Recovery {
    /// Nothing was in flight.
    None,
    /// The crash happened before the container was swapped. `:latest` still
    /// points at the old lineage and `start_project_container` will recreate
    /// from it unaided, so the only work is to clear the record.
    SelfHeal,
    /// The container was swapped but the migration never finished. The user
    /// must choose: resume, or roll back.
    OfferResumeOrRollback,
    /// The migration finished. The user must choose: confirm, or roll back.
    OfferConfirmOrRollback,
}

/// Decide the recovery action from the two independent signals.
///
/// The host-side state file says a migration was in flight; the container's
/// `triple-c.migration-state` label says whether the *swap* actually happened.
/// Neither alone is sufficient:
///
/// * state file but no labelled container → the crash predates the swap
///   (or the swapped container never got created), and everything self-heals.
/// * labelled container but no state file → a stale label from a migration that
///   was already confirmed; the label rides the final commit into the snapshot
///   image, so it can outlive its migration. It must not trigger anything.
///
/// `phase` is [`crate::models::MigrationState::phase`].
pub fn decide_recovery(phase: Option<&str>, container_has_in_progress_label: bool) -> Recovery {
    use crate::models::{
        MIGRATION_PHASE_AWAITING, MIGRATION_PHASE_INTERRUPTED, MIGRATION_PHASE_IN_PROGRESS,
    };
    match phase {
        None => Recovery::None,
        Some(MIGRATION_PHASE_AWAITING) => Recovery::OfferConfirmOrRollback,
        Some(MIGRATION_PHASE_IN_PROGRESS) | Some(MIGRATION_PHASE_INTERRUPTED) => {
            if container_has_in_progress_label {
                Recovery::OfferResumeOrRollback
            } else {
                Recovery::SelfHeal
            }
        }
        // An unrecognised phase is a record we cannot reason about. Treat it
        // like a finished migration awaiting a decision rather than silently
        // discarding it: the destructive option must always be the user's.
        Some(_) => Recovery::OfferConfirmOrRollback,
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Docker operations
// ─────────────────────────────────────────────────────────────────────────────

/// A throwaway container's stdout plus its exit code.
pub struct ThrowawayResult {
    pub stdout: String,
    pub stderr: String,
    pub exit_code: i64,
}

/// Run a shell program in a short-lived container off `image` and collect its
/// output.
///
/// The image's `ENTRYPOINT` is overridden — the Triple-C image's entrypoint
/// ends in `sleep infinity`, so leaving it in place would hang forever. The
/// container is removed on every path, including failure.
pub async fn run_throwaway(image: &str, script: &str) -> Result<ThrowawayResult, String> {
    let docker = get_docker()?;

    let config = Config {
        image: Some(image.to_string()),
        entrypoint: Some(vec!["/bin/sh".to_string()]),
        cmd: Some(vec!["-c".to_string(), script.to_string()]),
        user: Some("root".to_string()),
        working_dir: Some("/".to_string()),
        tty: Some(false),
        // Written explicitly rather than inherited: a probe container that
        // outlives a crash has to be findable, and nothing else in the app
        // labels these.
        labels: Some(HashMap::from([(
            LABEL_PROBE.to_string(),
            PROBE_LABEL_MIGRATION.to_string(),
        )])),
        host_config: Some(HostConfig {
            // No mounts on purpose: this must observe the *image*, not the
            // project's volumes, which are exactly the state migration does
            // not need to move.
            auto_remove: Some(false),
            ..Default::default()
        }),
        ..Default::default()
    };

    let created = docker
        .create_container(
            None::<CreateContainerOptions<String>>,
            config,
        )
        .await
        .map_err(|e| format!("Failed to create probe container for {}: {}", image, e))?;

    // From here on the container's removal is owned by a guard rather than by
    // the statement that used to sit after the await below. A plain statement
    // only runs if this future is *polled to completion*: an `Err(...)?` was
    // already handled, but a **dropped** future — the app quitting mid-flight,
    // a timeout, any `select!` that loses — skipped it silently and left a
    // container behind holding a multi-gigabyte base image open. That image is
    // then unsweepable (removal is deliberately unforced) and there is nothing
    // in the UI that would ever mention it.
    let guard = ProbeContainerGuard::new(created.id);

    let result = run_throwaway_inner(guard.id()).await;

    // The happy path still removes it *synchronously*, so a caller that goes on
    // to `docker rmi` the image it probed does not race the removal.
    guard.remove_now().await;

    result
}

/// Owns the lifetime of a probe container.
///
/// [`Self::remove_now`] is the normal path and awaits the removal. `Drop` is the
/// safety net for the abnormal one: it cannot await, so it hands the removal to
/// a detached task. That covers a dropped future while the process lives; it
/// cannot cover the process dying, which is what
/// [`reap_probe_containers`] is for.
struct ProbeContainerGuard {
    id: String,
    /// Cleared by `remove_now` so `Drop` does not queue a second removal.
    armed: bool,
}

impl ProbeContainerGuard {
    fn new(id: String) -> Self {
        Self { id, armed: true }
    }

    fn id(&self) -> &str {
        &self.id
    }

    /// **Disarm after the await, never before it.** Clearing `armed` first
    /// looked equivalent and was the exact inverse of this guard's purpose: on
    /// the one path it exists for — this future being dropped part-way through
    /// the removal — `Drop` then saw a disarmed guard and did nothing, so the
    /// container survived with no background removal queued behind it. Setting
    /// it afterwards means a cancelled `remove_now` falls back to `Drop`'s
    /// detached removal, and only a removal that actually completed disarms.
    async fn remove_now(mut self) {
        remove_probe_container(&self.id).await;
        self.armed = false;
    }
}

impl Drop for ProbeContainerGuard {
    fn drop(&mut self) {
        if !self.armed {
            return;
        }
        let id = std::mem::take(&mut self.id);
        log::warn!("Probe container {} was abandoned; removing it in the background", id);
        tauri::async_runtime::spawn(async move {
            remove_probe_container(&id).await;
        });
    }
}

/// Force-remove one probe container. Missing is success — the point is that the
/// container is gone.
async fn remove_probe_container(id: &str) {
    let Ok(docker) = get_docker() else {
        return;
    };
    match docker
        .remove_container(
            id,
            Some(RemoveContainerOptions {
                force: true,
                v: true,
                ..Default::default()
            }),
        )
        .await
    {
        Ok(())
        | Err(bollard::errors::Error::DockerResponseServerError {
            status_code: 404, ..
        }) => {}
        Err(e) => log::warn!("Failed to remove probe container {}: {}", id, e),
    }
}

/// Remove probe containers left behind by a previous run of the app.
///
/// A probe is labelled [`LABEL_PROBE`] precisely so it stays findable after a
/// crash, but until now nothing ever went looking. One leftover probe pins the
/// base image it was created from — several gigabytes that
/// `sweep_orphaned_snapshots` then reports as "in use" and correctly refuses to
/// touch, with no way for the user to find out why.
///
/// Safe to run at startup: a probe is a short-lived `/bin/sh` with no mounts
/// and no volumes, owned entirely by a `run_throwaway` call. If one is running
/// right now it belongs to this process — and this runs before any migration
/// can be started, so there is none to interrupt.
///
/// **Except that "belongs to this process" is not something this can know.**
/// The filter is a label, and labels are daemon-wide: a second copy of the app
/// migrating a project on the same daemon has probe containers carrying exactly
/// this label, and force-removing one mid-manifest-capture fails that
/// migration. In-process state cannot see the other instance, so the only
/// available brake is age — [`PROBE_REAP_MIN_AGE_SECS`]. A probe runs a `df`, an
/// `apt-get update` or a `find` over a root filesystem; none of those is a
/// multi-minute job, so anything younger than the gate is far more likely to be
/// someone's live probe than a leftover, and a leftover simply waits for the
/// next start.
pub async fn reap_probe_containers() {
    let Ok(docker) = get_docker() else {
        return;
    };

    let filters: HashMap<String, Vec<String>> = HashMap::from([(
        "label".to_string(),
        vec![format!("{}={}", LABEL_PROBE, PROBE_LABEL_MIGRATION)],
    )]);

    let containers = match docker
        .list_containers(Some(bollard::container::ListContainersOptions {
            all: true,
            filters,
            ..Default::default()
        }))
        .await
    {
        Ok(list) => list,
        Err(e) => {
            log::warn!("Could not list leftover probe containers: {}", e);
            return;
        }
    };

    let now = chrono::Utc::now().timestamp();
    for c in containers {
        // `created` is a unix timestamp; a summary without one is treated as
        // too young to touch, because unknown is never permission.
        let age = c.created.map(|created| now - created);
        match age {
            Some(age) if age >= PROBE_REAP_MIN_AGE_SECS => {}
            _ => {
                log::info!(
                    "Leaving migration probe container {} alone — it is younger than {} minutes, \
                     so it may belong to another Triple-C instance's live migration",
                    c.id.as_deref().unwrap_or("<unknown>"),
                    PROBE_REAP_MIN_AGE_SECS / 60
                );
                continue;
            }
        }
        if let Some(id) = c.id {
            log::info!("Removing leftover migration probe container {}", id);
            remove_probe_container(&id).await;
        }
    }
}

/// Remove throwaway images left behind by a staleness probe of a stopped
/// container — [`super::container::commit_container_for_probe`]'s commits.
///
/// **Load-bearing, not tidying.** A probe image is *tagged*, because bollard
/// gives no image id back from a commit and there has to be something to probe.
/// Tagged means not dangling, so [`super::container::sweep_orphaned_snapshots`]
/// — which collects every other kind of orphan this app can leave — will never
/// see one. Without this, a probe that dies between its commit and its own
/// cleanup (SIGKILL, a crash, a 409 from a concurrent remove) strands a
/// multi-gigabyte image that **no code path can ever reclaim**, and there is no
/// UI to find it either. That is the one leak in this app with no floor on it,
/// so this runs at startup beside [`reap_probe_containers`].
///
/// Age-gated for exactly the reason that one is: `reference=` is a daemon-wide
/// filter, so a second copy of the app probing a project on the same daemon has
/// images matching this glob, and removing one mid-capture fails that probe with
/// "No such image" — the bogus `probe_error` the staleness work exists to get
/// rid of. In-process state cannot see the other instance, so age is the only
/// brake, and [`PROBE_REAP_MIN_AGE_SECS`] is already the right one: a probe is a
/// `find` over a root filesystem, not a multi-minute job.
///
/// Never fails the caller. Housekeeping, like every other sweep here.
pub async fn reap_probe_images() {
    use bollard::image::{ListImagesOptions, RemoveImageOptions};

    let docker = match get_docker() {
        Ok(d) => d,
        Err(e) => {
            log::warn!("Could not reap leftover probe images: {}", e);
            return;
        }
    };

    let filters = HashMap::from([(
        "reference".to_string(),
        vec![format!("{}*", super::container::PROBE_IMAGE_PREFIX)],
    )]);
    let images = match docker
        .list_images(Some(ListImagesOptions {
            all: false,
            filters,
            ..Default::default()
        }))
        .await
    {
        Ok(images) => images,
        Err(e) => {
            log::warn!("Could not list leftover probe images: {}", e);
            return;
        }
    };

    let now = chrono::Utc::now().timestamp();
    for image in images {
        // Unlike a container summary, an image summary always carries a
        // `Created`, so there is no unknown-age case to defend against here.
        if now - image.created < PROBE_REAP_MIN_AGE_SECS {
            log::info!(
                "Leaving probe image {:?} alone — it is younger than {} minutes, so it may belong \
                 to another Triple-C instance's live probe",
                image.repo_tags,
                PROBE_REAP_MIN_AGE_SECS / 60
            );
            continue;
        }
        // By **tag**, never by image id. A `force` removal by id untags an
        // image everywhere, so an id that happens to carry another name loses
        // that name too — which is how a test fixture that tagged
        // `alpine:latest` into this namespace deleted the user's alpine. A real
        // leftover has exactly the one probe tag, so removing the tag removes
        // the image; anything else keeps whatever other names it has.
        for tag in image
            .repo_tags
            .iter()
            .filter(|t| t.starts_with(super::container::PROBE_IMAGE_PREFIX))
        {
            log::info!("Removing leftover probe image {}", tag);
            if let Err(e) = docker
                .remove_image(
                    tag,
                    Some(RemoveImageOptions {
                        force: true,
                        noprune: false,
                    }),
                    None,
                )
                .await
            {
                log::warn!("Could not remove leftover probe image {}: {}", tag, e);
            }
        }
    }
}

/// How old a `triple-c.probe=migration` container must be before
/// [`reap_probe_containers`] will force-remove it, in seconds.
///
/// The label is daemon-wide and this process cannot tell its own leftovers from
/// another instance's live probe, so this is the whole guard. Generous against
/// the longest probe there is (an `apt-get update` inside a throwaway container
/// on a slow link) and still short enough that a crashed run's probe stops
/// pinning a multi-gigabyte base image within the hour.
pub const PROBE_REAP_MIN_AGE_SECS: i64 = 30 * 60;

async fn run_throwaway_inner(id: &str) -> Result<ThrowawayResult, String> {
    let docker = get_docker()?;

    docker
        .start_container(id, None::<StartContainerOptions<String>>)
        .await
        .map_err(|e| format!("Failed to start probe container: {}", e))?;

    let mut wait = docker.wait_container(
        id,
        Some(WaitContainerOptions {
            condition: "not-running",
        }),
    );
    let mut exit_code: i64 = -1;
    while let Some(msg) = wait.next().await {
        match msg {
            Ok(r) => exit_code = r.status_code,
            // A non-zero exit is delivered as an Err by bollard; the status
            // code is still what we want, and the logs below carry the detail.
            Err(bollard::errors::Error::DockerContainerWaitError { code, .. }) => exit_code = code,
            Err(e) => return Err(format!("Probe container wait failed: {}", e)),
        }
    }

    let mut logs = docker.logs(
        id,
        Some(LogsOptions::<String> {
            stdout: true,
            stderr: true,
            follow: false,
            ..Default::default()
        }),
    );
    let mut stdout = String::new();
    let mut stderr = String::new();
    while let Some(chunk) = logs.next().await {
        match chunk {
            Ok(LogOutput::StdOut { message }) => {
                stdout.push_str(&String::from_utf8_lossy(&message))
            }
            Ok(LogOutput::StdErr { message }) => {
                stderr.push_str(&String::from_utf8_lossy(&message))
            }
            Ok(other) => stdout.push_str(&String::from_utf8_lossy(&other.into_bytes())),
            Err(e) => return Err(format!("Probe container log stream failed: {}", e)),
        }
    }

    Ok(ThrowawayResult {
        stdout,
        stderr,
        exit_code,
    })
}

/// Capture a [`Manifest`] from an image, via a throwaway container.
pub async fn manifest_from_image(image: &str) -> Result<Manifest, String> {
    let out = run_throwaway(image, &manifest_script()).await?;
    if !out.stdout.contains("###END") {
        return Err(format!(
            "Probe of image {} did not complete (exit {}){}",
            image,
            out.exit_code,
            if out.stderr.trim().is_empty() {
                String::new()
            } else {
                format!(": {}", out.stderr.trim())
            }
        ));
    }
    Ok(parse_manifest(&out.stdout))
}

/// Capture a [`Manifest`] from a *running* container.
///
/// Preferred over [`manifest_from_image`] for the "from" side whenever the
/// project is up: the snapshot image can lag the container by everything
/// installed since the last commit, and a verbatim set computed from a stale
/// manifest would silently fail to carry that work across.
pub async fn manifest_from_container(container_id: &str) -> Result<Manifest, String> {
    let (out, code) = super::exec::exec_oneshot_as(
        container_id,
        "root",
        vec!["/bin/sh".to_string(), "-c".to_string(), manifest_script()],
        Vec::new(),
    )
    .await?;
    if !out.contains("###END") {
        return Err(format!(
            "Probe of the running container did not complete (exit {})",
            code
        ));
    }
    Ok(parse_manifest(&out))
}

/// Cached stopped-container manifests, keyed by container id, each paired with
/// the container's `FinishedAt` at the time it was captured.
///
/// **Sound because a stopped container's writable layer cannot change.** Nothing
/// can write to it while it is not running, so a manifest captured after it
/// stopped stays true until it is started again — and `FinishedAt` moves on
/// every stop, which is what makes the key exact rather than merely plausible.
///
/// This exists because `get_container_staleness` is called from a `useEffect`
/// that fires whenever the container settles, so simply opening a stopped
/// project's Overview probes it. Uncached that meant a `docker commit` of the
/// whole writable layer per visit — measured at 44 s on a real project — where
/// before this feature the same visit cost one throwaway container or nothing at
/// all. A regression like that is not worth the answer it buys.
///
/// Capped, because a `Manifest` of a real container is a few MB: this only has
/// to serve "the project whose page is open", so a handful of entries is the
/// whole working set and the oldest is dropped past that.
static STOPPED_MANIFEST_CACHE: std::sync::Mutex<
    Option<Vec<(String, String, Manifest)>>,
> = std::sync::Mutex::new(None);

/// How many stopped-container manifests [`STOPPED_MANIFEST_CACHE`] keeps.
const STOPPED_MANIFEST_CACHE_MAX: usize = 4;

/// `FinishedAt` for a container, the cache's validity token. `None` when it
/// cannot be read, which is never treated as a hit.
async fn container_finished_at(container_id: &str) -> Option<String> {
    let docker = get_docker().ok()?;
    docker
        .inspect_container(container_id, None)
        .await
        .ok()?
        .state?
        .finished_at
        .filter(|s| !s.is_empty())
}

/// Capture a [`Manifest`] from a **stopped** container, reusing a cached one
/// when the container has not been started since it was taken.
///
/// See [`STOPPED_MANIFEST_CACHE`] for why this is exact and why it is needed.
pub async fn manifest_from_stopped_container_cached(
    container_id: &str,
) -> Result<Manifest, String> {
    let finished_at = container_finished_at(container_id).await;

    if let Some(token) = &finished_at {
        let guard = STOPPED_MANIFEST_CACHE.lock();
        if let Ok(cache) = guard {
            if let Some(entries) = cache.as_ref() {
                if let Some((_, _, manifest)) = entries
                    .iter()
                    .find(|(id, tok, _)| id == container_id && tok == token)
                {
                    log::debug!(
                        "Reusing the cached manifest for stopped container {}",
                        container_id
                    );
                    return Ok(manifest.clone());
                }
            }
        }
    }

    let manifest = manifest_from_stopped_container(container_id).await?;

    // Only cacheable if the container's state could be read at all; an unknown
    // `FinishedAt` means there is no token that could later be compared.
    if let Some(token) = finished_at {
        if let Ok(mut cache) = STOPPED_MANIFEST_CACHE.lock() {
            let entries = cache.get_or_insert_with(Vec::new);
            entries.retain(|(id, _, _)| id != container_id);
            entries.push((container_id.to_string(), token, manifest.clone()));
            while entries.len() > STOPPED_MANIFEST_CACHE_MAX {
                entries.remove(0);
            }
        }
    }

    Ok(manifest)
}

/// Capture a [`Manifest`] from a **stopped** container.
///
/// Commits the container's writable layer to a throwaway image, probes that,
/// and removes it. This is as current as [`manifest_from_container`] — it reads
/// the same filesystem — and it is why a stopped project no longer has to fall
/// back to its snapshot image, which may not exist at all and lags the
/// container by everything installed since the last commit when it does.
///
/// The image is removed on every path, including a failed probe. See
/// [`super::container::commit_container_for_probe`] for what a crash in the
/// window between the two costs, and why it is bounded.
pub async fn manifest_from_stopped_container(container_id: &str) -> Result<Manifest, String> {
    let image = super::container::commit_container_for_probe(container_id).await?;

    let manifest = manifest_from_image(&image)
        .await
        .map_err(|e| format!("Probe of the stopped container did not complete: {}", e));

    if let Err(e) = super::container::remove_image_by_name(&image).await {
        log::warn!(
            "Could not remove the staleness probe's throwaway image {}: {} — `reap_probe_images` \
             collects it at the next app start; the orphan sweep never will, because it is tagged",
            image,
            e
        );
    }

    manifest
}

/// The image ID (`sha256:…`) of a local image, or `None` if it is not present.
///
/// Deliberately the **ID**, not a repo digest: locally built images and custom
/// images have no `RepoDigests` entry at all, so a digest-based identity would
/// silently be empty for exactly the users most likely to change their base.
pub async fn image_id(image: &str) -> Result<Option<String>, String> {
    let docker = get_docker()?;
    match docker.inspect_image(image).await {
        Ok(info) => Ok(info.id.filter(|s| !s.is_empty())),
        Err(bollard::errors::Error::DockerResponseServerError {
            status_code: 404, ..
        }) => Ok(None),
        Err(e) => Err(format!("Failed to inspect image {}: {}", image, e)),
    }
}

/// An image's labels, or an empty map when it does not exist.
pub async fn image_labels(image: &str) -> HashMap<String, String> {
    let docker = match get_docker() {
        Ok(d) => d,
        Err(_) => return HashMap::new(),
    };
    match docker.inspect_image(image).await {
        Ok(info) => info
            .config
            .and_then(|c| c.labels)
            .unwrap_or_default(),
        Err(_) => HashMap::new(),
    }
}

/// An image's `Created` timestamp, if it exists.
pub async fn image_created(image: &str) -> Option<String> {
    let docker = get_docker().ok()?;
    docker.inspect_image(image).await.ok().and_then(|i| i.created)
}

/// Point a second tag at an existing image.
///
/// Free in both time and space — a 5.49 GB image was measured at 0.036 s and
/// 0 bytes — which is what makes keeping a rollback pin the default-safe
/// choice. (The *image* it pins is not free: snapshots share only 3 of 31
/// layers with the current base, so a retained rollback holds roughly its full
/// size on disk. That is the trade `MigrationOptions::keep_rollback` exposes.)
pub async fn tag_image(source: &str, repo: &str, tag: &str) -> Result<(), String> {
    let docker = get_docker()?;
    docker
        .tag_image(source, Some(TagImageOptions { repo, tag }))
        .await
        .map_err(|e| format!("Failed to tag {} as {}:{}: {}", source, repo, tag, e))
}

/// Remove an image tag. Missing is success — a rollback tag that is already
/// gone is the state the caller wanted.
pub async fn untag_image(reference: &str) -> Result<(), String> {
    let docker = get_docker()?;
    match docker
        .remove_image(
            reference,
            Some(bollard::image::RemoveImageOptions {
                force: false,
                noprune: false,
            }),
            None,
        )
        .await
    {
        Ok(_) => Ok(()),
        Err(bollard::errors::Error::DockerResponseServerError {
            status_code: 404, ..
        }) => Ok(()),
        Err(e) => Err(format!("Failed to remove image tag {}: {}", reference, e)),
    }
}

/// A pre-migration rollback tag for a project's snapshot repo.
pub fn rollback_tag(now: &chrono::DateTime<chrono::Utc>) -> String {
    format!("pre-migration-{}", now.format("%Y%m%d-%H%M%S"))
}

/// How long a rollback pin may sit with no migration record behind it before
/// [`reap_stale_migration_pins`] drops the tag.
///
/// Two weeks, chosen to be far longer than anyone deliberates over a base
/// update and far shorter than "forever", which is what it was.
///
/// **Measured from when the record went missing, not from the tag.** See
/// [`pin_is_reapable`] and
/// [`crate::storage::migration_store::note_ownerless_since`].
pub const STALE_PIN_MAX_AGE_DAYS: i64 = 14;

/// Recover the timestamp encoded in a tag produced by [`rollback_tag`].
///
/// `None` for anything that is not one of ours — a tag that merely *starts*
/// with `pre-migration-` but does not carry a parseable timestamp is left alone
/// rather than guessed at, because the consequence of guessing wrong is
/// deleting the only copy of somebody's system layer.
pub fn parse_rollback_tag(tag: &str) -> Option<chrono::DateTime<chrono::Utc>> {
    let stamp = tag.strip_prefix("pre-migration-")?;
    let naive = chrono::NaiveDateTime::parse_from_str(stamp, "%Y%m%d-%H%M%S").ok()?;
    Some(naive.and_utc())
}

/// Split `triple-c-snapshot-<projectId>:<tag>` into the project id and the tag.
///
/// `None` when the reference is not a snapshot repo at all.
pub fn parse_snapshot_reference(reference: &str) -> Option<(String, String)> {
    let (repo, tag) = split_image_ref(reference);
    let project_id = repo.strip_prefix("triple-c-snapshot-")?.to_string();
    if project_id.is_empty() {
        return None;
    }
    Some((project_id, tag))
}

/// Whether a rollback pin is safe to drop, given whether the project it belongs
/// to still has a migration record and how long it has been without one.
///
/// Pure so the decision can be tested without a daemon. The order of the
/// conditions is the point: **a pin whose migration is still awaiting
/// confirmation is never reaped at any age**, because it is the only copy of
/// the rollback target and the user has not yet said they are happy with the
/// new base.
///
/// ## `ownerless_since`, and why it is not the tag's timestamp
///
/// This used to compute the age from `parse_rollback_tag(tag)` — the instant
/// the migration *started*. A migration is allowed to sit at
/// `awaiting-confirmation` for as long as the user likes; that is what
/// `keep_rollback` is for. A project parked there for a month whose record is
/// then lost had a tag a month old, so the pin was reapable on the very next
/// check and the startup sweep deleted the image immediately after. The
/// fourteen days were nominal: the real grace period for the case the constant
/// was written for was zero.
///
/// So the clock starts when the claim was lost, which is recorded by
/// [`crate::storage::migration_store::note_ownerless_since`] the first time a
/// reaper notices. `None` means no reaper has recorded a sighting yet, and that
/// is **not** "sighted now": returning false there is what gives a pin its
/// first full fourteen days instead of none.
///
/// ## Clock skew
///
/// A `now` earlier than `ownerless_since` — a host clock that ran fast and was
/// corrected, or a data directory carried between machines — yields a negative
/// elapsed time. That is treated as not reapable, and the marker writer
/// re-anchors it, rather than letting a negative `num_days()` mean "never" or
/// an inflated one mean "immediately".
pub fn pin_is_reapable(
    tag: &str,
    has_migration_record: bool,
    ownerless_since: Option<chrono::DateTime<chrono::Utc>>,
    now: &chrono::DateTime<chrono::Utc>,
) -> bool {
    if has_migration_record {
        return false;
    }
    // Still required: the tag has to be one of ours. A hand-made
    // `pre-migration-keepme` is somebody's deliberate pin and is never guessed
    // at, whatever a marker beside it says.
    if parse_rollback_tag(tag).is_none() {
        return false;
    }
    let Some(since) = ownerless_since else {
        return false;
    };
    let elapsed = *now - since;
    elapsed >= chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS)
}

/// Drop `triple-c-snapshot-*:pre-migration-*` tags that no migration record
/// claims any more, so the images behind them become sweepable.
///
/// ## Why this scans tags instead of reading the records
///
/// Every other path to a rollback pin starts from
/// `migration_store::load`, and `load` reports an unparseable state file as
/// *absent* — so a single corrupt record used to strand a 4–12 GB image that no
/// code could ever name again. Confirming or rolling back both remove the
/// record and drop the tag together, so a `pre-migration-*` tag with no record
/// beside it is by definition one that lost its owner: a crash between the two,
/// a record that was deleted by hand, or the corrupt-file case.
///
/// Scanning the *tag pattern* is the only way to find those. `load` moving a
/// corrupt record aside (see `migration_store::load`) is what stops that case
/// from being permanently invisible here too.
///
/// ## Why it only untags
///
/// Dropping the tag turns the image dangling, and it is already labelled
/// `triple-c.managed=true` because `docker commit` created it — so
/// `sweep_orphaned_snapshots` collects it on the same pass, under the same two
/// safety conditions, with the daemon's "still in use by a container" refusal
/// still in front of it. Nothing here calls `docker rmi` on a reachable image.
pub async fn reap_stale_migration_pins() -> usize {
    use bollard::image::ListImagesOptions;

    let Ok(docker) = get_docker() else {
        return 0;
    };

    // `reference` matches against `repo:tag`, so this asks the daemon for
    // exactly the shape [`rollback_tag`] produces and nothing else.
    let filters: HashMap<String, Vec<String>> = HashMap::from([(
        "reference".to_string(),
        vec!["triple-c-snapshot-*:pre-migration-*".to_string()],
    )]);

    let images = match docker
        .list_images(Some(ListImagesOptions {
            all: false,
            filters,
            ..Default::default()
        }))
        .await
    {
        Ok(images) => images,
        Err(e) => {
            log::warn!("Could not list rollback pins: {}", e);
            return 0;
        }
    };

    let now = chrono::Utc::now();
    let mut reaped = 0usize;

    for summary in images {
        for reference in &summary.repo_tags {
            let Some((project_id, tag)) = parse_snapshot_reference(reference) else {
                continue;
            };
            // Filesystem presence, not `load`: a record we cannot parse must
            // still count as "somebody may want this back".
            let has_record =
                crate::storage::migration_store::has_record(&project_id).unwrap_or(true);
            if has_record {
                // Owned again (or still owned): throw away any grace clock a
                // previous pass started, so a pin that loses its record twice
                // gets a fresh fourteen days rather than inheriting a stale one.
                crate::storage::migration_store::clear_ownerless(&project_id, &tag);
                continue;
            }
            // Only a *well-formed* pin gets a marker written for it — a tag
            // that is not one of ours is left entirely alone, files included.
            if parse_rollback_tag(&tag).is_none() {
                continue;
            }
            // Records the first sighting when there is none, which is why this
            // returns `None` on that pass and the pin survives it.
            //
            // A `save` can land between the `has_record` above and this write,
            // which would plant a tombstone dated *now* behind a perfectly
            // valid record — invisible until that record is legitimately lost,
            // at which point the pin is already past its grace period and is
            // reaped on the first check. `note_ownerless_since` re-asks
            // `has_record` after the write and removes the marker again; the
            // reasoning for why that closes the window is on it.
            let ownerless_since =
                crate::storage::migration_store::note_ownerless_since(&project_id, &tag, &now);
            if !pin_is_reapable(&tag, has_record, ownerless_since, &now) {
                continue;
            }
            match untag_image(reference).await {
                Ok(()) => {
                    crate::storage::migration_store::clear_ownerless(&project_id, &tag);
                    log::info!(
                        "Dropped stale rollback pin {} ({:.2} GB) — no migration record has claimed it since {}, more than {} days",
                        reference,
                        summary.size as f64 / 1_073_741_824.0,
                        ownerless_since
                            .map(|t| t.to_rfc3339())
                            .unwrap_or_else(|| "unknown".to_string()),
                        STALE_PIN_MAX_AGE_DAYS,
                    );
                    reaped += 1;
                }
                Err(e) => log::warn!("Could not drop stale rollback pin {}: {}", reference, e),
            }
        }
    }

    reaped
}

/// Split `repo:tag` into its parts, defaulting the tag to `latest`.
pub fn split_image_ref(image: &str) -> (String, String) {
    match image.rsplit_once(':') {
        // A colon in the *registry host* part is a port, not a tag.
        Some((repo, tag)) if !tag.contains('/') => (repo.to_string(), tag.to_string()),
        _ => (image.to_string(), "latest".to_string()),
    }
}

/// Pre-flight environment checks, run against the **new base** in a throwaway
/// container before anything destructive happens.
pub struct PreflightEnvironment {
    /// `apt-get update` succeeded, so package replay has a chance.
    pub network_ok: bool,
    pub network_detail: String,
    /// Bytes available on Docker's storage backend.
    ///
    /// Measured with `df` **inside a container**, not with a host `statvfs`:
    /// on Windows the Docker root lives inside the WSL2 VM and is not a path
    /// the Tauri process can stat at all.
    pub available_bytes: u64,
}

/// Run the network and disk pre-flight checks in one throwaway container.
pub async fn preflight_environment(base_image: &str) -> Result<PreflightEnvironment, String> {
    let script = r#"
echo '###DF'
df -P / | tail -n 1
echo '###NET'
if apt-get -o Acquire::Retries=2 update >/dev/null 2>&1; then
  echo ok
else
  echo failed
fi
echo '###END'
exit 0
"#;
    let out = run_throwaway(base_image, script).await?;
    if !out.stdout.contains("###END") {
        return Err(format!(
            "Pre-flight probe of {} did not complete (exit {}){}",
            base_image,
            out.exit_code,
            if out.stderr.trim().is_empty() {
                String::new()
            } else {
                format!(": {}", out.stderr.trim())
            }
        ));
    }
    Ok(parse_preflight(&out.stdout))
}

/// Parse the pre-flight probe output. `df -P` reports 1024-byte blocks.
pub fn parse_preflight(raw: &str) -> PreflightEnvironment {
    let mut section = "";
    let mut available_bytes = 0u64;
    let mut network_ok = false;
    let mut network_detail = "not checked".to_string();
    for line in raw.lines() {
        if let Some(name) = line.strip_prefix("###") {
            section = name;
            continue;
        }
        match section {
            "DF" => {
                // Filesystem 1024-blocks Used Available Capacity Mounted-on
                let cols: Vec<&str> = line.split_whitespace().collect();
                if cols.len() >= 4 {
                    if let Ok(kb) = cols[cols.len() - 3].parse::<u64>() {
                        available_bytes = kb.saturating_mul(1024);
                    }
                }
            }
            "NET" => {
                if line.trim() == "ok" {
                    network_ok = true;
                    network_detail = "apt-get update succeeded".to_string();
                } else if line.trim() == "failed" {
                    network_ok = false;
                    network_detail =
                        "apt-get update failed — package replay will be skipped".to_string();
                }
            }
            _ => {}
        }
    }
    PreflightEnvironment {
        network_ok,
        network_detail,
        available_bytes,
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Tests
// ─────────────────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {

    /// The mount filter and the migration's exclusion list must agree.
    ///
    /// `project_path_mounts` skips a row with an empty `host_path` so a legacy
    /// row cannot brick the create. That makes `/workspace/<name>` ordinary
    /// writable-layer content rather than a bind mount — and if this function
    /// still excluded it, `compute_verbatim_paths` would skip staging it and
    /// the container swap would destroy whatever is there. A migration eating a
    /// directory is the quietest kind of data loss there is.
    #[test]
    fn an_unmountable_row_is_not_excluded_from_the_migration_payload() {
        let paths = vec![
            ProjectPath { host_path: "/home/u/code".into(), mount_name: "code".into() },
            // Legacy shapes that `project_path_mounts` skips.
            ProjectPath { host_path: "".into(), mount_name: "data".into() },
            ProjectPath { host_path: "/home/u/x".into(), mount_name: "  ".into() },
        ];
        let excluded = bind_mount_exclusions(&paths);
        assert_eq!(
            excluded,
            vec!["/workspace/code".to_string()],
            "only rows that are actually mounted may be excluded from staging"
        );
    }
    use super::*;
    use crate::models::{
        MIGRATION_PHASE_AWAITING, MIGRATION_PHASE_INTERRUPTED, MIGRATION_PHASE_IN_PROGRESS,
    };

    fn manifest(paths: &[(char, u64, &str)], dpkg: &[&str], base_features: &[&str]) -> Manifest {
        Manifest {
            paths: paths
                .iter()
                .map(|(k, s, p)| ManifestEntry {
                    kind: *k,
                    size: *s,
                    path: p.to_string(),
                })
                .collect(),
            dpkg_owned: dpkg.iter().map(|s| s.to_string()).collect(),
            features: base_features.iter().map(|s| s.to_string()).collect(),
            ..Default::default()
        }
    }

    fn strs(v: &[&str]) -> BTreeSet<String> {
        v.iter().map(|s| s.to_string()).collect()
    }

    // ── Manifest parsing ────────────────────────────────────────────────────

    #[test]
    fn the_manifest_parser_reads_every_section() {
        let raw = "###PATHS\n\
                   d\t4096\t/usr/local/bin\n\
                   f\t128\t/usr/local/bin/mytool\n\
                   ###DPKG\n\
                   /usr/local/lib/pkgfile\n\
                   ###APT\n\
                   socat\n\
                   postgresql-client\n\
                   ###NPM\n\
                   /usr/lib/node_modules\n\
                   /usr/lib/node_modules/pnpm\n\
                   /usr/lib/node_modules/@scope/tool\n\
                   ###FEATURES\n\
                   /usr/bin/socat\n\
                   ###ETC\n\
                   f\t10\t/etc/hosts\n\
                   ###PKGVER\n\
                   curl\t8.5.0-2ubuntu10.6\n\
                   ###END\n";
        let m = parse_manifest(raw);
        assert_eq!(m.paths.len(), 2);
        assert_eq!(m.paths[1].size, 128);
        assert!(m.dpkg_owned.contains("/usr/local/lib/pkgfile"));
        assert_eq!(m.apt_manual, strs(&["socat", "postgresql-client"]));
        // The prefix line has no `/node_modules/` segment and is dropped;
        // the scoped name survives intact.
        assert_eq!(m.npm_global, strs(&["pnpm", "@scope/tool"]));
        assert!(m.features.contains("/usr/bin/socat"));
        assert!(m.etc_paths.contains("/etc/hosts"));
        assert_eq!(
            m.dpkg_versions.get("curl").map(String::as_str),
            Some("8.5.0-2ubuntu10.6")
        );
    }

    #[test]
    fn malformed_manifest_lines_are_skipped_not_fatal() {
        let raw = "###PATHS\ngarbage\nf\tnotanumber\t/x\nf\t1\trelative/path\nf\t2\t/ok\n###END\n";
        let m = parse_manifest(raw);
        assert_eq!(m.paths.len(), 1);
        assert_eq!(m.paths[0].path, "/ok");
    }

    #[test]
    fn unknown_sections_do_not_leak_into_the_previous_one() {
        let raw = "###APT\nsocat\n###SOMETHINGNEW\nnoise\nmore-noise\n###END\n";
        let m = parse_manifest(raw);
        assert_eq!(m.apt_manual, strs(&["socat"]));
    }

    // ── Deltas ──────────────────────────────────────────────────────────────

    #[test]
    fn the_apt_delta_is_the_containers_manual_set_minus_the_bases() {
        let from = strs(&["socat", "postgresql-client", "redis-tools", "curl"]);
        let base = strs(&["socat", "curl", "git"]);
        assert_eq!(
            set_delta(&from, &base),
            vec!["postgresql-client".to_string(), "redis-tools".to_string()]
        );
        // A base that gained packages does not produce a negative delta.
        assert!(set_delta(&base, &from).contains(&"git".to_string()));
    }

    #[test]
    fn an_identical_package_set_produces_no_delta() {
        let s = strs(&["a", "b"]);
        assert!(set_delta(&s, &s).is_empty());
    }

    #[test]
    fn outdated_packages_count_version_differences_and_absences() {
        let mut from = Manifest::default();
        from.dpkg_versions.insert("curl".into(), "8.5.0-1".into());
        from.dpkg_versions.insert("git".into(), "2.43.0".into());
        from.dpkg_versions.insert("gone".into(), "1.0".into());
        let mut base = Manifest::default();
        base.dpkg_versions.insert("curl".into(), "8.5.0-2".into()); // newer
        base.dpkg_versions.insert("git".into(), "2.43.0".into()); // same
        base.dpkg_versions.insert("brandnew".into(), "1.0".into()); // absent
        // curl differs + brandnew is missing = 2. `gone` is only in the
        // container and is not drift against the base.
        assert_eq!(outdated_package_count(&from, &base), 2);
    }

    // ── dpkg ownership filter ───────────────────────────────────────────────

    #[test]
    fn dpkg_owned_paths_are_never_treated_as_user_authored() {
        // The real-world failure this guards: a path that exists only in the
        // container looks user-authored until you notice a package owns it.
        let from = manifest(
            &[
                ('f', 10, "/usr/local/lib/libowned.so"),
                ('f', 10, "/usr/local/bin/mytool"),
            ],
            &["/usr/local/lib/libowned.so"],
            &[],
        );
        let base = Manifest::default();
        assert_eq!(
            compute_verbatim_paths(&from, &base, &[]),
            vec!["/usr/local/bin/mytool".to_string()]
        );
    }

    #[test]
    fn ownership_recorded_only_in_the_base_still_filters() {
        // A package that moved into the base since the snapshot was taken owns
        // the path there but not in the container's older dpkg database.
        let from = manifest(&[('f', 10, "/opt/tool/bin/x")], &[], &[]);
        let mut base = Manifest::default();
        base.dpkg_owned.insert("/opt/tool/bin/x".to_string());
        assert!(compute_verbatim_paths(&from, &base, &[]).is_empty());
    }

    // ── Verbatim set ────────────────────────────────────────────────────────

    #[test]
    fn the_bases_own_content_is_never_copied_forward() {
        // /usr/local/aws-cli and /opt/mission-control are shipped by the
        // Dockerfile. Carrying the old copies over would pin the new base to
        // the old base's versions — the exact thing migration fixes.
        let from = manifest(
            &[
                ('d', 4096, "/usr/local/aws-cli"),
                ('f', 10, "/usr/local/aws-cli/v2/current/bin/aws"),
                ('d', 4096, "/opt/mission-control"),
                ('f', 10, "/opt/mission-control/README.md"),
                ('d', 4096, "/opt/mine"),
                ('f', 10, "/opt/mine/keep.txt"),
            ],
            &[],
            &[],
        );
        assert_eq!(
            compute_verbatim_paths(&from, &Manifest::default(), &[]),
            vec!["/opt/mine".to_string()]
        );
    }

    #[test]
    fn a_path_the_new_base_already_has_is_left_to_the_base() {
        let from = manifest(
            &[
                ('f', 10, "/usr/local/bin/triple-c-open"),
                ('f', 10, "/usr/local/bin/mytool"),
            ],
            &[],
            &[],
        );
        let base = manifest(&[('f', 20, "/usr/local/bin/triple-c-open")], &[], &[]);
        assert_eq!(
            compute_verbatim_paths(&from, &base, &[]),
            vec!["/usr/local/bin/mytool".to_string()]
        );
    }

    #[test]
    fn copy_roots_are_narrower_than_the_manifest_roots() {
        // /usr/local/etc is walked by the manifest but is not a copy root:
        // configuration is the base's to own.
        let from = manifest(
            &[
                ('f', 10, "/usr/local/etc/some.conf"),
                ('f', 10, "/usr/local/bin/mytool"),
            ],
            &[],
            &[],
        );
        assert_eq!(
            compute_verbatim_paths(&from, &Manifest::default(), &[]),
            vec!["/usr/local/bin/mytool".to_string()]
        );
    }

    #[test]
    fn empty_directory_trees_are_not_carried_across() {
        // Measured on a real project: /usr/local/share/{fonts,sgml,xml} exist
        // in the snapshot, do not exist in the current base, and are owned by
        // no package — a postinst made them. They carry nothing.
        let from = manifest(
            &[
                ('d', 4096, "/usr/local/share/fonts"),
                ('d', 4096, "/usr/local/share/sgml"),
                ('d', 4096, "/usr/local/share/sgml/nested"),
                ('d', 4096, "/opt/real"),
                ('f', 10, "/opt/real/thing"),
            ],
            &[],
            &[],
        );
        assert_eq!(
            compute_verbatim_paths(&from, &Manifest::default(), &[]),
            vec!["/opt/real".to_string()]
        );
    }

    #[test]
    fn an_empty_verbatim_set_is_the_normal_case() {
        // The measured reality: essentially nothing under these roots is
        // user-authored, so the copy step must be skippable.
        let from = manifest(&[('d', 4096, "/usr/local/bin"), ('d', 4096, "/opt")], &[], &[]);
        assert!(compute_verbatim_paths(&from, &Manifest::default(), &[]).is_empty());
    }

    #[test]
    fn subtrees_are_pruned_to_their_root() {
        let from = manifest(
            &[
                ('d', 4096, "/opt/mytool"),
                ('d', 4096, "/opt/mytool/bin"),
                ('f', 100, "/opt/mytool/bin/run"),
                ('f', 100, "/opt/mytool/LICENSE"),
                ('f', 100, "/srv/other.txt"),
            ],
            &[],
            &[],
        );
        assert_eq!(
            compute_verbatim_paths(&from, &Manifest::default(), &[]),
            vec!["/opt/mytool".to_string(), "/srv/other.txt".to_string()]
        );
    }

    #[test]
    fn pruning_keeps_siblings_that_share_a_name_prefix() {
        // "/opt/tool2" starts with "/opt/tool" as a *string* but is not under
        // it as a *path*.
        let set = strs(&["/opt/tool", "/opt/tool2", "/opt/tool/inner"]);
        assert_eq!(
            prune_to_roots(&set),
            vec!["/opt/tool".to_string(), "/opt/tool2".to_string()]
        );
    }

    #[test]
    fn payload_size_sums_files_under_the_pruned_roots_only() {
        let from = manifest(
            &[
                ('d', 4096, "/opt/mytool"),
                ('f', 100, "/opt/mytool/a"),
                ('f', 200, "/opt/mytool/b"),
                ('f', 999, "/opt/mission-control/big"),
            ],
            &[],
            &[],
        );
        let verbatim = compute_verbatim_paths(&from, &Manifest::default(), &[]);
        // Directories contribute their inode size on disk, not their contents;
        // counting them would double-count. Excluded subtrees contribute zero.
        assert_eq!(verbatim_payload_bytes(&from, &verbatim), 300);
    }

    // ── Data that migration destroys and cannot restore ─────────────────────

    #[test]
    fn a_database_under_var_lib_is_reported_because_nothing_replays_it() {
        // The regression this exists for: replaying `postgresql` onto the new
        // base reinstalls the package and gets an empty cluster. The ordinary
        // recreate path keeps /var because it creates from the snapshot, so a
        // silent migration would be *more* destructive than the thing it
        // replaces.
        let from = manifest(
            &[
                ('d', 4096, "/var/lib/postgresql"),
                ('d', 4096, "/var/lib/postgresql/16/main"),
                ('f', 8192, "/var/lib/postgresql/16/main/PG_VERSION"),
                ('f', 1024, "/var/lib/postgresql/16/main/base/1/2"),
                ('d', 4096, "/var/www"),
                ('d', 4096, "/var/www/site"),
                ('f', 500, "/var/www/site/index.html"),
            ],
            &[],
            &[],
        );
        let got = unpreserved_data(&from, &Manifest::default());
        assert_eq!(
            got.iter().map(|d| d.path.as_str()).collect::<Vec<_>>(),
            vec!["/var/lib/postgresql", "/var/www/site"]
        );
        assert_eq!(got[0].bytes, 9216);
        assert_eq!(got[0].file_count, 2);
        // And it is emphatically not in the copy set — reporting is the whole
        // answer here, not a half-working copy of a live database.
        assert!(!COPY_ROOTS.iter().any(|r| is_under("/var/lib/postgresql", r)));
    }

    #[test]
    fn package_machinery_under_var_is_never_reported_as_data_at_risk() {
        // /var/lib/apt exists in the base too, so it is the base's to own —
        // the same rule /etc gets. Reporting apt's lists would bury the one
        // line that matters under noise on every single migration.
        let from = manifest(
            &[
                ('d', 4096, "/var/lib/apt"),
                ('f', 900_000, "/var/lib/apt/lists/some.mirror_InRelease"),
                ('d', 4096, "/var/lib/dpkg"),
                ('f', 4096, "/var/lib/dpkg/status"),
            ],
            &[],
            &[],
        );
        let base = manifest(
            &[
                ('d', 4096, "/var/lib/apt"),
                ('d', 4096, "/var/lib/dpkg"),
                ('f', 4096, "/var/lib/dpkg/status"),
            ],
            &[],
            &[],
        );
        assert!(unpreserved_data(&from, &base).is_empty());
    }

    #[test]
    fn a_packages_own_scaffolding_under_var_is_not_data() {
        // nginx-common ships /var/www/html/index.nginx-debian.html. The apt
        // replay puts that back; only what the user wrote is at risk.
        let from = manifest(
            &[
                ('d', 4096, "/var/www/html"),
                ('f', 612, "/var/www/html/index.nginx-debian.html"),
            ],
            &["/var/www/html/index.nginx-debian.html"],
            &[],
        );
        assert!(unpreserved_data(&from, &Manifest::default()).is_empty());
    }

    #[test]
    fn data_is_reported_per_directory_not_per_file() {
        assert_eq!(
            data_unit("/var/lib/mysql/ibdata1").as_deref(),
            Some("/var/lib/mysql")
        );
        // A first-level directory is its own unit.
        assert_eq!(
            data_unit("/var/lib/mysql").as_deref(),
            Some("/var/lib/mysql")
        );
        // The root itself is not: it exists in every image.
        assert_eq!(data_unit("/var/lib").as_deref(), None);
        assert_eq!(data_unit("/var/www").as_deref(), None);
        assert_eq!(data_unit("/usr/local/bin/tool"), None);
    }

    // ── Bind-mount exclusion ────────────────────────────────────────────────

    fn pp(mount: &str) -> ProjectPath {
        ProjectPath {
            host_path: format!("/host/{}", mount),
            mount_name: mount.to_string(),
        }
    }

    #[test]
    fn bind_mount_targets_are_derived_from_the_projects_own_mount_names() {
        let paths = vec![pp("repo"), pp("docs"), pp("repo")];
        assert_eq!(
            bind_mount_exclusions(&paths),
            vec!["/workspace/docs".to_string(), "/workspace/repo".to_string()]
        );
        assert!(bind_mount_exclusions(&[]).is_empty());
    }

    #[test]
    fn workspace_content_on_a_bind_mount_is_excluded_but_loose_files_are_not() {
        // The whole reason /workspace is a copy root: `scratch.md` at the
        // workspace root is in the writable layer and is lost today.
        let from = manifest(
            &[
                ('d', 4096, "/workspace/repo"),
                ('f', 10, "/workspace/repo/src/main.rs"),
                ('f', 10, "/workspace/scratch.md"),
                ('d', 4096, "/workspace/notes"),
                ('f', 10, "/workspace/notes/todo.txt"),
            ],
            &[],
            &[],
        );
        let excl = bind_mount_exclusions(&[pp("repo")]);
        assert_eq!(
            compute_verbatim_paths(&from, &Manifest::default(), &excl),
            vec![
                "/workspace/notes".to_string(),
                "/workspace/scratch.md".to_string()
            ]
        );
    }

    #[test]
    fn a_mount_name_that_prefixes_another_does_not_over_exclude() {
        let from = manifest(
            &[
                ('d', 4096, "/workspace/app"),
                ('f', 10, "/workspace/app/x"),
                ('d', 4096, "/workspace/app-notes"),
                ('f', 10, "/workspace/app-notes/y"),
            ],
            &[],
            &[],
        );
        let excl = bind_mount_exclusions(&[pp("app")]);
        assert_eq!(
            compute_verbatim_paths(&from, &Manifest::default(), &excl),
            vec!["/workspace/app-notes".to_string()]
        );
    }

    #[test]
    fn tar_member_names_are_relative_so_the_archive_extracts_at_root() {
        assert_eq!(
            tar_member_names(&["/opt/mytool".to_string(), "/srv".to_string()]),
            vec!["opt/mytool".to_string(), "srv".to_string()]
        );
        assert!(tar_member_names(&["/".to_string()]).is_empty());
    }

    // ── Missing features ────────────────────────────────────────────────────

    #[test]
    fn a_feature_is_missing_only_when_the_new_base_actually_has_it() {
        let from = manifest(&[], &[], &["/usr/bin/jq"]);
        let base = manifest(&[], &[], &["/usr/bin/jq", "/usr/bin/socat"]);
        let (paths, labels) = missing_features(&from, &base);
        assert_eq!(paths, vec!["/usr/bin/socat".to_string()]);
        assert_eq!(labels, vec!["Auth bridge tunnel (socat)".to_string()]);

        // A capability the base dropped is never advertised as a reason to
        // migrate, even though the container "differs" from the base.
        let (paths, _) = missing_features(&base, &from);
        assert!(paths.is_empty());
    }

    // ── /etc ────────────────────────────────────────────────────────────────

    #[test]
    fn etc_deltas_surface_the_nodesource_rename_rather_than_copying_it() {
        let mut from = Manifest::default();
        from.etc_paths
            .insert("/etc/apt/sources.list.d/nodesource.sources".into());
        let mut base = Manifest::default();
        base.etc_paths
            .insert("/etc/apt/sources.list.d/nodesource.list".into());
        let (only_container, only_base) = etc_deltas(&from, &base);
        assert_eq!(
            only_container,
            vec!["/etc/apt/sources.list.d/nodesource.sources".to_string()]
        );
        assert_eq!(
            only_base,
            vec!["/etc/apt/sources.list.d/nodesource.list".to_string()]
        );
        // And /etc is not a copy root, so neither can be carried across —
        // having both would break every apt-get update with a duplicate source.
        assert!(!COPY_ROOTS.iter().any(|r| is_under("/etc/apt", r)));
    }

    // ── Crash-state machine ─────────────────────────────────────────────────

    #[test]
    fn no_state_file_means_no_recovery() {
        assert_eq!(decide_recovery(None, false), Recovery::None);
        // A stale in-progress label with no state file is a label that rode the
        // final commit into the snapshot image. It must not trigger anything.
        assert_eq!(decide_recovery(None, true), Recovery::None);
    }

    #[test]
    fn a_crash_before_the_container_swap_self_heals() {
        // `:latest` still points at the old lineage, so start_project_container
        // recreates from it unaided.
        assert_eq!(
            decide_recovery(Some(MIGRATION_PHASE_IN_PROGRESS), false),
            Recovery::SelfHeal
        );
        assert_eq!(
            decide_recovery(Some(MIGRATION_PHASE_INTERRUPTED), false),
            Recovery::SelfHeal
        );
    }

    #[test]
    fn a_crash_after_the_container_swap_needs_a_decision() {
        assert_eq!(
            decide_recovery(Some(MIGRATION_PHASE_IN_PROGRESS), true),
            Recovery::OfferResumeOrRollback
        );
        assert_eq!(
            decide_recovery(Some(MIGRATION_PHASE_INTERRUPTED), true),
            Recovery::OfferResumeOrRollback
        );
    }

    #[test]
    fn a_finished_migration_waits_for_confirm_or_rollback_either_way() {
        // The label is irrelevant once the final commit landed: the phase alone
        // decides, because the container is the migrated one by definition.
        for labelled in [true, false] {
            assert_eq!(
                decide_recovery(Some(MIGRATION_PHASE_AWAITING), labelled),
                Recovery::OfferConfirmOrRollback
            );
        }
    }

    #[test]
    fn an_unrecognised_phase_never_silently_discards_the_record() {
        assert_eq!(
            decide_recovery(Some("who-knows"), false),
            Recovery::OfferConfirmOrRollback
        );
    }

    // ── Misc ────────────────────────────────────────────────────────────────

    #[test]
    fn image_refs_split_on_the_tag_not_a_registry_port() {
        assert_eq!(
            split_image_ref("triple-c-snapshot-abc:latest"),
            ("triple-c-snapshot-abc".to_string(), "latest".to_string())
        );
        assert_eq!(
            split_image_ref("ghcr.io/shadowdao/triple-c-sandbox:latest"),
            (
                "ghcr.io/shadowdao/triple-c-sandbox".to_string(),
                "latest".to_string()
            )
        );
        assert_eq!(
            split_image_ref("registry:5000/img"),
            ("registry:5000/img".to_string(), "latest".to_string())
        );
    }

    #[test]
    fn rollback_tags_are_sortable_and_docker_legal() {
        let t = rollback_tag(
            &chrono::DateTime::parse_from_rfc3339("2026-08-09T17:04:05Z")
                .unwrap()
                .with_timezone(&chrono::Utc),
        );
        assert_eq!(t, "pre-migration-20260809-170405");
        assert!(t
            .chars()
            .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.'));
    }

    #[test]
    fn the_preflight_parser_reads_df_blocks_as_kibibytes() {
        let raw = "###DF\n/dev/sdc 1055762868 12345 900000000 2% /\n###NET\nok\n###END\n";
        let p = parse_preflight(raw);
        assert_eq!(p.available_bytes, 900_000_000u64 * 1024);
        assert!(p.network_ok);

        let raw = "###DF\n###NET\nfailed\n###END\n";
        let p = parse_preflight(raw);
        assert_eq!(p.available_bytes, 0);
        assert!(!p.network_ok);
    }

    #[test]
    fn the_manifest_script_emits_every_section_the_parser_expects() {
        let s = manifest_script();
        for section in [
            "###PATHS", "###DPKG", "###APT", "###NPM", "###FEATURES", "###ETC", "###PKGVER",
            "###END",
        ] {
            assert!(s.contains(section), "script is missing {}", section);
        }
        // Every probed feature path must reach the script, or the missing-
        // feature report would silently under-report.
        for (path, _) in FEATURE_PROBES {
            assert!(s.contains(path), "script is missing probe {}", path);
        }
    }

    #[test]
    fn shell_quoting_survives_an_apostrophe() {
        assert_eq!(shell_single_quote("/opt/a'b"), r#"'/opt/a'\''b'"#);
    }


    // ── Stale rollback pins (A5) ─────────────────────────────────────────────

    fn at(y: i32, m: u32, d: u32) -> chrono::DateTime<chrono::Utc> {
        chrono::NaiveDate::from_ymd_opt(y, m, d)
            .unwrap()
            .and_hms_opt(12, 0, 0)
            .unwrap()
            .and_utc()
    }

    #[test]
    fn a_rollback_tag_round_trips_through_its_parser() {
        let made = at(2026, 3, 14);
        assert_eq!(parse_rollback_tag(&rollback_tag(&made)), Some(made));
    }

    #[test]
    fn only_a_real_rollback_tag_parses() {
        assert_eq!(parse_rollback_tag("latest"), None);
        assert_eq!(parse_rollback_tag("pre-migration-"), None);
        // Looks like ours but carries no timestamp we produced. Guessing here
        // would mean deleting the only copy of somebody's system layer.
        assert_eq!(parse_rollback_tag("pre-migration-keepme"), None);
        assert_eq!(parse_rollback_tag("pre-migration-20260231-000000"), None);
    }

    #[test]
    fn a_snapshot_reference_yields_its_project_id() {
        assert_eq!(
            parse_snapshot_reference("triple-c-snapshot-abc-123:pre-migration-20260101-101500"),
            Some(("abc-123".to_string(), "pre-migration-20260101-101500".to_string()))
        );
        // Not ours: a base image, and a repo that merely shares a prefix.
        assert_eq!(parse_snapshot_reference("triple-c-sandbox:latest"), None);
        assert_eq!(parse_snapshot_reference("triple-c-snapshot-:latest"), None);
    }

    #[test]
    fn a_pin_awaiting_confirmation_is_never_reaped_at_any_age() {
        // The one rule that cannot bend: while a migration record exists, this
        // image is the only copy of the rollback target and the user has not
        // yet said they are happy on the new base.
        let ancient = rollback_tag(&at(2020, 1, 1));
        assert!(!pin_is_reapable(
            &ancient,
            true,
            Some(at(2020, 1, 1)),
            &at(2026, 8, 23)
        ));
    }

    #[test]
    fn an_unclaimed_pin_is_reaped_only_once_it_is_old() {
        let tag = rollback_tag(&at(2026, 8, 1));
        // The clock runs from when the record went missing, which here is well
        // after the migration started.
        let lost = at(2026, 8, 10);
        assert!(!pin_is_reapable(&tag, false, Some(lost), &at(2026, 8, 11)));
        assert!(!pin_is_reapable(
            &tag,
            false,
            Some(lost),
            &(lost + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS) - chrono::Duration::seconds(1))
        ));
        assert!(pin_is_reapable(
            &tag,
            false,
            Some(lost),
            &(lost + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS))
        ));
    }

    #[test]
    fn the_grace_period_runs_from_the_lost_record_not_from_the_tag() {
        // The bug this replaced, stated as a test. A migration parked at
        // `awaiting-confirmation` for a month — supported, that is what
        // `keep_rollback` is for — whose record is then lost had a
        // month-old tag, so the old rule made its pin reapable on the very
        // next app start with the startup sweep deleting the image two lines
        // later. Zero grace, on the one case the fourteen days exist for.
        let started = at(2026, 6, 1);
        let tag = rollback_tag(&started);
        let record_lost = at(2026, 7, 1);
        let noticed_immediately_after = record_lost + chrono::Duration::minutes(5);
        assert!(
            !pin_is_reapable(&tag, false, Some(record_lost), &noticed_immediately_after),
            "a tag a month old must still get its full grace period once orphaned"
        );
    }

    #[test]
    fn an_unsighted_pin_is_never_reaped_on_the_pass_that_first_sees_it() {
        // `None` means no reaper has recorded a sighting. Treating that as
        // "sighted now" would be harmless; treating it as "sighted long ago"
        // would not, and neither is what it means — the marker is written on
        // this pass and the pin becomes reapable fourteen days later.
        let tag = rollback_tag(&at(2020, 1, 1));
        assert!(!pin_is_reapable(&tag, false, None, &at(2026, 8, 23)));
    }

    #[test]
    fn a_clock_that_ran_backwards_neither_reaps_nor_strands() {
        // A marker dated after `now`: the host clock was fast and got
        // corrected, or the data directory came from another machine. A
        // negative elapsed time must read as "not yet", not as a huge age.
        let tag = rollback_tag(&at(2026, 1, 1));
        let marker = at(2026, 9, 1);
        assert!(!pin_is_reapable(&tag, false, Some(marker), &at(2026, 8, 1)));
        // The other direction is bounded by the marker rather than by the tag:
        // a wildly future `now` can only expire a clock that was actually
        // started, and a pin with no marker (the case above) still cannot be
        // reaped at all — which is what stops a fast host clock from making
        // *every* pin on the daemon instantly collectable.
        assert!(pin_is_reapable(
            &tag,
            false,
            Some(at(2026, 8, 20)),
            &at(2030, 1, 1)
        ));
        assert!(!pin_is_reapable(&tag, false, None, &at(2030, 1, 1)));
    }

    #[test]
    fn a_tag_we_cannot_date_is_left_alone() {
        // Even with an ancient ownerless marker sitting beside it: a tag that
        // merely *starts* `pre-migration-` is somebody's deliberate pin, and
        // the reaper never writes a marker for one in the first place.
        let ancient = Some(at(2020, 1, 1));
        let now = at(2026, 8, 23);
        assert!(!pin_is_reapable("pre-migration-handmade", false, ancient, &now));
        assert!(!pin_is_reapable("latest", false, ancient, &now));
    }

    // ── Live Docker ─────────────────────────────────────────────────────────

    /// The cache serves a second read of an unchanged stopped container, and —
    /// the half that matters — stops serving it the moment the container is
    /// started and stopped again. If invalidation were wrong this would report a
    /// filesystem the project no longer has, and a migration would be planned
    /// against it.
    ///
    /// ```text
    /// cargo test -- --ignored --nocapture stopped_manifest_cache
    /// ```
    #[cfg(unix)]
    #[tokio::test]
    #[ignore = "needs a Docker daemon; creates, commits and removes a throwaway container"]
    async fn the_stopped_manifest_cache_survives_a_reread_but_not_a_restart() {
        fn docker_cli(args: &[&str]) -> String {
            let out = std::process::Command::new("docker")
                .args(args)
                .output()
                .expect("docker CLI");
            assert!(
                out.status.success(),
                "docker {:?} failed: {}",
                args,
                String::from_utf8_lossy(&out.stderr)
            );
            String::from_utf8_lossy(&out.stdout).trim().to_string()
        }

        let image = std::env::var("TRIPLE_C_TEST_IMAGE")
            .unwrap_or_else(|_| "ghcr.io/shadowdao/triple-c-sandbox:latest".to_string());
        let first = format!("/opt/cache-marker-a-{}", std::process::id());
        let second = format!("/opt/cache-marker-b-{}", std::process::id());

        let id = docker_cli(&[
            "run", "-d", "--label", "triple-c.managed=true",
            "--entrypoint", "/bin/sh",
            &image, "-c", "sleep 600",
        ]);
        let cleanup = || {
            let _ = std::process::Command::new("docker")
                .args(["rm", "-f", &id])
                .output();
        };

        docker_cli(&["exec", &id, "mkdir", "-p", &first]);
        docker_cli(&["stop", "-t", "1", &id]);

        let t0 = std::time::Instant::now();
        let cold = manifest_from_stopped_container_cached(&id).await;
        let cold_ms = t0.elapsed().as_millis();

        let t1 = std::time::Instant::now();
        let warm = manifest_from_stopped_container_cached(&id).await;
        let warm_ms = t1.elapsed().as_millis();

        // Restart, change the filesystem, stop again — `FinishedAt` moves.
        docker_cli(&["start", &id]);
        docker_cli(&["exec", &id, "mkdir", "-p", &second]);
        docker_cli(&["stop", "-t", "1", &id]);
        let after_restart = manifest_from_stopped_container_cached(&id).await;

        cleanup();

        let has = |m: &Manifest, p: &str| m.paths.iter().any(|e| e.path == p && e.is_dir());

        let cold = cold.expect("cold read");
        let warm = warm.expect("warm read");
        let after_restart = after_restart.expect("read after restart");

        assert!(has(&cold, &first), "cold read missed {}", first);
        assert!(has(&warm, &first), "warm read missed {}", first);
        println!("cold {} ms, warm {} ms", cold_ms, warm_ms);
        assert!(
            warm_ms * 5 < cold_ms.max(5),
            "the second read cost {} ms against a cold {} ms — it re-committed \
             instead of using the cache",
            warm_ms,
            cold_ms
        );

        // The restart must have invalidated it: the new directory has to show up.
        assert!(
            has(&after_restart, &second),
            "a restart did not invalidate the cache — {} is missing, so this is \
             a stale manifest of a filesystem the container no longer has",
            second
        );
        assert!(has(&after_restart, &first), "the restart lost {}", first);
    }

    /// The reaper finds a leftover probe image by prefix and — crucially —
    /// refuses to remove a young one, because that image may be another
    /// Triple-C instance's live probe. Only a real daemon can say whether the
    /// `reference=` glob matches the names `get_probe_image_name` produces.
    ///
    /// The fixture is **committed**, not tagged and not built. An image's
    /// `Created` is its own, not its tag's, so tagging something already on disk
    /// into this namespace yields a fixture the reaper is right to call ancient
    /// — and BuildKit stamps a fixed epoch on `docker build` output, so a built
    /// one looks ancient too. A commit stamps *now*, verified against Engine
    /// 29.6, which is also how real probe images get their age.
    ///
    /// Both of those mistakes were made here first, and one of them deleted an
    /// unrelated `alpine:latest` — which is why `reap_probe_images` removes by
    /// tag rather than by image id.
    ///
    /// ```text
    /// cargo test -- --ignored --nocapture reaper_spares
    /// ```
    #[cfg(unix)]
    #[tokio::test]
    #[ignore = "needs a Docker daemon; builds and removes a throwaway image"]
    async fn the_reaper_spares_a_probe_image_young_enough_to_be_someone_elses() {
        use std::process::Command;

        fn docker_out(args: &[&str]) -> std::process::Output {
            Command::new("docker").args(args).output().expect("docker CLI")
        }

        let base = std::env::var("TRIPLE_C_TEST_IMAGE")
            .unwrap_or_else(|_| "alpine:latest".to_string());
        let name = crate::docker::container::get_probe_image_name("reapertest01234");

        // A never-started container is enough to commit from, and leaves the
        // daemon's run state alone entirely.
        let created = docker_out(&["create", &base, "true"]);
        assert!(
            created.status.success(),
            "could not create the fixture container from {}: {}",
            base,
            String::from_utf8_lossy(&created.stderr)
        );
        let cid = String::from_utf8_lossy(&created.stdout).trim().to_string();

        let committed = docker_out(&["commit", "--pause=false", &cid, &name]);
        let _ = docker_out(&["rm", "-f", &cid]);
        assert!(
            committed.status.success(),
            "could not commit the fixture image: {}",
            String::from_utf8_lossy(&committed.stderr)
        );

        reap_probe_images().await;

        let still_there = Command::new("docker")
            .args(["image", "inspect", &name])
            .output()
            .expect("docker image inspect")
            .status
            .success();

        let _ = Command::new("docker").args(["rmi", &name]).output();

        assert!(
            still_there,
            "a probe image committed seconds ago was reaped — that is another \
             instance's live probe being broken, see PROBE_REAP_MIN_AGE_SECS"
        );
    }

    /// A *stopped* container is readable, and what comes back is its writable
    /// layer rather than the image it was created from. This is the whole point
    /// of the function: the base image cannot answer it, and the project may
    /// well have no snapshot image at all.
    ///
    /// Also asserts the throwaway commit leaves nothing behind, which no unit
    /// test can. It has to assert on the `triple-c-probe-*` tags specifically:
    /// the probe image is *tagged*, so a leak never shows up as a dangling
    /// image and a dangling-set assertion here would pass either way.
    ///
    /// Ignored because it needs Docker and commits a container; run it with
    ///
    /// ```text
    /// cargo test -- --ignored --nocapture stopped_container
    /// ```
    #[cfg(unix)]
    #[tokio::test]
    #[ignore = "needs a Docker daemon; creates, commits and removes a throwaway container"]
    async fn a_stopped_container_is_read_from_its_writable_layer() {
        fn docker_cli(args: &[&str]) -> String {
            let out = std::process::Command::new("docker")
                .args(args)
                .output()
                .expect("docker CLI");
            assert!(
                out.status.success(),
                "docker {:?} failed: {}",
                args,
                String::from_utf8_lossy(&out.stderr)
            );
            String::from_utf8_lossy(&out.stdout).trim().to_string()
        }
        fn probe_images() -> Vec<String> {
            let mut ids: Vec<String> = docker_cli(&[
                "images", "-q",
                "--filter",
                &format!("reference={}*", crate::docker::container::PROBE_IMAGE_PREFIX),
            ])
            .lines()
            .map(|l| l.trim().to_string())
            .filter(|l| !l.is_empty())
            .collect();
            ids.sort();
            ids
        }

        let image = std::env::var("TRIPLE_C_TEST_IMAGE")
            .unwrap_or_else(|_| "ghcr.io/shadowdao/triple-c-sandbox:latest".to_string());
        // A marker only the writable layer can carry, under a MANIFEST_ROOTS root.
        let marker = format!("/opt/probe-marker-{}", std::process::id());

        // Another instance's live probe images are allowed to exist; what must
        // hold is that this probe adds none of its own.
        let before = probe_images();

        let id = docker_cli(&[
            "run", "-d", "--label", "triple-c.managed=true",
            "--entrypoint", "/bin/sh",
            &image, "-c", "sleep 300",
        ]);
        let cleanup = |id: &str| {
            let _ = std::process::Command::new("docker")
                .args(["rm", "-f", id])
                .output();
        };

        docker_cli(&["exec", &id, "mkdir", "-p", &marker]);
        docker_cli(&["stop", "-t", "1", &id]);

        let result = manifest_from_stopped_container(&id).await;

        cleanup(&id);

        let manifest = result.expect("a stopped container must be probeable");
        assert!(
            manifest.paths.iter().any(|e| e.path == marker && e.is_dir()),
            "the probe read the image, not the container's writable layer: {} missing",
            marker
        );
        // Non-empty package sets prove the probe script really ran, rather than
        // parsing an empty transcript into an empty-but-Ok manifest.
        assert!(
            !manifest.apt_manual.is_empty(),
            "apt-mark showmanual came back empty, so the probe did not run"
        );

        assert_eq!(
            probe_images(),
            before,
            "the throwaway probe image was not cleaned up"
        );
    }
}