Merge branch 'fix/disk' into integration/round-1

This commit is contained in:
2026-08-23 11:46:17 -07:00
8 changed files with 2371 additions and 342 deletions
@@ -227,58 +227,40 @@ async fn container_label(container_id: &str, label: &str) -> Option<String> {
// Migrate // Migrate
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
/// Project ids with a migration running **in this process right now**.
///
/// Two things need it. `reconcile_project_statuses` is callable from the
/// frontend at any time, not only at startup, and a live migration looks
/// exactly like a crashed one from the outside (state file says `in-progress`,
/// container carries the label) — without this guard a reconcile mid-run would
/// rewrite the phase to `interrupted` underneath a migration that is fine.
/// It also makes a second concurrent `migrate_project_to_base` for the same
/// project impossible.
static ACTIVE_MIGRATIONS: std::sync::OnceLock<
std::sync::Mutex<std::collections::HashSet<String>>,
> = std::sync::OnceLock::new();
fn active_migrations() -> &'static std::sync::Mutex<std::collections::HashSet<String>> {
ACTIVE_MIGRATIONS.get_or_init(|| std::sync::Mutex::new(std::collections::HashSet::new()))
}
/// Whether a migration for this project is running **in this process right /// Whether a migration for this project is running **in this process right
/// now**. Every command that stops, removes or recreates the project's /// now**. Every command that stops, removes or recreates the project's
/// container has to consult it: the window between `remove_container` and the /// container has to consult it: the window between `remove_container` and the
/// create that follows looks exactly like "no container", and an ordinary /// create that follows looks exactly like "no container", and an ordinary
/// Start landing in it creates a second container under the same name. /// Start landing in it creates a second container under the same name.
///
/// **This is now a view onto [`crate::project_lock`], not a set of its own.**
/// It used to be the app's only mutual-exclusion primitive, and it was one-way:
/// a migration claimed a project, everything else merely polled this once at
/// entry and never claimed anything. Two non-migration writers of
/// `triple-c-snapshot-{id}:latest` — a compaction and a recreate — could not
/// see each other at all. Folding the set into the shared registry means there
/// is exactly one answer to "is something happening to this project", and this
/// function is the specialisation of it that reconcile still needs: a *live*
/// migration is indistinguishable from a crashed one from the outside, and only
/// this process knows which it is looking at.
pub(crate) fn is_migrating(project_id: &str) -> bool { pub(crate) fn is_migrating(project_id: &str) -> bool {
active_migrations() crate::project_lock::is_held_by(project_id, crate::project_lock::ProjectOp::Migration)
.lock()
.unwrap_or_else(|e| e.into_inner())
.contains(project_id)
} }
/// RAII marker: removes the project from [`ACTIVE_MIGRATIONS`] however the /// RAII marker: releases the project's [`crate::project_lock`] claim however
/// migration ends, including an early `?`. /// the migration ends, including an early `?`.
struct ActiveGuard(String); ///
/// Kept as a named type rather than using [`crate::project_lock::ProjectGuard`]
/// directly so the migration path keeps reading as "take the migration guard",
/// and so the one place that decides what a migration's claim *is* stays here.
struct ActiveGuard(#[allow(dead_code)] crate::project_lock::ProjectGuard);
impl ActiveGuard { impl ActiveGuard {
/// `None` when a migration is already running for this project. /// `None` when a migration — or anything else — already holds this project.
fn acquire(project_id: &str) -> Option<Self> { fn acquire(project_id: &str) -> Option<Self> {
let mut set = active_migrations() crate::project_lock::try_acquire(project_id, crate::project_lock::ProjectOp::Migration)
.lock() .ok()
.unwrap_or_else(|e| e.into_inner()); .map(Self)
if !set.insert(project_id.to_string()) {
return None;
}
Some(Self(project_id.to_string()))
}
}
impl Drop for ActiveGuard {
fn drop(&mut self) {
active_migrations()
.lock()
.unwrap_or_else(|e| e.into_inner())
.remove(&self.0);
} }
} }
@@ -1163,7 +1145,23 @@ pub(crate) async fn purge_migration_artifacts(project_id: &str) {
} }
} }
} }
Ok(None) => return, Ok(None) => {
// **`Ok(None)` is not the same as "no file".** `migration_store::load`
// now reports an *unparseable* record as absent while deliberately
// leaving it on disk, so that `has_record` goes on protecting the
// rollback pin it describes. Returning here on that would leave the
// file — and therefore a permanently "claimed" pin — behind a Reset
// that has just deleted the snapshot and both volumes the record
// could possibly refer to.
if !migration_store::has_record(project_id).unwrap_or(false) {
return;
}
log::warn!(
"Project {} has a migration record that could not be read; removing it anyway \
because a Reset supersedes it",
project_id
);
}
Err(e) => log::warn!( Err(e) => log::warn!(
"Could not read the migration record for {} while cleaning up: {}", "Could not read the migration record for {} while cleaning up: {}",
project_id, project_id,
@@ -1172,6 +1170,10 @@ pub(crate) async fn purge_migration_artifacts(project_id: &str) {
} }
let _ = migration_store::clear_staging(project_id); let _ = migration_store::clear_staging(project_id);
let _ = migration_store::clear(project_id); let _ = migration_store::clear(project_id);
// The pins this project had are gone with the snapshot; their grace clocks
// are meaningless and would otherwise sit in the migrations directory
// forever.
migration_store::clear_ownerless_for_project(project_id);
} }
fn default_docker_socket() -> String { fn default_docker_socket() -> String {
+53 -24
View File
@@ -221,20 +221,35 @@ pub async fn start_project_container(
app_handle: tauri::AppHandle, app_handle: tauri::AppHandle,
state: State<'_, AppState>, state: State<'_, AppState>,
) -> Result<Project, String> { ) -> Result<Project, String> {
// A migration removes the container and creates its replacement moments // **Acquired, not polled.** A migration removes the container and creates
// later. Starting in that window finds no container, creates a second one // its replacement moments later. Starting in that window finds no
// under the same name, and the migration's own create then fails on the // container, creates a second one under the same name, and the migration's
// name conflict — which sends it into an auto-rollback that also cannot // own create then fails on the name conflict — which sends it into an
// create. The UI already refuses (`canMigrate` gates on the container being // auto-rollback that also cannot create. This used to be a one-shot
// stopped and no run being in flight); this is the same gate on the side // `is_migrating` read, which covered that case and no other: a start also
// that actually owns the invariant. // commits `triple-c-snapshot-{id}:latest`, so it races a *compaction*
if crate::commands::migration_commands::is_migrating(&project_id) { // committing the same tag with nothing between them. The claim is held for
return Err( // the whole start rather than checked at its door.
"A container base update is running for this project. Wait for it to finish, then start the project." //
.to_string(), // The UI already refuses (`canMigrate` gates on the container being stopped
); // and no run being in flight); this is the same gate on the side that
// actually owns the invariant.
let _guard =
crate::project_lock::try_acquire(&project_id, crate::project_lock::ProjectOp::Recreate)?;
start_project_container_locked(project_id, app_handle, state).await
} }
/// The body of [`start_project_container`], with **no claim of its own**.
///
/// Split out for exactly one caller: [`rebuild_project_container`] already
/// holds the project under [`crate::project_lock::ProjectOp::Reset`] for its
/// whole run, and a Reset that then went through the public command would be
/// refused by its own guard. Every other path must go through the command.
async fn start_project_container_locked(
project_id: String,
app_handle: tauri::AppHandle,
state: State<'_, AppState>,
) -> Result<Project, String> {
let mut project = state let mut project = state
.projects_store .projects_store
.get(&project_id) .get(&project_id)
@@ -539,6 +554,14 @@ pub async fn stop_project_container(
app_handle: tauri::AppHandle, app_handle: tauri::AppHandle,
state: State<'_, AppState>, state: State<'_, AppState>,
) -> Result<(), String> { ) -> Result<(), String> {
// Stop had **no** exclusion at all, which is the one gap CLAUDE.md's "every
// command that stops, removes or recreates the container consults
// `is_migrating`" rule already named and this function did not honour. A
// stop lands on the container a migration is mid-swap on, and it closes the
// exec sessions a compaction's config replay is not expecting to lose.
let _guard =
crate::project_lock::try_acquire(&project_id, crate::project_lock::ProjectOp::Recreate)?;
let project = state let project = state
.projects_store .projects_store
.get(&project_id) .get(&project_id)
@@ -571,13 +594,13 @@ pub async fn rebuild_project_container(
) -> Result<Project, String> { ) -> Result<Project, String> {
// Reset deletes both volumes and the snapshot image. Doing that while a // Reset deletes both volumes and the snapshot image. Doing that while a
// migration is mid-flight pulls the ground out from under it and leaves an // migration is mid-flight pulls the ground out from under it and leaves an
// orphan migration record pointing at images that no longer exist. // orphan migration record pointing at images that no longer exist — and
if crate::commands::migration_commands::is_migrating(&project_id) { // doing it while a *compaction* is mid-flight is worse, because the
return Err( // compaction then commits `flat(old)` back over the `:latest` this just
"A container base update is running for this project. Wait for it to finish before resetting." // destroyed and resurrects the system layer the user asked to be rid of.
.to_string(), // Held for the whole Reset, including the start at the end of it.
); let _guard =
} crate::project_lock::try_acquire(&project_id, crate::project_lock::ProjectOp::Reset)?;
let project = state let project = state
.projects_store .projects_store
@@ -611,8 +634,9 @@ pub async fn rebuild_project_container(
log::warn!("Failed to remove project volumes for project {}: {}", project_id, e); log::warn!("Failed to remove project volumes for project {}: {}", project_id, e);
} }
// Start fresh // Start fresh. The locked variant, because `_guard` above is this project's
start_project_container(project_id, app_handle, state).await // claim and the public command would be refused by it.
start_project_container_locked(project_id, app_handle, state).await
} }
/// Reconcile project statuses against actual Docker container state. /// Reconcile project statuses against actual Docker container state.
@@ -656,9 +680,14 @@ pub async fn reconcile_project_statuses(
) { ) {
continue; continue;
} }
// ...but never for a project this process is actively migrating: the // ...but never for a project this process is actively working on. A
// container is legitimately absent for part of that run. // migration's container is legitimately absent between the
if crate::commands::migration_commands::is_migrating(&project.id) { // `remove_container` and the create that follows, and so is a Reset's,
// and so is a start's before its create returns. Reconciling into any
// of those windows writes `Stopped` over a project that is mid-run.
// Broadened from `is_migrating` to the whole lock for exactly that
// reason — the migration was never the only operation with a gap.
if crate::project_lock::held(&project.id).is_some() {
continue; continue;
} }
File diff suppressed because it is too large Load Diff
+565 -8
View File
@@ -26,9 +26,6 @@ fn all_reclaim_targets() -> Vec<ReclaimTarget> {
ReclaimTarget::MigrationStaging, ReclaimTarget::MigrationStaging,
ReclaimTarget::ProbeContainers, ReclaimTarget::ProbeContainers,
ReclaimTarget::ScrubContainers, ReclaimTarget::ScrubContainers,
ReclaimTarget::OrphanVolume {
name: "triple-c-home-gone".to_string(),
},
ReclaimTarget::CompactSnapshot { ReclaimTarget::CompactSnapshot {
project_id: "p1".to_string(), project_id: "p1".to_string(),
}, },
@@ -54,7 +51,7 @@ fn every_variant_is_covered_by_the_safety_walk() {
.collect(); .collect();
assert_eq!( assert_eq!(
discriminants.len(), discriminants.len(),
10, 9,
"a ReclaimTarget variant was added or removed; update all_reclaim_targets() and check its \ "a ReclaimTarget variant was added or removed; update all_reclaim_targets() and check its \
safety: {:?}", safety: {:?}",
discriminants discriminants
@@ -358,6 +355,44 @@ fn a_compaction_container_is_never_matched_by_the_scrub_bucket() {
assert!(!is_compaction_container(&summary(&["/my-triple-c-compact-notes"], &[]))); assert!(!is_compaction_container(&summary(&["/my-triple-c-compact-notes"], &[])));
} }
#[test]
fn the_daemon_wide_buckets_leave_a_young_container_alone() {
// Both of these buckets are `Safety::Safe` — one tick, no confirmation —
// and both reach *every* matching container on the daemon, because a label
// and a name prefix are daemon-wide and `ReclaimTarget::project_id()` is
// `None` for them. So a second app instance's live migration probe, and a
// live secret rewrite's scratch container, are both in range. Age is the
// only discriminator available from this side of the process boundary, and
// a container the daemon gave no creation time for is treated as young.
let now = chrono::Utc::now().timestamp();
let mut young_probe = summary(&["/nervous_curie"], &[(migration::LABEL_PROBE, "migration")]);
young_probe.created = Some(now - 30);
assert!(is_migration_probe(&young_probe));
assert!(!is_reapable_migration_probe(&young_probe));
let mut old_probe = young_probe.clone();
old_probe.created = Some(now - migration::PROBE_REAP_MIN_AGE_SECS - 1);
assert!(is_reapable_migration_probe(&old_probe));
let mut undated_probe = young_probe.clone();
undated_probe.created = None;
assert!(!is_reapable_migration_probe(&undated_probe));
// `triple-c-scrub-*` is a **live** name: `rewrite_image_without_secrets`
// creates its scratch container under it, and killing that between the
// create and the commit leaves a revoked OAuth token baked into the
// snapshot's Config.Env — the exact thing that function exists to remove.
let mut young_scrub = summary(&["/triple-c-scrub-abc123"], &[]);
young_scrub.created = Some(now - 5);
assert!(is_scrub_container(&young_scrub));
assert!(!is_reapable_scrub_container(&young_scrub));
let mut old_scrub = young_scrub.clone();
old_scrub.created = Some(now - SCRATCH_CONTAINER_MIN_AGE_SECS - 1);
assert!(is_reapable_scrub_container(&old_scrub));
}
#[test] #[test]
fn a_probe_container_is_matched_on_its_label_not_on_the_daemons_filter() { fn a_probe_container_is_matched_on_its_label_not_on_the_daemons_filter() {
// The `label=triple-c.probe=migration` filter is an exact match and would // The `label=triple-c.probe=migration` filter is an exact match and would
@@ -393,7 +428,7 @@ fn project(id: &str, name: &str) -> Project {
#[test] #[test]
fn an_unreadable_projects_json_is_never_trusted() { fn an_unreadable_projects_json_is_never_trusted() {
let err = project_store_trust(&[project("a", "api")], true, false).unwrap_err(); let err = project_store_trust(&[project("a", "api")], true, None).unwrap_err();
assert!(err.contains("could not be read"), "{}", err); assert!(err.contains("could not be read"), "{}", err);
} }
@@ -403,20 +438,44 @@ fn an_empty_list_from_an_existing_file_is_treated_as_a_failed_load() {
// up and starts empty. That is right for the app and catastrophic here, so // up and starts empty. That is right for the app and catastrophic here, so
// the combination "empty list + file present" is refused rather than read as // the combination "empty list + file present" is refused rather than read as
// "the user has no projects". // "the user has no projects".
let err = project_store_trust(&[], true, true).unwrap_err(); let err = project_store_trust(&[], true, Some(&[])).unwrap_err();
assert!(err.contains("suppressed"), "{}", err); assert!(err.contains("suppressed"), "{}", err);
// No file at all is a genuine fresh install, and there is nothing on the // No file at all is a genuine fresh install, and there is nothing on the
// daemon to mis-attribute in that state. // daemon to mis-attribute in that state.
assert!(project_store_trust(&[], false, true).unwrap().is_empty()); assert!(project_store_trust(&[], false, Some(&[])).unwrap().is_empty());
} }
#[test] #[test]
fn a_healthy_store_yields_its_ids() { fn a_healthy_store_yields_its_ids() {
let ids = project_store_trust(&[project("a", "api"), project("b", "web")], true, true).unwrap(); let ids =
project_store_trust(&[project("a", "api"), project("b", "web")], true, Some(&[])).unwrap();
assert_eq!(ids, HashSet::from(["a".to_string(), "b".to_string()])); assert_eq!(ids, HashSet::from(["a".to_string(), "b".to_string()]));
} }
#[test]
fn a_project_only_the_file_knows_about_still_counts_as_live() {
// M6: `projects` is this process's in-memory list. A project added by a
// *second* copy of the app is in `projects.json` and not in that list, and
// its live home and config volumes then matched "no project claims this".
// The union is what closes the gap; the file is authoritative for
// everything this instance has not heard about.
let ids = project_store_trust(
&[project("a", "api")],
true,
Some(&["a".to_string(), "b-from-the-other-window".to_string()]),
)
.unwrap();
assert!(
ids.contains("b-from-the-other-window"),
"a project only the file knows about must not look orphaned: {:?}",
ids
);
// And the reverse: a project this instance just added is live whether or
// not the file has caught up.
assert!(ids.contains("a"), "{:?}", ids);
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Layer accounting — the number the whole UI exists to show // Layer accounting — the number the whole UI exists to show
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -647,6 +706,94 @@ fn the_compaction_dockerfile_reuses_the_one_scrub_list() {
assert!(!run_lines[0].contains('\n')); assert!(!run_lines[0].contains('\n'));
} }
#[test]
fn the_compaction_run_carries_the_scrub_script_byte_for_byte() {
// **This is the assertion the old one should have been.** The previous
// version checked only that the `RUN` was a single line — which the broken
// space-fold satisfied perfectly, while producing
// `… for p in …; do [ -e "$p" ] || continue sz=$(…) …`, i.e. shell that
// `sh` refuses to parse. Every compaction ever attempted failed on the
// build's first stage.
//
// Nothing is folded now: the script goes into the JSON exec form verbatim,
// so the strong statement is available — what reaches `/bin/sh -c` is
// exactly what `snapshot_scrub_script()` returned, newlines included.
let script = container::snapshot_scrub_script();
let df = compaction_dockerfile("triple-c-snapshot-p1:latest", &script);
let run = df
.lines()
.find(|l| l.starts_with("RUN "))
.expect("no RUN line");
let recovered = script_from_run_line(run).expect("the RUN is not a parseable exec form");
assert_eq!(
recovered, script,
"the compaction must run the scrub script unmodified"
);
// The exec form names the shell itself, because `RUN [...]` does not go
// through one.
let argv: Vec<String> = serde_json::from_str(run.trim_start_matches("RUN ")).unwrap();
assert_eq!(argv[0], "/bin/sh");
assert_eq!(argv[1], "-c");
}
#[test]
fn the_compaction_run_is_valid_shell() {
// The test that would have caught H1, and the only kind that can: hand the
// exact program the daemon will execute to a real shell and ask it to
// parse. `sh -n` reads and parses without running anything.
//
// It is run against the live `snapshot_scrub_script()` rather than a fixture
// precisely because that script is not this module's to own — it is free to
// grow a `case`, an `if` or a function, and this must keep holding when it
// does.
let df = compaction_dockerfile("triple-c-snapshot-p1:latest", &container::snapshot_scrub_script());
let run = df.lines().find(|l| l.starts_with("RUN ")).unwrap();
let script = script_from_run_line(run).unwrap();
assert_shell_parses(&script, "the compaction scrub");
}
#[test]
fn a_multi_line_script_with_blocks_survives_the_run_encoding() {
// The property the old fold did not have, stated directly: a script with a
// `for`/`do`, an `if`/`then`, a `case` and a quote-heavy line has to survive
// whatever this module does to it. Joining lines with a space breaks the
// first three; joining with `;` breaks `if x; then; y`. Encoding the string
// breaks none of them.
let awkward = "total=0\n for p in /tmp/a* /tmp/b*; do\n \t[ -e \"$p\" ] || continue\n \tcase \"$p\" in\n \t\t*.keep) continue ;;\n \tesac\n \tif [ -d \"$p\" ]; then\n \t\trm -rf -- \"$p\"\n \tfi\n done\n echo \"done: $total\"\n";
let df = compaction_dockerfile("x:latest", awkward);
let run = df.lines().find(|l| l.starts_with("RUN ")).unwrap();
assert!(!run.contains('\n'), "the RUN must still be one Dockerfile line");
assert_eq!(script_from_run_line(run).unwrap(), awkward);
assert_shell_parses(awkward, "an awkward but legal script");
}
/// Run `sh -n` over a program and fail with the shell's own diagnostic.
///
/// Skipped, loudly, on a host with no `/bin/sh` — which is not a case any
/// developer machine or CI image this repo targets is in, but a silent pass
/// would be worse than a skipped test.
fn assert_shell_parses(script: &str, what: &str) {
let output = match std::process::Command::new("/bin/sh")
.arg("-n")
.arg("-c")
.arg(script)
.output()
{
Ok(output) => output,
Err(e) => {
eprintln!("skipping the shell syntax check for {}: {}", what, e);
return;
}
};
assert!(
output.status.success(),
"{} is not valid shell:\n{}\n--- script ---\n{}",
what,
String::from_utf8_lossy(&output.stderr),
script
);
}
#[test] #[test]
fn the_compaction_build_is_labelled_so_the_sweep_can_collect_it() { fn the_compaction_build_is_labelled_so_the_sweep_can_collect_it() {
// Everything that cleans up after this build — the discard path when the // Everything that cleans up after this build — the discard path when the
@@ -672,6 +819,13 @@ fn the_compaction_dockerfile_never_reaches_a_bind_mount() {
// directories, mounted from the host. Nothing in a scrub may name one, and // directories, mounted from the host. Nothing in a scrub may name one, and
// the two read-only host mounts under /tmp are dot-prefixed so no glob // the two read-only host mounts under /tmp are dot-prefixed so no glob
// reaches them either. // reaches them either.
//
// **Necessary, not sufficient, and do not read it as coverage.** A path not
// appearing as a literal says nothing about where a glob or a symlink
// resolves to at runtime; the containment property itself is tested in
// `container.rs`, which owns the path list and the script. This assertion
// is kept because a literal `/workspace` appearing here would be an
// unambiguous mistake, and that is all it detects.
assert!(!df.contains("/workspace"), "{}", df); assert!(!df.contains("/workspace"), "{}", df);
assert!(!df.contains(".host-ca"), "{}", df); assert!(!df.contains(".host-ca"), "{}", df);
assert!(!df.contains(".host-aws"), "{}", df); assert!(!df.contains(".host-aws"), "{}", df);
@@ -936,3 +1090,406 @@ fn the_report_serialises_as_snake_case_like_every_other_ipc_struct() {
// type is `number | null`, matching every other optional in `types.ts`. // type is `number | null`, matching every other optional in `types.ts`.
assert!(json["projects"][0]["snapshot_above_base_bytes"].is_null()); assert!(json["projects"][0]["snapshot_above_base_bytes"].is_null());
} }
// ---------------------------------------------------------------------------
// Rollback pins — the guards the safe bucket was missing
// ---------------------------------------------------------------------------
fn moment(y: i32, m: u32, d: u32) -> chrono::DateTime<chrono::Utc> {
chrono::NaiveDate::from_ymd_opt(y, m, d)
.unwrap()
.and_hms_opt(12, 0, 0)
.unwrap()
.and_utc()
}
#[test]
fn the_safe_pin_bucket_applies_every_guard_the_other_paths_do() {
let now = moment(2026, 8, 23);
let ours = migration::rollback_tag(&moment(2026, 1, 1));
// 1. A tag that merely *starts* `pre-migration-`. `destroy` refuses it with
// a comment explaining that `tag: "latest"` would otherwise name the
// project's live snapshot; the safe bucket used to accept anything.
assert_eq!(
pin_disposition("pre-migration-keepme", false, Some(moment(2020, 1, 1)), &now),
PinDisposition::NotOurs
);
assert_eq!(
pin_disposition("latest", false, Some(moment(2020, 1, 1)), &now),
PinDisposition::NotOurs
);
// 2. A record still claims it. Never reaped at any age, and this is the one
// guard the old code did have.
assert_eq!(
pin_disposition(&ours, true, Some(moment(2020, 1, 1)), &now),
PinDisposition::Claimed
);
// 3. Ownerless but inside the grace period — including "no reaper has seen
// it yet", which is where the clock starts. The old code untagged this
// immediately, and `sweep_orphaned_snapshots` four lines later turned
// the untag into a deletion.
assert_eq!(
pin_disposition(&ours, false, None, &now),
PinDisposition::WithinGrace
);
assert_eq!(
pin_disposition(&ours, false, Some(now - chrono::Duration::days(1)), &now),
PinDisposition::WithinGrace
);
// 4. Ownerless and past its grace period. The only case that is collectable
// without a typed confirmation.
assert_eq!(
pin_disposition(
&ours,
false,
Some(now - chrono::Duration::days(migration::STALE_PIN_MAX_AGE_DAYS)),
&now
),
PinDisposition::Reapable
);
}
#[test]
fn the_safe_bucket_and_the_startup_reaper_cannot_disagree() {
// Both go through `migration::pin_is_reapable`. The bug was that the safe
// bucket did not: `reap_stale_migration_pins` required a parseable tag and
// an age, the reclaim button required neither, and the button is the path
// with no confirmation in front of it.
let now = moment(2026, 8, 23);
let ours = migration::rollback_tag(&moment(2026, 1, 1));
for since in [
None,
Some(now - chrono::Duration::days(1)),
Some(now - chrono::Duration::days(migration::STALE_PIN_MAX_AGE_DAYS)),
] {
for has_record in [true, false] {
assert_eq!(
pin_disposition(&ours, has_record, since, &now) == PinDisposition::Reapable,
migration::pin_is_reapable(&ours, has_record, since, &now),
"disposition and the reaper's own predicate disagreed for {:?}/{}",
since,
has_record
);
}
}
}
#[test]
fn an_orphaned_volume_cannot_be_reached_without_a_typed_confirmation() {
// M5: it was a `ReclaimTarget` at `Safety::Safe` — a tick and the group
// button. `reclaim` cannot be handed a `DestructiveTarget` at all, which is
// the type-level half of the guarantee; this pins the other half, that no
// reclaim variant names a volume any more.
for target in all_reclaim_targets() {
let wire = serde_json::to_value(&target).unwrap();
let kind = wire["kind"].as_str().unwrap();
assert!(
!kind.contains("volume"),
"{} can reach a volume from the safe path",
kind
);
}
// And the confirmation subject is the *volume* name, because an orphan has
// no project in the store whose name could be typed.
let target = DestructiveTarget::OrphanVolume {
name: "triple-c-claude-config-gone".to_string(),
project_id: "gone".to_string(),
};
assert!(confirmation_matches(
"triple-c-claude-config-gone",
" triple-c-claude-config-gone "
));
assert!(!confirmation_matches("triple-c-claude-config-gone", "gone"));
assert_eq!(target.project_id(), "gone");
}
// ---------------------------------------------------------------------------
// The numbers, which the user reads as facts
// ---------------------------------------------------------------------------
#[test]
fn the_snapshot_column_and_the_total_come_from_one_rule() {
// Case 1: the daemon measured the sharing. Its figure wins.
assert_eq!(snapshot_attribution(5_000_000_000, 4_700_000_000, Some(1)), 300_000_000);
// Case 2: no shared size, but the lineage is known. The layer arithmetic is
// the honest answer, and it is what the Snapshot column already showed —
// while `total_bytes` used `size - shared` (i.e. the whole image, base
// included) and `triple_c_total_bytes` then added the base again as its own
// row. That double count is the exact thing the comment beside the
// subtraction says it prevents.
assert_eq!(snapshot_attribution(5_000_000_000, 0, Some(300_000_000)), 300_000_000);
// Case 3: nothing known. The full size, not zero — an image that shares
// nothing measurable really does cost all of it, and a flattened snapshot
// is exactly that shape.
assert_eq!(snapshot_attribution(5_000_000_000, 0, None), 5_000_000_000);
// A daemon that reports `-1` for "not computed" must not be read as a
// 1-byte saving, and a negative result is never returned.
assert_eq!(snapshot_attribution(1_000, -1, None), 1_000);
assert_eq!(snapshot_attribution(1_000, 4_000, Some(0)), 0);
}
#[test]
fn a_row_adds_up() {
// The property the fix exists for, stated arithmetically: whatever the
// Snapshot column shows is what the Total is built from.
for (size, shared, above) in [
(5_000_000_000i64, 4_700_000_000i64, Some(1i64)),
(5_000_000_000, 0, Some(300_000_000)),
(5_000_000_000, 0, None),
(0, 0, None),
] {
let snapshot = snapshot_attribution(size, shared, above);
let (writable, home, config) = (10i64, 20i64, 30i64);
let total = snapshot + writable + home + config;
assert_eq!(
total - (writable + home + config),
snapshot,
"the Total column has to reconcile with the Snapshot column"
);
}
}
#[test]
fn human_never_prints_a_unit_the_ladder_forbids() {
// The 999,999 → "1000.0 KB" bug, which is the one `formatBytes.ts` was
// written to fix on the frontend. This function's output is rendered on the
// same line as `formatBytes`' in a compaction message, so the two
// disagreeing is visible in a single sentence.
assert_eq!(human(999_999), "1.0 MB");
assert_eq!(human(999_999_999), "1.0 GB");
assert_eq!(human(999_999_999_999), "1.0 TB");
// The ordinary cases still read the way `docker system df` prints them,
// base 1000.
assert_eq!(human(0), "0 B");
assert_eq!(human(999), "999 B");
assert_eq!(human(1_000), "1.0 KB");
assert_eq!(human(1_500_000), "1.5 MB");
assert_eq!(human(4_700_000_000), "4.7 GB");
// Rounding that does *not* cross the boundary is untouched.
assert_eq!(human(999_400), "999.4 KB");
}
#[test]
fn no_output_of_human_is_ever_a_four_digit_mantissa() {
// A sweep rather than a handful of cases: every power-of-ten boundary and
// its neighbours, which is where the bug lived.
let mut value = 1i64;
for _ in 0..19 {
for candidate in [value - 1, value, value + 1] {
if candidate < 0 {
continue;
}
let rendered = human(candidate);
let mantissa = rendered.split(' ').next().unwrap();
let numeric: f64 = mantissa.parse().unwrap();
assert!(
numeric < 1000.0 || rendered.ends_with(" PB"),
"{} rendered as {}, which the unit ladder is supposed to make impossible",
candidate,
rendered
);
}
value = match value.checked_mul(10) {
Some(v) => v,
None => break,
};
}
}
// ---------------------------------------------------------------------------
// End-to-end compaction, against a real daemon
// ---------------------------------------------------------------------------
/// The whole compaction mechanism, run for real.
///
/// `#[ignore]` because it needs a Docker daemon, pulls `busybox`, and takes
/// tens of seconds — `cargo test` has to stay daemon-free. Run it with
/// `cargo test -- --ignored compaction_end_to_end`.
///
/// It exists because H1 was invisible to every unit test in this file: the
/// generated Dockerfile looked right, the `RUN` was one line as asserted, and
/// the build failed on `/bin/sh: line 0: syntax error: unexpected "do"` every
/// single time. The only test that could have caught it is one that hands the
/// Dockerfile to a daemon.
///
/// It exercises the production functions — [`compaction_dockerfile`],
/// [`build_from_dockerfile`], [`restore_image_config`] — rather than
/// [`compact_snapshot`] itself, so it never has to create a
/// `triple-c-snapshot-*` tag that the app's own sweeps might reach.
#[tokio::test]
#[ignore]
async fn compaction_end_to_end_against_a_real_image() {
let stem = format!("compaction-e2e-{}", uuid::Uuid::new_v4().simple());
let source = format!("{}:source", stem);
let staging = format!("{}:compacting", stem);
let final_ref = format!("{}:latest", stem);
// A snapshot-shaped source: four stacked layers, each superseding the last,
// plus the scrub's own targets and a setuid bit that `COPY --from` has to
// preserve. The env var carries a newline and a double quote, which is what
// `restore_image_config` exists for.
let source_dockerfile = format!(
"FROM busybox:1.36\n\
RUN mkdir -p /tmp/claude-1000 /var/cache/apt/archives /var/log/apt /var/lib/apt/lists && \
dd if=/dev/urandom of=/big-a bs=1M count=40 2>/dev/null\n\
RUN dd if=/dev/urandom of=/big-b bs=1M count=40 2>/dev/null && rm -f /big-a\n\
RUN dd if=/dev/urandom of=/tmp/claude-1000/scratch bs=1M count=25 2>/dev/null && \
dd if=/dev/urandom of=/var/cache/apt/archives/x.deb bs=1M count=15 2>/dev/null && \
touch /var/log/dpkg.log\n\
RUN dd if=/dev/urandom of=/big-c bs=1M count=30 2>/dev/null && rm -f /big-b && \
touch /keepme && chmod 4755 /keepme\n\
LABEL {LABEL_MANAGED}=true\n\
WORKDIR /keep-this-dir\n"
);
// Force, unlike anything in production: `untag_image` is deliberately
// unforced because it runs against a user's images, and here the leftovers
// are this test's own and must not be left on the developer's daemon
// whatever refuses them.
async fn cleanup(refs: Vec<String>) {
let Ok(docker) = get_docker() else {
return;
};
for r in refs {
if let Err(e) = docker
.remove_image(
&r,
Some(RemoveImageOptions {
force: true,
noprune: false,
}),
None,
)
.await
{
eprintln!("could not clean up {}: {}", r, e);
}
}
}
build_from_dockerfile(&source_dockerfile, &source)
.await
.expect("could not build the throwaway source image");
let docker = get_docker().expect("no docker");
let before = docker.inspect_image(&source).await.expect("inspect source");
let before_size = before.size.unwrap_or(0);
let mut config = before.config.clone().expect("source has no config");
// A genuinely multi-line env var, injected here rather than through the
// Dockerfile because `ENV` cannot express a literal newline. This is the
// case `restore_image_config` exists for: it is why the config is replayed
// through a create-and-commit instead of being rendered back into
// Dockerfile instructions, where a newline and a `"` would not survive.
config
.env
.get_or_insert_with(Vec::new)
.push("E2E_MULTILINE=first\nsecond \"quoted\"".to_string());
// --- the thing under test ------------------------------------------------
let dockerfile = compaction_dockerfile(&source, &container::snapshot_scrub_script());
let built = build_from_dockerfile(&dockerfile, &staging).await;
assert!(
built.is_ok(),
"the compaction build failed, which is exactly the H1 regression: {:?}\n{}",
built,
dockerfile
);
restore_image_config(&staging, &final_ref, config)
.await
.expect("could not replay the image config");
let after = docker
.inspect_image(&final_ref)
.await
.expect("inspect compacted");
let after_size = after.size.unwrap_or(0);
let history = docker.image_history(&final_ref).await.expect("history");
let after_config = after.config.clone().expect("no config after");
// The scrub actually ran — the assertion H1 made impossible. `sh -c` inside
// the compacted image, so the answer comes from the filesystem rather than
// from the build log.
let probe = migration::run_throwaway(
&final_ref,
"ls /keepme >/dev/null 2>&1 && echo KEEP-OK\n\
[ -e /tmp/claude-1000/scratch ] && echo SCRATCH-LEFT\n\
[ -e /var/cache/apt/archives/x.deb ] && echo DEB-LEFT\n\
[ -e /var/log/dpkg.log ] && echo DPKGLOG-LEFT\n\
[ -e /big-a ] && echo BIGA-LEFT\n\
[ -e /big-c ] || echo BIGC-MISSING\n\
ls -l /keepme\n\
echo PROBE-END\n\
exit 0\n",
)
.await
.expect("could not probe the compacted image");
// **Everything is measured before anything is asserted**, so a failing
// assertion below does not leave several hundred megabytes of throwaway
// images on the developer's daemon.
cleanup(vec![final_ref, staging, source]).await;
// 1. It is smaller. The three superseded 3040 MB layers and the ~40 MB of
// scrub targets are the whole point.
assert!(
after_size < before_size,
"compaction did not shrink the image: {} -> {}",
before_size,
after_size
);
// 2. One layer of content.
let content_layers = history.iter().filter(|e| e.size > 0).count();
assert_eq!(content_layers, 1, "expected one content layer: {:?}", history);
// 3. The config round-tripped, newline and quote included.
let env = after_config.env.clone().unwrap_or_default();
assert!(
env.iter()
.any(|e| e == "E2E_MULTILINE=first\nsecond \"quoted\""),
"the multi-line env var did not survive: {:?}",
env
);
assert_eq!(after_config.working_dir.as_deref(), Some("/keep-this-dir"));
// 4. The scrub actually ran.
assert!(probe.stdout.contains("PROBE-END"), "{}", probe.stdout);
assert!(probe.stdout.contains("KEEP-OK"), "{}", probe.stdout);
assert!(
!probe.stdout.contains("SCRATCH-LEFT"),
"the agent scratchpad survived the scrub:\n{}",
probe.stdout
);
assert!(
!probe.stdout.contains("DEB-LEFT"),
"apt archives survived the scrub:\n{}",
probe.stdout
);
assert!(
!probe.stdout.contains("DPKGLOG-LEFT"),
"dpkg.log survived the scrub:\n{}",
probe.stdout
);
assert!(
!probe.stdout.contains("BIGC-MISSING"),
"the live payload was lost, which is worse than not compacting:\n{}",
probe.stdout
);
// `COPY --from` has to keep the setuid bit; a compaction that dropped it
// would break sudo inside every migrated project.
assert!(
probe.stdout.contains("-rwsr-xr-x"),
"the setuid bit did not survive the flatten:\n{}",
probe.stdout
);
}
+181 -16
View File
@@ -754,9 +754,16 @@ impl ProbeContainerGuard {
&self.id &self.id
} }
/// **Disarm after the await, never before it.** Clearing `armed` first
/// looked equivalent and was the exact inverse of this guard's purpose: on
/// the one path it exists for — this future being dropped part-way through
/// the removal — `Drop` then saw a disarmed guard and did nothing, so the
/// container survived with no background removal queued behind it. Setting
/// it afterwards means a cancelled `remove_now` falls back to `Drop`'s
/// detached removal, and only a removal that actually completed disarms.
async fn remove_now(mut self) { async fn remove_now(mut self) {
self.armed = false;
remove_probe_container(&self.id).await; remove_probe_container(&self.id).await;
self.armed = false;
} }
} }
@@ -810,6 +817,17 @@ async fn remove_probe_container(id: &str) {
/// and no volumes, owned entirely by a `run_throwaway` call. If one is running /// and no volumes, owned entirely by a `run_throwaway` call. If one is running
/// right now it belongs to this process — and this runs before any migration /// right now it belongs to this process — and this runs before any migration
/// can be started, so there is none to interrupt. /// can be started, so there is none to interrupt.
///
/// **Except that "belongs to this process" is not something this can know.**
/// The filter is a label, and labels are daemon-wide: a second copy of the app
/// migrating a project on the same daemon has probe containers carrying exactly
/// this label, and force-removing one mid-manifest-capture fails that
/// migration. In-process state cannot see the other instance, so the only
/// available brake is age — [`PROBE_REAP_MIN_AGE_SECS`]. A probe runs a `df`, an
/// `apt-get update` or a `find` over a root filesystem; none of those is a
/// multi-minute job, so anything younger than the gate is far more likely to be
/// someone's live probe than a leftover, and a leftover simply waits for the
/// next start.
pub async fn reap_probe_containers() { pub async fn reap_probe_containers() {
let Ok(docker) = get_docker() else { let Ok(docker) = get_docker() else {
return; return;
@@ -835,7 +853,23 @@ pub async fn reap_probe_containers() {
} }
}; };
let now = chrono::Utc::now().timestamp();
for c in containers { for c in containers {
// `created` is a unix timestamp; a summary without one is treated as
// too young to touch, because unknown is never permission.
let age = c.created.map(|created| now - created);
match age {
Some(age) if age >= PROBE_REAP_MIN_AGE_SECS => {}
_ => {
log::info!(
"Leaving migration probe container {} alone — it is younger than {} minutes, \
so it may belong to another Triple-C instance's live migration",
c.id.as_deref().unwrap_or("<unknown>"),
PROBE_REAP_MIN_AGE_SECS / 60
);
continue;
}
}
if let Some(id) = c.id { if let Some(id) = c.id {
log::info!("Removing leftover migration probe container {}", id); log::info!("Removing leftover migration probe container {}", id);
remove_probe_container(&id).await; remove_probe_container(&id).await;
@@ -843,6 +877,16 @@ pub async fn reap_probe_containers() {
} }
} }
/// How old a `triple-c.probe=migration` container must be before
/// [`reap_probe_containers`] will force-remove it, in seconds.
///
/// The label is daemon-wide and this process cannot tell its own leftovers from
/// another instance's live probe, so this is the whole guard. Generous against
/// the longest probe there is (an `apt-get update` inside a throwaway container
/// on a slow link) and still short enough that a crashed run's probe stops
/// pinning a multi-gigabyte base image within the hour.
pub const PROBE_REAP_MIN_AGE_SECS: i64 = 30 * 60;
async fn run_throwaway_inner(id: &str) -> Result<ThrowawayResult, String> { async fn run_throwaway_inner(id: &str) -> Result<ThrowawayResult, String> {
let docker = get_docker()?; let docker = get_docker()?;
@@ -1025,6 +1069,10 @@ pub fn rollback_tag(now: &chrono::DateTime<chrono::Utc>) -> String {
/// ///
/// Two weeks, chosen to be far longer than anyone deliberates over a base /// Two weeks, chosen to be far longer than anyone deliberates over a base
/// update and far shorter than "forever", which is what it was. /// update and far shorter than "forever", which is what it was.
///
/// **Measured from when the record went missing, not from the tag.** See
/// [`pin_is_reapable`] and
/// [`crate::storage::migration_store::note_ownerless_since`].
pub const STALE_PIN_MAX_AGE_DAYS: i64 = 14; pub const STALE_PIN_MAX_AGE_DAYS: i64 = 14;
/// Recover the timestamp encoded in a tag produced by [`rollback_tag`]. /// Recover the timestamp encoded in a tag produced by [`rollback_tag`].
@@ -1051,26 +1099,59 @@ pub fn parse_snapshot_reference(reference: &str) -> Option<(String, String)> {
Some((project_id, tag)) Some((project_id, tag))
} }
/// Whether a rollback pin is safe to drop, given how old it is and whether the /// Whether a rollback pin is safe to drop, given whether the project it belongs
/// project it belongs to still has a migration record. /// to still has a migration record and how long it has been without one.
/// ///
/// Pure so the decision can be tested without a daemon. The order of the two /// Pure so the decision can be tested without a daemon. The order of the
/// conditions is the point: **a pin whose migration is still awaiting /// conditions is the point: **a pin whose migration is still awaiting
/// confirmation is never reaped at any age**, because it is the only copy of /// confirmation is never reaped at any age**, because it is the only copy of
/// the rollback target and the user has not yet said they are happy with the /// the rollback target and the user has not yet said they are happy with the
/// new base. /// new base.
///
/// ## `ownerless_since`, and why it is not the tag's timestamp
///
/// This used to compute the age from `parse_rollback_tag(tag)` — the instant
/// the migration *started*. A migration is allowed to sit at
/// `awaiting-confirmation` for as long as the user likes; that is what
/// `keep_rollback` is for. A project parked there for a month whose record is
/// then lost had a tag a month old, so the pin was reapable on the very next
/// check and the startup sweep deleted the image immediately after. The
/// fourteen days were nominal: the real grace period for the case the constant
/// was written for was zero.
///
/// So the clock starts when the claim was lost, which is recorded by
/// [`crate::storage::migration_store::note_ownerless_since`] the first time a
/// reaper notices. `None` means no reaper has recorded a sighting yet, and that
/// is **not** "sighted now": returning false there is what gives a pin its
/// first full fourteen days instead of none.
///
/// ## Clock skew
///
/// A `now` earlier than `ownerless_since` — a host clock that ran fast and was
/// corrected, or a data directory carried between machines — yields a negative
/// elapsed time. That is treated as not reapable, and the marker writer
/// re-anchors it, rather than letting a negative `num_days()` mean "never" or
/// an inflated one mean "immediately".
pub fn pin_is_reapable( pub fn pin_is_reapable(
tag: &str, tag: &str,
has_migration_record: bool, has_migration_record: bool,
ownerless_since: Option<chrono::DateTime<chrono::Utc>>,
now: &chrono::DateTime<chrono::Utc>, now: &chrono::DateTime<chrono::Utc>,
) -> bool { ) -> bool {
if has_migration_record { if has_migration_record {
return false; return false;
} }
let Some(created) = parse_rollback_tag(tag) else { // Still required: the tag has to be one of ours. A hand-made
// `pre-migration-keepme` is somebody's deliberate pin and is never guessed
// at, whatever a marker beside it says.
if parse_rollback_tag(tag).is_none() {
return false;
}
let Some(since) = ownerless_since else {
return false; return false;
}; };
(*now - created).num_days() >= STALE_PIN_MAX_AGE_DAYS let elapsed = *now - since;
elapsed >= chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS)
} }
/// Drop `triple-c-snapshot-*:pre-migration-*` tags that no migration record /// Drop `triple-c-snapshot-*:pre-migration-*` tags that no migration record
@@ -1138,15 +1219,35 @@ pub async fn reap_stale_migration_pins() -> usize {
// still count as "somebody may want this back". // still count as "somebody may want this back".
let has_record = let has_record =
crate::storage::migration_store::has_record(&project_id).unwrap_or(true); crate::storage::migration_store::has_record(&project_id).unwrap_or(true);
if !pin_is_reapable(&tag, has_record, &now) { if has_record {
// Owned again (or still owned): throw away any grace clock a
// previous pass started, so a pin that loses its record twice
// gets a fresh fourteen days rather than inheriting a stale one.
crate::storage::migration_store::clear_ownerless(&project_id, &tag);
continue;
}
// Only a *well-formed* pin gets a marker written for it — a tag
// that is not one of ours is left entirely alone, files included.
if parse_rollback_tag(&tag).is_none() {
continue;
}
// Records the first sighting when there is none, which is why this
// returns `None` on that pass and the pin survives it.
let ownerless_since =
crate::storage::migration_store::note_ownerless_since(&project_id, &tag, &now);
if !pin_is_reapable(&tag, has_record, ownerless_since, &now) {
continue; continue;
} }
match untag_image(reference).await { match untag_image(reference).await {
Ok(()) => { Ok(()) => {
crate::storage::migration_store::clear_ownerless(&project_id, &tag);
log::info!( log::info!(
"Dropped stale rollback pin {} ({:.2} GB) — no migration record has claimed it for {} days", "Dropped stale rollback pin {} ({:.2} GB) — no migration record has claimed it since {}, more than {} days",
reference, reference,
summary.size as f64 / 1_073_741_824.0, summary.size as f64 / 1_073_741_824.0,
ownerless_since
.map(|t| t.to_rfc3339())
.unwrap_or_else(|| "unknown".to_string()),
STALE_PIN_MAX_AGE_DAYS, STALE_PIN_MAX_AGE_DAYS,
); );
reaped += 1; reaped += 1;
@@ -1915,29 +2016,93 @@ mod tests {
// image is the only copy of the rollback target and the user has not // image is the only copy of the rollback target and the user has not
// yet said they are happy on the new base. // yet said they are happy on the new base.
let ancient = rollback_tag(&at(2020, 1, 1)); let ancient = rollback_tag(&at(2020, 1, 1));
assert!(!pin_is_reapable(&ancient, true, &at(2026, 8, 23))); assert!(!pin_is_reapable(
&ancient,
true,
Some(at(2020, 1, 1)),
&at(2026, 8, 23)
));
} }
#[test] #[test]
fn an_unclaimed_pin_is_reaped_only_once_it_is_old() { fn an_unclaimed_pin_is_reaped_only_once_it_is_old() {
let made = at(2026, 8, 1); let tag = rollback_tag(&at(2026, 8, 1));
let tag = rollback_tag(&made); // The clock runs from when the record went missing, which here is well
assert!(!pin_is_reapable(&tag, false, &at(2026, 8, 2))); // after the migration started.
let lost = at(2026, 8, 10);
assert!(!pin_is_reapable(&tag, false, Some(lost), &at(2026, 8, 11)));
assert!(!pin_is_reapable( assert!(!pin_is_reapable(
&tag, &tag,
false, false,
&(made + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS - 1)) Some(lost),
&(lost + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS) - chrono::Duration::seconds(1))
)); ));
assert!(pin_is_reapable( assert!(pin_is_reapable(
&tag, &tag,
false, false,
&(made + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS)) Some(lost),
&(lost + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS))
)); ));
} }
#[test]
fn the_grace_period_runs_from_the_lost_record_not_from_the_tag() {
// The bug this replaced, stated as a test. A migration parked at
// `awaiting-confirmation` for a month — supported, that is what
// `keep_rollback` is for — whose record is then lost had a
// month-old tag, so the old rule made its pin reapable on the very
// next app start with the startup sweep deleting the image two lines
// later. Zero grace, on the one case the fourteen days exist for.
let started = at(2026, 6, 1);
let tag = rollback_tag(&started);
let record_lost = at(2026, 7, 1);
let noticed_immediately_after = record_lost + chrono::Duration::minutes(5);
assert!(
!pin_is_reapable(&tag, false, Some(record_lost), &noticed_immediately_after),
"a tag a month old must still get its full grace period once orphaned"
);
}
#[test]
fn an_unsighted_pin_is_never_reaped_on_the_pass_that_first_sees_it() {
// `None` means no reaper has recorded a sighting. Treating that as
// "sighted now" would be harmless; treating it as "sighted long ago"
// would not, and neither is what it means — the marker is written on
// this pass and the pin becomes reapable fourteen days later.
let tag = rollback_tag(&at(2020, 1, 1));
assert!(!pin_is_reapable(&tag, false, None, &at(2026, 8, 23)));
}
#[test]
fn a_clock_that_ran_backwards_neither_reaps_nor_strands() {
// A marker dated after `now`: the host clock was fast and got
// corrected, or the data directory came from another machine. A
// negative elapsed time must read as "not yet", not as a huge age.
let tag = rollback_tag(&at(2026, 1, 1));
let marker = at(2026, 9, 1);
assert!(!pin_is_reapable(&tag, false, Some(marker), &at(2026, 8, 1)));
// The other direction is bounded by the marker rather than by the tag:
// a wildly future `now` can only expire a clock that was actually
// started, and a pin with no marker (the case above) still cannot be
// reaped at all — which is what stops a fast host clock from making
// *every* pin on the daemon instantly collectable.
assert!(pin_is_reapable(
&tag,
false,
Some(at(2026, 8, 20)),
&at(2030, 1, 1)
));
assert!(!pin_is_reapable(&tag, false, None, &at(2030, 1, 1)));
}
#[test] #[test]
fn a_tag_we_cannot_date_is_left_alone() { fn a_tag_we_cannot_date_is_left_alone() {
assert!(!pin_is_reapable("pre-migration-handmade", false, &at(2026, 8, 23))); // Even with an ancient ownerless marker sitting beside it: a tag that
assert!(!pin_is_reapable("latest", false, &at(2026, 8, 23))); // merely *starts* `pre-migration-` is somebody's deliberate pin, and
// the reaper never writes a marker for one in the first place.
let ancient = Some(at(2020, 1, 1));
let now = at(2026, 8, 23);
assert!(!pin_is_reapable("pre-migration-handmade", false, ancient, &now));
assert!(!pin_is_reapable("latest", false, ancient, &now));
} }
} }
+14
View File
@@ -5,6 +5,7 @@ mod docker;
mod install_helper; mod install_helper;
mod logging; mod logging;
mod models; mod models;
mod project_lock;
mod storage; mod storage;
pub mod web_terminal; pub mod web_terminal;
@@ -263,6 +264,19 @@ pub fn run() {
let drag_temp_dir = app.path().temp_dir().ok(); let drag_temp_dir = app.path().temp_dir().ok();
tauri::async_runtime::spawn(async move { tauri::async_runtime::spawn(async move {
crate::docker::reap_probe_containers().await; crate::docker::reap_probe_containers().await;
// Before the sweep, and for the same reason the pins are:
// `triple-c-snapshot-*:compacting` is a *tagged* image, so the
// sweep's `dangling=true` filter cannot see it, and the
// `triple-c-compact-*` container a crashed compaction leaves
// behind pins that image open. Untagging first is what turns
// both into something the sweep can collect on the same pass.
let stranded = crate::docker::disk::reap_stale_compaction_artifacts().await;
if stranded > 0 {
log::info!(
"Startup housekeeping dropped {} stranded compaction staging tag(s)",
stranded
);
}
let reaped = crate::docker::reap_stale_migration_pins().await; let reaped = crate::docker::reap_stale_migration_pins().await;
if reaped > 0 { if reaped > 0 {
log::info!("Startup housekeeping dropped {} stale rollback pin(s)", reaped); log::info!("Startup housekeeping dropped {} stale rollback pin(s)", reaped);
+338
View File
@@ -0,0 +1,338 @@
//! Per-project mutual exclusion for everything that rewrites a project's
//! container or its snapshot image.
//!
//! ## Why polling was not enough
//!
//! Until this module existed the app had exactly one mutual-exclusion
//! primitive — the `ACTIVE_MIGRATIONS` set behind
//! `migration_commands::is_migrating` — and it was **one-way**. A migration
//! took a guard for its whole run; everything else merely *asked once, at
//! entry*, whether a migration was in flight and then proceeded with no claim
//! of its own. Two non-migration operations on the same project could not see
//! each other at all, and a migration could start underneath one that was
//! already halfway through.
//!
//! That is not a theoretical gap. Compaction resolves
//! `triple-c-snapshot-{id}:latest` when its build starts and commits back over
//! that same tag minutes later, and the Settings panel is a sidebar rather than
//! a modal — so Project Home stays live with Start, Stop, Reset and Migrate all
//! clickable while a compaction runs. Three interleavings were reproduced:
//!
//! * Compaction commits `flat(A)` over `:latest` after a migration has already
//! moved that tag to a new lineage. The migration is silently reverted, the
//! config replay lands twice, and the migration record says
//! `awaiting-confirmation` against a base the tag no longer points at.
//! * Compaction resolves A, the user starts the project and works for an hour,
//! a recreate commits D over `:latest`, and the compaction then overwrites it
//! with `flat(A)` — orphaning an hour of system-layer work while reporting
//! success and a byte saving.
//! * Compaction resurrects the system layer a Reset had just destroyed.
//!
//! Every one of those is "two writers of `:latest`, neither holding anything".
//! So this registry replaces the polling with an actual claim: an operation
//! **acquires** a [`ProjectGuard`] and holds it for its whole run, and a second
//! operation on the same project is refused with a message naming the holder.
//!
//! ## What this does NOT protect against, stated plainly
//!
//! **This is in-process state.** Two copies of the app pointed at the same
//! Docker daemon share nothing here: instance A's compaction and instance B's
//! migration will both acquire happily and then race exactly as before.
//! `reap_probe_containers` and the `triple-c-compact-*` / `triple-c-scrub-*`
//! sweeps are worse than that — they are daemon-wide force-removals driven by
//! a name or a label, so instance B can destroy a container instance A is
//! mid-commit against.
//!
//! A daemon-visible lock was considered and rejected for now, and the reasoning
//! is recorded here so it is not re-derived from scratch:
//!
//! * A **lock container** would work — container names are unique daemon-wide
//! and `create` fails atomically on a name conflict — but a container has to
//! be created *from an image*, and that pins the image. A lock on
//! `triple-c-snapshot-{id}` would block the very `rmi`/sweep paths it guards,
//! and a leaked lock container would pin multiple gigabytes forever.
//! * A **named volume** is not usable: `create_volume` on an existing name
//! returns the existing volume rather than failing, so it cannot be a
//! test-and-set.
//! * A **label on the snapshot image** is not atomic — read/modify/commit has
//! the same race it would be trying to close.
//!
//! So the cross-process case is **documented, not solved**. What this module
//! does do about it is bound the damage: [`any_held_excluding`] lets the daemon-wide
//! reapers skip work while this process is mid-operation, and the reapers
//! themselves gained age gates so a young container belonging to somebody else
//! is left alone (see `docker::disk::reap_stale_compaction_artifacts` and
//! `docker::migration::reap_probe_containers`).
//!
//! ## Refuse, do not queue
//!
//! [`try_acquire`] never waits. Every caller is a user-initiated action behind
//! a button, and a button that blocks for the four minutes a compaction takes
//! is worse than one that says what is running. The refusal string is written
//! for the user and names the holder.
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
/// The operations that claim a project.
///
/// One variant per *class of writer*, not per command: `Recreate` covers Start
/// as well, because Start's create-and-commit path is the same writer of
/// `triple-c-snapshot-{id}:latest` that a recreate is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProjectOp {
/// `migrate_project_to_base`, `resume_migration`, `rollback_migration`,
/// `confirm_migration`.
Migration,
/// `disk::compact_snapshot` — the long one, and the reason this exists.
Compaction,
/// Start / stop / recreate. Anything in `start_project_container`'s path.
Recreate,
/// `rebuild_project_container` — deletes both volumes and the snapshot.
Reset,
/// `disk::destroy` — a volume, a snapshot image, or a rollback pin.
Destroy,
/// `disk::clear_caches` — an exec into the live container. It does not
/// write `:latest`, but it must not run while the container is being
/// removed out from under it.
CacheClear,
}
impl ProjectOp {
/// What is happening, phrased for the message a user reads.
pub fn describe(self) -> &'static str {
match self {
ProjectOp::Migration => "A container base update is running for this project",
ProjectOp::Compaction => "This project's snapshot is being compacted",
ProjectOp::Recreate => "This project's container is being started or recreated",
ProjectOp::Reset => "This project is being reset",
ProjectOp::Destroy => "Something of this project's is being deleted",
ProjectOp::CacheClear => "This project's caches are being cleared",
}
}
/// What the *refused* caller was trying to do, for the tail of the message.
fn blocked_action(self) -> &'static str {
match self {
ProjectOp::Migration => "starting a base update",
ProjectOp::Compaction => "compacting its snapshot",
ProjectOp::Recreate => "starting or recreating its container",
ProjectOp::Reset => "resetting it",
ProjectOp::Destroy => "deleting anything of its",
ProjectOp::CacheClear => "clearing its caches",
}
}
}
/// Project id → the operation currently holding it.
///
/// A `std::sync::Mutex` rather than a `tokio` one on purpose: it is only ever
/// held for the length of a `HashMap` insert or remove, never across an await,
/// and [`is_held_by`] has to be callable from the synchronous helpers in
/// `disk.rs` that already ask this question.
static HOLDERS: OnceLock<Mutex<HashMap<String, ProjectOp>>> = OnceLock::new();
fn holders() -> &'static Mutex<HashMap<String, ProjectOp>> {
HOLDERS.get_or_init(|| Mutex::new(HashMap::new()))
}
/// A claim on one project, released on drop.
///
/// RAII rather than an explicit release for the reason [`ProjectOp::Migration`]'s
/// predecessor already learned: a plain release statement is skipped by an
/// early `?`, by a panic, and by the future simply being dropped. A guard is
/// not.
#[derive(Debug)]
pub struct ProjectGuard {
project_id: String,
}
impl Drop for ProjectGuard {
fn drop(&mut self) {
// `into_inner` on a poisoned lock: a panic while some other thread held
// this map for the duration of one insert cannot have left it
// inconsistent, and refusing to release afterwards would strand the
// project as permanently busy.
holders()
.lock()
.unwrap_or_else(|e| e.into_inner())
.remove(&self.project_id);
}
}
/// Claim a project for `op`, or say who has it.
///
/// The error is user-facing copy, not a debug string — it goes straight back
/// over IPC to a toast.
pub fn try_acquire(project_id: &str, op: ProjectOp) -> Result<ProjectGuard, String> {
let mut map = holders().lock().unwrap_or_else(|e| e.into_inner());
if let Some(holder) = map.get(project_id).copied() {
return Err(format!(
"{}. Wait for it to finish before {}.",
holder.describe(),
op.blocked_action()
));
}
map.insert(project_id.to_string(), op);
Ok(ProjectGuard {
project_id: project_id.to_string(),
})
}
/// Which operation holds this project, if any.
pub fn held(project_id: &str) -> Option<ProjectOp> {
holders()
.lock()
.unwrap_or_else(|e| e.into_inner())
.get(project_id)
.copied()
}
/// Whether this project is held by exactly `op`.
///
/// `migration_commands::is_migrating` is this, specialised — which is the whole
/// point of folding `ACTIVE_MIGRATIONS` into this registry: there is now one
/// answer to "is something happening to this project", not two that can
/// disagree.
pub fn is_held_by(project_id: &str, op: ProjectOp) -> bool {
held(project_id) == Some(op)
}
/// Whether any project **other than** `exclude_project_id` is currently held by
/// `op`. Pass an empty id to ask about every project.
///
/// Used by the daemon-wide reapers, which cannot tell which project a
/// `triple-c-compact-*` container belongs to — the name carries a random uuid,
/// not a project id — so "is this process compacting anything right now" is the
/// only in-process question they can ask before force-removing one. The
/// exclusion is for the reaper that runs *inside* a compaction, which is
/// already holding a claim of its own and would otherwise see it and skip.
pub fn any_held_excluding(op: ProjectOp, exclude_project_id: &str) -> bool {
holders()
.lock()
.unwrap_or_else(|e| e.into_inner())
.iter()
.any(|(project_id, held)| *held == op && project_id != exclude_project_id)
}
#[cfg(test)]
mod tests {
use super::*;
/// Ids are namespaced per test: the registry is process-global, and
/// `cargo test` runs these on several threads at once.
fn id(name: &str) -> String {
format!("project-lock-test-{}", name)
}
#[test]
fn a_second_acquire_on_the_same_project_is_refused() {
let p = id("second-acquire");
let first = try_acquire(&p, ProjectOp::Compaction).expect("first claim");
let second = try_acquire(&p, ProjectOp::Recreate);
let err = second.expect_err("a second claim must be refused, not queued");
// The refusal has to name the holder — "busy" alone leaves the user
// with nothing to wait for.
assert!(err.contains("snapshot is being compacted"), "{}", err);
assert!(err.contains("starting or recreating"), "{}", err);
drop(first);
// And it has to be retakeable the moment the holder goes away.
try_acquire(&p, ProjectOp::Recreate).expect("released on drop");
}
#[test]
fn the_guard_releases_on_an_early_return() {
let p = id("early-return");
fn bails(project_id: &str) -> Result<(), String> {
let _guard = try_acquire(project_id, ProjectOp::Reset)?;
Err("something failed".to_string())
}
assert!(bails(&p).is_err());
assert_eq!(held(&p), None, "an early `?` must not strand the claim");
}
#[test]
fn the_guard_releases_on_a_panic() {
let p = id("panic");
let result = std::panic::catch_unwind(|| {
let _guard = try_acquire(&id("panic"), ProjectOp::Migration).unwrap();
panic!("boom");
});
assert!(result.is_err());
assert_eq!(held(&p), None, "a panic must not strand the claim either");
}
#[test]
fn two_projects_do_not_block_each_other() {
let a = id("independent-a");
let b = id("independent-b");
let _one = try_acquire(&a, ProjectOp::Compaction).expect("a");
let _two = try_acquire(&b, ProjectOp::Compaction).expect("b");
assert!(is_held_by(&a, ProjectOp::Compaction));
assert!(is_held_by(&b, ProjectOp::Compaction));
}
#[test]
fn is_held_by_distinguishes_the_operation() {
let p = id("which-op");
let _guard = try_acquire(&p, ProjectOp::Compaction).unwrap();
assert!(is_held_by(&p, ProjectOp::Compaction));
assert!(
!is_held_by(&p, ProjectOp::Migration),
"a compaction is not a migration — `is_migrating` is built on this"
);
}
#[test]
fn any_held_sees_across_projects() {
let p = id("any-held");
assert!(!any_held_excluding(ProjectOp::Destroy, ""));
let _guard = try_acquire(&p, ProjectOp::Destroy).unwrap();
assert!(any_held_excluding(ProjectOp::Destroy, ""));
// …and a holder can ask the question without its own claim answering
// it, which is what lets a compaction sweep leftovers before it starts.
assert!(!any_held_excluding(ProjectOp::Destroy, &p));
}
/// Concurrency, not just sequencing: N threads racing for one project must
/// produce exactly one winner.
#[test]
fn exactly_one_of_many_racing_threads_wins() {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
let p = id("race");
let start = Arc::new(std::sync::Barrier::new(8));
// The second barrier is what makes this deterministic rather than
// merely likely: no winner releases until every thread has had its
// turn, so "only one got in" cannot be an artefact of a loser arriving
// after the winner already left.
let attempted = Arc::new(std::sync::Barrier::new(8));
let won = Arc::new(AtomicUsize::new(0));
let mut handles = Vec::new();
for _ in 0..8 {
let start = Arc::clone(&start);
let attempted = Arc::clone(&attempted);
let won = Arc::clone(&won);
let p = p.clone();
handles.push(std::thread::spawn(move || {
start.wait();
let claim = try_acquire(&p, ProjectOp::Compaction);
if claim.is_ok() {
won.fetch_add(1, Ordering::SeqCst);
}
attempted.wait();
drop(claim);
}));
}
for handle in handles {
handle.join().unwrap();
}
assert_eq!(
won.load(Ordering::SeqCst),
1,
"eight threads raced for one project and more than one got in"
);
assert_eq!(held(&p), None);
}
}
+224 -17
View File
@@ -50,15 +50,28 @@ fn sanitize(project_id: &str) -> String {
/// flight; an unparseable file is treated the same way (and logged) rather than /// flight; an unparseable file is treated the same way (and logged) rather than
/// blocking every future migration on a corrupt record. /// blocking every future migration on a corrupt record.
/// ///
/// **A corrupt record is moved aside, not merely ignored.** Reporting "absent" /// **A corrupt record is copied aside and left in place.** An earlier version
/// while leaving the file in place strands the rollback pin it describes: the /// *renamed* it to `.bak`, on the reasoning that a file nothing can parse
/// `:pre-migration-*` tag holding a 412 GB image stays on disk, no code path /// should stop making the project look busy. That destroyed the one signal
/// can find it again (every one of them starts here and is told there is no /// [`has_record`] exists to carry. The chain, in order:
/// migration), and the leftover file goes on making the project look like it ///
/// has a migration in flight to anything that checks for the file rather than /// 1. The rename makes the file vanish, so `has_record` — pure filesystem
/// parsing it — including [`has_record`], which the pin reaper relies on. /// presence — flips to false.
/// Renaming to `.bak` follows the same convention as `projects.json`: the /// 2. `reconcile_migration` calls this, gets `Ok(None)`, and returns. An
/// user's bytes are kept, but they stop pinning gigabytes. /// in-flight or interrupted migration becomes invisible: no resume offer, no
/// rollback offer, and the phase is never normalised.
/// 3. Both pin reapers use `has_record` as their conservative guard, so the
/// project's `:pre-migration-*` tag — the only copy of its pre-migration
/// system layer — is now "ownerless" to both of them, and the startup sweep
/// turns the untag into a deletion.
///
/// A record that cannot be parsed is exactly the case where the *most*
/// conservative answer is wanted, not the least. So the bytes are copied to a
/// **uniquely named** backup (a fixed `.bak` meant a second corruption silently
/// overwrote the first, and nothing ever read either back) and the original
/// stays where it is. The pin it describes then ages out through the ownerless
/// tombstone in `docker::migration::reap_stale_migration_pins` rather than
/// being reaped on the next app start.
pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> { pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
let path = state_path(project_id)?; let path = state_path(project_id)?;
if !path.exists() { if !path.exists() {
@@ -69,15 +82,22 @@ pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
match serde_json::from_str::<MigrationState>(&data) { match serde_json::from_str::<MigrationState>(&data) {
Ok(state) => Ok(Some(state)), Ok(state) => Ok(Some(state)),
Err(e) => { Err(e) => {
let backup = path.with_extension("json.bak"); let backup = corrupt_backup_path(&path, &chrono::Utc::now());
let moved = fs::rename(&path, &backup); let copied = if backup.exists() {
// Already kept a copy of this exact corruption this second;
// nothing to add.
Ok(())
} else {
fs::copy(&path, &backup).map(|_| ())
};
log::error!( log::error!(
"Failed to parse migration state for project {}: {} — treating as absent{}", "Failed to parse migration state for project {}: {} — treating as absent, but \
the record is left in place so `has_record` still protects its rollback pin{}",
project_id, project_id,
e, e,
match moved { match copied {
Ok(()) => format!(" and moved the record to {}", backup.display()), Ok(()) => format!(" (a copy was kept at {})", backup.display()),
Err(ref e) => format!(" (could not move the record aside: {})", e), Err(ref e) => format!(" (could not keep a copy: {})", e),
} }
); );
Ok(None) Ok(None)
@@ -85,6 +105,15 @@ pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
} }
} }
/// Where a copy of an unparseable record is kept.
///
/// Timestamped rather than a fixed `.bak`: a second corruption used to
/// overwrite the first, so the one case where the user's bytes matter most was
/// the case where they were most likely to be gone.
fn corrupt_backup_path(path: &std::path::Path, now: &chrono::DateTime<chrono::Utc>) -> PathBuf {
path.with_extension(format!("json.corrupt-{}.bak", now.format("%Y%m%d-%H%M%S")))
}
/// Whether a project has a migration record on disk *at all*, without parsing /// Whether a project has a migration record on disk *at all*, without parsing
/// it. /// it.
/// ///
@@ -97,17 +126,195 @@ pub fn has_record(project_id: &str) -> Result<bool, String> {
Ok(state_path(project_id)?.exists()) Ok(state_path(project_id)?.exists())
} }
/// Atomically write a project's migration state. /// Atomically **and durably** write a project's migration state.
///
/// Write-temp-then-rename alone is only half of it, and the missing half is the
/// half this record exists for. `fs::write` returns once the bytes are in the
/// page cache; a rename over them is atomic *with respect to other readers*,
/// not with respect to power loss. Losing power in that window leaves the
/// rename applied and the data not yet written — i.e. a 0-byte or truncated
/// `{id}.json` — which is precisely the corrupt-record case above, produced by
/// the code whose job is to make that case impossible.
///
/// So: fsync the file before the rename, and fsync the *directory* after it,
/// because the rename itself is directory metadata and is not durable until the
/// directory is synced. A sync that fails is reported rather than swallowed —
/// this is the crash record, and "probably written" is not a state it may be
/// in.
pub fn save(project_id: &str, state: &MigrationState) -> Result<(), String> { pub fn save(project_id: &str, state: &MigrationState) -> Result<(), String> {
let path = state_path(project_id)?; let path = state_path(project_id)?;
let data = serde_json::to_string_pretty(state) let data = serde_json::to_string_pretty(state)
.map_err(|e| format!("Failed to serialize migration state: {}", e))?; .map_err(|e| format!("Failed to serialize migration state: {}", e))?;
let tmp = path.with_extension("json.tmp"); let tmp = path.with_extension("json.tmp");
fs::write(&tmp, data).map_err(|e| format!("Failed to write migration state: {}", e))?;
{
use std::io::Write;
let mut file = fs::File::create(&tmp)
.map_err(|e| format!("Failed to write migration state: {}", e))?;
file.write_all(data.as_bytes())
.map_err(|e| format!("Failed to write migration state: {}", e))?;
file.sync_all()
.map_err(|e| format!("Failed to flush migration state to disk: {}", e))?;
}
fs::rename(&tmp, &path).map_err(|e| format!("Failed to commit migration state: {}", e))?; fs::rename(&tmp, &path).map_err(|e| format!("Failed to commit migration state: {}", e))?;
sync_dir(&path);
// A project with a record is not ownerless, whatever a reaper concluded
// before this write — so the grace clock is thrown away rather than left to
// expire against a pin that now has an owner again.
clear_ownerless_for_project(project_id);
Ok(()) Ok(())
} }
/// fsync the directory holding `path`, so a rename into it survives power loss.
///
/// Best effort *only* on the platforms where it is meaningless: Windows has no
/// directory handle to sync and returns an error for the attempt, so a failure
/// is logged rather than propagated. The file's own `sync_all` above is the
/// part that carries the data, and it is not best effort.
fn sync_dir(path: &std::path::Path) {
let Some(dir) = path.parent() else {
return;
};
match fs::File::open(dir).and_then(|d| d.sync_all()) {
Ok(()) => {}
Err(e) => log::debug!(
"Could not fsync the migrations directory {}: {} — the record itself was flushed",
dir.display(),
e
),
}
}
// ---------------------------------------------------------------------------
// Ownerless-pin tombstones
// ---------------------------------------------------------------------------
/// Marker recording **when a rollback pin was first seen with no record behind
/// it**.
///
/// ## Why the grace period cannot be measured from the tag
///
/// `docker::migration::pin_is_reapable` used to date a pin from the timestamp
/// encoded in `pre-migration-<YYYYmmdd-HHMMSS>` — i.e. from when the migration
/// *started*. That is the wrong epoch by a whole feature. A migration is
/// allowed to sit at `awaiting-confirmation` indefinitely; `keep_rollback`
/// exists precisely so a user can run on the new base for a month before
/// deciding. If that project's record is then lost — a corrupt file, a deleted
/// state file, a half-restored data directory — the pin is fourteen days old on
/// the very first check, so it is untagged on the next app start and the
/// startup sweep deletes the image two lines later. The fourteen-day grace
/// period the constant promises is zero in the only situation it was written
/// for.
///
/// The clock has to start when the *claim* was lost, and nothing on the daemon
/// records that moment. So it is written down here, the first time a reaper
/// notices, and the age is measured from the marker.
///
/// One file per `(project_id, tag)` in the migrations directory, holding an
/// RFC3339 instant. Tiny, and losing one costs a fresh fourteen days rather
/// than a deletion — the failure direction that keeps somebody's only rollback
/// copy.
fn ownerless_marker_path(project_id: &str, tag: &str) -> Result<PathBuf, String> {
Ok(migrations_dir()?.join(format!(
"{}.{}.ownerless",
sanitize(project_id),
sanitize(tag)
)))
}
/// When this pin was first observed ownerless, **without recording anything**.
///
/// For the survey paths, which describe the world and must not change it.
/// `None` means "no reaper has seen it yet", which is not the same as "seen
/// just now" and must not be treated as a start date.
pub fn peek_ownerless_since(
project_id: &str,
tag: &str,
) -> Option<chrono::DateTime<chrono::Utc>> {
let path = ownerless_marker_path(project_id, tag).ok()?;
let raw = fs::read_to_string(path).ok()?;
chrono::DateTime::parse_from_rfc3339(raw.trim())
.ok()
.map(|t| t.with_timezone(&chrono::Utc))
}
/// Read the first-observed instant for a pin, creating the marker if this is
/// the first sighting. Returns `None` when the clock has not started yet.
///
/// **Clock skew is handled here rather than at the comparison.** A host clock
/// that was running fast when the marker was written leaves a timestamp in the
/// future; measured naively that is a negative age, which a `num_days() >= 14`
/// test reads as "never reapable" — a pin that can never be collected, forever.
/// A marker dated after `now` is therefore rewritten to `now`, restarting the
/// grace period. The other direction — a clock jumping forward — cannot shorten
/// the period below what has actually elapsed on the *marker's* terms, because
/// there is nothing to compare against but wall time; what it cannot do any
/// more is make every pin instantly reapable, which dating from the tag did.
pub fn note_ownerless_since(
project_id: &str,
tag: &str,
now: &chrono::DateTime<chrono::Utc>,
) -> Option<chrono::DateTime<chrono::Utc>> {
let path = ownerless_marker_path(project_id, tag).ok()?;
let existing = fs::read_to_string(&path).ok().and_then(|raw| {
chrono::DateTime::parse_from_rfc3339(raw.trim())
.ok()
.map(|t| t.with_timezone(&chrono::Utc))
});
match existing {
Some(seen) if seen <= *now => Some(seen),
// Absent, unparseable, or dated in the future: (re)start the clock.
_ => {
if let Err(e) = fs::write(&path, now.to_rfc3339()) {
log::warn!(
"Could not record that rollback pin {}:{} is ownerless: {} — its grace \
period restarts on the next check",
project_id,
tag,
e
);
}
None
}
}
}
/// Forget a pin's ownerless marker. Missing is success.
///
/// Called when the pin is untagged, and when a record reappears for the
/// project — a re-migrated project must not inherit the previous run's clock.
pub fn clear_ownerless(project_id: &str, tag: &str) {
let Ok(path) = ownerless_marker_path(project_id, tag) else {
return;
};
match fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => log::warn!("Could not remove {}: {}", path.display(), e),
}
}
/// Drop every ownerless marker belonging to one project.
///
/// A project that has a record again is by definition not ownerless, whatever
/// a reaper concluded before.
pub fn clear_ownerless_for_project(project_id: &str) {
let Ok(dir) = migrations_dir() else {
return;
};
let prefix = format!("{}.", sanitize(project_id));
let Ok(entries) = fs::read_dir(&dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name().to_string_lossy().to_string();
if name.starts_with(&prefix) && name.ends_with(".ownerless") {
let _ = fs::remove_file(entry.path());
}
}
}
/// Remove a project's migration state file. Missing is success. /// Remove a project's migration state file. Missing is success.
pub fn clear(project_id: &str) -> Result<(), String> { pub fn clear(project_id: &str) -> Result<(), String> {
let path = state_path(project_id)?; let path = state_path(project_id)?;