Merge branch 'fix/disk' into integration/round-1
This commit is contained in:
@@ -227,58 +227,40 @@ async fn container_label(container_id: &str, label: &str) -> Option<String> {
|
|||||||
// Migrate
|
// Migrate
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Project ids with a migration running **in this process right now**.
|
|
||||||
///
|
|
||||||
/// Two things need it. `reconcile_project_statuses` is callable from the
|
|
||||||
/// frontend at any time, not only at startup, and a live migration looks
|
|
||||||
/// exactly like a crashed one from the outside (state file says `in-progress`,
|
|
||||||
/// container carries the label) — without this guard a reconcile mid-run would
|
|
||||||
/// rewrite the phase to `interrupted` underneath a migration that is fine.
|
|
||||||
/// It also makes a second concurrent `migrate_project_to_base` for the same
|
|
||||||
/// project impossible.
|
|
||||||
static ACTIVE_MIGRATIONS: std::sync::OnceLock<
|
|
||||||
std::sync::Mutex<std::collections::HashSet<String>>,
|
|
||||||
> = std::sync::OnceLock::new();
|
|
||||||
|
|
||||||
fn active_migrations() -> &'static std::sync::Mutex<std::collections::HashSet<String>> {
|
|
||||||
ACTIVE_MIGRATIONS.get_or_init(|| std::sync::Mutex::new(std::collections::HashSet::new()))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether a migration for this project is running **in this process right
|
/// Whether a migration for this project is running **in this process right
|
||||||
/// now**. Every command that stops, removes or recreates the project's
|
/// now**. Every command that stops, removes or recreates the project's
|
||||||
/// container has to consult it: the window between `remove_container` and the
|
/// container has to consult it: the window between `remove_container` and the
|
||||||
/// create that follows looks exactly like "no container", and an ordinary
|
/// create that follows looks exactly like "no container", and an ordinary
|
||||||
/// Start landing in it creates a second container under the same name.
|
/// Start landing in it creates a second container under the same name.
|
||||||
|
///
|
||||||
|
/// **This is now a view onto [`crate::project_lock`], not a set of its own.**
|
||||||
|
/// It used to be the app's only mutual-exclusion primitive, and it was one-way:
|
||||||
|
/// a migration claimed a project, everything else merely polled this once at
|
||||||
|
/// entry and never claimed anything. Two non-migration writers of
|
||||||
|
/// `triple-c-snapshot-{id}:latest` — a compaction and a recreate — could not
|
||||||
|
/// see each other at all. Folding the set into the shared registry means there
|
||||||
|
/// is exactly one answer to "is something happening to this project", and this
|
||||||
|
/// function is the specialisation of it that reconcile still needs: a *live*
|
||||||
|
/// migration is indistinguishable from a crashed one from the outside, and only
|
||||||
|
/// this process knows which it is looking at.
|
||||||
pub(crate) fn is_migrating(project_id: &str) -> bool {
|
pub(crate) fn is_migrating(project_id: &str) -> bool {
|
||||||
active_migrations()
|
crate::project_lock::is_held_by(project_id, crate::project_lock::ProjectOp::Migration)
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(|e| e.into_inner())
|
|
||||||
.contains(project_id)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// RAII marker: removes the project from [`ACTIVE_MIGRATIONS`] however the
|
/// RAII marker: releases the project's [`crate::project_lock`] claim however
|
||||||
/// migration ends, including an early `?`.
|
/// the migration ends, including an early `?`.
|
||||||
struct ActiveGuard(String);
|
///
|
||||||
|
/// Kept as a named type rather than using [`crate::project_lock::ProjectGuard`]
|
||||||
|
/// directly so the migration path keeps reading as "take the migration guard",
|
||||||
|
/// and so the one place that decides what a migration's claim *is* stays here.
|
||||||
|
struct ActiveGuard(#[allow(dead_code)] crate::project_lock::ProjectGuard);
|
||||||
|
|
||||||
impl ActiveGuard {
|
impl ActiveGuard {
|
||||||
/// `None` when a migration is already running for this project.
|
/// `None` when a migration — or anything else — already holds this project.
|
||||||
fn acquire(project_id: &str) -> Option<Self> {
|
fn acquire(project_id: &str) -> Option<Self> {
|
||||||
let mut set = active_migrations()
|
crate::project_lock::try_acquire(project_id, crate::project_lock::ProjectOp::Migration)
|
||||||
.lock()
|
.ok()
|
||||||
.unwrap_or_else(|e| e.into_inner());
|
.map(Self)
|
||||||
if !set.insert(project_id.to_string()) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some(Self(project_id.to_string()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for ActiveGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
active_migrations()
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(|e| e.into_inner())
|
|
||||||
.remove(&self.0);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1163,7 +1145,23 @@ pub(crate) async fn purge_migration_artifacts(project_id: &str) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(None) => return,
|
Ok(None) => {
|
||||||
|
// **`Ok(None)` is not the same as "no file".** `migration_store::load`
|
||||||
|
// now reports an *unparseable* record as absent while deliberately
|
||||||
|
// leaving it on disk, so that `has_record` goes on protecting the
|
||||||
|
// rollback pin it describes. Returning here on that would leave the
|
||||||
|
// file — and therefore a permanently "claimed" pin — behind a Reset
|
||||||
|
// that has just deleted the snapshot and both volumes the record
|
||||||
|
// could possibly refer to.
|
||||||
|
if !migration_store::has_record(project_id).unwrap_or(false) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log::warn!(
|
||||||
|
"Project {} has a migration record that could not be read; removing it anyway \
|
||||||
|
because a Reset supersedes it",
|
||||||
|
project_id
|
||||||
|
);
|
||||||
|
}
|
||||||
Err(e) => log::warn!(
|
Err(e) => log::warn!(
|
||||||
"Could not read the migration record for {} while cleaning up: {}",
|
"Could not read the migration record for {} while cleaning up: {}",
|
||||||
project_id,
|
project_id,
|
||||||
@@ -1172,6 +1170,10 @@ pub(crate) async fn purge_migration_artifacts(project_id: &str) {
|
|||||||
}
|
}
|
||||||
let _ = migration_store::clear_staging(project_id);
|
let _ = migration_store::clear_staging(project_id);
|
||||||
let _ = migration_store::clear(project_id);
|
let _ = migration_store::clear(project_id);
|
||||||
|
// The pins this project had are gone with the snapshot; their grace clocks
|
||||||
|
// are meaningless and would otherwise sit in the migrations directory
|
||||||
|
// forever.
|
||||||
|
migration_store::clear_ownerless_for_project(project_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_docker_socket() -> String {
|
fn default_docker_socket() -> String {
|
||||||
|
|||||||
@@ -221,20 +221,35 @@ pub async fn start_project_container(
|
|||||||
app_handle: tauri::AppHandle,
|
app_handle: tauri::AppHandle,
|
||||||
state: State<'_, AppState>,
|
state: State<'_, AppState>,
|
||||||
) -> Result<Project, String> {
|
) -> Result<Project, String> {
|
||||||
// A migration removes the container and creates its replacement moments
|
// **Acquired, not polled.** A migration removes the container and creates
|
||||||
// later. Starting in that window finds no container, creates a second one
|
// its replacement moments later. Starting in that window finds no
|
||||||
// under the same name, and the migration's own create then fails on the
|
// container, creates a second one under the same name, and the migration's
|
||||||
// name conflict — which sends it into an auto-rollback that also cannot
|
// own create then fails on the name conflict — which sends it into an
|
||||||
// create. The UI already refuses (`canMigrate` gates on the container being
|
// auto-rollback that also cannot create. This used to be a one-shot
|
||||||
// stopped and no run being in flight); this is the same gate on the side
|
// `is_migrating` read, which covered that case and no other: a start also
|
||||||
// that actually owns the invariant.
|
// commits `triple-c-snapshot-{id}:latest`, so it races a *compaction*
|
||||||
if crate::commands::migration_commands::is_migrating(&project_id) {
|
// committing the same tag with nothing between them. The claim is held for
|
||||||
return Err(
|
// the whole start rather than checked at its door.
|
||||||
"A container base update is running for this project. Wait for it to finish, then start the project."
|
//
|
||||||
.to_string(),
|
// The UI already refuses (`canMigrate` gates on the container being stopped
|
||||||
);
|
// and no run being in flight); this is the same gate on the side that
|
||||||
}
|
// actually owns the invariant.
|
||||||
|
let _guard =
|
||||||
|
crate::project_lock::try_acquire(&project_id, crate::project_lock::ProjectOp::Recreate)?;
|
||||||
|
start_project_container_locked(project_id, app_handle, state).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The body of [`start_project_container`], with **no claim of its own**.
|
||||||
|
///
|
||||||
|
/// Split out for exactly one caller: [`rebuild_project_container`] already
|
||||||
|
/// holds the project under [`crate::project_lock::ProjectOp::Reset`] for its
|
||||||
|
/// whole run, and a Reset that then went through the public command would be
|
||||||
|
/// refused by its own guard. Every other path must go through the command.
|
||||||
|
async fn start_project_container_locked(
|
||||||
|
project_id: String,
|
||||||
|
app_handle: tauri::AppHandle,
|
||||||
|
state: State<'_, AppState>,
|
||||||
|
) -> Result<Project, String> {
|
||||||
let mut project = state
|
let mut project = state
|
||||||
.projects_store
|
.projects_store
|
||||||
.get(&project_id)
|
.get(&project_id)
|
||||||
@@ -539,6 +554,14 @@ pub async fn stop_project_container(
|
|||||||
app_handle: tauri::AppHandle,
|
app_handle: tauri::AppHandle,
|
||||||
state: State<'_, AppState>,
|
state: State<'_, AppState>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
|
// Stop had **no** exclusion at all, which is the one gap CLAUDE.md's "every
|
||||||
|
// command that stops, removes or recreates the container consults
|
||||||
|
// `is_migrating`" rule already named and this function did not honour. A
|
||||||
|
// stop lands on the container a migration is mid-swap on, and it closes the
|
||||||
|
// exec sessions a compaction's config replay is not expecting to lose.
|
||||||
|
let _guard =
|
||||||
|
crate::project_lock::try_acquire(&project_id, crate::project_lock::ProjectOp::Recreate)?;
|
||||||
|
|
||||||
let project = state
|
let project = state
|
||||||
.projects_store
|
.projects_store
|
||||||
.get(&project_id)
|
.get(&project_id)
|
||||||
@@ -571,13 +594,13 @@ pub async fn rebuild_project_container(
|
|||||||
) -> Result<Project, String> {
|
) -> Result<Project, String> {
|
||||||
// Reset deletes both volumes and the snapshot image. Doing that while a
|
// Reset deletes both volumes and the snapshot image. Doing that while a
|
||||||
// migration is mid-flight pulls the ground out from under it and leaves an
|
// migration is mid-flight pulls the ground out from under it and leaves an
|
||||||
// orphan migration record pointing at images that no longer exist.
|
// orphan migration record pointing at images that no longer exist — and
|
||||||
if crate::commands::migration_commands::is_migrating(&project_id) {
|
// doing it while a *compaction* is mid-flight is worse, because the
|
||||||
return Err(
|
// compaction then commits `flat(old)` back over the `:latest` this just
|
||||||
"A container base update is running for this project. Wait for it to finish before resetting."
|
// destroyed and resurrects the system layer the user asked to be rid of.
|
||||||
.to_string(),
|
// Held for the whole Reset, including the start at the end of it.
|
||||||
);
|
let _guard =
|
||||||
}
|
crate::project_lock::try_acquire(&project_id, crate::project_lock::ProjectOp::Reset)?;
|
||||||
|
|
||||||
let project = state
|
let project = state
|
||||||
.projects_store
|
.projects_store
|
||||||
@@ -611,8 +634,9 @@ pub async fn rebuild_project_container(
|
|||||||
log::warn!("Failed to remove project volumes for project {}: {}", project_id, e);
|
log::warn!("Failed to remove project volumes for project {}: {}", project_id, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start fresh
|
// Start fresh. The locked variant, because `_guard` above is this project's
|
||||||
start_project_container(project_id, app_handle, state).await
|
// claim and the public command would be refused by it.
|
||||||
|
start_project_container_locked(project_id, app_handle, state).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reconcile project statuses against actual Docker container state.
|
/// Reconcile project statuses against actual Docker container state.
|
||||||
@@ -656,9 +680,14 @@ pub async fn reconcile_project_statuses(
|
|||||||
) {
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// ...but never for a project this process is actively migrating: the
|
// ...but never for a project this process is actively working on. A
|
||||||
// container is legitimately absent for part of that run.
|
// migration's container is legitimately absent between the
|
||||||
if crate::commands::migration_commands::is_migrating(&project.id) {
|
// `remove_container` and the create that follows, and so is a Reset's,
|
||||||
|
// and so is a start's before its create returns. Reconciling into any
|
||||||
|
// of those windows writes `Stopped` over a project that is mid-run.
|
||||||
|
// Broadened from `is_migrating` to the whole lock for exactly that
|
||||||
|
// reason — the migration was never the only operation with a gap.
|
||||||
|
if crate::project_lock::held(&project.id).is_some() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+951
-234
File diff suppressed because it is too large
Load Diff
@@ -26,9 +26,6 @@ fn all_reclaim_targets() -> Vec<ReclaimTarget> {
|
|||||||
ReclaimTarget::MigrationStaging,
|
ReclaimTarget::MigrationStaging,
|
||||||
ReclaimTarget::ProbeContainers,
|
ReclaimTarget::ProbeContainers,
|
||||||
ReclaimTarget::ScrubContainers,
|
ReclaimTarget::ScrubContainers,
|
||||||
ReclaimTarget::OrphanVolume {
|
|
||||||
name: "triple-c-home-gone".to_string(),
|
|
||||||
},
|
|
||||||
ReclaimTarget::CompactSnapshot {
|
ReclaimTarget::CompactSnapshot {
|
||||||
project_id: "p1".to_string(),
|
project_id: "p1".to_string(),
|
||||||
},
|
},
|
||||||
@@ -54,7 +51,7 @@ fn every_variant_is_covered_by_the_safety_walk() {
|
|||||||
.collect();
|
.collect();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
discriminants.len(),
|
discriminants.len(),
|
||||||
10,
|
9,
|
||||||
"a ReclaimTarget variant was added or removed; update all_reclaim_targets() and check its \
|
"a ReclaimTarget variant was added or removed; update all_reclaim_targets() and check its \
|
||||||
safety: {:?}",
|
safety: {:?}",
|
||||||
discriminants
|
discriminants
|
||||||
@@ -358,6 +355,44 @@ fn a_compaction_container_is_never_matched_by_the_scrub_bucket() {
|
|||||||
assert!(!is_compaction_container(&summary(&["/my-triple-c-compact-notes"], &[])));
|
assert!(!is_compaction_container(&summary(&["/my-triple-c-compact-notes"], &[])));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_daemon_wide_buckets_leave_a_young_container_alone() {
|
||||||
|
// Both of these buckets are `Safety::Safe` — one tick, no confirmation —
|
||||||
|
// and both reach *every* matching container on the daemon, because a label
|
||||||
|
// and a name prefix are daemon-wide and `ReclaimTarget::project_id()` is
|
||||||
|
// `None` for them. So a second app instance's live migration probe, and a
|
||||||
|
// live secret rewrite's scratch container, are both in range. Age is the
|
||||||
|
// only discriminator available from this side of the process boundary, and
|
||||||
|
// a container the daemon gave no creation time for is treated as young.
|
||||||
|
let now = chrono::Utc::now().timestamp();
|
||||||
|
|
||||||
|
let mut young_probe = summary(&["/nervous_curie"], &[(migration::LABEL_PROBE, "migration")]);
|
||||||
|
young_probe.created = Some(now - 30);
|
||||||
|
assert!(is_migration_probe(&young_probe));
|
||||||
|
assert!(!is_reapable_migration_probe(&young_probe));
|
||||||
|
|
||||||
|
let mut old_probe = young_probe.clone();
|
||||||
|
old_probe.created = Some(now - migration::PROBE_REAP_MIN_AGE_SECS - 1);
|
||||||
|
assert!(is_reapable_migration_probe(&old_probe));
|
||||||
|
|
||||||
|
let mut undated_probe = young_probe.clone();
|
||||||
|
undated_probe.created = None;
|
||||||
|
assert!(!is_reapable_migration_probe(&undated_probe));
|
||||||
|
|
||||||
|
// `triple-c-scrub-*` is a **live** name: `rewrite_image_without_secrets`
|
||||||
|
// creates its scratch container under it, and killing that between the
|
||||||
|
// create and the commit leaves a revoked OAuth token baked into the
|
||||||
|
// snapshot's Config.Env — the exact thing that function exists to remove.
|
||||||
|
let mut young_scrub = summary(&["/triple-c-scrub-abc123"], &[]);
|
||||||
|
young_scrub.created = Some(now - 5);
|
||||||
|
assert!(is_scrub_container(&young_scrub));
|
||||||
|
assert!(!is_reapable_scrub_container(&young_scrub));
|
||||||
|
|
||||||
|
let mut old_scrub = young_scrub.clone();
|
||||||
|
old_scrub.created = Some(now - SCRATCH_CONTAINER_MIN_AGE_SECS - 1);
|
||||||
|
assert!(is_reapable_scrub_container(&old_scrub));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_probe_container_is_matched_on_its_label_not_on_the_daemons_filter() {
|
fn a_probe_container_is_matched_on_its_label_not_on_the_daemons_filter() {
|
||||||
// The `label=triple-c.probe=migration` filter is an exact match and would
|
// The `label=triple-c.probe=migration` filter is an exact match and would
|
||||||
@@ -393,7 +428,7 @@ fn project(id: &str, name: &str) -> Project {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_unreadable_projects_json_is_never_trusted() {
|
fn an_unreadable_projects_json_is_never_trusted() {
|
||||||
let err = project_store_trust(&[project("a", "api")], true, false).unwrap_err();
|
let err = project_store_trust(&[project("a", "api")], true, None).unwrap_err();
|
||||||
assert!(err.contains("could not be read"), "{}", err);
|
assert!(err.contains("could not be read"), "{}", err);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -403,20 +438,44 @@ fn an_empty_list_from_an_existing_file_is_treated_as_a_failed_load() {
|
|||||||
// up and starts empty. That is right for the app and catastrophic here, so
|
// up and starts empty. That is right for the app and catastrophic here, so
|
||||||
// the combination "empty list + file present" is refused rather than read as
|
// the combination "empty list + file present" is refused rather than read as
|
||||||
// "the user has no projects".
|
// "the user has no projects".
|
||||||
let err = project_store_trust(&[], true, true).unwrap_err();
|
let err = project_store_trust(&[], true, Some(&[])).unwrap_err();
|
||||||
assert!(err.contains("suppressed"), "{}", err);
|
assert!(err.contains("suppressed"), "{}", err);
|
||||||
|
|
||||||
// No file at all is a genuine fresh install, and there is nothing on the
|
// No file at all is a genuine fresh install, and there is nothing on the
|
||||||
// daemon to mis-attribute in that state.
|
// daemon to mis-attribute in that state.
|
||||||
assert!(project_store_trust(&[], false, true).unwrap().is_empty());
|
assert!(project_store_trust(&[], false, Some(&[])).unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_healthy_store_yields_its_ids() {
|
fn a_healthy_store_yields_its_ids() {
|
||||||
let ids = project_store_trust(&[project("a", "api"), project("b", "web")], true, true).unwrap();
|
let ids =
|
||||||
|
project_store_trust(&[project("a", "api"), project("b", "web")], true, Some(&[])).unwrap();
|
||||||
assert_eq!(ids, HashSet::from(["a".to_string(), "b".to_string()]));
|
assert_eq!(ids, HashSet::from(["a".to_string(), "b".to_string()]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_project_only_the_file_knows_about_still_counts_as_live() {
|
||||||
|
// M6: `projects` is this process's in-memory list. A project added by a
|
||||||
|
// *second* copy of the app is in `projects.json` and not in that list, and
|
||||||
|
// its live home and config volumes then matched "no project claims this".
|
||||||
|
// The union is what closes the gap; the file is authoritative for
|
||||||
|
// everything this instance has not heard about.
|
||||||
|
let ids = project_store_trust(
|
||||||
|
&[project("a", "api")],
|
||||||
|
true,
|
||||||
|
Some(&["a".to_string(), "b-from-the-other-window".to_string()]),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
ids.contains("b-from-the-other-window"),
|
||||||
|
"a project only the file knows about must not look orphaned: {:?}",
|
||||||
|
ids
|
||||||
|
);
|
||||||
|
// And the reverse: a project this instance just added is live whether or
|
||||||
|
// not the file has caught up.
|
||||||
|
assert!(ids.contains("a"), "{:?}", ids);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Layer accounting — the number the whole UI exists to show
|
// Layer accounting — the number the whole UI exists to show
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -647,6 +706,94 @@ fn the_compaction_dockerfile_reuses_the_one_scrub_list() {
|
|||||||
assert!(!run_lines[0].contains('\n'));
|
assert!(!run_lines[0].contains('\n'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_compaction_run_carries_the_scrub_script_byte_for_byte() {
|
||||||
|
// **This is the assertion the old one should have been.** The previous
|
||||||
|
// version checked only that the `RUN` was a single line — which the broken
|
||||||
|
// space-fold satisfied perfectly, while producing
|
||||||
|
// `… for p in …; do [ -e "$p" ] || continue sz=$(…) …`, i.e. shell that
|
||||||
|
// `sh` refuses to parse. Every compaction ever attempted failed on the
|
||||||
|
// build's first stage.
|
||||||
|
//
|
||||||
|
// Nothing is folded now: the script goes into the JSON exec form verbatim,
|
||||||
|
// so the strong statement is available — what reaches `/bin/sh -c` is
|
||||||
|
// exactly what `snapshot_scrub_script()` returned, newlines included.
|
||||||
|
let script = container::snapshot_scrub_script();
|
||||||
|
let df = compaction_dockerfile("triple-c-snapshot-p1:latest", &script);
|
||||||
|
let run = df
|
||||||
|
.lines()
|
||||||
|
.find(|l| l.starts_with("RUN "))
|
||||||
|
.expect("no RUN line");
|
||||||
|
let recovered = script_from_run_line(run).expect("the RUN is not a parseable exec form");
|
||||||
|
assert_eq!(
|
||||||
|
recovered, script,
|
||||||
|
"the compaction must run the scrub script unmodified"
|
||||||
|
);
|
||||||
|
// The exec form names the shell itself, because `RUN [...]` does not go
|
||||||
|
// through one.
|
||||||
|
let argv: Vec<String> = serde_json::from_str(run.trim_start_matches("RUN ")).unwrap();
|
||||||
|
assert_eq!(argv[0], "/bin/sh");
|
||||||
|
assert_eq!(argv[1], "-c");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_compaction_run_is_valid_shell() {
|
||||||
|
// The test that would have caught H1, and the only kind that can: hand the
|
||||||
|
// exact program the daemon will execute to a real shell and ask it to
|
||||||
|
// parse. `sh -n` reads and parses without running anything.
|
||||||
|
//
|
||||||
|
// It is run against the live `snapshot_scrub_script()` rather than a fixture
|
||||||
|
// precisely because that script is not this module's to own — it is free to
|
||||||
|
// grow a `case`, an `if` or a function, and this must keep holding when it
|
||||||
|
// does.
|
||||||
|
let df = compaction_dockerfile("triple-c-snapshot-p1:latest", &container::snapshot_scrub_script());
|
||||||
|
let run = df.lines().find(|l| l.starts_with("RUN ")).unwrap();
|
||||||
|
let script = script_from_run_line(run).unwrap();
|
||||||
|
assert_shell_parses(&script, "the compaction scrub");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_multi_line_script_with_blocks_survives_the_run_encoding() {
|
||||||
|
// The property the old fold did not have, stated directly: a script with a
|
||||||
|
// `for`/`do`, an `if`/`then`, a `case` and a quote-heavy line has to survive
|
||||||
|
// whatever this module does to it. Joining lines with a space breaks the
|
||||||
|
// first three; joining with `;` breaks `if x; then; y`. Encoding the string
|
||||||
|
// breaks none of them.
|
||||||
|
let awkward = "total=0\n for p in /tmp/a* /tmp/b*; do\n \t[ -e \"$p\" ] || continue\n \tcase \"$p\" in\n \t\t*.keep) continue ;;\n \tesac\n \tif [ -d \"$p\" ]; then\n \t\trm -rf -- \"$p\"\n \tfi\n done\n echo \"done: $total\"\n";
|
||||||
|
let df = compaction_dockerfile("x:latest", awkward);
|
||||||
|
let run = df.lines().find(|l| l.starts_with("RUN ")).unwrap();
|
||||||
|
assert!(!run.contains('\n'), "the RUN must still be one Dockerfile line");
|
||||||
|
assert_eq!(script_from_run_line(run).unwrap(), awkward);
|
||||||
|
assert_shell_parses(awkward, "an awkward but legal script");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run `sh -n` over a program and fail with the shell's own diagnostic.
|
||||||
|
///
|
||||||
|
/// Skipped, loudly, on a host with no `/bin/sh` — which is not a case any
|
||||||
|
/// developer machine or CI image this repo targets is in, but a silent pass
|
||||||
|
/// would be worse than a skipped test.
|
||||||
|
fn assert_shell_parses(script: &str, what: &str) {
|
||||||
|
let output = match std::process::Command::new("/bin/sh")
|
||||||
|
.arg("-n")
|
||||||
|
.arg("-c")
|
||||||
|
.arg(script)
|
||||||
|
.output()
|
||||||
|
{
|
||||||
|
Ok(output) => output,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("skipping the shell syntax check for {}: {}", what, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"{} is not valid shell:\n{}\n--- script ---\n{}",
|
||||||
|
what,
|
||||||
|
String::from_utf8_lossy(&output.stderr),
|
||||||
|
script
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn the_compaction_build_is_labelled_so_the_sweep_can_collect_it() {
|
fn the_compaction_build_is_labelled_so_the_sweep_can_collect_it() {
|
||||||
// Everything that cleans up after this build — the discard path when the
|
// Everything that cleans up after this build — the discard path when the
|
||||||
@@ -672,6 +819,13 @@ fn the_compaction_dockerfile_never_reaches_a_bind_mount() {
|
|||||||
// directories, mounted from the host. Nothing in a scrub may name one, and
|
// directories, mounted from the host. Nothing in a scrub may name one, and
|
||||||
// the two read-only host mounts under /tmp are dot-prefixed so no glob
|
// the two read-only host mounts under /tmp are dot-prefixed so no glob
|
||||||
// reaches them either.
|
// reaches them either.
|
||||||
|
//
|
||||||
|
// **Necessary, not sufficient, and do not read it as coverage.** A path not
|
||||||
|
// appearing as a literal says nothing about where a glob or a symlink
|
||||||
|
// resolves to at runtime; the containment property itself is tested in
|
||||||
|
// `container.rs`, which owns the path list and the script. This assertion
|
||||||
|
// is kept because a literal `/workspace` appearing here would be an
|
||||||
|
// unambiguous mistake, and that is all it detects.
|
||||||
assert!(!df.contains("/workspace"), "{}", df);
|
assert!(!df.contains("/workspace"), "{}", df);
|
||||||
assert!(!df.contains(".host-ca"), "{}", df);
|
assert!(!df.contains(".host-ca"), "{}", df);
|
||||||
assert!(!df.contains(".host-aws"), "{}", df);
|
assert!(!df.contains(".host-aws"), "{}", df);
|
||||||
@@ -936,3 +1090,406 @@ fn the_report_serialises_as_snake_case_like_every_other_ipc_struct() {
|
|||||||
// type is `number | null`, matching every other optional in `types.ts`.
|
// type is `number | null`, matching every other optional in `types.ts`.
|
||||||
assert!(json["projects"][0]["snapshot_above_base_bytes"].is_null());
|
assert!(json["projects"][0]["snapshot_above_base_bytes"].is_null());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Rollback pins — the guards the safe bucket was missing
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn moment(y: i32, m: u32, d: u32) -> chrono::DateTime<chrono::Utc> {
|
||||||
|
chrono::NaiveDate::from_ymd_opt(y, m, d)
|
||||||
|
.unwrap()
|
||||||
|
.and_hms_opt(12, 0, 0)
|
||||||
|
.unwrap()
|
||||||
|
.and_utc()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_safe_pin_bucket_applies_every_guard_the_other_paths_do() {
|
||||||
|
let now = moment(2026, 8, 23);
|
||||||
|
let ours = migration::rollback_tag(&moment(2026, 1, 1));
|
||||||
|
|
||||||
|
// 1. A tag that merely *starts* `pre-migration-`. `destroy` refuses it with
|
||||||
|
// a comment explaining that `tag: "latest"` would otherwise name the
|
||||||
|
// project's live snapshot; the safe bucket used to accept anything.
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition("pre-migration-keepme", false, Some(moment(2020, 1, 1)), &now),
|
||||||
|
PinDisposition::NotOurs
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition("latest", false, Some(moment(2020, 1, 1)), &now),
|
||||||
|
PinDisposition::NotOurs
|
||||||
|
);
|
||||||
|
|
||||||
|
// 2. A record still claims it. Never reaped at any age, and this is the one
|
||||||
|
// guard the old code did have.
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition(&ours, true, Some(moment(2020, 1, 1)), &now),
|
||||||
|
PinDisposition::Claimed
|
||||||
|
);
|
||||||
|
|
||||||
|
// 3. Ownerless but inside the grace period — including "no reaper has seen
|
||||||
|
// it yet", which is where the clock starts. The old code untagged this
|
||||||
|
// immediately, and `sweep_orphaned_snapshots` four lines later turned
|
||||||
|
// the untag into a deletion.
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition(&ours, false, None, &now),
|
||||||
|
PinDisposition::WithinGrace
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition(&ours, false, Some(now - chrono::Duration::days(1)), &now),
|
||||||
|
PinDisposition::WithinGrace
|
||||||
|
);
|
||||||
|
|
||||||
|
// 4. Ownerless and past its grace period. The only case that is collectable
|
||||||
|
// without a typed confirmation.
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition(
|
||||||
|
&ours,
|
||||||
|
false,
|
||||||
|
Some(now - chrono::Duration::days(migration::STALE_PIN_MAX_AGE_DAYS)),
|
||||||
|
&now
|
||||||
|
),
|
||||||
|
PinDisposition::Reapable
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_safe_bucket_and_the_startup_reaper_cannot_disagree() {
|
||||||
|
// Both go through `migration::pin_is_reapable`. The bug was that the safe
|
||||||
|
// bucket did not: `reap_stale_migration_pins` required a parseable tag and
|
||||||
|
// an age, the reclaim button required neither, and the button is the path
|
||||||
|
// with no confirmation in front of it.
|
||||||
|
let now = moment(2026, 8, 23);
|
||||||
|
let ours = migration::rollback_tag(&moment(2026, 1, 1));
|
||||||
|
for since in [
|
||||||
|
None,
|
||||||
|
Some(now - chrono::Duration::days(1)),
|
||||||
|
Some(now - chrono::Duration::days(migration::STALE_PIN_MAX_AGE_DAYS)),
|
||||||
|
] {
|
||||||
|
for has_record in [true, false] {
|
||||||
|
assert_eq!(
|
||||||
|
pin_disposition(&ours, has_record, since, &now) == PinDisposition::Reapable,
|
||||||
|
migration::pin_is_reapable(&ours, has_record, since, &now),
|
||||||
|
"disposition and the reaper's own predicate disagreed for {:?}/{}",
|
||||||
|
since,
|
||||||
|
has_record
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_orphaned_volume_cannot_be_reached_without_a_typed_confirmation() {
|
||||||
|
// M5: it was a `ReclaimTarget` at `Safety::Safe` — a tick and the group
|
||||||
|
// button. `reclaim` cannot be handed a `DestructiveTarget` at all, which is
|
||||||
|
// the type-level half of the guarantee; this pins the other half, that no
|
||||||
|
// reclaim variant names a volume any more.
|
||||||
|
for target in all_reclaim_targets() {
|
||||||
|
let wire = serde_json::to_value(&target).unwrap();
|
||||||
|
let kind = wire["kind"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
!kind.contains("volume"),
|
||||||
|
"{} can reach a volume from the safe path",
|
||||||
|
kind
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the confirmation subject is the *volume* name, because an orphan has
|
||||||
|
// no project in the store whose name could be typed.
|
||||||
|
let target = DestructiveTarget::OrphanVolume {
|
||||||
|
name: "triple-c-claude-config-gone".to_string(),
|
||||||
|
project_id: "gone".to_string(),
|
||||||
|
};
|
||||||
|
assert!(confirmation_matches(
|
||||||
|
"triple-c-claude-config-gone",
|
||||||
|
" triple-c-claude-config-gone "
|
||||||
|
));
|
||||||
|
assert!(!confirmation_matches("triple-c-claude-config-gone", "gone"));
|
||||||
|
assert_eq!(target.project_id(), "gone");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// The numbers, which the user reads as facts
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_snapshot_column_and_the_total_come_from_one_rule() {
|
||||||
|
// Case 1: the daemon measured the sharing. Its figure wins.
|
||||||
|
assert_eq!(snapshot_attribution(5_000_000_000, 4_700_000_000, Some(1)), 300_000_000);
|
||||||
|
|
||||||
|
// Case 2: no shared size, but the lineage is known. The layer arithmetic is
|
||||||
|
// the honest answer, and it is what the Snapshot column already showed —
|
||||||
|
// while `total_bytes` used `size - shared` (i.e. the whole image, base
|
||||||
|
// included) and `triple_c_total_bytes` then added the base again as its own
|
||||||
|
// row. That double count is the exact thing the comment beside the
|
||||||
|
// subtraction says it prevents.
|
||||||
|
assert_eq!(snapshot_attribution(5_000_000_000, 0, Some(300_000_000)), 300_000_000);
|
||||||
|
|
||||||
|
// Case 3: nothing known. The full size, not zero — an image that shares
|
||||||
|
// nothing measurable really does cost all of it, and a flattened snapshot
|
||||||
|
// is exactly that shape.
|
||||||
|
assert_eq!(snapshot_attribution(5_000_000_000, 0, None), 5_000_000_000);
|
||||||
|
|
||||||
|
// A daemon that reports `-1` for "not computed" must not be read as a
|
||||||
|
// 1-byte saving, and a negative result is never returned.
|
||||||
|
assert_eq!(snapshot_attribution(1_000, -1, None), 1_000);
|
||||||
|
assert_eq!(snapshot_attribution(1_000, 4_000, Some(0)), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_row_adds_up() {
|
||||||
|
// The property the fix exists for, stated arithmetically: whatever the
|
||||||
|
// Snapshot column shows is what the Total is built from.
|
||||||
|
for (size, shared, above) in [
|
||||||
|
(5_000_000_000i64, 4_700_000_000i64, Some(1i64)),
|
||||||
|
(5_000_000_000, 0, Some(300_000_000)),
|
||||||
|
(5_000_000_000, 0, None),
|
||||||
|
(0, 0, None),
|
||||||
|
] {
|
||||||
|
let snapshot = snapshot_attribution(size, shared, above);
|
||||||
|
let (writable, home, config) = (10i64, 20i64, 30i64);
|
||||||
|
let total = snapshot + writable + home + config;
|
||||||
|
assert_eq!(
|
||||||
|
total - (writable + home + config),
|
||||||
|
snapshot,
|
||||||
|
"the Total column has to reconcile with the Snapshot column"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn human_never_prints_a_unit_the_ladder_forbids() {
|
||||||
|
// The 999,999 → "1000.0 KB" bug, which is the one `formatBytes.ts` was
|
||||||
|
// written to fix on the frontend. This function's output is rendered on the
|
||||||
|
// same line as `formatBytes`' in a compaction message, so the two
|
||||||
|
// disagreeing is visible in a single sentence.
|
||||||
|
assert_eq!(human(999_999), "1.0 MB");
|
||||||
|
assert_eq!(human(999_999_999), "1.0 GB");
|
||||||
|
assert_eq!(human(999_999_999_999), "1.0 TB");
|
||||||
|
|
||||||
|
// The ordinary cases still read the way `docker system df` prints them,
|
||||||
|
// base 1000.
|
||||||
|
assert_eq!(human(0), "0 B");
|
||||||
|
assert_eq!(human(999), "999 B");
|
||||||
|
assert_eq!(human(1_000), "1.0 KB");
|
||||||
|
assert_eq!(human(1_500_000), "1.5 MB");
|
||||||
|
assert_eq!(human(4_700_000_000), "4.7 GB");
|
||||||
|
// Rounding that does *not* cross the boundary is untouched.
|
||||||
|
assert_eq!(human(999_400), "999.4 KB");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_output_of_human_is_ever_a_four_digit_mantissa() {
|
||||||
|
// A sweep rather than a handful of cases: every power-of-ten boundary and
|
||||||
|
// its neighbours, which is where the bug lived.
|
||||||
|
let mut value = 1i64;
|
||||||
|
for _ in 0..19 {
|
||||||
|
for candidate in [value - 1, value, value + 1] {
|
||||||
|
if candidate < 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let rendered = human(candidate);
|
||||||
|
let mantissa = rendered.split(' ').next().unwrap();
|
||||||
|
let numeric: f64 = mantissa.parse().unwrap();
|
||||||
|
assert!(
|
||||||
|
numeric < 1000.0 || rendered.ends_with(" PB"),
|
||||||
|
"{} rendered as {}, which the unit ladder is supposed to make impossible",
|
||||||
|
candidate,
|
||||||
|
rendered
|
||||||
|
);
|
||||||
|
}
|
||||||
|
value = match value.checked_mul(10) {
|
||||||
|
Some(v) => v,
|
||||||
|
None => break,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// End-to-end compaction, against a real daemon
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// The whole compaction mechanism, run for real.
|
||||||
|
///
|
||||||
|
/// `#[ignore]` because it needs a Docker daemon, pulls `busybox`, and takes
|
||||||
|
/// tens of seconds — `cargo test` has to stay daemon-free. Run it with
|
||||||
|
/// `cargo test -- --ignored compaction_end_to_end`.
|
||||||
|
///
|
||||||
|
/// It exists because H1 was invisible to every unit test in this file: the
|
||||||
|
/// generated Dockerfile looked right, the `RUN` was one line as asserted, and
|
||||||
|
/// the build failed on `/bin/sh: line 0: syntax error: unexpected "do"` every
|
||||||
|
/// single time. The only test that could have caught it is one that hands the
|
||||||
|
/// Dockerfile to a daemon.
|
||||||
|
///
|
||||||
|
/// It exercises the production functions — [`compaction_dockerfile`],
|
||||||
|
/// [`build_from_dockerfile`], [`restore_image_config`] — rather than
|
||||||
|
/// [`compact_snapshot`] itself, so it never has to create a
|
||||||
|
/// `triple-c-snapshot-*` tag that the app's own sweeps might reach.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore]
|
||||||
|
async fn compaction_end_to_end_against_a_real_image() {
|
||||||
|
let stem = format!("compaction-e2e-{}", uuid::Uuid::new_v4().simple());
|
||||||
|
let source = format!("{}:source", stem);
|
||||||
|
let staging = format!("{}:compacting", stem);
|
||||||
|
let final_ref = format!("{}:latest", stem);
|
||||||
|
|
||||||
|
// A snapshot-shaped source: four stacked layers, each superseding the last,
|
||||||
|
// plus the scrub's own targets and a setuid bit that `COPY --from` has to
|
||||||
|
// preserve. The env var carries a newline and a double quote, which is what
|
||||||
|
// `restore_image_config` exists for.
|
||||||
|
let source_dockerfile = format!(
|
||||||
|
"FROM busybox:1.36\n\
|
||||||
|
RUN mkdir -p /tmp/claude-1000 /var/cache/apt/archives /var/log/apt /var/lib/apt/lists && \
|
||||||
|
dd if=/dev/urandom of=/big-a bs=1M count=40 2>/dev/null\n\
|
||||||
|
RUN dd if=/dev/urandom of=/big-b bs=1M count=40 2>/dev/null && rm -f /big-a\n\
|
||||||
|
RUN dd if=/dev/urandom of=/tmp/claude-1000/scratch bs=1M count=25 2>/dev/null && \
|
||||||
|
dd if=/dev/urandom of=/var/cache/apt/archives/x.deb bs=1M count=15 2>/dev/null && \
|
||||||
|
touch /var/log/dpkg.log\n\
|
||||||
|
RUN dd if=/dev/urandom of=/big-c bs=1M count=30 2>/dev/null && rm -f /big-b && \
|
||||||
|
touch /keepme && chmod 4755 /keepme\n\
|
||||||
|
LABEL {LABEL_MANAGED}=true\n\
|
||||||
|
WORKDIR /keep-this-dir\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Force, unlike anything in production: `untag_image` is deliberately
|
||||||
|
// unforced because it runs against a user's images, and here the leftovers
|
||||||
|
// are this test's own and must not be left on the developer's daemon
|
||||||
|
// whatever refuses them.
|
||||||
|
async fn cleanup(refs: Vec<String>) {
|
||||||
|
let Ok(docker) = get_docker() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for r in refs {
|
||||||
|
if let Err(e) = docker
|
||||||
|
.remove_image(
|
||||||
|
&r,
|
||||||
|
Some(RemoveImageOptions {
|
||||||
|
force: true,
|
||||||
|
noprune: false,
|
||||||
|
}),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("could not clean up {}: {}", r, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
build_from_dockerfile(&source_dockerfile, &source)
|
||||||
|
.await
|
||||||
|
.expect("could not build the throwaway source image");
|
||||||
|
|
||||||
|
let docker = get_docker().expect("no docker");
|
||||||
|
let before = docker.inspect_image(&source).await.expect("inspect source");
|
||||||
|
let before_size = before.size.unwrap_or(0);
|
||||||
|
let mut config = before.config.clone().expect("source has no config");
|
||||||
|
// A genuinely multi-line env var, injected here rather than through the
|
||||||
|
// Dockerfile because `ENV` cannot express a literal newline. This is the
|
||||||
|
// case `restore_image_config` exists for: it is why the config is replayed
|
||||||
|
// through a create-and-commit instead of being rendered back into
|
||||||
|
// Dockerfile instructions, where a newline and a `"` would not survive.
|
||||||
|
config
|
||||||
|
.env
|
||||||
|
.get_or_insert_with(Vec::new)
|
||||||
|
.push("E2E_MULTILINE=first\nsecond \"quoted\"".to_string());
|
||||||
|
|
||||||
|
// --- the thing under test ------------------------------------------------
|
||||||
|
let dockerfile = compaction_dockerfile(&source, &container::snapshot_scrub_script());
|
||||||
|
let built = build_from_dockerfile(&dockerfile, &staging).await;
|
||||||
|
assert!(
|
||||||
|
built.is_ok(),
|
||||||
|
"the compaction build failed, which is exactly the H1 regression: {:?}\n{}",
|
||||||
|
built,
|
||||||
|
dockerfile
|
||||||
|
);
|
||||||
|
|
||||||
|
restore_image_config(&staging, &final_ref, config)
|
||||||
|
.await
|
||||||
|
.expect("could not replay the image config");
|
||||||
|
|
||||||
|
let after = docker
|
||||||
|
.inspect_image(&final_ref)
|
||||||
|
.await
|
||||||
|
.expect("inspect compacted");
|
||||||
|
let after_size = after.size.unwrap_or(0);
|
||||||
|
let history = docker.image_history(&final_ref).await.expect("history");
|
||||||
|
let after_config = after.config.clone().expect("no config after");
|
||||||
|
|
||||||
|
// The scrub actually ran — the assertion H1 made impossible. `sh -c` inside
|
||||||
|
// the compacted image, so the answer comes from the filesystem rather than
|
||||||
|
// from the build log.
|
||||||
|
let probe = migration::run_throwaway(
|
||||||
|
&final_ref,
|
||||||
|
"ls /keepme >/dev/null 2>&1 && echo KEEP-OK\n\
|
||||||
|
[ -e /tmp/claude-1000/scratch ] && echo SCRATCH-LEFT\n\
|
||||||
|
[ -e /var/cache/apt/archives/x.deb ] && echo DEB-LEFT\n\
|
||||||
|
[ -e /var/log/dpkg.log ] && echo DPKGLOG-LEFT\n\
|
||||||
|
[ -e /big-a ] && echo BIGA-LEFT\n\
|
||||||
|
[ -e /big-c ] || echo BIGC-MISSING\n\
|
||||||
|
ls -l /keepme\n\
|
||||||
|
echo PROBE-END\n\
|
||||||
|
exit 0\n",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("could not probe the compacted image");
|
||||||
|
|
||||||
|
// **Everything is measured before anything is asserted**, so a failing
|
||||||
|
// assertion below does not leave several hundred megabytes of throwaway
|
||||||
|
// images on the developer's daemon.
|
||||||
|
cleanup(vec![final_ref, staging, source]).await;
|
||||||
|
|
||||||
|
// 1. It is smaller. The three superseded 30–40 MB layers and the ~40 MB of
|
||||||
|
// scrub targets are the whole point.
|
||||||
|
assert!(
|
||||||
|
after_size < before_size,
|
||||||
|
"compaction did not shrink the image: {} -> {}",
|
||||||
|
before_size,
|
||||||
|
after_size
|
||||||
|
);
|
||||||
|
|
||||||
|
// 2. One layer of content.
|
||||||
|
let content_layers = history.iter().filter(|e| e.size > 0).count();
|
||||||
|
assert_eq!(content_layers, 1, "expected one content layer: {:?}", history);
|
||||||
|
|
||||||
|
// 3. The config round-tripped, newline and quote included.
|
||||||
|
let env = after_config.env.clone().unwrap_or_default();
|
||||||
|
assert!(
|
||||||
|
env.iter()
|
||||||
|
.any(|e| e == "E2E_MULTILINE=first\nsecond \"quoted\""),
|
||||||
|
"the multi-line env var did not survive: {:?}",
|
||||||
|
env
|
||||||
|
);
|
||||||
|
assert_eq!(after_config.working_dir.as_deref(), Some("/keep-this-dir"));
|
||||||
|
|
||||||
|
// 4. The scrub actually ran.
|
||||||
|
assert!(probe.stdout.contains("PROBE-END"), "{}", probe.stdout);
|
||||||
|
assert!(probe.stdout.contains("KEEP-OK"), "{}", probe.stdout);
|
||||||
|
assert!(
|
||||||
|
!probe.stdout.contains("SCRATCH-LEFT"),
|
||||||
|
"the agent scratchpad survived the scrub:\n{}",
|
||||||
|
probe.stdout
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!probe.stdout.contains("DEB-LEFT"),
|
||||||
|
"apt archives survived the scrub:\n{}",
|
||||||
|
probe.stdout
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!probe.stdout.contains("DPKGLOG-LEFT"),
|
||||||
|
"dpkg.log survived the scrub:\n{}",
|
||||||
|
probe.stdout
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!probe.stdout.contains("BIGC-MISSING"),
|
||||||
|
"the live payload was lost, which is worse than not compacting:\n{}",
|
||||||
|
probe.stdout
|
||||||
|
);
|
||||||
|
// `COPY --from` has to keep the setuid bit; a compaction that dropped it
|
||||||
|
// would break sudo inside every migrated project.
|
||||||
|
assert!(
|
||||||
|
probe.stdout.contains("-rwsr-xr-x"),
|
||||||
|
"the setuid bit did not survive the flatten:\n{}",
|
||||||
|
probe.stdout
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -754,9 +754,16 @@ impl ProbeContainerGuard {
|
|||||||
&self.id
|
&self.id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// **Disarm after the await, never before it.** Clearing `armed` first
|
||||||
|
/// looked equivalent and was the exact inverse of this guard's purpose: on
|
||||||
|
/// the one path it exists for — this future being dropped part-way through
|
||||||
|
/// the removal — `Drop` then saw a disarmed guard and did nothing, so the
|
||||||
|
/// container survived with no background removal queued behind it. Setting
|
||||||
|
/// it afterwards means a cancelled `remove_now` falls back to `Drop`'s
|
||||||
|
/// detached removal, and only a removal that actually completed disarms.
|
||||||
async fn remove_now(mut self) {
|
async fn remove_now(mut self) {
|
||||||
self.armed = false;
|
|
||||||
remove_probe_container(&self.id).await;
|
remove_probe_container(&self.id).await;
|
||||||
|
self.armed = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -810,6 +817,17 @@ async fn remove_probe_container(id: &str) {
|
|||||||
/// and no volumes, owned entirely by a `run_throwaway` call. If one is running
|
/// and no volumes, owned entirely by a `run_throwaway` call. If one is running
|
||||||
/// right now it belongs to this process — and this runs before any migration
|
/// right now it belongs to this process — and this runs before any migration
|
||||||
/// can be started, so there is none to interrupt.
|
/// can be started, so there is none to interrupt.
|
||||||
|
///
|
||||||
|
/// **Except that "belongs to this process" is not something this can know.**
|
||||||
|
/// The filter is a label, and labels are daemon-wide: a second copy of the app
|
||||||
|
/// migrating a project on the same daemon has probe containers carrying exactly
|
||||||
|
/// this label, and force-removing one mid-manifest-capture fails that
|
||||||
|
/// migration. In-process state cannot see the other instance, so the only
|
||||||
|
/// available brake is age — [`PROBE_REAP_MIN_AGE_SECS`]. A probe runs a `df`, an
|
||||||
|
/// `apt-get update` or a `find` over a root filesystem; none of those is a
|
||||||
|
/// multi-minute job, so anything younger than the gate is far more likely to be
|
||||||
|
/// someone's live probe than a leftover, and a leftover simply waits for the
|
||||||
|
/// next start.
|
||||||
pub async fn reap_probe_containers() {
|
pub async fn reap_probe_containers() {
|
||||||
let Ok(docker) = get_docker() else {
|
let Ok(docker) = get_docker() else {
|
||||||
return;
|
return;
|
||||||
@@ -835,7 +853,23 @@ pub async fn reap_probe_containers() {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let now = chrono::Utc::now().timestamp();
|
||||||
for c in containers {
|
for c in containers {
|
||||||
|
// `created` is a unix timestamp; a summary without one is treated as
|
||||||
|
// too young to touch, because unknown is never permission.
|
||||||
|
let age = c.created.map(|created| now - created);
|
||||||
|
match age {
|
||||||
|
Some(age) if age >= PROBE_REAP_MIN_AGE_SECS => {}
|
||||||
|
_ => {
|
||||||
|
log::info!(
|
||||||
|
"Leaving migration probe container {} alone — it is younger than {} minutes, \
|
||||||
|
so it may belong to another Triple-C instance's live migration",
|
||||||
|
c.id.as_deref().unwrap_or("<unknown>"),
|
||||||
|
PROBE_REAP_MIN_AGE_SECS / 60
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
if let Some(id) = c.id {
|
if let Some(id) = c.id {
|
||||||
log::info!("Removing leftover migration probe container {}", id);
|
log::info!("Removing leftover migration probe container {}", id);
|
||||||
remove_probe_container(&id).await;
|
remove_probe_container(&id).await;
|
||||||
@@ -843,6 +877,16 @@ pub async fn reap_probe_containers() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// How old a `triple-c.probe=migration` container must be before
|
||||||
|
/// [`reap_probe_containers`] will force-remove it, in seconds.
|
||||||
|
///
|
||||||
|
/// The label is daemon-wide and this process cannot tell its own leftovers from
|
||||||
|
/// another instance's live probe, so this is the whole guard. Generous against
|
||||||
|
/// the longest probe there is (an `apt-get update` inside a throwaway container
|
||||||
|
/// on a slow link) and still short enough that a crashed run's probe stops
|
||||||
|
/// pinning a multi-gigabyte base image within the hour.
|
||||||
|
pub const PROBE_REAP_MIN_AGE_SECS: i64 = 30 * 60;
|
||||||
|
|
||||||
async fn run_throwaway_inner(id: &str) -> Result<ThrowawayResult, String> {
|
async fn run_throwaway_inner(id: &str) -> Result<ThrowawayResult, String> {
|
||||||
let docker = get_docker()?;
|
let docker = get_docker()?;
|
||||||
|
|
||||||
@@ -1025,6 +1069,10 @@ pub fn rollback_tag(now: &chrono::DateTime<chrono::Utc>) -> String {
|
|||||||
///
|
///
|
||||||
/// Two weeks, chosen to be far longer than anyone deliberates over a base
|
/// Two weeks, chosen to be far longer than anyone deliberates over a base
|
||||||
/// update and far shorter than "forever", which is what it was.
|
/// update and far shorter than "forever", which is what it was.
|
||||||
|
///
|
||||||
|
/// **Measured from when the record went missing, not from the tag.** See
|
||||||
|
/// [`pin_is_reapable`] and
|
||||||
|
/// [`crate::storage::migration_store::note_ownerless_since`].
|
||||||
pub const STALE_PIN_MAX_AGE_DAYS: i64 = 14;
|
pub const STALE_PIN_MAX_AGE_DAYS: i64 = 14;
|
||||||
|
|
||||||
/// Recover the timestamp encoded in a tag produced by [`rollback_tag`].
|
/// Recover the timestamp encoded in a tag produced by [`rollback_tag`].
|
||||||
@@ -1051,26 +1099,59 @@ pub fn parse_snapshot_reference(reference: &str) -> Option<(String, String)> {
|
|||||||
Some((project_id, tag))
|
Some((project_id, tag))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether a rollback pin is safe to drop, given how old it is and whether the
|
/// Whether a rollback pin is safe to drop, given whether the project it belongs
|
||||||
/// project it belongs to still has a migration record.
|
/// to still has a migration record and how long it has been without one.
|
||||||
///
|
///
|
||||||
/// Pure so the decision can be tested without a daemon. The order of the two
|
/// Pure so the decision can be tested without a daemon. The order of the
|
||||||
/// conditions is the point: **a pin whose migration is still awaiting
|
/// conditions is the point: **a pin whose migration is still awaiting
|
||||||
/// confirmation is never reaped at any age**, because it is the only copy of
|
/// confirmation is never reaped at any age**, because it is the only copy of
|
||||||
/// the rollback target and the user has not yet said they are happy with the
|
/// the rollback target and the user has not yet said they are happy with the
|
||||||
/// new base.
|
/// new base.
|
||||||
|
///
|
||||||
|
/// ## `ownerless_since`, and why it is not the tag's timestamp
|
||||||
|
///
|
||||||
|
/// This used to compute the age from `parse_rollback_tag(tag)` — the instant
|
||||||
|
/// the migration *started*. A migration is allowed to sit at
|
||||||
|
/// `awaiting-confirmation` for as long as the user likes; that is what
|
||||||
|
/// `keep_rollback` is for. A project parked there for a month whose record is
|
||||||
|
/// then lost had a tag a month old, so the pin was reapable on the very next
|
||||||
|
/// check and the startup sweep deleted the image immediately after. The
|
||||||
|
/// fourteen days were nominal: the real grace period for the case the constant
|
||||||
|
/// was written for was zero.
|
||||||
|
///
|
||||||
|
/// So the clock starts when the claim was lost, which is recorded by
|
||||||
|
/// [`crate::storage::migration_store::note_ownerless_since`] the first time a
|
||||||
|
/// reaper notices. `None` means no reaper has recorded a sighting yet, and that
|
||||||
|
/// is **not** "sighted now": returning false there is what gives a pin its
|
||||||
|
/// first full fourteen days instead of none.
|
||||||
|
///
|
||||||
|
/// ## Clock skew
|
||||||
|
///
|
||||||
|
/// A `now` earlier than `ownerless_since` — a host clock that ran fast and was
|
||||||
|
/// corrected, or a data directory carried between machines — yields a negative
|
||||||
|
/// elapsed time. That is treated as not reapable, and the marker writer
|
||||||
|
/// re-anchors it, rather than letting a negative `num_days()` mean "never" or
|
||||||
|
/// an inflated one mean "immediately".
|
||||||
pub fn pin_is_reapable(
|
pub fn pin_is_reapable(
|
||||||
tag: &str,
|
tag: &str,
|
||||||
has_migration_record: bool,
|
has_migration_record: bool,
|
||||||
|
ownerless_since: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
now: &chrono::DateTime<chrono::Utc>,
|
now: &chrono::DateTime<chrono::Utc>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
if has_migration_record {
|
if has_migration_record {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
let Some(created) = parse_rollback_tag(tag) else {
|
// Still required: the tag has to be one of ours. A hand-made
|
||||||
|
// `pre-migration-keepme` is somebody's deliberate pin and is never guessed
|
||||||
|
// at, whatever a marker beside it says.
|
||||||
|
if parse_rollback_tag(tag).is_none() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Some(since) = ownerless_since else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
(*now - created).num_days() >= STALE_PIN_MAX_AGE_DAYS
|
let elapsed = *now - since;
|
||||||
|
elapsed >= chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drop `triple-c-snapshot-*:pre-migration-*` tags that no migration record
|
/// Drop `triple-c-snapshot-*:pre-migration-*` tags that no migration record
|
||||||
@@ -1138,15 +1219,35 @@ pub async fn reap_stale_migration_pins() -> usize {
|
|||||||
// still count as "somebody may want this back".
|
// still count as "somebody may want this back".
|
||||||
let has_record =
|
let has_record =
|
||||||
crate::storage::migration_store::has_record(&project_id).unwrap_or(true);
|
crate::storage::migration_store::has_record(&project_id).unwrap_or(true);
|
||||||
if !pin_is_reapable(&tag, has_record, &now) {
|
if has_record {
|
||||||
|
// Owned again (or still owned): throw away any grace clock a
|
||||||
|
// previous pass started, so a pin that loses its record twice
|
||||||
|
// gets a fresh fourteen days rather than inheriting a stale one.
|
||||||
|
crate::storage::migration_store::clear_ownerless(&project_id, &tag);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Only a *well-formed* pin gets a marker written for it — a tag
|
||||||
|
// that is not one of ours is left entirely alone, files included.
|
||||||
|
if parse_rollback_tag(&tag).is_none() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Records the first sighting when there is none, which is why this
|
||||||
|
// returns `None` on that pass and the pin survives it.
|
||||||
|
let ownerless_since =
|
||||||
|
crate::storage::migration_store::note_ownerless_since(&project_id, &tag, &now);
|
||||||
|
if !pin_is_reapable(&tag, has_record, ownerless_since, &now) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
match untag_image(reference).await {
|
match untag_image(reference).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
|
crate::storage::migration_store::clear_ownerless(&project_id, &tag);
|
||||||
log::info!(
|
log::info!(
|
||||||
"Dropped stale rollback pin {} ({:.2} GB) — no migration record has claimed it for {} days",
|
"Dropped stale rollback pin {} ({:.2} GB) — no migration record has claimed it since {}, more than {} days",
|
||||||
reference,
|
reference,
|
||||||
summary.size as f64 / 1_073_741_824.0,
|
summary.size as f64 / 1_073_741_824.0,
|
||||||
|
ownerless_since
|
||||||
|
.map(|t| t.to_rfc3339())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string()),
|
||||||
STALE_PIN_MAX_AGE_DAYS,
|
STALE_PIN_MAX_AGE_DAYS,
|
||||||
);
|
);
|
||||||
reaped += 1;
|
reaped += 1;
|
||||||
@@ -1915,29 +2016,93 @@ mod tests {
|
|||||||
// image is the only copy of the rollback target and the user has not
|
// image is the only copy of the rollback target and the user has not
|
||||||
// yet said they are happy on the new base.
|
// yet said they are happy on the new base.
|
||||||
let ancient = rollback_tag(&at(2020, 1, 1));
|
let ancient = rollback_tag(&at(2020, 1, 1));
|
||||||
assert!(!pin_is_reapable(&ancient, true, &at(2026, 8, 23)));
|
assert!(!pin_is_reapable(
|
||||||
|
&ancient,
|
||||||
|
true,
|
||||||
|
Some(at(2020, 1, 1)),
|
||||||
|
&at(2026, 8, 23)
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_unclaimed_pin_is_reaped_only_once_it_is_old() {
|
fn an_unclaimed_pin_is_reaped_only_once_it_is_old() {
|
||||||
let made = at(2026, 8, 1);
|
let tag = rollback_tag(&at(2026, 8, 1));
|
||||||
let tag = rollback_tag(&made);
|
// The clock runs from when the record went missing, which here is well
|
||||||
assert!(!pin_is_reapable(&tag, false, &at(2026, 8, 2)));
|
// after the migration started.
|
||||||
|
let lost = at(2026, 8, 10);
|
||||||
|
assert!(!pin_is_reapable(&tag, false, Some(lost), &at(2026, 8, 11)));
|
||||||
assert!(!pin_is_reapable(
|
assert!(!pin_is_reapable(
|
||||||
&tag,
|
&tag,
|
||||||
false,
|
false,
|
||||||
&(made + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS - 1))
|
Some(lost),
|
||||||
|
&(lost + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS) - chrono::Duration::seconds(1))
|
||||||
));
|
));
|
||||||
assert!(pin_is_reapable(
|
assert!(pin_is_reapable(
|
||||||
&tag,
|
&tag,
|
||||||
false,
|
false,
|
||||||
&(made + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS))
|
Some(lost),
|
||||||
|
&(lost + chrono::Duration::days(STALE_PIN_MAX_AGE_DAYS))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_grace_period_runs_from_the_lost_record_not_from_the_tag() {
|
||||||
|
// The bug this replaced, stated as a test. A migration parked at
|
||||||
|
// `awaiting-confirmation` for a month — supported, that is what
|
||||||
|
// `keep_rollback` is for — whose record is then lost had a
|
||||||
|
// month-old tag, so the old rule made its pin reapable on the very
|
||||||
|
// next app start with the startup sweep deleting the image two lines
|
||||||
|
// later. Zero grace, on the one case the fourteen days exist for.
|
||||||
|
let started = at(2026, 6, 1);
|
||||||
|
let tag = rollback_tag(&started);
|
||||||
|
let record_lost = at(2026, 7, 1);
|
||||||
|
let noticed_immediately_after = record_lost + chrono::Duration::minutes(5);
|
||||||
|
assert!(
|
||||||
|
!pin_is_reapable(&tag, false, Some(record_lost), ¬iced_immediately_after),
|
||||||
|
"a tag a month old must still get its full grace period once orphaned"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unsighted_pin_is_never_reaped_on_the_pass_that_first_sees_it() {
|
||||||
|
// `None` means no reaper has recorded a sighting. Treating that as
|
||||||
|
// "sighted now" would be harmless; treating it as "sighted long ago"
|
||||||
|
// would not, and neither is what it means — the marker is written on
|
||||||
|
// this pass and the pin becomes reapable fourteen days later.
|
||||||
|
let tag = rollback_tag(&at(2020, 1, 1));
|
||||||
|
assert!(!pin_is_reapable(&tag, false, None, &at(2026, 8, 23)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_clock_that_ran_backwards_neither_reaps_nor_strands() {
|
||||||
|
// A marker dated after `now`: the host clock was fast and got
|
||||||
|
// corrected, or the data directory came from another machine. A
|
||||||
|
// negative elapsed time must read as "not yet", not as a huge age.
|
||||||
|
let tag = rollback_tag(&at(2026, 1, 1));
|
||||||
|
let marker = at(2026, 9, 1);
|
||||||
|
assert!(!pin_is_reapable(&tag, false, Some(marker), &at(2026, 8, 1)));
|
||||||
|
// The other direction is bounded by the marker rather than by the tag:
|
||||||
|
// a wildly future `now` can only expire a clock that was actually
|
||||||
|
// started, and a pin with no marker (the case above) still cannot be
|
||||||
|
// reaped at all — which is what stops a fast host clock from making
|
||||||
|
// *every* pin on the daemon instantly collectable.
|
||||||
|
assert!(pin_is_reapable(
|
||||||
|
&tag,
|
||||||
|
false,
|
||||||
|
Some(at(2026, 8, 20)),
|
||||||
|
&at(2030, 1, 1)
|
||||||
|
));
|
||||||
|
assert!(!pin_is_reapable(&tag, false, None, &at(2030, 1, 1)));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_tag_we_cannot_date_is_left_alone() {
|
fn a_tag_we_cannot_date_is_left_alone() {
|
||||||
assert!(!pin_is_reapable("pre-migration-handmade", false, &at(2026, 8, 23)));
|
// Even with an ancient ownerless marker sitting beside it: a tag that
|
||||||
assert!(!pin_is_reapable("latest", false, &at(2026, 8, 23)));
|
// merely *starts* `pre-migration-` is somebody's deliberate pin, and
|
||||||
|
// the reaper never writes a marker for one in the first place.
|
||||||
|
let ancient = Some(at(2020, 1, 1));
|
||||||
|
let now = at(2026, 8, 23);
|
||||||
|
assert!(!pin_is_reapable("pre-migration-handmade", false, ancient, &now));
|
||||||
|
assert!(!pin_is_reapable("latest", false, ancient, &now));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ mod docker;
|
|||||||
mod install_helper;
|
mod install_helper;
|
||||||
mod logging;
|
mod logging;
|
||||||
mod models;
|
mod models;
|
||||||
|
mod project_lock;
|
||||||
mod storage;
|
mod storage;
|
||||||
pub mod web_terminal;
|
pub mod web_terminal;
|
||||||
|
|
||||||
@@ -263,6 +264,19 @@ pub fn run() {
|
|||||||
let drag_temp_dir = app.path().temp_dir().ok();
|
let drag_temp_dir = app.path().temp_dir().ok();
|
||||||
tauri::async_runtime::spawn(async move {
|
tauri::async_runtime::spawn(async move {
|
||||||
crate::docker::reap_probe_containers().await;
|
crate::docker::reap_probe_containers().await;
|
||||||
|
// Before the sweep, and for the same reason the pins are:
|
||||||
|
// `triple-c-snapshot-*:compacting` is a *tagged* image, so the
|
||||||
|
// sweep's `dangling=true` filter cannot see it, and the
|
||||||
|
// `triple-c-compact-*` container a crashed compaction leaves
|
||||||
|
// behind pins that image open. Untagging first is what turns
|
||||||
|
// both into something the sweep can collect on the same pass.
|
||||||
|
let stranded = crate::docker::disk::reap_stale_compaction_artifacts().await;
|
||||||
|
if stranded > 0 {
|
||||||
|
log::info!(
|
||||||
|
"Startup housekeeping dropped {} stranded compaction staging tag(s)",
|
||||||
|
stranded
|
||||||
|
);
|
||||||
|
}
|
||||||
let reaped = crate::docker::reap_stale_migration_pins().await;
|
let reaped = crate::docker::reap_stale_migration_pins().await;
|
||||||
if reaped > 0 {
|
if reaped > 0 {
|
||||||
log::info!("Startup housekeeping dropped {} stale rollback pin(s)", reaped);
|
log::info!("Startup housekeeping dropped {} stale rollback pin(s)", reaped);
|
||||||
|
|||||||
@@ -0,0 +1,338 @@
|
|||||||
|
//! Per-project mutual exclusion for everything that rewrites a project's
|
||||||
|
//! container or its snapshot image.
|
||||||
|
//!
|
||||||
|
//! ## Why polling was not enough
|
||||||
|
//!
|
||||||
|
//! Until this module existed the app had exactly one mutual-exclusion
|
||||||
|
//! primitive — the `ACTIVE_MIGRATIONS` set behind
|
||||||
|
//! `migration_commands::is_migrating` — and it was **one-way**. A migration
|
||||||
|
//! took a guard for its whole run; everything else merely *asked once, at
|
||||||
|
//! entry*, whether a migration was in flight and then proceeded with no claim
|
||||||
|
//! of its own. Two non-migration operations on the same project could not see
|
||||||
|
//! each other at all, and a migration could start underneath one that was
|
||||||
|
//! already halfway through.
|
||||||
|
//!
|
||||||
|
//! That is not a theoretical gap. Compaction resolves
|
||||||
|
//! `triple-c-snapshot-{id}:latest` when its build starts and commits back over
|
||||||
|
//! that same tag minutes later, and the Settings panel is a sidebar rather than
|
||||||
|
//! a modal — so Project Home stays live with Start, Stop, Reset and Migrate all
|
||||||
|
//! clickable while a compaction runs. Three interleavings were reproduced:
|
||||||
|
//!
|
||||||
|
//! * Compaction commits `flat(A)` over `:latest` after a migration has already
|
||||||
|
//! moved that tag to a new lineage. The migration is silently reverted, the
|
||||||
|
//! config replay lands twice, and the migration record says
|
||||||
|
//! `awaiting-confirmation` against a base the tag no longer points at.
|
||||||
|
//! * Compaction resolves A, the user starts the project and works for an hour,
|
||||||
|
//! a recreate commits D over `:latest`, and the compaction then overwrites it
|
||||||
|
//! with `flat(A)` — orphaning an hour of system-layer work while reporting
|
||||||
|
//! success and a byte saving.
|
||||||
|
//! * Compaction resurrects the system layer a Reset had just destroyed.
|
||||||
|
//!
|
||||||
|
//! Every one of those is "two writers of `:latest`, neither holding anything".
|
||||||
|
//! So this registry replaces the polling with an actual claim: an operation
|
||||||
|
//! **acquires** a [`ProjectGuard`] and holds it for its whole run, and a second
|
||||||
|
//! operation on the same project is refused with a message naming the holder.
|
||||||
|
//!
|
||||||
|
//! ## What this does NOT protect against, stated plainly
|
||||||
|
//!
|
||||||
|
//! **This is in-process state.** Two copies of the app pointed at the same
|
||||||
|
//! Docker daemon share nothing here: instance A's compaction and instance B's
|
||||||
|
//! migration will both acquire happily and then race exactly as before.
|
||||||
|
//! `reap_probe_containers` and the `triple-c-compact-*` / `triple-c-scrub-*`
|
||||||
|
//! sweeps are worse than that — they are daemon-wide force-removals driven by
|
||||||
|
//! a name or a label, so instance B can destroy a container instance A is
|
||||||
|
//! mid-commit against.
|
||||||
|
//!
|
||||||
|
//! A daemon-visible lock was considered and rejected for now, and the reasoning
|
||||||
|
//! is recorded here so it is not re-derived from scratch:
|
||||||
|
//!
|
||||||
|
//! * A **lock container** would work — container names are unique daemon-wide
|
||||||
|
//! and `create` fails atomically on a name conflict — but a container has to
|
||||||
|
//! be created *from an image*, and that pins the image. A lock on
|
||||||
|
//! `triple-c-snapshot-{id}` would block the very `rmi`/sweep paths it guards,
|
||||||
|
//! and a leaked lock container would pin multiple gigabytes forever.
|
||||||
|
//! * A **named volume** is not usable: `create_volume` on an existing name
|
||||||
|
//! returns the existing volume rather than failing, so it cannot be a
|
||||||
|
//! test-and-set.
|
||||||
|
//! * A **label on the snapshot image** is not atomic — read/modify/commit has
|
||||||
|
//! the same race it would be trying to close.
|
||||||
|
//!
|
||||||
|
//! So the cross-process case is **documented, not solved**. What this module
|
||||||
|
//! does do about it is bound the damage: [`any_held_excluding`] lets the daemon-wide
|
||||||
|
//! reapers skip work while this process is mid-operation, and the reapers
|
||||||
|
//! themselves gained age gates so a young container belonging to somebody else
|
||||||
|
//! is left alone (see `docker::disk::reap_stale_compaction_artifacts` and
|
||||||
|
//! `docker::migration::reap_probe_containers`).
|
||||||
|
//!
|
||||||
|
//! ## Refuse, do not queue
|
||||||
|
//!
|
||||||
|
//! [`try_acquire`] never waits. Every caller is a user-initiated action behind
|
||||||
|
//! a button, and a button that blocks for the four minutes a compaction takes
|
||||||
|
//! is worse than one that says what is running. The refusal string is written
|
||||||
|
//! for the user and names the holder.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
/// The operations that claim a project.
|
||||||
|
///
|
||||||
|
/// One variant per *class of writer*, not per command: `Recreate` covers Start
|
||||||
|
/// as well, because Start's create-and-commit path is the same writer of
|
||||||
|
/// `triple-c-snapshot-{id}:latest` that a recreate is.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ProjectOp {
|
||||||
|
/// `migrate_project_to_base`, `resume_migration`, `rollback_migration`,
|
||||||
|
/// `confirm_migration`.
|
||||||
|
Migration,
|
||||||
|
/// `disk::compact_snapshot` — the long one, and the reason this exists.
|
||||||
|
Compaction,
|
||||||
|
/// Start / stop / recreate. Anything in `start_project_container`'s path.
|
||||||
|
Recreate,
|
||||||
|
/// `rebuild_project_container` — deletes both volumes and the snapshot.
|
||||||
|
Reset,
|
||||||
|
/// `disk::destroy` — a volume, a snapshot image, or a rollback pin.
|
||||||
|
Destroy,
|
||||||
|
/// `disk::clear_caches` — an exec into the live container. It does not
|
||||||
|
/// write `:latest`, but it must not run while the container is being
|
||||||
|
/// removed out from under it.
|
||||||
|
CacheClear,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProjectOp {
|
||||||
|
/// What is happening, phrased for the message a user reads.
|
||||||
|
pub fn describe(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
ProjectOp::Migration => "A container base update is running for this project",
|
||||||
|
ProjectOp::Compaction => "This project's snapshot is being compacted",
|
||||||
|
ProjectOp::Recreate => "This project's container is being started or recreated",
|
||||||
|
ProjectOp::Reset => "This project is being reset",
|
||||||
|
ProjectOp::Destroy => "Something of this project's is being deleted",
|
||||||
|
ProjectOp::CacheClear => "This project's caches are being cleared",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the *refused* caller was trying to do, for the tail of the message.
|
||||||
|
fn blocked_action(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
ProjectOp::Migration => "starting a base update",
|
||||||
|
ProjectOp::Compaction => "compacting its snapshot",
|
||||||
|
ProjectOp::Recreate => "starting or recreating its container",
|
||||||
|
ProjectOp::Reset => "resetting it",
|
||||||
|
ProjectOp::Destroy => "deleting anything of its",
|
||||||
|
ProjectOp::CacheClear => "clearing its caches",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Project id → the operation currently holding it.
|
||||||
|
///
|
||||||
|
/// A `std::sync::Mutex` rather than a `tokio` one on purpose: it is only ever
|
||||||
|
/// held for the length of a `HashMap` insert or remove, never across an await,
|
||||||
|
/// and [`is_held_by`] has to be callable from the synchronous helpers in
|
||||||
|
/// `disk.rs` that already ask this question.
|
||||||
|
static HOLDERS: OnceLock<Mutex<HashMap<String, ProjectOp>>> = OnceLock::new();
|
||||||
|
|
||||||
|
fn holders() -> &'static Mutex<HashMap<String, ProjectOp>> {
|
||||||
|
HOLDERS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A claim on one project, released on drop.
|
||||||
|
///
|
||||||
|
/// RAII rather than an explicit release for the reason [`ProjectOp::Migration`]'s
|
||||||
|
/// predecessor already learned: a plain release statement is skipped by an
|
||||||
|
/// early `?`, by a panic, and by the future simply being dropped. A guard is
|
||||||
|
/// not.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct ProjectGuard {
|
||||||
|
project_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ProjectGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
// `into_inner` on a poisoned lock: a panic while some other thread held
|
||||||
|
// this map for the duration of one insert cannot have left it
|
||||||
|
// inconsistent, and refusing to release afterwards would strand the
|
||||||
|
// project as permanently busy.
|
||||||
|
holders()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|e| e.into_inner())
|
||||||
|
.remove(&self.project_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim a project for `op`, or say who has it.
|
||||||
|
///
|
||||||
|
/// The error is user-facing copy, not a debug string — it goes straight back
|
||||||
|
/// over IPC to a toast.
|
||||||
|
pub fn try_acquire(project_id: &str, op: ProjectOp) -> Result<ProjectGuard, String> {
|
||||||
|
let mut map = holders().lock().unwrap_or_else(|e| e.into_inner());
|
||||||
|
if let Some(holder) = map.get(project_id).copied() {
|
||||||
|
return Err(format!(
|
||||||
|
"{}. Wait for it to finish before {}.",
|
||||||
|
holder.describe(),
|
||||||
|
op.blocked_action()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
map.insert(project_id.to_string(), op);
|
||||||
|
Ok(ProjectGuard {
|
||||||
|
project_id: project_id.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which operation holds this project, if any.
|
||||||
|
pub fn held(project_id: &str) -> Option<ProjectOp> {
|
||||||
|
holders()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|e| e.into_inner())
|
||||||
|
.get(project_id)
|
||||||
|
.copied()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether this project is held by exactly `op`.
|
||||||
|
///
|
||||||
|
/// `migration_commands::is_migrating` is this, specialised — which is the whole
|
||||||
|
/// point of folding `ACTIVE_MIGRATIONS` into this registry: there is now one
|
||||||
|
/// answer to "is something happening to this project", not two that can
|
||||||
|
/// disagree.
|
||||||
|
pub fn is_held_by(project_id: &str, op: ProjectOp) -> bool {
|
||||||
|
held(project_id) == Some(op)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether any project **other than** `exclude_project_id` is currently held by
|
||||||
|
/// `op`. Pass an empty id to ask about every project.
|
||||||
|
///
|
||||||
|
/// Used by the daemon-wide reapers, which cannot tell which project a
|
||||||
|
/// `triple-c-compact-*` container belongs to — the name carries a random uuid,
|
||||||
|
/// not a project id — so "is this process compacting anything right now" is the
|
||||||
|
/// only in-process question they can ask before force-removing one. The
|
||||||
|
/// exclusion is for the reaper that runs *inside* a compaction, which is
|
||||||
|
/// already holding a claim of its own and would otherwise see it and skip.
|
||||||
|
pub fn any_held_excluding(op: ProjectOp, exclude_project_id: &str) -> bool {
|
||||||
|
holders()
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|e| e.into_inner())
|
||||||
|
.iter()
|
||||||
|
.any(|(project_id, held)| *held == op && project_id != exclude_project_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Ids are namespaced per test: the registry is process-global, and
|
||||||
|
/// `cargo test` runs these on several threads at once.
|
||||||
|
fn id(name: &str) -> String {
|
||||||
|
format!("project-lock-test-{}", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_second_acquire_on_the_same_project_is_refused() {
|
||||||
|
let p = id("second-acquire");
|
||||||
|
let first = try_acquire(&p, ProjectOp::Compaction).expect("first claim");
|
||||||
|
let second = try_acquire(&p, ProjectOp::Recreate);
|
||||||
|
let err = second.expect_err("a second claim must be refused, not queued");
|
||||||
|
// The refusal has to name the holder — "busy" alone leaves the user
|
||||||
|
// with nothing to wait for.
|
||||||
|
assert!(err.contains("snapshot is being compacted"), "{}", err);
|
||||||
|
assert!(err.contains("starting or recreating"), "{}", err);
|
||||||
|
drop(first);
|
||||||
|
// And it has to be retakeable the moment the holder goes away.
|
||||||
|
try_acquire(&p, ProjectOp::Recreate).expect("released on drop");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_guard_releases_on_an_early_return() {
|
||||||
|
let p = id("early-return");
|
||||||
|
fn bails(project_id: &str) -> Result<(), String> {
|
||||||
|
let _guard = try_acquire(project_id, ProjectOp::Reset)?;
|
||||||
|
Err("something failed".to_string())
|
||||||
|
}
|
||||||
|
assert!(bails(&p).is_err());
|
||||||
|
assert_eq!(held(&p), None, "an early `?` must not strand the claim");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_guard_releases_on_a_panic() {
|
||||||
|
let p = id("panic");
|
||||||
|
let result = std::panic::catch_unwind(|| {
|
||||||
|
let _guard = try_acquire(&id("panic"), ProjectOp::Migration).unwrap();
|
||||||
|
panic!("boom");
|
||||||
|
});
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(held(&p), None, "a panic must not strand the claim either");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn two_projects_do_not_block_each_other() {
|
||||||
|
let a = id("independent-a");
|
||||||
|
let b = id("independent-b");
|
||||||
|
let _one = try_acquire(&a, ProjectOp::Compaction).expect("a");
|
||||||
|
let _two = try_acquire(&b, ProjectOp::Compaction).expect("b");
|
||||||
|
assert!(is_held_by(&a, ProjectOp::Compaction));
|
||||||
|
assert!(is_held_by(&b, ProjectOp::Compaction));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn is_held_by_distinguishes_the_operation() {
|
||||||
|
let p = id("which-op");
|
||||||
|
let _guard = try_acquire(&p, ProjectOp::Compaction).unwrap();
|
||||||
|
assert!(is_held_by(&p, ProjectOp::Compaction));
|
||||||
|
assert!(
|
||||||
|
!is_held_by(&p, ProjectOp::Migration),
|
||||||
|
"a compaction is not a migration — `is_migrating` is built on this"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn any_held_sees_across_projects() {
|
||||||
|
let p = id("any-held");
|
||||||
|
assert!(!any_held_excluding(ProjectOp::Destroy, ""));
|
||||||
|
let _guard = try_acquire(&p, ProjectOp::Destroy).unwrap();
|
||||||
|
assert!(any_held_excluding(ProjectOp::Destroy, ""));
|
||||||
|
// …and a holder can ask the question without its own claim answering
|
||||||
|
// it, which is what lets a compaction sweep leftovers before it starts.
|
||||||
|
assert!(!any_held_excluding(ProjectOp::Destroy, &p));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Concurrency, not just sequencing: N threads racing for one project must
|
||||||
|
/// produce exactly one winner.
|
||||||
|
#[test]
|
||||||
|
fn exactly_one_of_many_racing_threads_wins() {
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
let p = id("race");
|
||||||
|
let start = Arc::new(std::sync::Barrier::new(8));
|
||||||
|
// The second barrier is what makes this deterministic rather than
|
||||||
|
// merely likely: no winner releases until every thread has had its
|
||||||
|
// turn, so "only one got in" cannot be an artefact of a loser arriving
|
||||||
|
// after the winner already left.
|
||||||
|
let attempted = Arc::new(std::sync::Barrier::new(8));
|
||||||
|
let won = Arc::new(AtomicUsize::new(0));
|
||||||
|
let mut handles = Vec::new();
|
||||||
|
for _ in 0..8 {
|
||||||
|
let start = Arc::clone(&start);
|
||||||
|
let attempted = Arc::clone(&attempted);
|
||||||
|
let won = Arc::clone(&won);
|
||||||
|
let p = p.clone();
|
||||||
|
handles.push(std::thread::spawn(move || {
|
||||||
|
start.wait();
|
||||||
|
let claim = try_acquire(&p, ProjectOp::Compaction);
|
||||||
|
if claim.is_ok() {
|
||||||
|
won.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
attempted.wait();
|
||||||
|
drop(claim);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
for handle in handles {
|
||||||
|
handle.join().unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
won.load(Ordering::SeqCst),
|
||||||
|
1,
|
||||||
|
"eight threads raced for one project and more than one got in"
|
||||||
|
);
|
||||||
|
assert_eq!(held(&p), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,15 +50,28 @@ fn sanitize(project_id: &str) -> String {
|
|||||||
/// flight; an unparseable file is treated the same way (and logged) rather than
|
/// flight; an unparseable file is treated the same way (and logged) rather than
|
||||||
/// blocking every future migration on a corrupt record.
|
/// blocking every future migration on a corrupt record.
|
||||||
///
|
///
|
||||||
/// **A corrupt record is moved aside, not merely ignored.** Reporting "absent"
|
/// **A corrupt record is copied aside and left in place.** An earlier version
|
||||||
/// while leaving the file in place strands the rollback pin it describes: the
|
/// *renamed* it to `.bak`, on the reasoning that a file nothing can parse
|
||||||
/// `:pre-migration-*` tag holding a 4–12 GB image stays on disk, no code path
|
/// should stop making the project look busy. That destroyed the one signal
|
||||||
/// can find it again (every one of them starts here and is told there is no
|
/// [`has_record`] exists to carry. The chain, in order:
|
||||||
/// migration), and the leftover file goes on making the project look like it
|
///
|
||||||
/// has a migration in flight to anything that checks for the file rather than
|
/// 1. The rename makes the file vanish, so `has_record` — pure filesystem
|
||||||
/// parsing it — including [`has_record`], which the pin reaper relies on.
|
/// presence — flips to false.
|
||||||
/// Renaming to `.bak` follows the same convention as `projects.json`: the
|
/// 2. `reconcile_migration` calls this, gets `Ok(None)`, and returns. An
|
||||||
/// user's bytes are kept, but they stop pinning gigabytes.
|
/// in-flight or interrupted migration becomes invisible: no resume offer, no
|
||||||
|
/// rollback offer, and the phase is never normalised.
|
||||||
|
/// 3. Both pin reapers use `has_record` as their conservative guard, so the
|
||||||
|
/// project's `:pre-migration-*` tag — the only copy of its pre-migration
|
||||||
|
/// system layer — is now "ownerless" to both of them, and the startup sweep
|
||||||
|
/// turns the untag into a deletion.
|
||||||
|
///
|
||||||
|
/// A record that cannot be parsed is exactly the case where the *most*
|
||||||
|
/// conservative answer is wanted, not the least. So the bytes are copied to a
|
||||||
|
/// **uniquely named** backup (a fixed `.bak` meant a second corruption silently
|
||||||
|
/// overwrote the first, and nothing ever read either back) and the original
|
||||||
|
/// stays where it is. The pin it describes then ages out through the ownerless
|
||||||
|
/// tombstone in `docker::migration::reap_stale_migration_pins` rather than
|
||||||
|
/// being reaped on the next app start.
|
||||||
pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
|
pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
|
||||||
let path = state_path(project_id)?;
|
let path = state_path(project_id)?;
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
@@ -69,15 +82,22 @@ pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
|
|||||||
match serde_json::from_str::<MigrationState>(&data) {
|
match serde_json::from_str::<MigrationState>(&data) {
|
||||||
Ok(state) => Ok(Some(state)),
|
Ok(state) => Ok(Some(state)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let backup = path.with_extension("json.bak");
|
let backup = corrupt_backup_path(&path, &chrono::Utc::now());
|
||||||
let moved = fs::rename(&path, &backup);
|
let copied = if backup.exists() {
|
||||||
|
// Already kept a copy of this exact corruption this second;
|
||||||
|
// nothing to add.
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
fs::copy(&path, &backup).map(|_| ())
|
||||||
|
};
|
||||||
log::error!(
|
log::error!(
|
||||||
"Failed to parse migration state for project {}: {} — treating as absent{}",
|
"Failed to parse migration state for project {}: {} — treating as absent, but \
|
||||||
|
the record is left in place so `has_record` still protects its rollback pin{}",
|
||||||
project_id,
|
project_id,
|
||||||
e,
|
e,
|
||||||
match moved {
|
match copied {
|
||||||
Ok(()) => format!(" and moved the record to {}", backup.display()),
|
Ok(()) => format!(" (a copy was kept at {})", backup.display()),
|
||||||
Err(ref e) => format!(" (could not move the record aside: {})", e),
|
Err(ref e) => format!(" (could not keep a copy: {})", e),
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
Ok(None)
|
Ok(None)
|
||||||
@@ -85,6 +105,15 @@ pub fn load(project_id: &str) -> Result<Option<MigrationState>, String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Where a copy of an unparseable record is kept.
|
||||||
|
///
|
||||||
|
/// Timestamped rather than a fixed `.bak`: a second corruption used to
|
||||||
|
/// overwrite the first, so the one case where the user's bytes matter most was
|
||||||
|
/// the case where they were most likely to be gone.
|
||||||
|
fn corrupt_backup_path(path: &std::path::Path, now: &chrono::DateTime<chrono::Utc>) -> PathBuf {
|
||||||
|
path.with_extension(format!("json.corrupt-{}.bak", now.format("%Y%m%d-%H%M%S")))
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether a project has a migration record on disk *at all*, without parsing
|
/// Whether a project has a migration record on disk *at all*, without parsing
|
||||||
/// it.
|
/// it.
|
||||||
///
|
///
|
||||||
@@ -97,17 +126,195 @@ pub fn has_record(project_id: &str) -> Result<bool, String> {
|
|||||||
Ok(state_path(project_id)?.exists())
|
Ok(state_path(project_id)?.exists())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Atomically write a project's migration state.
|
/// Atomically **and durably** write a project's migration state.
|
||||||
|
///
|
||||||
|
/// Write-temp-then-rename alone is only half of it, and the missing half is the
|
||||||
|
/// half this record exists for. `fs::write` returns once the bytes are in the
|
||||||
|
/// page cache; a rename over them is atomic *with respect to other readers*,
|
||||||
|
/// not with respect to power loss. Losing power in that window leaves the
|
||||||
|
/// rename applied and the data not yet written — i.e. a 0-byte or truncated
|
||||||
|
/// `{id}.json` — which is precisely the corrupt-record case above, produced by
|
||||||
|
/// the code whose job is to make that case impossible.
|
||||||
|
///
|
||||||
|
/// So: fsync the file before the rename, and fsync the *directory* after it,
|
||||||
|
/// because the rename itself is directory metadata and is not durable until the
|
||||||
|
/// directory is synced. A sync that fails is reported rather than swallowed —
|
||||||
|
/// this is the crash record, and "probably written" is not a state it may be
|
||||||
|
/// in.
|
||||||
pub fn save(project_id: &str, state: &MigrationState) -> Result<(), String> {
|
pub fn save(project_id: &str, state: &MigrationState) -> Result<(), String> {
|
||||||
let path = state_path(project_id)?;
|
let path = state_path(project_id)?;
|
||||||
let data = serde_json::to_string_pretty(state)
|
let data = serde_json::to_string_pretty(state)
|
||||||
.map_err(|e| format!("Failed to serialize migration state: {}", e))?;
|
.map_err(|e| format!("Failed to serialize migration state: {}", e))?;
|
||||||
let tmp = path.with_extension("json.tmp");
|
let tmp = path.with_extension("json.tmp");
|
||||||
fs::write(&tmp, data).map_err(|e| format!("Failed to write migration state: {}", e))?;
|
|
||||||
|
{
|
||||||
|
use std::io::Write;
|
||||||
|
let mut file = fs::File::create(&tmp)
|
||||||
|
.map_err(|e| format!("Failed to write migration state: {}", e))?;
|
||||||
|
file.write_all(data.as_bytes())
|
||||||
|
.map_err(|e| format!("Failed to write migration state: {}", e))?;
|
||||||
|
file.sync_all()
|
||||||
|
.map_err(|e| format!("Failed to flush migration state to disk: {}", e))?;
|
||||||
|
}
|
||||||
|
|
||||||
fs::rename(&tmp, &path).map_err(|e| format!("Failed to commit migration state: {}", e))?;
|
fs::rename(&tmp, &path).map_err(|e| format!("Failed to commit migration state: {}", e))?;
|
||||||
|
sync_dir(&path);
|
||||||
|
// A project with a record is not ownerless, whatever a reaper concluded
|
||||||
|
// before this write — so the grace clock is thrown away rather than left to
|
||||||
|
// expire against a pin that now has an owner again.
|
||||||
|
clear_ownerless_for_project(project_id);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// fsync the directory holding `path`, so a rename into it survives power loss.
|
||||||
|
///
|
||||||
|
/// Best effort *only* on the platforms where it is meaningless: Windows has no
|
||||||
|
/// directory handle to sync and returns an error for the attempt, so a failure
|
||||||
|
/// is logged rather than propagated. The file's own `sync_all` above is the
|
||||||
|
/// part that carries the data, and it is not best effort.
|
||||||
|
fn sync_dir(path: &std::path::Path) {
|
||||||
|
let Some(dir) = path.parent() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match fs::File::open(dir).and_then(|d| d.sync_all()) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(e) => log::debug!(
|
||||||
|
"Could not fsync the migrations directory {}: {} — the record itself was flushed",
|
||||||
|
dir.display(),
|
||||||
|
e
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Ownerless-pin tombstones
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Marker recording **when a rollback pin was first seen with no record behind
|
||||||
|
/// it**.
|
||||||
|
///
|
||||||
|
/// ## Why the grace period cannot be measured from the tag
|
||||||
|
///
|
||||||
|
/// `docker::migration::pin_is_reapable` used to date a pin from the timestamp
|
||||||
|
/// encoded in `pre-migration-<YYYYmmdd-HHMMSS>` — i.e. from when the migration
|
||||||
|
/// *started*. That is the wrong epoch by a whole feature. A migration is
|
||||||
|
/// allowed to sit at `awaiting-confirmation` indefinitely; `keep_rollback`
|
||||||
|
/// exists precisely so a user can run on the new base for a month before
|
||||||
|
/// deciding. If that project's record is then lost — a corrupt file, a deleted
|
||||||
|
/// state file, a half-restored data directory — the pin is fourteen days old on
|
||||||
|
/// the very first check, so it is untagged on the next app start and the
|
||||||
|
/// startup sweep deletes the image two lines later. The fourteen-day grace
|
||||||
|
/// period the constant promises is zero in the only situation it was written
|
||||||
|
/// for.
|
||||||
|
///
|
||||||
|
/// The clock has to start when the *claim* was lost, and nothing on the daemon
|
||||||
|
/// records that moment. So it is written down here, the first time a reaper
|
||||||
|
/// notices, and the age is measured from the marker.
|
||||||
|
///
|
||||||
|
/// One file per `(project_id, tag)` in the migrations directory, holding an
|
||||||
|
/// RFC3339 instant. Tiny, and losing one costs a fresh fourteen days rather
|
||||||
|
/// than a deletion — the failure direction that keeps somebody's only rollback
|
||||||
|
/// copy.
|
||||||
|
fn ownerless_marker_path(project_id: &str, tag: &str) -> Result<PathBuf, String> {
|
||||||
|
Ok(migrations_dir()?.join(format!(
|
||||||
|
"{}.{}.ownerless",
|
||||||
|
sanitize(project_id),
|
||||||
|
sanitize(tag)
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// When this pin was first observed ownerless, **without recording anything**.
|
||||||
|
///
|
||||||
|
/// For the survey paths, which describe the world and must not change it.
|
||||||
|
/// `None` means "no reaper has seen it yet", which is not the same as "seen
|
||||||
|
/// just now" and must not be treated as a start date.
|
||||||
|
pub fn peek_ownerless_since(
|
||||||
|
project_id: &str,
|
||||||
|
tag: &str,
|
||||||
|
) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||||
|
let path = ownerless_marker_path(project_id, tag).ok()?;
|
||||||
|
let raw = fs::read_to_string(path).ok()?;
|
||||||
|
chrono::DateTime::parse_from_rfc3339(raw.trim())
|
||||||
|
.ok()
|
||||||
|
.map(|t| t.with_timezone(&chrono::Utc))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the first-observed instant for a pin, creating the marker if this is
|
||||||
|
/// the first sighting. Returns `None` when the clock has not started yet.
|
||||||
|
///
|
||||||
|
/// **Clock skew is handled here rather than at the comparison.** A host clock
|
||||||
|
/// that was running fast when the marker was written leaves a timestamp in the
|
||||||
|
/// future; measured naively that is a negative age, which a `num_days() >= 14`
|
||||||
|
/// test reads as "never reapable" — a pin that can never be collected, forever.
|
||||||
|
/// A marker dated after `now` is therefore rewritten to `now`, restarting the
|
||||||
|
/// grace period. The other direction — a clock jumping forward — cannot shorten
|
||||||
|
/// the period below what has actually elapsed on the *marker's* terms, because
|
||||||
|
/// there is nothing to compare against but wall time; what it cannot do any
|
||||||
|
/// more is make every pin instantly reapable, which dating from the tag did.
|
||||||
|
pub fn note_ownerless_since(
|
||||||
|
project_id: &str,
|
||||||
|
tag: &str,
|
||||||
|
now: &chrono::DateTime<chrono::Utc>,
|
||||||
|
) -> Option<chrono::DateTime<chrono::Utc>> {
|
||||||
|
let path = ownerless_marker_path(project_id, tag).ok()?;
|
||||||
|
let existing = fs::read_to_string(&path).ok().and_then(|raw| {
|
||||||
|
chrono::DateTime::parse_from_rfc3339(raw.trim())
|
||||||
|
.ok()
|
||||||
|
.map(|t| t.with_timezone(&chrono::Utc))
|
||||||
|
});
|
||||||
|
match existing {
|
||||||
|
Some(seen) if seen <= *now => Some(seen),
|
||||||
|
// Absent, unparseable, or dated in the future: (re)start the clock.
|
||||||
|
_ => {
|
||||||
|
if let Err(e) = fs::write(&path, now.to_rfc3339()) {
|
||||||
|
log::warn!(
|
||||||
|
"Could not record that rollback pin {}:{} is ownerless: {} — its grace \
|
||||||
|
period restarts on the next check",
|
||||||
|
project_id,
|
||||||
|
tag,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Forget a pin's ownerless marker. Missing is success.
|
||||||
|
///
|
||||||
|
/// Called when the pin is untagged, and when a record reappears for the
|
||||||
|
/// project — a re-migrated project must not inherit the previous run's clock.
|
||||||
|
pub fn clear_ownerless(project_id: &str, tag: &str) {
|
||||||
|
let Ok(path) = ownerless_marker_path(project_id, tag) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match fs::remove_file(&path) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||||
|
Err(e) => log::warn!("Could not remove {}: {}", path.display(), e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop every ownerless marker belonging to one project.
|
||||||
|
///
|
||||||
|
/// A project that has a record again is by definition not ownerless, whatever
|
||||||
|
/// a reaper concluded before.
|
||||||
|
pub fn clear_ownerless_for_project(project_id: &str) {
|
||||||
|
let Ok(dir) = migrations_dir() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let prefix = format!("{}.", sanitize(project_id));
|
||||||
|
let Ok(entries) = fs::read_dir(&dir) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
if name.starts_with(&prefix) && name.ends_with(".ownerless") {
|
||||||
|
let _ = fs::remove_file(entry.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Remove a project's migration state file. Missing is success.
|
/// Remove a project's migration state file. Missing is success.
|
||||||
pub fn clear(project_id: &str) -> Result<(), String> {
|
pub fn clear(project_id: &str) -> Result<(), String> {
|
||||||
let path = state_path(project_id)?;
|
let path = state_path(project_id)?;
|
||||||
|
|||||||
Reference in New Issue
Block a user