Report and retry Docker resources remove_project could not delete #36

Merged
jknapp merged 4 commits from fix/remove-project-cleanup-reporting into main 2026-08-27 16:59:57 +00:00
9 changed files with 220 additions and 18 deletions
Showing only changes of commit 61bdbc4a5b - Show all commits
+74 -4
View File
@@ -751,6 +751,20 @@ pub async fn remove_project(
// `remove_volumes_by_name`'s fail-closed handling of the same // `remove_volumes_by_name`'s fail-closed handling of the same
// situation — the alternative silently drops the one resource most // situation — the alternative silently drops the one resource most
// likely to block everything else if it does exist. // likely to block everything else if it does exist.
//
// Exec sessions are closed for `project.container_id` unconditionally,
// before the lookup above and regardless of whether it succeeds —
// that is host-side state with no Docker dependency, so it must not
// wait on a daemon that might not answer. Resolving through
// `find_existing_container` instead of using it directly would leave
// these open in exactly the two cases this whole change exists to
// handle: Docker unreachable (no id resolved, no way to ever close
// them again once the project record is gone) and the stale-id race
// (sessions were opened against the container that actually exists,
// which is what gets resolved below, not the stored id).
if let Some(ref stored_id) = project.container_id {
state.exec_manager.close_sessions_for_container(stored_id).await;
}
let container_ref = match docker::find_existing_container(project).await { let container_ref = match docker::find_existing_container(project).await {
Ok(found) => found, Ok(found) => found,
Err(e) => { Err(e) => {
@@ -763,7 +777,9 @@ pub async fn remove_project(
} }
}; };
if let Some(ref container_id) = container_ref { if let Some(ref container_id) = container_ref {
state.exec_manager.close_sessions_for_container(container_id).await; if project.container_id.as_deref() != Some(container_id.as_str()) {
state.exec_manager.close_sessions_for_container(container_id).await;
}
let _ = docker::stop_container(container_id).await; let _ = docker::stop_container(container_id).await;
if let Err(e) = docker::remove_container(container_id).await { if let Err(e) = docker::remove_container(container_id).await {
log::warn!( log::warn!(
@@ -832,9 +848,24 @@ pub async fn remove_project(
// record would tell startup housekeeping to delete its container and // record would tell startup housekeeping to delete its container and
// volumes out from under it. Roll the record back rather than leaving // volumes out from under it. Roll the record back rather than leaving
// that mismatch for the retry to discover the hard way. // that mismatch for the retry to discover the hard way.
//
// This is a second, narrower line of defence, not the only one — a crash
// between the `save` above and the `remove` below leaves exactly the same
// mismatch with no error for either side to catch, which is why
// `retry_pending_cleanup_logged` also refuses to act on a record whose
// project is still listed in `projects.json`. Belt and suspenders: a
// caught failure here is handled immediately rather than waiting for the
// next launch to notice.
if let Err(e) = state.projects_store.remove(&project_id) { if let Err(e) = state.projects_store.remove(&project_id) {
if !report.is_clean() { if !report.is_clean() {
let _ = crate::storage::pending_cleanup::clear(&project_id); if let Err(clear_err) = crate::storage::pending_cleanup::clear(&project_id) {
log::error!(
"Project {} was not removed ({}), and its pending-cleanup record could not \
be rolled back either ({}) — it will name this still-live project until \
startup housekeeping's own guard clears it",
project_id, e, clear_err
);
}
} }
return Err(e); return Err(e);
} }
@@ -853,7 +884,21 @@ pub async fn remove_project(
/// reasoning), so there is no IPC contract to keep. A record that still has /// reasoning), so there is no IPC contract to keep. A record that still has
/// leftovers after this is written back so the next run does not lose track /// leftovers after this is written back so the next run does not lose track
/// of what changed; one that is now empty is deleted. /// of what changed; one that is now empty is deleted.
pub async fn retry_pending_cleanup_logged() { ///
/// Takes the `ProjectsStore` so it can refuse to touch a project that is
/// still live: `remove_project` writes a pending-cleanup record durably
/// (fsync'd) *before* it asks the store to drop the project, and that
/// store write is a plain `fs::write` with no fsync of its own. A crash or
/// power loss in the gap between the two — or the store write failing
/// outright, on top of the round-2 fix that only rolls the record back when
/// that failure is caught in-process — can leave a record on disk pointing
/// at a project `projects.json` still lists. Without this check, the very
/// first retry after such a crash deletes that project's container,
/// snapshot image and *both volumes, including the one holding the OAuth
/// credential and every session transcript*, out from under a project the
/// user still sees in the sidebar. A record whose project still exists is
/// therefore always stale — cleared without touching Docker, not retried.
pub async fn retry_pending_cleanup_logged(projects_store: &crate::storage::projects_store::ProjectsStore) {
let records = crate::storage::pending_cleanup::list(); let records = crate::storage::pending_cleanup::list();
if records.is_empty() { if records.is_empty() {
return; return;
@@ -863,6 +908,23 @@ pub async fn retry_pending_cleanup_logged() {
let mut still_pending = 0usize; let mut still_pending = 0usize;
for mut record in records { for mut record in records {
if projects_store.get(&record.project_id).is_some() {
log::warn!(
"Pending cleanup record for project {} ({}) names a project that still exists — \
clearing the record without touching Docker rather than risk deleting a live \
project's resources",
record.project_id, record.project_name
);
if let Err(e) = crate::storage::pending_cleanup::clear(&record.project_id) {
log::error!(
"Could not clear the stale pending-cleanup record for still-live project {} \
({}): {}",
record.project_id, record.project_name, e
);
}
continue;
}
if let Some(container_id) = record.container_id.take() { if let Some(container_id) = record.container_id.take() {
match docker::remove_container(&container_id).await { match docker::remove_container(&container_id).await {
Ok(()) => {} Ok(()) => {}
@@ -1472,9 +1534,17 @@ pub async fn rebuild_project_container(
// handling of the same lookup failing, `?` here aborts Reset outright: // handling of the same lookup failing, `?` here aborts Reset outright:
// every step after this one needs Docker too, so there is no useful // every step after this one needs Docker too, so there is no useful
// partial progress to make without it. // partial progress to make without it.
// Closed for the stored id unconditionally, then again for the resolved
// one if it differs — see the matching comment in `remove_project` for
// why the stale-id race can leave sessions open under either identity.
if let Some(ref stored_id) = project.container_id {
state.exec_manager.close_sessions_for_container(stored_id).await;
}
let container_ref = docker::find_existing_container(&project).await?; let container_ref = docker::find_existing_container(&project).await?;
if let Some(ref container_id) = container_ref { if let Some(ref container_id) = container_ref {
state.exec_manager.close_sessions_for_container(container_id).await; if project.container_id.as_deref() != Some(container_id.as_str()) {
state.exec_manager.close_sessions_for_container(container_id).await;
}
let _ = docker::stop_container(container_id).await; let _ = docker::stop_container(container_id).await;
docker::remove_container(container_id).await?; docker::remove_container(container_id).await?;
state.projects_store.set_container_id(&project_id, None)?; state.projects_store.set_container_id(&project_id, None)?;
+8 -2
View File
@@ -250,6 +250,7 @@ pub fn run() {
// an image open and the sweep will not force; pins are untagged // an image open and the sweep will not force; pins are untagged
// second so the images they were holding are dangling by the time // second so the images they were holding are dangling by the time
// the sweep lists them; the sweep runs last and collects both. // the sweep lists them; the sweep runs last and collects both.
let projects_store_for_cleanup = projects_store_setup.clone();
tauri::async_runtime::spawn(async move { tauri::async_runtime::spawn(async move {
crate::docker::reap_probe_containers().await; crate::docker::reap_probe_containers().await;
let reaped = crate::docker::reap_stale_migration_pins().await; let reaped = crate::docker::reap_stale_migration_pins().await;
@@ -259,8 +260,13 @@ pub fn run() {
crate::docker::sweep_orphaned_snapshots_logged("startup").await; crate::docker::sweep_orphaned_snapshots_logged("startup").await;
// A container/image/volume `remove_project` could not delete // A container/image/volume `remove_project` could not delete
// is recorded rather than lost — see triple-c#31 — and this is // is recorded rather than lost — see triple-c#31 — and this is
// the only place anything ever retries it. // the only place anything ever retries it. Takes the store so
crate::commands::project_commands::retry_pending_cleanup_logged().await; // it can refuse to touch a project that turns out to still be
// live — see the long comment on the function itself.
crate::commands::project_commands::retry_pending_cleanup_logged(
&projects_store_for_cleanup,
)
.await;
}); });
// Auto-start web terminal server if enabled in settings // Auto-start web terminal server if enabled in settings
@@ -295,6 +295,28 @@ mod tests {
fs::remove_dir_all(&dir).ok(); fs::remove_dir_all(&dir).ok();
} }
/// A record that fails to parse is moved aside once, rather than left in
/// place to be re-warned about — and re-warned about — on every future
/// launch forever.
#[test]
fn an_unparseable_record_is_moved_aside_exactly_once() {
let dir = temp_dir("corrupt-aside");
let bad = dir.join("bad.json");
fs::write(&bad, "{ not json").unwrap();
list_in(&dir);
assert!(!bad.exists(), "the bad file should have been moved aside");
let corrupt = dir.join("bad.json.corrupt");
assert!(corrupt.exists(), "and the moved copy should be at .json.corrupt");
// A second pass must not warn about `bad.json` again — it is gone —
// and must not choke on `.json.corrupt` already being there.
assert!(list_in(&dir).is_empty());
assert!(corrupt.exists(), "the aside copy is not itself deleted");
fs::remove_dir_all(&dir).ok();
}
/// `list_in` must not pick up the `.json.tmp` staging file `save_in` /// `list_in` must not pick up the `.json.tmp` staging file `save_in`
/// leaves behind if a crash lands between the write and the rename — the /// leaves behind if a crash lands between the write and the rename — the
/// whole point of the temp-then-rename dance is that only the renamed /// whole point of the temp-then-rename dance is that only the renamed
@@ -19,7 +19,7 @@ import ConfigTab from "./ConfigTab";
import FilesTab from "./FilesTab"; import FilesTab from "./FilesTab";
import BrowserTab from "./BrowserTab"; import BrowserTab from "./BrowserTab";
import { formatUptime } from "./format"; import { formatUptime } from "./format";
import { describeLeftovers, leftoverVerb } from "./removalReport"; import { describeLeftovers, leftoverPronoun, leftoverVerb } from "./removalReport";
const TABS = [ const TABS = [
{ id: "overview", label: "Overview" }, { id: "overview", label: "Overview" },
@@ -299,7 +299,7 @@ export default function ProjectHome({ projectId, active }: Props) {
useAppState.getState().pushToast({ useAppState.getState().pushToast({
kind: "error", kind: "error",
message: `${project.name}” was removed, but Triple-C could not confirm its Docker resources were removed`, message: `${project.name}” was removed, but Triple-C could not confirm its Docker resources were removed`,
detail: `Triple-C could not confirm ${describeLeftovers(report)} ${verb} removed, and could not record this for a retry. You may need to remove them manually (\`docker rm\` / \`docker rmi\` / \`docker volume rm\`).`, detail: `Triple-C could not confirm ${describeLeftovers(report)} ${verb} removed, and could not record this for a retry. You may need to remove ${leftoverPronoun(report)} manually (\`docker rm\` / \`docker rmi\` / \`docker volume rm\`).`,
}); });
} }
} }
@@ -18,9 +18,22 @@ export function describeLeftovers(report: ProjectRemovalReport): string {
return parts.join(", "); return parts.join(", ");
} }
/** How many distinct things `describeLeftovers` is describing — a container
* and an image each count as one, however many volumes are named. Shared by
* `leftoverVerb` and `leftoverPronoun` so the two can never disagree about
* singular vs. plural. */
function leftoverCount(report: ProjectRemovalReport): number {
return (report.container ? 1 : 0) + (report.image ? 1 : 0) + report.volumes.length;
}
/** Verb agreement for `describeLeftovers`'s output — "its container" needs /** Verb agreement for `describeLeftovers`'s output — "its container" needs
* "was", "its container, a volume" needs "were". */ * "was", "its container, a volume" needs "were". */
export function leftoverVerb(report: ProjectRemovalReport): "was" | "were" { export function leftoverVerb(report: ProjectRemovalReport): "was" | "were" {
const count = (report.container ? 1 : 0) + (report.image ? 1 : 0) + report.volumes.length; return leftoverCount(report) === 1 ? "was" : "were";
return count === 1 ? "was" : "were"; }
/** Pronoun agreement for referring back to `describeLeftovers`'s output —
* "remove it manually" for one thing, "remove them manually" for more. */
export function leftoverPronoun(report: ProjectRemovalReport): "it" | "them" {
return leftoverCount(report) === 1 ? "it" : "them";
} }
+4 -8
View File
@@ -3,6 +3,7 @@ import { save } from "@tauri-apps/plugin-dialog";
import type { Project } from "../lib/types"; import type { Project } from "../lib/types";
import * as commands from "../lib/tauri-commands"; import * as commands from "../lib/tauri-commands";
import { formatBytes } from "../lib/formatBytes"; import { formatBytes } from "../lib/formatBytes";
import { describeResetLeftovers, resetLeftoverPronoun } from "../lib/resetOutcome";
import { useAppState } from "../store/appState"; import { useAppState } from "../store/appState";
import { useProjects } from "./useProjects"; import { useProjects } from "./useProjects";
import { useTerminal } from "./useTerminal"; import { useTerminal } from "./useTerminal";
@@ -58,12 +59,7 @@ export function useProjectActions(project: Project) {
() => () =>
run("Reset", async () => { run("Reset", async () => {
const outcome = await rebuild(project.id); const outcome = await rebuild(project.id);
const { leftover_image, leftover_volumes } = outcome; if (outcome.leftover_image || outcome.leftover_volumes.length > 0) {
if (leftover_image || leftover_volumes.length > 0) {
const parts: string[] = [];
if (leftover_image) parts.push("its previous container image");
if (leftover_volumes.length === 1) parts.push("a volume");
else if (leftover_volumes.length > 1) parts.push(`${leftover_volumes.length} volumes`);
// Not "run `docker volume rm`" — by the time this renders, the new // Not "run `docker volume rm`" — by the time this renders, the new
// container this same call just started already has the leftover // container this same call just started already has the leftover
// volume mounted, so that command would just hit the same 409 // volume mounted, so that command would just hit the same 409
@@ -71,9 +67,9 @@ export function useProjectActions(project: Project) {
pushToast({ pushToast({
kind: "error", kind: "error",
message: `Reset for “${project.name}” did not fully clean up`, message: `Reset for “${project.name}” did not fully clean up`,
detail: `Triple-C could not remove ${parts.join(" and ")} from before the reset, so \ detail: `Triple-C could not remove ${describeResetLeftovers(outcome)} from before the reset, so \
the new container may still be built from, or contain, old data. Stop the project, then try \ the new container may still be built from, or contain, old data. Stop the project, then try \
Reset again, or remove ${parts.length > 1 ? "them" : "it"} manually once stopped.`, Reset again, or remove ${resetLeftoverPronoun(outcome)} manually once stopped.`,
}); });
} }
return outcome; return outcome;
+24
View File
@@ -140,3 +140,27 @@ describe("useProjects puts the status back when a refused command never ran", ()
expect(statusOf()).toBe("stopped"); expect(statusOf()).toBe("stopped");
}); });
}); });
describe("useProjects.rebuild on success", () => {
it("puts the outcome's project, not the whole outcome, into the list", async () => {
const rebuilt = project("running");
rebuildProjectContainer.mockResolvedValue({
project: rebuilt,
leftover_image: null,
leftover_volumes: [],
});
const { result } = renderHook(() => useProjects());
let outcome!: Awaited<ReturnType<typeof result.current.rebuild>>;
await act(async () => {
outcome = await result.current.rebuild("p1");
});
// A regression here would put the `{ project, leftover_image,
// leftover_volumes }` wrapper into the projects list instead of the
// `Project` it wraps — a shape mismatch `tsc` would not catch inside a
// callback typed to take `unknown` per Tauri's `invoke`.
expect(useAppState.getState().projects.find((p) => p.id === "p1")).toEqual(rebuilt);
expect(outcome.leftover_volumes).toEqual([]);
});
});
+39
View File
@@ -0,0 +1,39 @@
import { describe, it, expect } from "vitest";
import { describeResetLeftovers, resetLeftoverPronoun } from "./resetOutcome";
import type { ProjectResetOutcome } from "./types";
function outcome(overrides: Partial<ProjectResetOutcome> = {}): ProjectResetOutcome {
return {
project: {} as ProjectResetOutcome["project"],
leftover_image: null,
leftover_volumes: [],
...overrides,
};
}
describe("describeResetLeftovers", () => {
it("names the image first, then the volumes", () => {
expect(describeResetLeftovers(outcome({ leftover_image: "x" }))).toBe(
"its previous container image",
);
expect(describeResetLeftovers(outcome({ leftover_volumes: ["v1"] }))).toBe("a volume");
expect(describeResetLeftovers(outcome({ leftover_volumes: ["v1", "v2"] }))).toBe("2 volumes");
expect(
describeResetLeftovers(outcome({ leftover_image: "x", leftover_volumes: ["v1", "v2"] })),
).toBe("its previous container image and 2 volumes");
});
});
describe("resetLeftoverPronoun", () => {
it("is singular for exactly one leftover", () => {
expect(resetLeftoverPronoun(outcome({ leftover_image: "x" }))).toBe("it");
expect(resetLeftoverPronoun(outcome({ leftover_volumes: ["v1"] }))).toBe("it");
});
it("is plural once more than one thing survived", () => {
expect(resetLeftoverPronoun(outcome({ leftover_image: "x", leftover_volumes: ["v1"] }))).toBe(
"them",
);
expect(resetLeftoverPronoun(outcome({ leftover_volumes: ["v1", "v2"] }))).toBe("them");
});
});
+32
View File
@@ -0,0 +1,32 @@
import type { ProjectResetOutcome } from "./types";
/**
* Names what a `ProjectResetOutcome` says Reset could not clear, for
* `useProjectActions`'s Reset toast.
*
* The image is named first and phrased as "its previous container image"
* rather than folded in with the volumes — it is the more serious of the
* two: the new container is built from it whenever it exists, so a
* surviving image means Reset silently rebuilt the exact system layer it
* was asked to discard, while a surviving volume only means old data rides
* along.
*/
export function describeResetLeftovers(outcome: ProjectResetOutcome): string {
const parts: string[] = [];
if (outcome.leftover_image) parts.push("its previous container image");
if (outcome.leftover_volumes.length === 1) parts.push("a volume");
else if (outcome.leftover_volumes.length > 1) parts.push(`${outcome.leftover_volumes.length} volumes`);
return parts.join(" and ");
}
/** How many distinct things `describeResetLeftovers` is describing — the
* image counts as one, however many volumes are named alongside it. */
function resetLeftoverCount(outcome: ProjectResetOutcome): number {
return (outcome.leftover_image ? 1 : 0) + outcome.leftover_volumes.length;
}
/** Pronoun agreement for referring back to `describeResetLeftovers`'s
* output — "remove it manually" for one thing, "remove them" for more. */
export function resetLeftoverPronoun(outcome: ProjectResetOutcome): "it" | "them" {
return resetLeftoverCount(outcome) === 1 ? "it" : "them";
}