From 9c888caf13bb6b6cddf5184efdaa596f169e7e19 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Mon, 16 Nov 2026 08:09:51 -0800 Subject: [PATCH] fix(aws): follow clone and setup in the background, after the start Ruling R5: the start ends after Connecting (Running, lock released) and a background follower (cancelled on Stop, Remove, tunnel close, app exit) reports only a failure it saw or one on a task this start ran. A stale Starting/Stopping AWS project loads as Error, and reconcile also checks a Stopped one that still has a task ARN, so a crash never hides a live task. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 12 +- app/src-tauri/src/aws/workspace.rs | 23 ---- app/src-tauri/src/aws/workspace/tests.rs | 102 +++++++++++++-- .../src/commands/project_commands.rs | 89 +++++++++---- app/src-tauri/src/host/aws_host.rs | 7 +- app/src-tauri/src/host/aws_host/status.rs | 123 +++++++++++++----- .../src/host/aws_host/status/tests.rs | 90 +++++++++++-- app/src-tauri/src/host/aws_host/tests.rs | 49 +++++++ app/src-tauri/src/host/tunnels.rs | 81 ++++++++++++ app/src-tauri/src/storage/projects_store.rs | 62 ++++++--- .../projects/home/AwsWorkspaceStatusCard.tsx | 7 +- 11 files changed, 518 insertions(+), 127 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 46bf36c..1385467 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -422,10 +422,14 @@ docker exec stdout → tokio task → emit("terminal-output-{sessionId}") → li subshell; `status` is `running` from the start until `ok` / `failed ` (a chain that dies early writes `failed`, so the idle stop is never blocked). `triple-c-workspace-status` (python, no secrets) prints `{clone:{done,repos},setup:{state,code,tail}}`. - The start command follows it after "Connecting…" (`host/aws_host/status.rs`: background exec - every 2 s, a progress line only on change, at most 35 min, quiet stop when the tunnel goes); - a failed clone/setup never fails the start — it arrives as the `workspace-warning` event (warning - toast), and Project Home's Overview polls `aws_workspace_status` every 5 s while visible. + **The start does not wait for them** (it ends after "Connecting…": Running, lock released); + `aws_host::spawn_follow` follows in a background task (`host/aws_host/status.rs`: background exec + every 2 s, at most 35 min, holds no project lock; ended by `tunnels::cancel_follow` on Stop/Remove + and by any tunnel close or app exit). Its only output is a `workspace-warning` event (warning + toast) for a failure it saw happen or one on a task this start ran; progress shows in Project + Home's Overview card, which polls `aws_workspace_status` every 5 s while Running and visible. + At load, a Starting/Stopping AWS project becomes Error (not Stopped — a crash mid-start may have + left a billed task), and `reconcile_aws_projects` also checks a Stopped one still holding a task ARN. **Lifetime cap** (`TRIPLEC_MAX_LIFETIME_MINUTES`, advisory — sudo can kill it): the idle watcher (started when idle stop *or* a non-zero cap is set) pushes best effort, records `lifetime` and sends TERM; the hook exits 76 (`aws::tasks::LIFETIME_EXIT_CODE` → `StopKind::LifetimeCap` → diff --git a/app/src-tauri/src/aws/workspace.rs b/app/src-tauri/src/aws/workspace.rs index 797f0ff..8224282 100644 --- a/app/src-tauri/src/aws/workspace.rs +++ b/app/src-tauri/src/aws/workspace.rs @@ -286,29 +286,6 @@ fn serialize_payload(m: &BTreeMap) -> Result { /// `TRIPLEC_SETUP_SCRIPT` when set, `TRIPLEC_MAX_LIFETIME_MINUTES` when the /// environment caps it), keys sorted. Fails when it would exceed 65,536 bytes. #[cfg_attr(not(test), allow(dead_code))] -pub fn project_secret_payload( - env_pairs: &[(String, String)], - agent_token: &str, - files: &[(String, Vec)], - repos: &[RepoSource], - wip: bool, - idle: Option, - storage: StorageMode, -) -> Result { - project_secret_payload_with( - env_pairs, - agent_token, - files, - repos, - wip, - idle, - storage, - PayloadExtras::default(), - ) -} - -/// [`project_secret_payload`] with the phase 6 keys. -#[cfg_attr(not(test), allow(dead_code))] #[allow(clippy::too_many_arguments)] pub fn project_secret_payload_with( env_pairs: &[(String, String)], diff --git a/app/src-tauri/src/aws/workspace/tests.rs b/app/src-tauri/src/aws/workspace/tests.rs index 269702f..b6d7d78 100644 --- a/app/src-tauri/src/aws/workspace/tests.rs +++ b/app/src-tauri/src/aws/workspace/tests.rs @@ -469,7 +469,7 @@ fn payload_holds_all_project_data_with_sorted_keys() { ("ca/corp.pem".to_string(), b"PEM".to_vec()), ]; let repos = project(StorageMode::Hybrid).aws.unwrap().repos; - let json = project_secret_payload( + let json = project_secret_payload_with( &env, "tok", &files, @@ -477,6 +477,7 @@ fn payload_holds_all_project_data_with_sorted_keys() { true, Some(30), StorageMode::Hybrid, + PayloadExtras::default(), ) .unwrap(); let m = payload_map(&json); @@ -525,7 +526,16 @@ fn payload_holds_all_project_data_with_sorted_keys() { #[test] fn payload_off_values() { let json = - project_secret_payload(&[], "t", &[], &[], false, None, StorageMode::Ephemeral).unwrap(); + project_secret_payload_with( + &[], + "t", + &[], + &[], + false, + None, + StorageMode::Ephemeral, + PayloadExtras::default(), + ).unwrap(); let m = payload_map(&json); assert_eq!(m["TRIPLEC_WIP_PUSH"], "0"); assert_eq!(m["TRIPLEC_STORAGE"], "ephemeral"); @@ -538,8 +548,26 @@ fn payload_off_values() { fn payload_is_independent_of_env_pair_order() { let a = pairs(&[("B", "2"), ("A", "1"), ("C", "3")]); let b = pairs(&[("C", "3"), ("A", "1"), ("B", "2")]); - let pa = project_secret_payload(&a, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap(); - let pb = project_secret_payload(&b, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap(); + let pa = project_secret_payload_with( + &a, + "t", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ).unwrap(); + let pb = project_secret_payload_with( + &b, + "t", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ).unwrap(); assert_eq!(pa, pb); } @@ -549,7 +577,16 @@ fn payload_rejects_bad_file_names() { "ssh/", "ssh/.", "ssh/..", "ssh/a/b", "other/x", "x", "ca/../k", "ssh/a:b", ] { let files = vec![(name.to_string(), b"x".to_vec())]; - let r = project_secret_payload(&[], "t", &files, &[], true, None, StorageMode::Hybrid); + let r = project_secret_payload_with( + &[], + "t", + &files, + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ); if name == "ssh/a:b" { // A colon is a fine file name; only the secret's *keys* are restricted. assert!(r.is_ok()); @@ -563,7 +600,16 @@ fn payload_rejects_bad_file_names() { fn payload_rejects_env_names_a_secret_reference_cannot_carry() { for bad in ["A:B", "", "1ABC", "A-B", "A B"] { let env = pairs(&[(bad, "SENTINEL-VALUE-7f3a")]); - let r = project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid); + let r = project_secret_payload_with( + &env, + "t", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ); assert!(r.is_err(), "{bad:?} accepted"); // The error never echoes a value. assert!(!r.unwrap_err().contains("SENTINEL-VALUE-7f3a")); @@ -581,7 +627,16 @@ fn payload_keeps_its_own_keys_over_custom_env() { ("OK", "1"), ]); let m = payload_map( - &project_secret_payload(&env, "real", &[], &[], true, None, StorageMode::Hybrid).unwrap(), + &project_secret_payload_with( + &env, + "real", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ).unwrap(), ); assert_eq!(m["TRIPLEC_AGENT_TOKEN"], "real"); // The task definition's fixed environment owns these two names. @@ -597,7 +652,16 @@ fn payload_keeps_its_own_keys_over_custom_env() { fn payload_over_64_kib_fails() { let env = pairs(&[("BIG", &"x".repeat(65_536))]); let err = - project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap_err(); + project_secret_payload_with( + &env, + "t", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ).unwrap_err(); assert!(err.contains("65,536"), "{err}"); } @@ -643,7 +707,16 @@ fn payload_omits_setup_script_and_lifetime_when_unset() { fn payload_setup_script_and_lifetime_cannot_be_forged_by_custom_env() { let env = pairs(&[("TRIPLEC_SETUP_SCRIPT", "rm -rf /"), ("TRIPLEC_MAX_LIFETIME_MINUTES", "1")]); let m = payload_map( - &project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap(), + &project_secret_payload_with( + &env, + "t", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ).unwrap(), ); assert!(!m.contains_key("TRIPLEC_SETUP_SCRIPT")); assert!(!m.contains_key("TRIPLEC_MAX_LIFETIME_MINUTES")); @@ -679,7 +752,16 @@ fn payload_size_error_names_the_setup_script_when_it_is_the_large_part() { fn payload_size_error_names_a_large_custom_variable_by_name_only() { let env = pairs(&[("BIG_BLOB", &"SENTINEL".repeat(9000))]); let err = - project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap_err(); + project_secret_payload_with( + &env, + "t", + &[], + &[], + true, + None, + StorageMode::Hybrid, + PayloadExtras::default(), + ).unwrap_err(); assert!(err.contains("the environment variable BIG_BLOB (71 KiB)"), "{err}"); assert!(!err.contains("SENTINEL"), "{err}"); } diff --git a/app/src-tauri/src/commands/project_commands.rs b/app/src-tauri/src/commands/project_commands.rs index 8b6417b..ddd440b 100644 --- a/app/src-tauri/src/commands/project_commands.rs +++ b/app/src-tauri/src/commands/project_commands.rs @@ -1407,9 +1407,6 @@ async fn start_project_container_locked( // Wrap container operations so that any failure resets status to Stopped. let host = ProjectHost::for_project(&project); - // An AWS start's clone/setup problems: the start still succeeds, and this - // goes to the user as a `workspace-warning` event once it is Running. - let mut workspace_warning: Option = None; let result: Result = async { let target = host .ensure_started(&project, &settings, &state.projects_store, &|msg: &str| { @@ -1434,14 +1431,6 @@ async fn start_project_container_locked( target.clone(), ); - // AWS: follow the background clone and setup script until they are - // done (or 35 minutes pass), as progress lines (phase 6, Q4). A - // local project returns at once. - workspace_warning = crate::host::aws_host::follow_start(&project, &|msg: &str| { - emit_progress(&app_handle, &project_id, msg) - }) - .await; - Ok(target) }.await; @@ -1502,8 +1491,14 @@ async fn start_project_container_locked( .await; } - if let Some(detail) = workspace_warning { - emit_workspace_warning(&app_handle, &project_id, &project.name, &detail); + // AWS: the start is done (Running, lock released when this returns); + // the background clone and setup script are followed by a background + // task that only reports a failure (ruling R5). Never for Docker. + if project.is_aws() { + let (handle, id, name) = (app_handle.clone(), project_id.clone(), project.name.clone()); + crate::host::aws_host::spawn_follow(&project, move |detail| { + emit_workspace_warning(&handle, &id, &name, &detail) + }); } project.container_id = Some(container_id); @@ -1511,11 +1506,10 @@ async fn start_project_container_locked( Ok(project) } -/// The start's warning channel (phase 6, Q4): the start command returns the -/// stored `Project`, which has no place for a one-off warning, so a started -/// workspace whose clone or setup script failed is reported as a -/// `workspace-warning` event `{project_id, message, detail}`; the frontend -/// shows it as a warning toast. +/// The follower's warning channel (phase 6, Q4/R5): a started workspace +/// whose clone or setup script failed (or still ran after 35 minutes) is +/// reported as a `workspace-warning` event `{project_id, message, detail}`; +/// the frontend shows it as a warning toast. fn emit_workspace_warning(app_handle: &tauri::AppHandle, project_id: &str, name: &str, detail: &str) { let _ = app_handle.emit( "workspace-warning", @@ -1569,6 +1563,9 @@ pub async fn stop_project_container( } state.projects_store.update_status(&project_id, ProjectStatus::Stopping)?; + // AWS: the background clone/setup follower ends now (it holds no lock; + // closing the tunnel would end it too). + crate::host::tunnels::cancel_follow(&project_id); // AWS: the terminals and the auth bridge close *before* the stop guard's // push, so nothing written after its snapshot is silently dropped (the @@ -2014,6 +2011,17 @@ pub async fn reconcile_project_statuses( /// How many AWS projects reconcile at once. const AWS_RECONCILE_CONCURRENCY: usize = 4; +/// Whether an AWS project's status must be checked against ECS at startup: +/// anything but a clean Stopped — and a Stopped one that still has a task +/// ARN (Stop clears it, so the stored ARN means a start or stop never +/// finished and a billed task may be running). +pub(crate) fn needs_aws_reconcile(p: &Project) -> bool { + match p.status { + ProjectStatus::Running | ProjectStatus::Error | ProjectStatus::Starting | ProjectStatus::Stopping => true, + ProjectStatus::Stopped => p.aws.as_ref().is_some_and(|a| a.task_arn.is_some()), + } +} + /// Reconcile every AWS project against ECS (the authority), independently of /// Docker: an app started without Docker still learns which tasks run and /// re-opens their tunnels. One bounded DescribeTasks per project, a few at a @@ -2037,10 +2045,7 @@ pub async fn reconcile_aws_projects(state: State<'_, AppState>) -> Result(src: &'a str, name: &str) -> &'a str { + let start = src.find(&format!("fn {name}(")).unwrap_or_else(|| panic!("{name} missing")); + let rest = &src[start..]; + let end = rest.find("\n}\n").unwrap(); + &rest[..end] + } + + #[test] + fn the_workspace_follower_is_spawned_after_the_start_succeeds_and_only_for_aws() { + // Ruling R5: the start ends after "Connecting…" (Running, lock + // released when the command returns); the clone/setup follow runs + // in the background and never inside the start. + let src = include_str!("project_commands.rs"); + let start = fn_body(src, "start_project_container_locked"); + let running = start + .find("update_status(&project_id, ProjectStatus::Running)") + .expect("the start sets Running"); + let spawn = start.find("aws_host::spawn_follow(").expect("the start spawns the follower"); + assert!(spawn > running, "the follower is spawned only after the project is Running"); + assert_eq!(start.matches("spawn_follow(").count(), 1); + let guard = start[..spawn].rfind("if project.is_aws() {").expect("AWS-only"); + assert!(start[guard..spawn].lines().count() <= 4, "the spawn sits right in the AWS guard"); + assert!(!start.contains("follow_workspace_status"), "no follow inside the start"); + // Docker: `spawn_follow` itself also refuses a local project + // (`status::tests::a_local_project_has_no_workspace_status_and_no_follower`). + } + + #[test] + fn stop_ends_the_follower_before_it_stops_the_task() { + let src = include_str!("project_commands.rs"); + let stop = fn_body(src, "stop_project_container"); + let cancel = stop.find("tunnels::cancel_follow(&project_id)").expect("stop cancels the follower"); + let task = stop.find(".stop_project(&project, &settings)").expect("stop stops the task"); + assert!(cancel < task); + // Stop's own claim never waits on the follower: it holds none + // (`status::tests::the_follower_takes_no_project_lock`). + } + } diff --git a/app/src-tauri/src/host/aws_host.rs b/app/src-tauri/src/host/aws_host.rs index 6c3e66e..31d2082 100644 --- a/app/src-tauri/src/host/aws_host.rs +++ b/app/src-tauri/src/host/aws_host.rs @@ -82,7 +82,7 @@ const WATCH_CALL_TIMEOUT: Duration = Duration::from_secs(60); const STOP_TIMEOUT: Duration = Duration::from_secs(45); pub mod status; -pub use status::{follow_start, workspace_status, WorkspaceStatus}; +pub use status::{spawn_follow, workspace_status, WorkspaceStatus}; fn s(e: E) -> String { e.to_string() @@ -589,6 +589,7 @@ pub async fn ensure_started( // Already connected and answering: nothing to do. if let Some(ep) = tunnels::get_endpoint(&project.id) { if matches!(agent::is_running(&ep).await, Ok(true)) { + tunnels::mark_attached(&project.id); return Ok(ContainerTarget::Agent(ep)); } tunnels::close(&project.id).await; @@ -653,6 +654,7 @@ pub async fn ensure_started( access_point_id, workspace_access_point_id, task_definition_arn: started.task.task_definition_arn.clone(), + created: started.created, }, ) { old.forward.close().await; @@ -834,6 +836,7 @@ async fn task_arn_or_find( /// the stored task ARN, else whatever live task the project has (a crash /// between RunTask and the store write). Closes the tunnel. pub async fn stop_project(project: &Project, settings: &AppSettings) -> Result<(), String> { + tunnels::cancel_follow(&project.id); if let Some(ep) = tunnels::get_endpoint(&project.id) { if let Some(res) = stop_endpoint(&ep).await { return res; @@ -867,6 +870,7 @@ pub async fn remove_resources( settings: &AppSettings, discard_data: bool, ) -> Result<(), String> { + tunnels::cancel_follow(&project.id); let (_, region, sdk) = project_sdk(project, settings).await?; let who = caller_identity(&sdk).await.map_err(s)?; let ok = owner_key(®ion, &who.user_id); @@ -1104,6 +1108,7 @@ async fn reattach( .as_ref() .and_then(|a| a.workspace_access_point_id.clone()), task_definition_arn: task.task_definition_arn.clone(), + created: false, }; if !reattach_still_wanted(store, &project.id, &task.arn) { tunnel.forward.close().await; diff --git a/app/src-tauri/src/host/aws_host/status.rs b/app/src-tauri/src/host/aws_host/status.rs index 51de5fa..09294ea 100644 --- a/app/src-tauri/src/host/aws_host/status.rs +++ b/app/src-tauri/src/host/aws_host/status.rs @@ -3,18 +3,25 @@ //! //! The entrypoint clones in the background and then runs the setup script; //! `triple-c-workspace-status` (container) prints both as one JSON line, -//! mirrored by [`WorkspaceStatus`]. The start reads it through the agent every -//! [`FOLLOW_INTERVAL`] as a **background** exec (never activity for idle -//! stop), emits a progress line only when it changes, and stops when the -//! clone is done and no setup is running, after [`FOLLOW_LIMIT`], or quietly -//! when the agent is gone (tunnel dropped, old image without the script). -//! A failed clone or setup does not fail the start: [`follow_warning`] is -//! what the user is told instead. +//! mirrored by [`WorkspaceStatus`]. +//! +//! **The start does not wait for them** (ruling R5, amending Q4): it ends +//! after "Connecting…" as in phase 5 — project Running, project lock +//! released, terminals and Stop usable. [`spawn_follow`] then follows in the +//! background: a read through the agent every [`FOLLOW_INTERVAL`] as a +//! **background** exec (never activity for idle stop), a log line only when +//! it changes, until the clone is done and no setup runs, [`FOLLOW_LIMIT`] +//! passes, the agent is gone (tunnel dropped, task died, image without the +//! script) or its token is cancelled (`tunnels::cancel_follow` on Stop and +//! Remove; any tunnel close; app exit). Its only output is the final warning +//! ([`follow_warning`]); Project Home's Overview card polls the status itself +//! while the project is Running, so progress shows there. use std::future::Future; use std::time::Duration; use serde::{Deserialize, Serialize}; +use tokio_util::sync::CancellationToken; use super::super::{tunnels, ContainerTarget}; use crate::models::Project; @@ -281,16 +288,37 @@ impl Default for FollowTiming { /// How a follow ended. #[derive(Debug, Clone, PartialEq, Eq)] pub enum FollowEnd { - Settled(WorkspaceStatus), + Settled { + status: WorkspaceStatus, + /// The very first read was already settled: nothing happened in + /// this follow (an attach to a workspace that finished long ago). + first_read: bool, + }, /// [`FollowTiming::limit`] passed; the last status read, if any. TimedOut(Option), - /// The agent went away or kept failing; nothing to say. + /// The agent went away, kept failing, or the follow was cancelled: + /// nothing to say. Lost, } -/// Follow `src` until it settles, the limit passes or it is lost, calling -/// `progress` with each new progress line and each new setup log line. +/// Follow `src` until it settles, the limit passes, it is lost or `cancel` +/// fires, calling `progress` with each new progress line and each new setup +/// log line. pub async fn follow_workspace_status( + src: &S, + total_repos: usize, + timing: FollowTiming, + cancel: &CancellationToken, + progress: &(dyn Fn(&str) + Send + Sync), +) -> FollowEnd { + tokio::select! { + biased; + _ = cancel.cancelled() => FollowEnd::Lost, + end = follow_inner(src, total_repos, timing, progress) => end, + } +} + +async fn follow_inner( src: &S, total_repos: usize, timing: FollowTiming, @@ -301,12 +329,14 @@ pub async fn follow_workspace_status( let mut last_log: Option = None; let mut last_status: Option = None; let mut failures = 0u32; + let mut reads = 0u32; loop { match src.read().await { StatusRead::Status(s) => { failures = 0; + reads += 1; if s.settled() { - return FollowEnd::Settled(s); + return FollowEnd::Settled { status: s, first_read: reads == 1 }; } if let Some(line) = s.progress_line(total_repos) { if last_line.as_deref() != Some(line.as_str()) { @@ -341,11 +371,20 @@ pub async fn follow_workspace_status( } } -/// What the user is told after a follow; `None` when all went well (or the -/// follow was lost — Project Home still shows the status on demand). -pub fn follow_warning(end: &FollowEnd, limit: Duration) -> Option { +/// What the user is told after a follow; `None` when all went well, the +/// follow was lost, or the workspace had settled before this start — +/// a failure is reported only when this follow saw it happen, or the +/// task is one this start ran (`created`). Project Home still shows an old +/// failure on demand. +pub fn follow_warning(end: &FollowEnd, limit: Duration, created: bool) -> Option { match end { - FollowEnd::Settled(s) => s.problems(), + FollowEnd::Settled { status, first_read } => { + if *first_read && !created { + None + } else { + status.problems() + } + } FollowEnd::TimedOut(s) => { let what = match s { Some(s) if !s.clone.done => "still cloning its repos", @@ -366,25 +405,43 @@ pub fn follow_warning(end: &FollowEnd, limit: Duration) -> Option { } } -/// The start's follow of a just-connected AWS project; the warning to show, -/// if any. A local project returns `None` at once. -pub async fn follow_start( - project: &Project, - progress: &(dyn Fn(&str) + Send + Sync), -) -> Option { - if !project.is_aws() { - return None; +/// The follow of one project's workspace, registered with the tunnel +/// registry for cancellation. `on_warning` gets the warning, if any. +pub async fn run_follow( + src: &S, + project_id: &str, + total_repos: usize, + created: bool, + timing: FollowTiming, + on_warning: impl FnOnce(String), +) { + let (id, cancel) = tunnels::begin_follow(project_id); + let log_line = |m: &str| log::debug!("Project {project_id}: {m}"); + let end = follow_workspace_status(src, total_repos, timing, &cancel, &log_line).await; + tunnels::end_follow(project_id, id); + if let Some(w) = follow_warning(&end, timing.limit, created) { + on_warning(w); } +} + +/// After a successful start of an AWS project: follow its clone and setup +/// script in the background (see the module docs). `false` (nothing +/// spawned) for a local project or one with no tunnel. +pub fn spawn_follow( + project: &Project, + on_warning: impl FnOnce(String) + Send + 'static, +) -> bool { + if !project.is_aws() || tunnels::get_endpoint(&project.id).is_none() { + return false; + } + let project_id = project.id.clone(); let total = project.aws.as_ref().map_or(0, |a| a.repos.len()); - let timing = FollowTiming::default(); - let end = follow_workspace_status( - &TunnelStatusSource { project_id: &project.id }, - total, - timing, - progress, - ) - .await; - follow_warning(&end, timing.limit) + let created = tunnels::ran_task(&project_id); + tauri::async_runtime::spawn(async move { + let src = TunnelStatusSource { project_id: &project_id }; + run_follow(&src, &project_id, total, created, FollowTiming::default(), on_warning).await; + }); + true } #[cfg(test)] diff --git a/app/src-tauri/src/host/aws_host/status/tests.rs b/app/src-tauri/src/host/aws_host/status/tests.rs index b1064f6..7565d4f 100644 --- a/app/src-tauri/src/host/aws_host/status/tests.rs +++ b/app/src-tauri/src/host/aws_host/status/tests.rs @@ -1,6 +1,7 @@ use std::sync::Mutex; use super::*; +use crate::host::tunnels; /// Returns its scripted reads in order, then repeats the last one. struct Scripted { @@ -36,7 +37,8 @@ fn st(json: &str) -> StatusRead { async fn follow(src: &Scripted, total: usize) -> (FollowEnd, Vec) { let lines = Mutex::new(Vec::new()); let progress = |m: &str| lines.lock().unwrap().push(m.to_string()); - let end = follow_workspace_status(src, total, FollowTiming::default(), &progress).await; + let cancel = CancellationToken::new(); + let end = follow_workspace_status(src, total, FollowTiming::default(), &cancel, &progress).await; (end, lines.into_inner().unwrap()) } @@ -75,8 +77,9 @@ async fn follow_emits_each_change_once_and_stops_when_settled() { assert_eq!(src.count(), 8); // Seven waits of 2 s between eight reads. assert_eq!(started.elapsed(), FOLLOW_INTERVAL * 7); - let FollowEnd::Settled(s) = &end else { panic!("{end:?}") }; - let w = follow_warning(&end, FOLLOW_LIMIT).unwrap(); + let FollowEnd::Settled { status: s, first_read: false } = &end else { panic!("{end:?}") }; + // Seen failing in this follow: reported even on an attach. + let w = follow_warning(&end, FOLLOW_LIMIT, false).unwrap(); assert!(w.contains("Cloning failed for a repo: b."), "{w}"); assert!(w.contains("The setup script failed (exit 2)."), "{w}"); assert!(w.ends_with("Last lines:\nSetting up jq\nE: oops"), "{w}"); @@ -90,7 +93,20 @@ async fn a_workspace_already_settled_ends_at_once_without_a_warning() { let (end, lines) = follow(&src, 1).await; assert!(lines.is_empty()); assert_eq!(started.elapsed(), std::time::Duration::ZERO); - assert_eq!(follow_warning(&end, FOLLOW_LIMIT), None); + assert_eq!(follow_warning(&end, FOLLOW_LIMIT, true), None); +} + +#[tokio::test(start_paused = true)] +async fn an_old_failure_warns_only_for_a_task_this_start_ran() { + let src = Scripted::new(vec![st(SETUP_FAILED)]); + let (end, lines) = follow(&src, 2).await; + assert!(lines.is_empty()); + assert!(matches!(end, FollowEnd::Settled { first_read: true, .. }), "{end:?}"); + // Attached to a workspace that failed long ago: no toast. + assert_eq!(follow_warning(&end, FOLLOW_LIMIT, false), None); + // This start ran the task and it failed before the first read. + let w = follow_warning(&end, FOLLOW_LIMIT, true).unwrap(); + assert!(w.contains("The setup script failed (exit 2)."), "{w}"); } #[tokio::test(start_paused = true)] @@ -102,7 +118,7 @@ async fn follow_times_out_after_35_minutes() { assert!(matches!(end, FollowEnd::TimedOut(Some(_))), "{end:?}"); assert!(started.elapsed() <= FOLLOW_LIMIT); assert!(started.elapsed() > FOLLOW_LIMIT - FOLLOW_INTERVAL * 2); - let w = follow_warning(&end, FOLLOW_LIMIT).unwrap(); + let w = follow_warning(&end, FOLLOW_LIMIT, false).unwrap(); assert!(w.contains("still cloning its repos after 35 minutes"), "{w}"); } @@ -110,7 +126,7 @@ async fn follow_times_out_after_35_minutes() { async fn a_timed_out_setup_says_so_and_carries_any_clone_failure() { let src = Scripted::new(vec![st(SETUP_2)]); let (end, _) = follow(&src, 2).await; - let w = follow_warning(&end, FOLLOW_LIMIT).unwrap(); + let w = follow_warning(&end, FOLLOW_LIMIT, false).unwrap(); assert!(w.contains("still running its setup script"), "{w}"); assert!(w.contains("Cloning failed for a repo: b."), "{w}"); } @@ -121,7 +137,7 @@ async fn a_lost_agent_ends_the_follow_quietly() { let (end, lines) = follow(&src, 1).await; assert_eq!(end, FollowEnd::Lost); assert_eq!(lines, ["Cloning repos… (1/1)"]); - assert_eq!(follow_warning(&end, FOLLOW_LIMIT), None); + assert_eq!(follow_warning(&end, FOLLOW_LIMIT, false), None); } #[tokio::test(start_paused = true)] @@ -129,7 +145,7 @@ async fn failing_reads_are_retried_then_give_up_quietly() { let fail = || StatusRead::Failed("tunnel hiccup".into()); // Four failures, then a good read: the count resets. let src = Scripted::new(vec![fail(), fail(), fail(), fail(), st(ALL_OK)]); - assert!(matches!(follow(&src, 1).await.0, FollowEnd::Settled(_))); + assert!(matches!(follow(&src, 1).await.0, FollowEnd::Settled { .. })); let src = Scripted::new(vec![fail()]); let (end, _) = follow(&src, 1).await; assert_eq!(end, FollowEnd::Lost); @@ -183,8 +199,62 @@ fn the_status_command_reports_a_missing_script_as_127() { } #[tokio::test] -async fn a_local_project_has_no_workspace_status() { +async fn a_local_project_has_no_workspace_status_and_no_follower() { let p = Project::new("p".into(), vec![]); assert_eq!(workspace_status(&p).await, Ok(None)); - assert_eq!(follow_start(&p, &|_| {}).await, None); + assert!(!spawn_follow(&p, |_| panic!("no warning for Docker"))); + assert!(!tunnels::is_following(&p.id)); +} + +/// Never answers: a follow in progress until something ends it. +struct Hang; +impl StatusSource for Hang { + async fn read(&self) -> StatusRead { + std::future::pending().await + } +} + +#[tokio::test(start_paused = true)] +async fn stop_ends_a_follow_in_progress_without_a_warning() { + let id = "follow-stop-test"; + let task = tokio::spawn(async move { + let warned = std::sync::Arc::new(Mutex::new(false)); + let w = warned.clone(); + run_follow(&Hang, id, 1, true, FollowTiming::default(), move |_| *w.lock().unwrap() = true) + .await; + let v = *warned.lock().unwrap(); + v + }); + tokio::time::sleep(FOLLOW_INTERVAL * 3).await; + assert!(tunnels::is_following(id)); + // What Stop and Remove call (`stop_project_container`, `stop_project`, + // `remove_resources`); closing the tunnel does the same. + tunnels::cancel_follow(id); + let warned = tokio::time::timeout(std::time::Duration::from_secs(1), task) + .await + .expect("the follow ends at once") + .unwrap(); + assert!(!warned); + assert!(!tunnels::is_following(id)); +} + +#[tokio::test(start_paused = true)] +async fn a_new_follow_replaces_the_old_one() { + let id = "follow-replace-test"; + let first = tokio::spawn(run_follow(&Hang, id, 1, true, FollowTiming::default(), |_| {})); + tokio::time::sleep(FOLLOW_INTERVAL).await; + let (_, second) = tunnels::begin_follow(id); + tokio::time::timeout(std::time::Duration::from_secs(1), first).await.unwrap().unwrap(); + // The old follower's end does not unregister the new one. + assert!(tunnels::is_following(id)); + assert!(!second.is_cancelled()); + tunnels::cancel_follow(id); + assert!(second.is_cancelled()); +} + +#[test] +fn the_follower_takes_no_project_lock() { + // Stop must never wait for it (ruling R5). + let src = include_str!("../status.rs"); + assert!(!src.contains("project_lock")); } diff --git a/app/src-tauri/src/host/aws_host/tests.rs b/app/src-tauri/src/host/aws_host/tests.rs index fb625c9..a6ba67c 100644 --- a/app/src-tauri/src/host/aws_host/tests.rs +++ b/app/src-tauri/src/host/aws_host/tests.rs @@ -875,3 +875,52 @@ async fn an_unattached_aws_project_gets_a_target_id_no_tunnel_can_have() { let local = Project::new("l".into(), Vec::new()); assert_eq!(unattached_container_id(&local), None); } + +#[tokio::test] +async fn a_persisted_starting_aws_project_with_a_task_arn_is_reconciled_against_ecs_on_load() { + use crate::commands::project_commands::needs_aws_reconcile; + use crate::storage::projects_store::settle_stale_statuses; + + // The app quit mid-start: Starting, the task's ARN already recorded. + let mut p = aws_project(StorageMode::Hybrid); + p.status = ProjectStatus::Starting; + p.aws.as_mut().unwrap().task_arn = Some(task_arn("mid")); + let mut docker = Project::new("local".into(), vec![]); + docker.status = ProjectStatus::Starting; + + let mut loaded = vec![p.clone(), docker]; + assert!(settle_stale_statuses(&mut loaded)); + // Not Stopped (that would hide a billed task): Error, ARN kept. + assert_eq!(loaded[0].status, ProjectStatus::Error); + assert_eq!(loaded[0].aws.as_ref().unwrap().task_arn, Some(task_arn("mid"))); + assert_eq!(loaded[1].status, ProjectStatus::Stopped, "Docker is unchanged"); + assert!(needs_aws_reconcile(&loaded[0])); + + // ECS says it runs: Running. + let p = loaded[0].clone(); + let fake = Fake::default().with_task(task(&p, "mid", "RUNNING", true)); + let (_d, store) = store_with(&p); + assert!(matches!(reconcile_project(&fake, &store, &p, &find(&p)).await.unwrap(), Reconciled::Live(_))); + assert_eq!(store.get(&p.id).unwrap().status, ProjectStatus::Running); + // ECS says it is gone: Stopped, ARN cleared. + let (_d, store) = store_with(&p); + assert_eq!(reconcile_project(&Fake::default(), &store, &p, &find(&p)).await.unwrap(), Reconciled::Gone); + let after = store.get(&p.id).unwrap(); + assert_eq!(after.status, ProjectStatus::Stopped); + assert_eq!(after.aws.unwrap().task_arn, None); +} + +#[test] +fn a_stopped_aws_project_is_reconciled_only_when_it_still_has_a_task_arn() { + use crate::commands::project_commands::needs_aws_reconcile; + let mut p = aws_project(StorageMode::Hybrid); + p.status = ProjectStatus::Stopped; + assert!(!needs_aws_reconcile(&p), "a clean stop clears the ARN"); + p.aws.as_mut().unwrap().task_arn = Some(task_arn("left")); + assert!(needs_aws_reconcile(&p)); + for s in [ProjectStatus::Running, ProjectStatus::Error, ProjectStatus::Starting, ProjectStatus::Stopping] { + p.status = s; + p.aws.as_mut().unwrap().task_arn = None; + assert!(needs_aws_reconcile(&p)); + } +} diff --git a/app/src-tauri/src/host/tunnels.rs b/app/src-tauri/src/host/tunnels.rs index b989dbe..9f9d07a 100644 --- a/app/src-tauri/src/host/tunnels.rs +++ b/app/src-tauri/src/host/tunnels.rs @@ -18,6 +18,7 @@ use std::time::Duration; use aws_config::SdkConfig; use futures_util::future::BoxFuture; use tokio::sync::watch; +use tokio_util::sync::CancellationToken; use super::agent::AgentEndpoint; use crate::aws::tasks::StopKind; @@ -53,6 +54,9 @@ pub struct LiveTunnel { pub task_definition_arn: String, /// EFS storage's `/workspace` access point. pub workspace_access_point_id: Option, + /// The start that opened this tunnel ran the task (rather than + /// attaching to one already running, or re-attaching after a restart). + pub created: bool, } /// Where an endpoint's task is: `(connection, region, cluster, task ARN)`. @@ -70,6 +74,12 @@ struct Registry { /// start's own tunnel) and [`cancel_reattach`]: a background re-attach /// that began under an older generation must never register. generation: HashMap, + /// The background clone/setup follower per project (`aws_host::status`), + /// with the id [`begin_follow`] gave it. Cancelled by a new follower, + /// [`cancel_follow`] (Stop, Remove), and whenever the tunnel is taken + /// out or every tunnel closed. + followers: HashMap, + next_follower: u64, } fn registry() -> &'static Mutex { @@ -209,8 +219,60 @@ pub async fn close_if_task(project_id: &str, task_arn: &str) { } } +/// Whether the project's registered tunnel came from a start that ran its +/// task (see [`LiveTunnel::created`]). +pub fn ran_task(project_id: &str) -> bool { + lock().live.get(project_id).is_some_and(|t| t.created) +} + +/// A later start found the tunnel already up: it ran no task itself. +pub fn mark_attached(project_id: &str) { + if let Some(t) = lock().live.get_mut(project_id) { + t.created = false; + } +} + +/// Register a background follower for `project_id`, cancelling the one +/// before it. Returns its id (for [`end_follow`]) and its token. +pub fn begin_follow(project_id: &str) -> (u64, CancellationToken) { + let mut r = lock(); + r.next_follower += 1; + let id = r.next_follower; + let token = CancellationToken::new(); + if let Some((_, old)) = r.followers.insert(project_id.to_string(), (id, token.clone())) { + old.cancel(); + } + (id, token) +} + +/// A follower finished: forget it, unless a newer one has replaced it. +pub fn end_follow(project_id: &str, id: u64) { + let mut r = lock(); + if r.followers.get(project_id).is_some_and(|(i, _)| *i == id) { + r.followers.remove(project_id); + } +} + +/// End the project's follower, if one runs (Stop, Remove). +pub fn cancel_follow(project_id: &str) { + cancel_follow_locked(&mut lock(), project_id); +} + +fn cancel_follow_locked(r: &mut Registry, project_id: &str) { + if let Some((_, t)) = r.followers.remove(project_id) { + t.cancel(); + } +} + +/// Whether a follower runs for the project. +#[cfg(test)] +pub fn is_following(project_id: &str) -> bool { + lock().followers.contains_key(project_id) +} + /// Under the lock: invalidate a re-attach in flight and take the tunnel out. fn take(r: &mut Registry, project_id: &str) -> Option { + cancel_follow_locked(r, project_id); r.pending.remove(project_id); *r.generation.entry(project_id.to_string()).or_default() += 1; let t = r.live.remove(project_id); @@ -259,6 +321,9 @@ pub async fn close_all() { let mut r = lock(); r.pending.clear(); r.stopped.clear(); + for (_, (_, t)) in r.followers.drain() { + t.cancel(); + } for g in r.generation.values_mut() { *g += 1; } @@ -339,6 +404,7 @@ pub(crate) mod tests { access_point_id: Some("fsap-1".into()), task_definition_arn: "arn:td:1".into(), workspace_access_point_id: None, + created: true, } } @@ -364,6 +430,21 @@ pub(crate) mod tests { assert!(get_endpoint(&p).is_none() && task_of(&ep).is_none()); } + #[tokio::test] + async fn closing_the_tunnel_ends_its_follower_and_an_attach_clears_created() { + let closes = Arc::new(AtomicUsize::new(0)); + let p = pid("follow"); + register(&p, fake_tunnel(61009, closes.clone())); + assert!(ran_task(&p)); + mark_attached(&p); + assert!(!ran_task(&p)); + let (_, token) = begin_follow(&p); + assert!(is_following(&p) && !token.is_cancelled()); + close(&p).await; + assert!(token.is_cancelled()); + assert!(!is_following(&p)); + } + #[tokio::test] async fn re_registering_hands_back_the_old_tunnel_and_clears_a_stale_stop() { let closes = Arc::new(AtomicUsize::new(0)); diff --git a/app/src-tauri/src/storage/projects_store.rs b/app/src-tauri/src/storage/projects_store.rs index f0dda02..99aa688 100644 --- a/app/src-tauri/src/storage/projects_store.rs +++ b/app/src-tauri/src/storage/projects_store.rs @@ -93,6 +93,46 @@ fn record_corrupt_load(file_path: &Path, now: &chrono::DateTime) { } } +/// The load's rewrite of stale transient statuses; `true` when it changed +/// any. +pub(crate) fn settle_stale_statuses(projects: &mut [Project]) -> bool { + // Reconcile stale transient statuses: on a cold app start no Docker + // operations can be in flight, so Starting/Stopping are always stale. + // Running/Error are left as-is and reconciled against Docker later + // via the reconcile_project_statuses command. + // + // An AWS project goes to **Error**, not Stopped: an app that quit + // mid-start (or mid-stop) may have left a billed task running, with + // or without its ARN recorded yet. Error is what a failed start that + // left a task already uses; `reconcile_aws_projects` then asks ECS + // (by ARN, else by the project's tags) and settles it Running or + // Stopped. Stopped would hide the live task. + let mut changed = false; + for p in projects.iter_mut() { + match p.status { + crate::models::ProjectStatus::Starting | crate::models::ProjectStatus::Stopping => { + let to = if p.is_aws() { + crate::models::ProjectStatus::Error + } else { + crate::models::ProjectStatus::Stopped + }; + log::warn!( + "Reconciling stale '{}' status for project '{}' ({}) → {:?}", + serde_json::to_string(&p.status).unwrap_or_default().trim_matches('"'), + p.name, + p.id, + to + ); + p.status = to; + p.updated_at = chrono::Utc::now().to_rfc3339(); + changed = true; + } + _ => {} + } + } + changed +} + pub struct ProjectsStore { projects: Mutex>, file_path: PathBuf, @@ -155,28 +195,8 @@ impl ProjectsStore { (Vec::new(), false) }; - // Reconcile stale transient statuses: on a cold app start no Docker - // operations can be in flight, so Starting/Stopping are always stale. - // Running/Error are left as-is and reconciled against Docker later - // via the reconcile_project_statuses command. let mut projects = projects; - let mut needs_save = needs_save; - for p in projects.iter_mut() { - match p.status { - crate::models::ProjectStatus::Starting | crate::models::ProjectStatus::Stopping => { - log::warn!( - "Reconciling stale '{}' status for project '{}' ({}) → Stopped", - serde_json::to_string(&p.status).unwrap_or_default().trim_matches('"'), - p.name, - p.id - ); - p.status = crate::models::ProjectStatus::Stopped; - p.updated_at = chrono::Utc::now().to_rfc3339(); - needs_save = true; - } - _ => {} - } - } + let needs_save = settle_stale_statuses(&mut projects) || needs_save; let store = Self { projects: Mutex::new(projects), diff --git a/app/src/components/projects/home/AwsWorkspaceStatusCard.tsx b/app/src/components/projects/home/AwsWorkspaceStatusCard.tsx index 4eca4f6..06fe5ad 100644 --- a/app/src/components/projects/home/AwsWorkspaceStatusCard.tsx +++ b/app/src/components/projects/home/AwsWorkspaceStatusCard.tsx @@ -13,9 +13,10 @@ const REPO_STATE: Record = { }; /** - * An AWS workspace's background clone and setup script (phase 6, Q4): the - * same status the start followed, re-read every 5 s while the project is - * Running and the window is visible. Each read is a background exec, so the + * An AWS workspace's background clone and setup script (phase 6, Q4/R5): + * the start ends at "Connecting…" and this card is where their progress + * shows, re-read every 5 s while the project is Running and the window is + * visible. Each read is a background exec, so the * card never keeps an idle workspace up. Renders nothing for a local project, * a workspace that cannot be asked, or one with no repos and no setup script. */