From 6e24977cf06b02032593dde3c360b50bf64ff307 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Mon, 16 Nov 2026 18:16:22 -0800 Subject: [PATCH] fix(aws): replace an orphaned exec-off EC2 task instead of refusing for ever A found task with ECS Exec off that still waits for an instance is another start's only while one could still be waiting for it (created within the EC2 placement wait plus 2 minutes); an older one is a crashed start's orphan and is replaced (stopped with its own reason, checked, waited for). INSTANCE_RACE now mentions Stop. The rerun's race records the winner's task, as the first launch's race does; a first task another start replaced says so. TaskInfo gains created_unix. Co-Authored-By: Claude Opus 5.5 --- app/src-tauri/src/aws/runtime/tests.rs | 1 + app/src-tauri/src/aws/tasks.rs | 3 + app/src-tauri/src/aws/tasks/tests.rs | 1 + app/src-tauri/src/host/aws_host.rs | 73 ++++++++++++++++---- app/src-tauri/src/host/aws_host/tests.rs | 86 +++++++++++++++++++++++- 5 files changed, 151 insertions(+), 13 deletions(-) diff --git a/app/src-tauri/src/aws/runtime/tests.rs b/app/src-tauri/src/aws/runtime/tests.rs index f89dbe8..85fcf45 100644 --- a/app/src-tauri/src/aws/runtime/tests.rs +++ b/app/src-tauri/src/aws/runtime/tests.rs @@ -278,6 +278,7 @@ fn live(id: &str) -> TaskInfo { runtime_id: None, container_instance_arn: None, exec_enabled: true, + created_unix: None, exec_agent_running: false, exit_code: None, owner: Some(USER.into()), diff --git a/app/src-tauri/src/aws/tasks.rs b/app/src-tauri/src/aws/tasks.rs index 18c451f..e23abb5 100644 --- a/app/src-tauri/src/aws/tasks.rs +++ b/app/src-tauri/src/aws/tasks.rs @@ -71,6 +71,8 @@ pub struct TaskInfo { /// off on a RunTask that has to wait for an EC2 instance (verified live, /// phase 7), and such a task can never be reached. pub exec_enabled: bool, + /// When ECS created the task (Unix seconds), if it says. + pub created_unix: Option, /// The `workspace` container's exit code, once it has one. pub exit_code: Option, /// Its `triple-c:owner` / `triple-c:project` tags (DescribeTasks with @@ -121,6 +123,7 @@ pub(crate) fn task_info(t: &Task) -> TaskInfo { runtime_id: workspace.and_then(|c| c.runtime_id()).map(str::to_string), container_instance_arn: t.container_instance_arn().map(str::to_string), exec_enabled: t.enable_execute_command(), + created_unix: t.created_at().map(|d| d.secs()), exec_agent_running: workspace.is_some_and(|c| { c.managed_agents().iter().any(|a| { a.name() == Some(&ManagedAgentName::ExecuteCommandAgent) diff --git a/app/src-tauri/src/aws/tasks/tests.rs b/app/src-tauri/src/aws/tasks/tests.rs index f65133c..fbd7f22 100644 --- a/app/src-tauri/src/aws/tasks/tests.rs +++ b/app/src-tauri/src/aws/tasks/tests.rs @@ -66,6 +66,7 @@ fn info(id: &str, last: &str) -> TaskInfo { runtime_id: None, container_instance_arn: None, exec_enabled: true, + created_unix: None, exec_agent_running: false, exit_code: None, owner: Some(USER.into()), diff --git a/app/src-tauri/src/host/aws_host.rs b/app/src-tauri/src/host/aws_host.rs index 8bac289..038f2db 100644 --- a/app/src-tauri/src/host/aws_host.rs +++ b/app/src-tauri/src/host/aws_host.rs @@ -544,9 +544,39 @@ pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Tr const STOP_WAIT: Duration = Duration::from_secs(3 * 60); /// A start found the project's task with ECS Exec off and still waiting for -/// an instance: another start is bringing one up. +/// an instance, recently started: another start is bringing one up. pub const INSTANCE_RACE: &str = "Another start of this workspace is bringing up an EC2 \ - instance; try again in a minute (once it is up, Start connects to it)"; + instance; try again in a minute (once it is up, Start connects to it), or Stop the \ + workspace to cancel it"; + +/// The stop reason of a task a start replaced (one with ECS Exec off). +pub const REPLACED_REASON: &str = "Replaced by another Triple-C start"; + +/// How long past the EC2 placement wait an exec-off task still waiting for +/// an instance counts as another start's, before it is taken as orphaned. +const ORPHAN_MARGIN: Duration = Duration::from_secs(2 * 60); + +/// Whether an exec-off task still waiting for an instance is older than +/// any start could still be waiting for it (`ec2_wait` + a margin): a +/// crashed start's orphan. No creation time reads as fresh. +fn is_orphaned(t: &TaskInfo, ec2_wait: WaitTiming, now_unix: i64) -> bool { + let limit = (ec2_wait.timeout + ORPHAN_MARGIN).as_secs() as i64; + t.created_unix.is_some_and(|c| now_unix - c > limit) +} + +/// Why [`rerun_once_placed`] did not give a new task. +enum RerunError { + /// A concurrent start's live task was found: record it, report + /// [`START_RACE`]. + Race(TaskInfo), + Failed(String), +} + +impl From for RerunError { + fn from(e: String) -> Self { + RerunError::Failed(e) + } +} /// Wait (at most `min(timeout, STOP_WAIT)`) until `arn` is STOPPED or gone. async fn wait_stopped(tasks: &T, cluster: &str, arn: &str, timing: WaitTiming) -> Result<(), String> { @@ -593,7 +623,7 @@ async fn rerun_once_placed( task_def_arn: &str, timing: WaitTiming, progress: &(dyn Fn(&str) + Send + Sync), -) -> Result { +) -> Result { let cluster = compute.cluster_arn.as_str(); let deadline = tokio::time::Instant::now() + timing.timeout; let wanted = [first.to_string()]; @@ -607,13 +637,21 @@ async fn rerun_once_placed( .into_iter() .find(|t| t.arn == first) else { - return Err(format!("The task {first} is no longer known to ECS; start the project again")); + return Err(format!("The task {first} is no longer known to ECS; start the project again").into()); }; if t.desired_status == "STOPPED" || t.last_status == "STOPPED" { + if t.stopped_reason.as_deref() == Some(REPLACED_REASON) { + return Err(RerunError::Failed( + "Another start of this workspace replaced this one; Start again in a minute \ + to connect to it." + .into(), + )); + } return Err(format!( "The task stopped before an EC2 instance took it ({}); start the project again.", t.stopped_reason.as_deref().unwrap_or("no reason given") - )); + ) + .into()); } if t.container_instance_arn.is_some() && t.desired_status == "RUNNING" { break; @@ -627,7 +665,8 @@ async fn rerun_once_placed( why in the group's activity in the EC2 console).", timing.timeout.as_secs() / 60, if max == 1 { "" } else { "s" } - )); + ) + .into()); } tokio::time::sleep(timing.interval).await; } @@ -643,11 +682,11 @@ async fn rerun_once_placed( .await .map_err(|e| e.to_string())?; if start.reused { - return Err(START_RACE.into()); + return Err(RerunError::Race(start.task)); } if !start.task.exec_enabled { let _ = stop_created(tasks, cluster, &start.task.arn).await; - return Err(EXEC_OFF.into()); + return Err(RerunError::Failed(EXEC_OFF.into())); } Ok(start.task) } @@ -736,14 +775,19 @@ pub async fn start_core { - let msg = if e == START_RACE { e } else { with_log_tail(tasks, &logs, &arn, e).await }; + Err(RerunError::Race(winner)) => { + // As on the first launch: the winner's task is the project's. + record_task(&winner.arn); + return Err(START_RACE.into()); + } + Err(RerunError::Failed(e)) => { + let msg = with_log_tail(tasks, &logs, &arn, e).await; if stop_created(tasks, &cluster, &arn).await { (inp.on_task)(None); } diff --git a/app/src-tauri/src/host/aws_host/tests.rs b/app/src-tauri/src/host/aws_host/tests.rs index f1a3ded..dbb972b 100644 --- a/app/src-tauri/src/host/aws_host/tests.rs +++ b/app/src-tauri/src/host/aws_host/tests.rs @@ -58,6 +58,7 @@ fn task(p: &Project, id: &str, last: &str, ready: bool) -> TaskInfo { runtime_id: ready.then(|| "rt-1".into()), container_instance_arn: None, exec_enabled: true, + created_unix: None, exec_agent_running: ready, exit_code: None, owner: Some(USER.into()), @@ -98,6 +99,8 @@ struct Fake { rotated_by_loser: bool, /// StopTask is refused (expired credentials, say). stop_fails: bool, + /// StopTask is accepted but the task never reaches STOPPED. + stop_noop: bool, /// What `on_task` was told, in order. recorded: Mutex>>, /// `group stream limit` of each log read. @@ -175,6 +178,9 @@ impl TaskApi for Fake { return Err(AwsError::Other { message: "ExpiredToken".into() }); } // As ECS does: a stopped task is no longer live. + if self.stop_noop { + return Ok(()); + } if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) { t.desired_status = "STOPPED".into(); t.last_status = "STOPPED".into(); @@ -1518,6 +1524,8 @@ async fn the_cold_reruns_launch_never_doubles_another_starts_task() { let (r, _) = start(&fake, &p).await; assert_eq!(r.err().unwrap(), START_RACE); assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "{:?}", fake.calls()); + // The winner's task is recorded, as on the first launch's race. + assert_eq!(fake.recorded.lock().unwrap().last().cloned().flatten(), Some(ec2_task_arn("w1"))); } #[tokio::test] @@ -1538,6 +1546,82 @@ async fn an_execless_task_that_cannot_be_stopped_fails_the_start() { assert!(!fake.called("run")); } +fn now_unix() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64 +} + +/// An exec-off task still waiting for an instance, older than any start +/// could still be waiting (a crashed start's orphan), is replaced; a fresh +/// one is another start's. +#[tokio::test] +async fn an_orphaned_waiting_execless_task_is_replaced_and_a_fresh_one_is_left_alone() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old")); + let waiting = |created: i64| TaskInfo { + exec_enabled: false, + created_unix: Some(created), + ..ec2_task(&p, "old", "PROVISIONING", false, false) + }; + let fake_with = |t: TaskInfo| { + Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])), + run: Some(ec2_task(&p, "new", "RUNNING", true, true)), + ..Default::default() + } + .with_task(t) + }; + let fake = fake_with(waiting(now_unix() - 3600)); + let (r, _) = start(&fake, &p).await; + assert_eq!(r.unwrap().task.arn, ec2_task_arn("new")); + assert!(fake.calls().contains(&format!("stop {} {REPLACED_REASON}", ec2_task_arn("old"))), "{:?}", fake.calls()); + + let fake = fake_with(waiting(now_unix())); + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert_eq!(e, INSTANCE_RACE); + assert!(e.contains("Stop"), "{e}"); + assert!(!fake.called("run") && !fake.called("stop")); +} + +#[tokio::test] +async fn a_cold_start_whose_task_another_start_replaced_says_so() { + let p = ec2_project(); + let execless = |last: &str| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, false) }; + let fake = Fake { + user_record: Some(user_record_param()), + runs: Mutex::new(vec![execless("PROVISIONING")]), + ..Default::default() + }; + *fake.script.lock().unwrap() = vec![TaskInfo { + desired_status: "STOPPED".into(), + stopped_reason: Some(REPLACED_REASON.into()), + ..execless("PROVISIONING") + }]; + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.starts_with("Another start of this workspace replaced this one"), "{e}"); + assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1); +} + +#[tokio::test] +async fn an_execless_task_that_never_stops_fails_the_start_without_a_new_run() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old")); + let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) }; + let fake = Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])), + stop_noop: true, + ..Default::default() + } + .with_task(old); + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.contains("did not stop in time"), "{e}"); + assert!(fake.called("stop") && !fake.called("run")); +} + #[tokio::test] async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() { let mut p = ec2_project(); @@ -1553,7 +1637,7 @@ async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() { let (r, _) = start(&fake, &p).await; assert_eq!(r.unwrap().task.arn, ec2_task_arn("new")); let calls = fake.calls(); - let stop = calls.iter().position(|c| *c == format!("stop {} {STOP_REASON}", ec2_task_arn("old"))).expect("the old task is stopped"); + let stop = calls.iter().position(|c| *c == format!("stop {} {REPLACED_REASON}", ec2_task_arn("old"))).expect("the old task is stopped"); assert!(stop < calls.iter().position(|c| c == "run").unwrap(), "{calls:?}"); }