diff --git a/app/src-tauri/src/aws/runtime/tests.rs b/app/src-tauri/src/aws/runtime/tests.rs index f89dbe8..85fcf45 100644 --- a/app/src-tauri/src/aws/runtime/tests.rs +++ b/app/src-tauri/src/aws/runtime/tests.rs @@ -278,6 +278,7 @@ fn live(id: &str) -> TaskInfo { runtime_id: None, container_instance_arn: None, exec_enabled: true, + created_unix: None, exec_agent_running: false, exit_code: None, owner: Some(USER.into()), diff --git a/app/src-tauri/src/aws/tasks.rs b/app/src-tauri/src/aws/tasks.rs index 18c451f..e23abb5 100644 --- a/app/src-tauri/src/aws/tasks.rs +++ b/app/src-tauri/src/aws/tasks.rs @@ -71,6 +71,8 @@ pub struct TaskInfo { /// off on a RunTask that has to wait for an EC2 instance (verified live, /// phase 7), and such a task can never be reached. pub exec_enabled: bool, + /// When ECS created the task (Unix seconds), if it says. + pub created_unix: Option, /// The `workspace` container's exit code, once it has one. pub exit_code: Option, /// Its `triple-c:owner` / `triple-c:project` tags (DescribeTasks with @@ -121,6 +123,7 @@ pub(crate) fn task_info(t: &Task) -> TaskInfo { runtime_id: workspace.and_then(|c| c.runtime_id()).map(str::to_string), container_instance_arn: t.container_instance_arn().map(str::to_string), exec_enabled: t.enable_execute_command(), + created_unix: t.created_at().map(|d| d.secs()), exec_agent_running: workspace.is_some_and(|c| { c.managed_agents().iter().any(|a| { a.name() == Some(&ManagedAgentName::ExecuteCommandAgent) diff --git a/app/src-tauri/src/aws/tasks/tests.rs b/app/src-tauri/src/aws/tasks/tests.rs index f65133c..fbd7f22 100644 --- a/app/src-tauri/src/aws/tasks/tests.rs +++ b/app/src-tauri/src/aws/tasks/tests.rs @@ -66,6 +66,7 @@ fn info(id: &str, last: &str) -> TaskInfo { runtime_id: None, container_instance_arn: None, exec_enabled: true, + created_unix: None, exec_agent_running: false, exit_code: None, owner: Some(USER.into()), diff --git a/app/src-tauri/src/host/aws_host.rs b/app/src-tauri/src/host/aws_host.rs index 8bac289..038f2db 100644 --- a/app/src-tauri/src/host/aws_host.rs +++ b/app/src-tauri/src/host/aws_host.rs @@ -544,9 +544,39 @@ pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Tr const STOP_WAIT: Duration = Duration::from_secs(3 * 60); /// A start found the project's task with ECS Exec off and still waiting for -/// an instance: another start is bringing one up. +/// an instance, recently started: another start is bringing one up. pub const INSTANCE_RACE: &str = "Another start of this workspace is bringing up an EC2 \ - instance; try again in a minute (once it is up, Start connects to it)"; + instance; try again in a minute (once it is up, Start connects to it), or Stop the \ + workspace to cancel it"; + +/// The stop reason of a task a start replaced (one with ECS Exec off). +pub const REPLACED_REASON: &str = "Replaced by another Triple-C start"; + +/// How long past the EC2 placement wait an exec-off task still waiting for +/// an instance counts as another start's, before it is taken as orphaned. +const ORPHAN_MARGIN: Duration = Duration::from_secs(2 * 60); + +/// Whether an exec-off task still waiting for an instance is older than +/// any start could still be waiting for it (`ec2_wait` + a margin): a +/// crashed start's orphan. No creation time reads as fresh. +fn is_orphaned(t: &TaskInfo, ec2_wait: WaitTiming, now_unix: i64) -> bool { + let limit = (ec2_wait.timeout + ORPHAN_MARGIN).as_secs() as i64; + t.created_unix.is_some_and(|c| now_unix - c > limit) +} + +/// Why [`rerun_once_placed`] did not give a new task. +enum RerunError { + /// A concurrent start's live task was found: record it, report + /// [`START_RACE`]. + Race(TaskInfo), + Failed(String), +} + +impl From for RerunError { + fn from(e: String) -> Self { + RerunError::Failed(e) + } +} /// Wait (at most `min(timeout, STOP_WAIT)`) until `arn` is STOPPED or gone. async fn wait_stopped(tasks: &T, cluster: &str, arn: &str, timing: WaitTiming) -> Result<(), String> { @@ -593,7 +623,7 @@ async fn rerun_once_placed( task_def_arn: &str, timing: WaitTiming, progress: &(dyn Fn(&str) + Send + Sync), -) -> Result { +) -> Result { let cluster = compute.cluster_arn.as_str(); let deadline = tokio::time::Instant::now() + timing.timeout; let wanted = [first.to_string()]; @@ -607,13 +637,21 @@ async fn rerun_once_placed( .into_iter() .find(|t| t.arn == first) else { - return Err(format!("The task {first} is no longer known to ECS; start the project again")); + return Err(format!("The task {first} is no longer known to ECS; start the project again").into()); }; if t.desired_status == "STOPPED" || t.last_status == "STOPPED" { + if t.stopped_reason.as_deref() == Some(REPLACED_REASON) { + return Err(RerunError::Failed( + "Another start of this workspace replaced this one; Start again in a minute \ + to connect to it." + .into(), + )); + } return Err(format!( "The task stopped before an EC2 instance took it ({}); start the project again.", t.stopped_reason.as_deref().unwrap_or("no reason given") - )); + ) + .into()); } if t.container_instance_arn.is_some() && t.desired_status == "RUNNING" { break; @@ -627,7 +665,8 @@ async fn rerun_once_placed( why in the group's activity in the EC2 console).", timing.timeout.as_secs() / 60, if max == 1 { "" } else { "s" } - )); + ) + .into()); } tokio::time::sleep(timing.interval).await; } @@ -643,11 +682,11 @@ async fn rerun_once_placed( .await .map_err(|e| e.to_string())?; if start.reused { - return Err(START_RACE.into()); + return Err(RerunError::Race(start.task)); } if !start.task.exec_enabled { let _ = stop_created(tasks, cluster, &start.task.arn).await; - return Err(EXEC_OFF.into()); + return Err(RerunError::Failed(EXEC_OFF.into())); } Ok(start.task) } @@ -736,14 +775,19 @@ pub async fn start_core { - let msg = if e == START_RACE { e } else { with_log_tail(tasks, &logs, &arn, e).await }; + Err(RerunError::Race(winner)) => { + // As on the first launch: the winner's task is the project's. + record_task(&winner.arn); + return Err(START_RACE.into()); + } + Err(RerunError::Failed(e)) => { + let msg = with_log_tail(tasks, &logs, &arn, e).await; if stop_created(tasks, &cluster, &arn).await { (inp.on_task)(None); } diff --git a/app/src-tauri/src/host/aws_host/tests.rs b/app/src-tauri/src/host/aws_host/tests.rs index f1a3ded..dbb972b 100644 --- a/app/src-tauri/src/host/aws_host/tests.rs +++ b/app/src-tauri/src/host/aws_host/tests.rs @@ -58,6 +58,7 @@ fn task(p: &Project, id: &str, last: &str, ready: bool) -> TaskInfo { runtime_id: ready.then(|| "rt-1".into()), container_instance_arn: None, exec_enabled: true, + created_unix: None, exec_agent_running: ready, exit_code: None, owner: Some(USER.into()), @@ -98,6 +99,8 @@ struct Fake { rotated_by_loser: bool, /// StopTask is refused (expired credentials, say). stop_fails: bool, + /// StopTask is accepted but the task never reaches STOPPED. + stop_noop: bool, /// What `on_task` was told, in order. recorded: Mutex>>, /// `group stream limit` of each log read. @@ -175,6 +178,9 @@ impl TaskApi for Fake { return Err(AwsError::Other { message: "ExpiredToken".into() }); } // As ECS does: a stopped task is no longer live. + if self.stop_noop { + return Ok(()); + } if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) { t.desired_status = "STOPPED".into(); t.last_status = "STOPPED".into(); @@ -1518,6 +1524,8 @@ async fn the_cold_reruns_launch_never_doubles_another_starts_task() { let (r, _) = start(&fake, &p).await; assert_eq!(r.err().unwrap(), START_RACE); assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "{:?}", fake.calls()); + // The winner's task is recorded, as on the first launch's race. + assert_eq!(fake.recorded.lock().unwrap().last().cloned().flatten(), Some(ec2_task_arn("w1"))); } #[tokio::test] @@ -1538,6 +1546,82 @@ async fn an_execless_task_that_cannot_be_stopped_fails_the_start() { assert!(!fake.called("run")); } +fn now_unix() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64 +} + +/// An exec-off task still waiting for an instance, older than any start +/// could still be waiting (a crashed start's orphan), is replaced; a fresh +/// one is another start's. +#[tokio::test] +async fn an_orphaned_waiting_execless_task_is_replaced_and_a_fresh_one_is_left_alone() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old")); + let waiting = |created: i64| TaskInfo { + exec_enabled: false, + created_unix: Some(created), + ..ec2_task(&p, "old", "PROVISIONING", false, false) + }; + let fake_with = |t: TaskInfo| { + Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])), + run: Some(ec2_task(&p, "new", "RUNNING", true, true)), + ..Default::default() + } + .with_task(t) + }; + let fake = fake_with(waiting(now_unix() - 3600)); + let (r, _) = start(&fake, &p).await; + assert_eq!(r.unwrap().task.arn, ec2_task_arn("new")); + assert!(fake.calls().contains(&format!("stop {} {REPLACED_REASON}", ec2_task_arn("old"))), "{:?}", fake.calls()); + + let fake = fake_with(waiting(now_unix())); + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert_eq!(e, INSTANCE_RACE); + assert!(e.contains("Stop"), "{e}"); + assert!(!fake.called("run") && !fake.called("stop")); +} + +#[tokio::test] +async fn a_cold_start_whose_task_another_start_replaced_says_so() { + let p = ec2_project(); + let execless = |last: &str| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, false) }; + let fake = Fake { + user_record: Some(user_record_param()), + runs: Mutex::new(vec![execless("PROVISIONING")]), + ..Default::default() + }; + *fake.script.lock().unwrap() = vec![TaskInfo { + desired_status: "STOPPED".into(), + stopped_reason: Some(REPLACED_REASON.into()), + ..execless("PROVISIONING") + }]; + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.starts_with("Another start of this workspace replaced this one"), "{e}"); + assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1); +} + +#[tokio::test] +async fn an_execless_task_that_never_stops_fails_the_start_without_a_new_run() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old")); + let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) }; + let fake = Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])), + stop_noop: true, + ..Default::default() + } + .with_task(old); + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.contains("did not stop in time"), "{e}"); + assert!(fake.called("stop") && !fake.called("run")); +} + #[tokio::test] async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() { let mut p = ec2_project(); @@ -1553,7 +1637,7 @@ async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() { let (r, _) = start(&fake, &p).await; assert_eq!(r.unwrap().task.arn, ec2_task_arn("new")); let calls = fake.calls(); - let stop = calls.iter().position(|c| *c == format!("stop {} {STOP_REASON}", ec2_task_arn("old"))).expect("the old task is stopped"); + let stop = calls.iter().position(|c| *c == format!("stop {} {REPLACED_REASON}", ec2_task_arn("old"))).expect("the old task is stopped"); assert!(stop < calls.iter().position(|c| c == "run").unwrap(), "{calls:?}"); }