diff --git a/app/src-tauri/src/host/aws_host.rs b/app/src-tauri/src/host/aws_host.rs index 7047808..8bac289 100644 --- a/app/src-tauri/src/host/aws_host.rs +++ b/app/src-tauri/src/host/aws_host.rs @@ -539,21 +539,58 @@ async fn wait_task( pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Triple-C can't \ reach it. Start the project again."; +/// The most a start waits for a task it stopped to reach STOPPED before +/// running another (its own budget, apart from the placement wait). +const STOP_WAIT: Duration = Duration::from_secs(3 * 60); + +/// A start found the project's task with ECS Exec off and still waiting for +/// an instance: another start is bringing one up. +pub const INSTANCE_RACE: &str = "Another start of this workspace is bringing up an EC2 \ + instance; try again in a minute (once it is up, Start connects to it)"; + +/// Wait (at most `min(timeout, STOP_WAIT)`) until `arn` is STOPPED or gone. +async fn wait_stopped(tasks: &T, cluster: &str, arn: &str, timing: WaitTiming) -> Result<(), String> { + let deadline = tokio::time::Instant::now() + timing.timeout.min(STOP_WAIT); + let wanted = [arn.to_string()]; + loop { + let gone = tasks + .describe(cluster, &wanted) + .await + .map_err(|e| e.to_string())? + .into_iter() + .find(|t| t.arn == arn) + .is_none_or(|t| t.last_status == "STOPPED"); + if gone { + return Ok(()); + } + if tokio::time::Instant::now() + timing.interval > deadline { + return Err(format!( + "The task {arn} did not stop in time; start the project again in a minute." + )); + } + tokio::time::sleep(timing.interval).await; + } +} + /// A cold EC2 start's second RunTask. ECS turns `enableExecuteCommand` off /// on a RunTask that has to wait for an instance (PROVISIONING; verified /// live, phase 7) and the task keeps it off once placed, so it could never /// be reached. That first task still makes the capacity provider launch an -/// instance: once it is placed, it is stopped (it runs nothing yet) and the -/// task run again, now straight onto that instance with ECS Exec. Returns -/// the new task; the caller stops `first` itself if this fails before the -/// stop. +/// instance: once an instance has taken it (and it is still meant to run), +/// it is stopped and the task run again — through [`find_or_run`], so a +/// concurrent start's task is never doubled ([`START_RACE`]) — now straight +/// onto that instance with ECS Exec. A first task that stops by itself (a +/// user's Stop, a failed placement) ends the start with its reason; nothing +/// is run again. Returns the new task. #[allow(clippy::too_many_arguments)] async fn rerun_once_placed( tasks: &T, + env: &EnvironmentInfo, + owner: &str, + project_id: &str, compute: &crate::aws::environments::UserCompute, first: &str, task_def_arn: &str, - tags: &[(String, String)], timing: WaitTiming, progress: &(dyn Fn(&str) + Send + Sync), ) -> Result { @@ -561,17 +598,25 @@ async fn rerun_once_placed( let deadline = tokio::time::Instant::now() + timing.timeout; let wanted = [first.to_string()]; progress(PROGRESS_INSTANCE); - // Until an instance has taken the first task (or it stopped). + // Until an instance has taken the first task. loop { - let t = tasks + let Some(t) = tasks .describe(cluster, &wanted) .await .map_err(|e| e.to_string())? .into_iter() - .find(|t| t.arn == first); - match t { - Some(t) if waiting_for_instance(&t) => {} - _ => break, + .find(|t| t.arn == first) + else { + return Err(format!("The task {first} is no longer known to ECS; start the project again")); + }; + if t.desired_status == "STOPPED" || t.last_status == "STOPPED" { + return Err(format!( + "The task stopped before an EC2 instance took it ({}); start the project again.", + t.stopped_reason.as_deref().unwrap_or("no reason given") + )); + } + if t.container_instance_arn.is_some() && t.desired_status == "RUNNING" { + break; } if tokio::time::Instant::now() + timing.interval > deadline { let max = compute.max_instances; @@ -592,36 +637,19 @@ async fn rerun_once_placed( .stop_task(cluster, first, STOP_REASON) .await .map_err(|e| e.to_string())?; - loop { - let gone = tasks - .describe(cluster, &wanted) - .await - .map_err(|e| e.to_string())? - .into_iter() - .find(|t| t.arn == first) - .is_none_or(|t| t.last_status == "STOPPED"); - if gone { - break; - } - if tokio::time::Instant::now() + timing.interval > deadline { - return Err(format!( - "The first task on the new EC2 instance did not stop within {} minutes; \ - start the project again.", - timing.timeout.as_secs() / 60 - )); - } - tokio::time::sleep(timing.interval).await; - } + wait_stopped(tasks, cluster, first, timing).await?; progress(PROGRESS_WAITING); - let task = tasks - .run_task(crate::aws::tasks::ec2_run_task_request(compute, task_def_arn, tags)) + let start = find_or_run(tasks, env, owner, project_id, task_def_arn, RunOn::Ec2(compute)) .await - .map_err(|e| crate::aws::tasks::ec2_run_error(e).to_string())?; - if !task.exec_enabled { - let _ = stop_created(tasks, cluster, &task.arn).await; + .map_err(|e| e.to_string())?; + if start.reused { + return Err(START_RACE.into()); + } + if !start.task.exec_enabled { + let _ = stop_created(tasks, cluster, &start.task.arn).await; return Err(EXEC_OFF.into()); } - Ok(task) + Ok(start.task) } /// Where a project's task may be: the environment's cluster, the user's EC2 @@ -707,9 +735,18 @@ pub async fn start_core { arn = t.arn; record_task(&arn); } Err(e) => { - let msg = with_log_tail(tasks, &logs, &arn, e).await; + let msg = if e == START_RACE { e } else { with_log_tail(tasks, &logs, &arn, e).await }; if stop_created(tasks, &cluster, &arn).await { (inp.on_task)(None); } diff --git a/app/src-tauri/src/host/aws_host/tests.rs b/app/src-tauri/src/host/aws_host/tests.rs index b9358ae..f1a3ded 100644 --- a/app/src-tauri/src/host/aws_host/tests.rs +++ b/app/src-tauri/src/host/aws_host/tests.rs @@ -177,6 +177,7 @@ impl TaskApi for Fake { // As ECS does: a stopped task is no longer live. if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) { t.desired_status = "STOPPED".into(); + t.last_status = "STOPPED".into(); } Ok(()) } @@ -1461,6 +1462,82 @@ async fn a_second_task_without_ecs_exec_is_stopped_and_reported() { assert!(calls.contains(&format!("stop {} {STOP_REASON}", ec2_task_arn("e9"))), "{calls:?}"); } +#[tokio::test] +async fn a_cold_ec2_task_stopped_before_placement_ends_the_start_with_its_reason() { + let p = ec2_project(); + let execless = |last: &str| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, false) }; + let fake = Fake { + user_record: Some(user_record_param()), + runs: Mutex::new(vec![execless("PROVISIONING")]), + ..Default::default() + }; + *fake.script.lock().unwrap() = vec![ + execless("PROVISIONING"), + TaskInfo { + desired_status: "STOPPED".into(), + stopped_reason: Some("Task stopped by user".into()), + ..execless("PROVISIONING") + }, + ]; + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.contains("Task stopped by user"), "{e}"); + assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "no second RunTask"); +} + +#[tokio::test] +async fn an_execless_task_still_waiting_for_an_instance_is_another_starts() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("other")); + let other = TaskInfo { exec_enabled: false, ..ec2_task(&p, "other", "PROVISIONING", false, false) }; + let fake = Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("other")])])), + ..Default::default() + } + .with_task(other); + let (r, _) = start(&fake, &p).await; + assert_eq!(r.err().unwrap(), INSTANCE_RACE); + assert!(!fake.called("run") && !fake.called("stop"), "{:?}", fake.calls()); +} + +#[tokio::test] +async fn the_cold_reruns_launch_never_doubles_another_starts_task() { + let p = ec2_project(); + let execless = |last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, placed) }; + let winner = ec2_task(&p, "w1", "PENDING", false, true); + let fake = Fake { + user_record: Some(user_record_param()), + runs: Mutex::new(vec![execless("PROVISIONING", false)]), + // Nothing for the start's own finds; the winner's task by the rerun. + listed: Mutex::new(vec![vec![], vec![], vec![], vec![ec2_task_arn("w1")]]), + ..Default::default() + } + .with_task(winner); + *fake.script.lock().unwrap() = vec![execless("PENDING", true), execless("STOPPED", true)]; + let (r, _) = start(&fake, &p).await; + assert_eq!(r.err().unwrap(), START_RACE); + assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "{:?}", fake.calls()); +} + +#[tokio::test] +async fn an_execless_task_that_cannot_be_stopped_fails_the_start() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old")); + let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) }; + let fake = Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])), + stop_fails: true, + ..Default::default() + } + .with_task(old); + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.contains("Could not stop") && e.contains("ExpiredToken"), "{e}"); + assert!(!fake.called("run")); +} + #[tokio::test] async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() { let mut p = ec2_project();