diff --git a/app/src-tauri/src/aws/runtime/tests.rs b/app/src-tauri/src/aws/runtime/tests.rs index be73910..f89dbe8 100644 --- a/app/src-tauri/src/aws/runtime/tests.rs +++ b/app/src-tauri/src/aws/runtime/tests.rs @@ -277,6 +277,7 @@ fn live(id: &str) -> TaskInfo { ), runtime_id: None, container_instance_arn: None, + exec_enabled: true, exec_agent_running: false, exit_code: None, owner: Some(USER.into()), diff --git a/app/src-tauri/src/aws/tasks.rs b/app/src-tauri/src/aws/tasks.rs index b65db02..18c451f 100644 --- a/app/src-tauri/src/aws/tasks.rs +++ b/app/src-tauri/src/aws/tasks.rs @@ -67,6 +67,10 @@ pub struct TaskInfo { pub container_instance_arn: Option, /// The `workspace` container's ExecuteCommandAgent is RUNNING. pub exec_agent_running: bool, + /// ECS Exec is on for the task (`enableExecuteCommand`). ECS turns it + /// off on a RunTask that has to wait for an EC2 instance (verified live, + /// phase 7), and such a task can never be reached. + pub exec_enabled: bool, /// The `workspace` container's exit code, once it has one. pub exit_code: Option, /// Its `triple-c:owner` / `triple-c:project` tags (DescribeTasks with @@ -116,6 +120,7 @@ pub(crate) fn task_info(t: &Task) -> TaskInfo { task_definition_arn: t.task_definition_arn().unwrap_or_default().to_string(), runtime_id: workspace.and_then(|c| c.runtime_id()).map(str::to_string), container_instance_arn: t.container_instance_arn().map(str::to_string), + exec_enabled: t.enable_execute_command(), exec_agent_running: workspace.is_some_and(|c| { c.managed_agents().iter().any(|a| { a.name() == Some(&ManagedAgentName::ExecuteCommandAgent) diff --git a/app/src-tauri/src/aws/tasks/tests.rs b/app/src-tauri/src/aws/tasks/tests.rs index 23c321b..f65133c 100644 --- a/app/src-tauri/src/aws/tasks/tests.rs +++ b/app/src-tauri/src/aws/tasks/tests.rs @@ -65,6 +65,7 @@ fn info(id: &str, last: &str) -> TaskInfo { task_definition_arn: td_arn(3), runtime_id: None, container_instance_arn: None, + exec_enabled: true, exec_agent_running: false, exit_code: None, owner: Some(USER.into()), diff --git a/app/src-tauri/src/host/aws_host.rs b/app/src-tauri/src/host/aws_host.rs index 010daca..1dc4b6c 100644 --- a/app/src-tauri/src/host/aws_host.rs +++ b/app/src-tauri/src/host/aws_host.rs @@ -532,6 +532,96 @@ async fn wait_task( } } +/// What a start says when ECS ran its task with ECS Exec off even with an +/// instance up (it should not happen; the task can't be reached). +pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Triple-C can't \ + reach it. Start the project again."; + +/// A cold EC2 start's second RunTask. ECS turns `enableExecuteCommand` off +/// on a RunTask that has to wait for an instance (PROVISIONING; verified +/// live, phase 7) and the task keeps it off once placed, so it could never +/// be reached. That first task still makes the capacity provider launch an +/// instance: once it is placed, it is stopped (it runs nothing yet) and the +/// task run again, now straight onto that instance with ECS Exec. Returns +/// the new task; the caller stops `first` itself if this fails before the +/// stop. +#[allow(clippy::too_many_arguments)] +async fn rerun_once_placed( + tasks: &T, + compute: &crate::aws::environments::UserCompute, + first: &str, + task_def_arn: &str, + tags: &[(String, String)], + timing: WaitTiming, + progress: &(dyn Fn(&str) + Send + Sync), +) -> Result { + let cluster = compute.cluster_arn.as_str(); + let deadline = tokio::time::Instant::now() + timing.timeout; + let wanted = [first.to_string()]; + progress(PROGRESS_INSTANCE); + // Until an instance has taken the first task (or it stopped). + loop { + let t = tasks + .describe(cluster, &wanted) + .await + .map_err(|e| e.to_string())? + .into_iter() + .find(|t| t.arn == first); + match t { + Some(t) if waiting_for_instance(&t) => {} + _ => break, + } + if tokio::time::Instant::now() + timing.interval > deadline { + let max = compute.max_instances; + return Err(format!( + "No EC2 instance had room for the task within {} minutes. Your capacity may \ + already be running its maximum of {max} instance{} (stop another EC2 \ + workspace), or its Auto Scaling group could not launch one (an admin can see \ + why in the group's activity in the EC2 console).", + timing.timeout.as_secs() / 60, + if max == 1 { "" } else { "s" } + )); + } + tokio::time::sleep(timing.interval).await; + } + // Free its reservation before running again, or the second task would + // wait for an instance too. + tasks + .stop_task(cluster, first, STOP_REASON) + .await + .map_err(|e| e.to_string())?; + loop { + let gone = tasks + .describe(cluster, &wanted) + .await + .map_err(|e| e.to_string())? + .into_iter() + .find(|t| t.arn == first) + .is_none_or(|t| t.last_status == "STOPPED"); + if gone { + break; + } + if tokio::time::Instant::now() + timing.interval > deadline { + return Err(format!( + "The first task on the new EC2 instance did not stop within {} minutes; \ + start the project again.", + timing.timeout.as_secs() / 60 + )); + } + tokio::time::sleep(timing.interval).await; + } + progress(PROGRESS_WAITING); + let task = tasks + .run_task(crate::aws::tasks::ec2_run_task_request(compute, task_def_arn, tags)) + .await + .map_err(|e| crate::aws::tasks::ec2_run_error(e).to_string())?; + if !task.exec_enabled { + let _ = stop_created(tasks, cluster, &task.arn).await; + return Err(EXEC_OFF.into()); + } + Ok(task) +} + /// Where a project's task may be: the environment's cluster, the user's EC2 /// cluster from their record, and the per-user cluster a stored task ARN /// names (the record may be gone while that task runs), without repeats. @@ -613,6 +703,12 @@ pub async fn start_core { + arn = t.arn; + record_task(&arn); + } + Err(e) => { + let msg = with_log_tail(tasks, &logs, &arn, e).await; + if stop_created(tasks, &cluster, &arn).await { + (inp.on_task)(None); + } + return Err(msg); + } + } + } let ec2 = match run_on { RunOn::Ec2(c) => Some(c.max_instances), RunOn::Fargate { .. } => None, diff --git a/app/src-tauri/src/host/aws_host/tests.rs b/app/src-tauri/src/host/aws_host/tests.rs index 031a066..a45a0cb 100644 --- a/app/src-tauri/src/host/aws_host/tests.rs +++ b/app/src-tauri/src/host/aws_host/tests.rs @@ -57,6 +57,7 @@ fn task(p: &Project, id: &str, last: &str, ready: bool) -> TaskInfo { task_definition_arn: format!("arn:aws:ecs:{REGION}:{ACCOUNT}:task-definition/{}:1", family(p)), runtime_id: ready.then(|| "rt-1".into()), container_instance_arn: None, + exec_enabled: true, exec_agent_running: ready, exit_code: None, owner: Some(USER.into()), @@ -86,6 +87,8 @@ struct Fake { tasks: Mutex>, /// What RunTask starts. run: Option, + /// When non-empty, what successive RunTasks start (before `run`). + runs: Mutex>, logs: Vec, role: Option, connect_fails: bool, @@ -134,7 +137,11 @@ impl TaskApi for Fake { async fn run_task(&self, req: RunTaskInput) -> Result { self.log("run"); self.run_requests.lock().unwrap().push(req); - let t = self.run.clone().expect("a run was expected"); + let queued = { + let mut q = self.runs.lock().unwrap(); + (!q.is_empty()).then(|| q.remove(0)) + }; + let t = queued.or_else(|| self.run.clone()).expect("a run was expected"); self.tasks.lock().unwrap().insert(t.arn.clone(), t.clone()); Ok(t) } @@ -167,6 +174,10 @@ impl TaskApi for Fake { if self.stop_fails { return Err(AwsError::Other { message: "ExpiredToken".into() }); } + // As ECS does: a stopped task is no longer live. + if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) { + t.desired_status = "STOPPED".into(); + } Ok(()) } async fn log_tail(&self, g: &str, st: &str, l: i32) -> Result, AwsError> { @@ -1384,6 +1395,90 @@ async fn an_ec2_start_runs_on_the_users_capacity_and_says_when_it_waits_for_an_i assert!(fake.describe_clusters.lock().unwrap().iter().all(|c| *c == user_cluster_arn())); } +/// ECS turns ECS Exec off on a RunTask that has to wait for an instance +/// (verified live): that task only brings the instance up; once placed it +/// is stopped and the task run again, onto the instance, with ECS Exec. +#[tokio::test] +async fn a_cold_ec2_start_runs_the_task_again_once_an_instance_is_up() { + let p = ec2_project(); + let execless = |last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, placed) }; + let fake = Fake { + user_record: Some(user_record_param()), + runs: Mutex::new(vec![execless("PROVISIONING", false), ec2_task(&p, "e9", "PENDING", false, true)]), + ..Default::default() + }; + *fake.script.lock().unwrap() = vec![ + execless("PROVISIONING", false), + execless("PENDING", true), + TaskInfo { desired_status: "STOPPED".into(), ..execless("PENDING", true) }, + execless("STOPPED", true), + ec2_task(&p, "e9", "RUNNING", true, true), + ]; + let (r, steps) = start(&fake, &p).await; + let started = r.unwrap(); + assert_eq!(started.task.arn, ec2_task_arn("e9")); + let calls = fake.calls(); + assert_eq!(calls.iter().filter(|c| *c == "run").count(), 2, "{calls:?}"); + let stop = format!("stop {} {STOP_REASON}", ec2_task_arn("e1")); + let pos = |c: &str| calls.iter().rposition(|x| x == c).unwrap(); + assert!(calls.iter().position(|c| *c == stop).unwrap() < pos("run"), "{calls:?}"); + assert_eq!(*fake.stop_clusters.lock().unwrap(), [user_cluster_arn()]); + // Both requests ask for ECS Exec on the user's capacity. + for req in fake.run_requests.lock().unwrap().iter() { + assert_eq!(req.enable_execute_command(), Some(true)); + assert_eq!(req.capacity_provider_strategy()[0].capacity_provider(), USER_CLUSTER); + } + // The second task is the one recorded. + assert_eq!(*fake.recorded.lock().unwrap(), vec![Some(ec2_task_arn("e1")), Some(ec2_task_arn("e9"))]); + assert_eq!( + steps, + vec![ + PROGRESS_READING, + PROGRESS_PREPARING, + PROGRESS_STARTING, + PROGRESS_WAITING, + PROGRESS_INSTANCE, + PROGRESS_WAITING, + PROGRESS_CONNECTING + ] + ); +} + +#[tokio::test] +async fn a_second_task_without_ecs_exec_is_stopped_and_reported() { + let p = ec2_project(); + let execless = |id: &str, last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, id, last, false, placed) }; + let fake = Fake { + user_record: Some(user_record_param()), + runs: Mutex::new(vec![execless("e1", "PROVISIONING", false), execless("e9", "PENDING", true)]), + ..Default::default() + }; + *fake.script.lock().unwrap() = vec![execless("e1", "PENDING", true), execless("e1", "STOPPED", true)]; + let (r, _) = start(&fake, &p).await; + assert_eq!(r.err().unwrap().lines().next(), Some(EXEC_OFF)); + let calls = fake.calls(); + assert!(calls.contains(&format!("stop {} {STOP_REASON}", ec2_task_arn("e9"))), "{calls:?}"); +} + +#[tokio::test] +async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old")); + let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) }; + let fake = Fake { + user_record: Some(user_record_param()), + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])), + run: Some(ec2_task(&p, "new", "RUNNING", true, true)), + ..Default::default() + } + .with_task(old); + let (r, _) = start(&fake, &p).await; + assert_eq!(r.unwrap().task.arn, ec2_task_arn("new")); + let calls = fake.calls(); + let stop = calls.iter().position(|c| *c == format!("stop {} {STOP_REASON}", ec2_task_arn("old"))).expect("the old task is stopped"); + assert!(stop < calls.iter().position(|c| c == "run").unwrap(), "{calls:?}"); +} + #[tokio::test] async fn a_fargate_start_never_reads_the_ec2_record() { let p = aws_project(StorageMode::Ephemeral);