diff --git a/app/src-tauri/src/aws/runtime.rs b/app/src-tauri/src/aws/runtime.rs index 8ec989a..13fa50c 100644 --- a/app/src-tauri/src/aws/runtime.rs +++ b/app/src-tauri/src/aws/runtime.rs @@ -112,6 +112,28 @@ pub fn ec2_cluster_of_task(env: &str, task_arn: &str) -> Option { .and_then(|_| cluster_arn_of_task(task_arn)) } +/// Where a project's task may be: `env_cluster` (the environment's), then +/// the user's EC2 cluster from their record, then the per-user cluster +/// `stored_task_arn` names (the record may be gone while that task runs), +/// without repeats. +pub fn project_task_clusters( + env: &str, + env_cluster: String, + ec2_cluster: Option, + stored_task_arn: Option<&str>, +) -> Vec { + let mut out = vec![env_cluster]; + for c in [ec2_cluster, stored_task_arn.and_then(|a| ec2_cluster_of_task(env, a))] + .into_iter() + .flatten() + { + if !out.contains(&c) { + out.push(c); + } + } + out +} + /// Whether the caller's task role exists in the environment: the onboarding /// signal a workspace user can read. A denied lookup is not an answer, so /// it is let through (RunTask will say what is wrong). @@ -296,18 +318,12 @@ pub async fn remove_workspace_resources = Vec::new(); for c in &clusters { if let Some(t) = diff --git a/app/src-tauri/src/host/aws_host.rs b/app/src-tauri/src/host/aws_host.rs index 038f2db..4d96423 100644 --- a/app/src-tauri/src/host/aws_host.rs +++ b/app/src-tauri/src/host/aws_host.rs @@ -32,7 +32,7 @@ use super::ContainerTarget; use crate::aws::environments::EnvironmentInfo; use crate::aws::environments::UserRecord; use crate::aws::runtime::{ - check_onboarded, cluster_arn_of_task, cluster_of_task, ec2_cluster_of_task, + check_onboarded, cluster_arn_of_task, cluster_of_task, ec2_cluster_of_task, project_task_clusters, ensure_agent_token, read_agent_token, read_environment, read_user_record, remove_workspace_resources, wait_access_point, PollTiming, RemoveRequest, RuntimeApi, SdkRuntimeApi, @@ -554,7 +554,19 @@ pub const REPLACED_REASON: &str = "Replaced by another Triple-C start"; /// How long past the EC2 placement wait an exec-off task still waiting for /// an instance counts as another start's, before it is taken as orphaned. -const ORPHAN_MARGIN: Duration = Duration::from_secs(2 * 60); +/// The task's age is ECS's `createdAt` against this machine's clock, so the +/// margin also absorbs clock skew: a local clock more than this far ahead +/// could take another start's waiting task for an orphan and replace it (the +/// other start then says so: "Another start … replaced this one"); one far +/// behind only delays the replacement. NTP-synced clocks are well inside it. +const ORPHAN_MARGIN: Duration = Duration::from_secs(10 * 60); + +/// Seconds since the Unix epoch on this machine's clock (0 if it is before). +fn now_unix() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs() as i64) +} /// Whether an exec-off task still waiting for an instance is older than /// any start could still be waiting for it (`ec2_wait` + a margin): a @@ -691,23 +703,15 @@ async fn rerun_once_placed( Ok(start.task) } -/// Where a project's task may be: the environment's cluster, the user's EC2 -/// cluster from their record, and the per-user cluster a stored task ARN -/// names (the record may be gone while that task runs), without repeats. +/// Where a project's task may be ([`project_task_clusters`]): the +/// environment's cluster, the record's EC2 cluster, the stored task's. fn task_clusters(env: &EnvironmentInfo, record: Option<&UserRecord>, stored: Option<&str>) -> Vec { - let mut out = vec![env.cluster_arn.clone()]; - for c in [ + project_task_clusters( + &env.name, + env.cluster_arn.clone(), record.and_then(UserRecord::valid).map(|c| c.cluster_arn.clone()), - stored.and_then(|a| ec2_cluster_of_task(&env.name, a)), - ] - .into_iter() - .flatten() - { - if !out.contains(&c) { - out.push(c); - } - } - out + stored, + ) } /// The cluster (ARN) a task runs on, from its ARN. @@ -744,15 +748,19 @@ pub async fn start_core (Some(r), None), + Err(e) => { + log::warn!("Project {}: could not read the EC2 record ({e})", project.id); + (None, Some(format!("Could not read your EC2 capacity record: {e}"))) + } + } } else { - None + (None, None) }; // The EC2 wait (and its progress) for a task on a per-user cluster. let ec2_of = |task_arn: &str| { @@ -778,10 +786,7 @@ pub async fn start_core RunOn::Fargate { spot: cfg.spot }, Some(UserRecord::Valid(c)) => RunOn::Ec2(c), @@ -1314,16 +1323,8 @@ impl FindTask { .map(|p| p.environment.clone()) .unwrap_or_default(); let ok = owner_key(region, owner_user_id); - let mut clusters = vec![environment_cluster_name(&env)]; let stored = project.aws.as_ref().and_then(|a| a.task_arn.as_deref()); - for c in [ec2_cluster, stored.and_then(|a| ec2_cluster_of_task(&env, a))] - .into_iter() - .flatten() - { - if !clusters.contains(&c) { - clusters.push(c); - } - } + let clusters = project_task_clusters(&env, environment_cluster_name(&env), ec2_cluster, stored); FindTask { clusters, family: task_family(&env, &ok, &project.id), @@ -1461,9 +1462,20 @@ pub enum Reconciled { /// The task is up (or on its way): re-open the tunnel. Live(Box), /// The task stopped, or ECS no longer knows it: the project is Stopped. + /// Also an EC2 task with ECS Exec off that could never be attached to + /// (placed, or orphaned): reconcile stopped it. Gone, + /// An EC2 task with ECS Exec off, still waiting for an instance and + /// recent: a start (perhaps this app's, before a restart) was + /// interrupted mid cold start. The project is Error, its ARN kept so + /// Stop still reaches the task; [`INTERRUPTED`] says what to do. + Interrupted, } +/// What reconcile says about [`Reconciled::Interrupted`]. +pub const INTERRUPTED: &str = "A start of this workspace was interrupted while its EC2 \ + instance came up; Start again (or Stop it)."; + /// Ask ECS about a project's task: the stored ARN, else (none stored) its /// live task found by family and tags. pub async fn reconcile_task( @@ -1541,7 +1553,33 @@ pub async fn reconcile_project( find: &FindTask, ) -> Result { let arn = project.aws.as_ref().and_then(|a| a.task_arn.clone()); - let r = reconcile_task(api, arn.as_deref(), find).await?; + let mut r = reconcile_task(api, arn.as_deref(), find).await?; + // A task with ECS Exec off can never be attached to (a cold EC2 start + // the app was closed during): never Running. Placed, or too old for any + // start to still wait for it → stop it (then it's Gone); still waiting + // and recent → Interrupted, left for a Start or Stop. + if let Reconciled::Live(t) = &r { + if !t.exec_enabled { + if waiting_for_instance(t) && !is_orphaned(t, WaitTiming::ec2(), now_unix()) { + if arn.as_deref() != Some(t.arn.as_str()) { + record_task(store, &project.id, Some(&live_record(t))); + } + store.update_status(&project.id, ProjectStatus::Error)?; + return Ok(Reconciled::Interrupted); + } + log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn); + let cluster = cluster_of_task(&t.arn).ok_or_else(|| format!("{} is not a task ARN with a cluster", t.arn))?; + if let Err(e) = bounded_stop(api.stop_task(cluster, &t.arn, REPLACED_REASON)).await { + // It may still run: keep (or record) its ARN so Stop finds it. + if arn.as_deref() != Some(t.arn.as_str()) { + record_task(store, &project.id, Some(&live_record(t))); + } + store.update_status(&project.id, ProjectStatus::Error)?; + return Err(format!("Could not stop the workspace's unreachable task {}: {e}", t.arn)); + } + r = Reconciled::Gone; + } + } match &r { Reconciled::Gone => { store.update_status(&project.id, ProjectStatus::Stopped)?; @@ -1549,25 +1587,28 @@ pub async fn reconcile_project( } Reconciled::Live(t) => { if arn.as_deref() != Some(t.arn.as_str()) { - record_task( - store, - &project.id, - Some(&TaskRecord { - task_arn: t.arn.clone(), - access_point_id: None, - task_definition_arn: Some(t.task_definition_arn.clone()), - workspace_access_point_id: None, - }), - ); + record_task(store, &project.id, Some(&live_record(t))); } if project.status != ProjectStatus::Running { store.update_status(&project.id, ProjectStatus::Running)?; } } + Reconciled::Interrupted => {} } Ok(r) } +/// The store record of a live task reconcile found (no access points: the +/// stored ones stand). +fn live_record(t: &TaskInfo) -> TaskRecord { + TaskRecord { + task_arn: t.arn.clone(), + access_point_id: None, + task_definition_arn: Some(t.task_definition_arn.clone()), + workspace_access_point_id: None, + } +} + /// `reconcile_project_statuses` for an AWS project: decide, and when the /// task is live re-open its tunnel in the background (terminals wait for /// it through [`tunnels::wait_endpoint`]), then — copy-at-connect — copy the @@ -1630,6 +1671,10 @@ pub async fn reconcile( Ok(Reconciled::Gone) => { log::info!("Project '{}' ({}): its task is gone — set to Stopped", project.name, project.id) } + Ok(Reconciled::Interrupted) => { + log::info!("Project '{}' ({}): {INTERRUPTED}", project.name, project.id); + on_connect_warning(INTERRUPTED.to_string()); + } Err(e) => log::warn!("Project {}: could not check its task ({e}); status left as is", project.id), } } @@ -1679,6 +1724,11 @@ async fn reattach( let task = wait_ready(&api, &cluster, &task.arn, &logs, timing, |_| {}) .await .map_err(s)?; + // Reconcile never hands over an exec-off task; never wait on a tunnel + // ECS Exec can't open either. + if !task.exec_enabled { + return Err(format!("The task {} has ECS Exec off; Start the project again", task.arn)); + } let (forward, endpoint) = SsmConnector { sdk: ctx.sdk.clone() }.connect(&task, &token).await?; let tunnel = LiveTunnel { task: ecs_task(&task)?, diff --git a/app/src-tauri/src/host/aws_host/tests.rs b/app/src-tauri/src/host/aws_host/tests.rs index dbb972b..04b0b72 100644 --- a/app/src-tauri/src/host/aws_host/tests.rs +++ b/app/src-tauri/src/host/aws_host/tests.rs @@ -107,6 +107,8 @@ struct Fake { log_reads: Mutex>, /// The caller's EC2 record (absent by default). user_record: Option, + /// Reading the EC2 record fails (throttled, say). + user_record_fails: bool, /// RunTask requests, in order. run_requests: Mutex>, /// The clusters ListTasks / StopTask / DescribeTasks were asked about. @@ -263,6 +265,9 @@ impl RuntimeApi for Fake { } async fn user_record(&self, env: &str, _owner_key: &str) -> Result { self.log(format!("user-record {env}")); + if self.user_record_fails { + return Err(AwsError::Other { message: "Throttling".into() }); + } Ok(self.user_record.clone().unwrap_or(crate::aws::environments::Param::Missing)) } async fn role_exists(&self, _name: &str) -> Result { @@ -1815,3 +1820,94 @@ fn ec2_capacity_reads_as_host_settings_need_it() { assert_eq!(v, serde_json::json!({ "state": "unusable", "reason": "schema is not 1" })); assert_eq!(serde_json::to_value(ec2_capacity_from(None)).unwrap(), serde_json::json!({ "state": "unknown" })); } + +/// Reconcile never marks an exec-off task Running (an app restart during a +/// cold EC2 start): a placed or orphaned one is stopped (its ARN cleared +/// only once the stop succeeded); a recent one still waiting is Error with +/// its ARN kept. +#[tokio::test] +async fn reconcile_never_attaches_to_a_task_with_ecs_exec_off() { + let mut p = ec2_project(); + p.status = ProjectStatus::Running; + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("x")); + let execless = |last: &str, placed: bool, created: i64| TaskInfo { + exec_enabled: false, + created_unix: Some(created), + ..ec2_task(&p, "x", last, false, placed) + }; + let find = FindTask::new(&p, REGION, USER, Some(user_cluster_arn())); + let stopped_with_replaced = |f: &Fake| f.calls().contains(&format!("stop {} {REPLACED_REASON}", ec2_task_arn("x"))); + + // Placed: stopped, Stopped, ARN cleared. + let fake = Fake::default().with_task(execless("PENDING", true, now_unix())); + let (_d, store) = store_with(&p); + assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Gone); + assert!(stopped_with_replaced(&fake), "{:?}", fake.calls()); + let after = store.get(&p.id).unwrap(); + assert_eq!(after.status, ProjectStatus::Stopped); + assert_eq!(after.aws.unwrap().task_arn, None); + + // Orphaned (still waiting, but older than any start waits): the same. + let fake = Fake::default().with_task(execless("PROVISIONING", false, now_unix() - 3600)); + let (_d, store) = store_with(&p); + assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Gone); + assert!(stopped_with_replaced(&fake)); + assert_eq!(store.get(&p.id).unwrap().status, ProjectStatus::Stopped); + + // The stop fails: Error, the ARN kept. + let fake = Fake { stop_fails: true, ..Default::default() }.with_task(execless("RUNNING", true, now_unix())); + let (_d, store) = store_with(&p); + let e = reconcile_project(&fake, &store, &p, &find).await.unwrap_err(); + assert!(e.contains("Could not stop") && e.contains("ExpiredToken"), "{e}"); + let after = store.get(&p.id).unwrap(); + assert_eq!(after.status, ProjectStatus::Error); + assert_eq!(after.aws.unwrap().task_arn, Some(ec2_task_arn("x"))); + + // Recent and still waiting: Interrupted (Error), ARN kept, nothing stopped. + let fake = Fake::default().with_task(execless("PROVISIONING", false, now_unix() - 60)); + let (_d, store) = store_with(&p); + assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Interrupted); + assert!(!fake.called("stop")); + let after = store.get(&p.id).unwrap(); + assert_eq!(after.status, ProjectStatus::Error); + assert_eq!(after.aws.unwrap().task_arn, Some(ec2_task_arn("x"))); + assert!(INTERRUPTED.contains("Start again")); +} + +#[test] +fn an_orphan_is_older_than_the_ec2_wait_plus_ten_minutes() { + assert_eq!(ORPHAN_MARGIN, Duration::from_secs(10 * 60)); + let p = ec2_project(); + let at = |age: i64| TaskInfo { created_unix: Some(1_000_000 - age), ..ec2_task(&p, "o", "PROVISIONING", false, false) }; + let wait = WaitTiming::ec2(); + assert!(!is_orphaned(&at(19 * 60), wait, 1_000_000)); + assert!(is_orphaned(&at(21 * 60), wait, 1_000_000)); + assert!(!is_orphaned(&TaskInfo { created_unix: None, ..at(0) }, wait, 1_000_000)); +} + +/// A failed read of the EC2 record does not stop an attach: the stored +/// task's cluster is still searched. Only a new run needs the record. +#[tokio::test] +async fn an_ec2_record_read_error_still_attaches_and_fails_only_a_new_run() { + let mut p = ec2_project(); + p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("e8")); + let live = ec2_task(&p, "e8", "RUNNING", true, true); + let fake = Fake { + user_record_fails: true, + listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![live.arn.clone()])])), + ..Default::default() + } + .with_task(live); + let (r, _) = start(&fake, &p).await; + let started = r.unwrap(); + assert!(!started.created); + assert_eq!(started.task.arn, ec2_task_arn("e8")); + assert_eq!(*fake.list_clusters.lock().unwrap(), [cluster_arn(), user_cluster_arn()]); + + // Nothing live: the start fails with the read error, before any write. + let fake = Fake { user_record_fails: true, ..Default::default() }; + let (r, _) = start(&fake, &p).await; + let e = r.err().unwrap(); + assert!(e.contains("Could not read your EC2 capacity record") && e.contains("Throttling"), "{e}"); + assert!(!fake.called("put-secret") && !fake.called("run")); +}