fix(aws): never attach to an exec-off EC2 task after a restart; widen the orphan margin

Reconcile stops a placed or orphaned exec-off task (REPLACED_REASON,
checked; the ARN is cleared only once the stop succeeded) and marks a
recent one still waiting Error with its ARN kept and an 'interrupted'
warning. Reattach refuses an exec-off task. ORPHAN_MARGIN is 10 minutes
(clock skew). A failed EC2 record read no longer blocks attaching to a
live task. One helper builds the task cluster list.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 18:30:05 -08:00
1 parent 719a75b46d
commit 8c9ffd4c4c
3 files changed
+223 -61

No files matched your search

+27 -11
View File
@@ -112,6 +112,28 @@ pub fn ec2_cluster_of_task(env: &str, task_arn: &str) -> Option<String> {
.and_then(|_| cluster_arn_of_task(task_arn))
}
/// Where a project's task may be: `env_cluster` (the environment's), then
/// the user's EC2 cluster from their record, then the per-user cluster
/// `stored_task_arn` names (the record may be gone while that task runs),
/// without repeats.
pub fn project_task_clusters(
env: &str,
env_cluster: String,
ec2_cluster: Option<String>,
stored_task_arn: Option<&str>,
) -> Vec<String> {
let mut out = vec![env_cluster];
for c in [ec2_cluster, stored_task_arn.and_then(|a| ec2_cluster_of_task(env, a))]
.into_iter()
.flatten()
{
if !out.contains(&c) {
out.push(c);
}
}
out
}
/// Whether the caller's task role exists in the environment: the onboarding
/// signal a workspace user can read. A denied lookup is not an answer, so
/// it is let through (RunTask will say what is wrong).
@@ -296,18 +318,12 @@ pub async fn remove_workspace_resources<T: TaskApi, W: WorkspaceApi, R: RuntimeA
// 1. The task: in the environment's cluster, the user's EC2 cluster (the
// record's, and the stored task's when that is a per-user cluster of
// this environment — the record may be gone).
let mut clusters = vec![cluster.clone()];
for c in [
let clusters = project_task_clusters(
req.env,
cluster.clone(),
req.ec2_cluster.map(str::to_string),
req.stored_task_arn.and_then(|a| ec2_cluster_of_task(req.env, a)),
]
.into_iter()
.flatten()
{
if !clusters.contains(&c) {
clusters.push(c);
}
}
req.stored_task_arn,
);
let mut to_stop: Vec<(String, String)> = Vec::new();
for c in &clusters {
if let Some(t) =
+100 -50
View File
@@ -32,7 +32,7 @@ use super::ContainerTarget;
use crate::aws::environments::EnvironmentInfo;
use crate::aws::environments::UserRecord;
use crate::aws::runtime::{
check_onboarded, cluster_arn_of_task, cluster_of_task, ec2_cluster_of_task,
check_onboarded, cluster_arn_of_task, cluster_of_task, ec2_cluster_of_task, project_task_clusters,
ensure_agent_token, read_agent_token, read_environment, read_user_record,
remove_workspace_resources, wait_access_point, PollTiming, RemoveRequest, RuntimeApi,
SdkRuntimeApi,
@@ -554,7 +554,19 @@ pub const REPLACED_REASON: &str = "Replaced by another Triple-C start";
/// How long past the EC2 placement wait an exec-off task still waiting for
/// an instance counts as another start's, before it is taken as orphaned.
const ORPHAN_MARGIN: Duration = Duration::from_secs(2 * 60);
/// The task's age is ECS's `createdAt` against this machine's clock, so the
/// margin also absorbs clock skew: a local clock more than this far ahead
/// could take another start's waiting task for an orphan and replace it (the
/// other start then says so: "Another start … replaced this one"); one far
/// behind only delays the replacement. NTP-synced clocks are well inside it.
const ORPHAN_MARGIN: Duration = Duration::from_secs(10 * 60);
/// Seconds since the Unix epoch on this machine's clock (0 if it is before).
fn now_unix() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs() as i64)
}
/// Whether an exec-off task still waiting for an instance is older than
/// any start could still be waiting for it (`ec2_wait` + a margin): a
@@ -691,23 +703,15 @@ async fn rerun_once_placed<T: TaskApi>(
Ok(start.task)
}
/// Where a project's task may be: the environment's cluster, the user's EC2
/// cluster from their record, and the per-user cluster a stored task ARN
/// names (the record may be gone while that task runs), without repeats.
/// Where a project's task may be ([`project_task_clusters`]): the
/// environment's cluster, the record's EC2 cluster, the stored task's.
fn task_clusters(env: &EnvironmentInfo, record: Option<&UserRecord>, stored: Option<&str>) -> Vec<String> {
let mut out = vec![env.cluster_arn.clone()];
for c in [
project_task_clusters(
&env.name,
env.cluster_arn.clone(),
record.and_then(UserRecord::valid).map(|c| c.cluster_arn.clone()),
stored.and_then(|a| ec2_cluster_of_task(&env.name, a)),
]
.into_iter()
.flatten()
{
if !out.contains(&c) {
out.push(c);
}
}
out
stored,
)
}
/// The cluster (ARN) a task runs on, from its ARN.
@@ -744,15 +748,19 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
};
// EC2 (phase 7): the user's record says where new tasks run. Read it
// only for EC2 projects; a Fargate start asks exactly what it did.
let record = if cfg.launch_type == LaunchType::Ec2 {
Some(
read_user_record(rt, &env.name, &ok, &inp.who.account, region)
.await
.map_err(s)?,
)
// only for EC2 projects; a Fargate start asks exactly what it did. A
// failed read still lets a live task (found in the environment's
// cluster or the stored task's) be attached; only a new run needs it.
let (record, record_err) = if cfg.launch_type == LaunchType::Ec2 {
match read_user_record(rt, &env.name, &ok, &inp.who.account, region).await {
Ok(r) => (Some(r), None),
Err(e) => {
log::warn!("Project {}: could not read the EC2 record ({e})", project.id);
(None, Some(format!("Could not read your EC2 capacity record: {e}")))
}
}
} else {
None
(None, None)
};
// The EC2 wait (and its progress) for a task on a per-user cluster.
let ec2_of = |task_arn: &str| {
@@ -778,10 +786,7 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
// An old one no start can still be waiting for is an orphan (a crashed
// start): it is replaced like a placed one.
if let Some(t) = existing.take_if(|t| !t.exec_enabled) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs() as i64);
if waiting_for_instance(&t) && !is_orphaned(&t, inp.ec2_wait, now) {
if waiting_for_instance(&t) && !is_orphaned(&t, inp.ec2_wait, now_unix()) {
return Err(INSTANCE_RACE.into());
}
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
@@ -813,7 +818,11 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
(task, token, None, None, false)
} else {
// Where the new task runs; an EC2 project without usable capacity
// fails here, before anything is written.
// (or whose record could not be read) fails here, before anything
// is written.
if let Some(e) = record_err {
return Err(e);
}
let run_on = match &record {
None => RunOn::Fargate { spot: cfg.spot },
Some(UserRecord::Valid(c)) => RunOn::Ec2(c),
@@ -1314,16 +1323,8 @@ impl FindTask {
.map(|p| p.environment.clone())
.unwrap_or_default();
let ok = owner_key(region, owner_user_id);
let mut clusters = vec![environment_cluster_name(&env)];
let stored = project.aws.as_ref().and_then(|a| a.task_arn.as_deref());
for c in [ec2_cluster, stored.and_then(|a| ec2_cluster_of_task(&env, a))]
.into_iter()
.flatten()
{
if !clusters.contains(&c) {
clusters.push(c);
}
}
let clusters = project_task_clusters(&env, environment_cluster_name(&env), ec2_cluster, stored);
FindTask {
clusters,
family: task_family(&env, &ok, &project.id),
@@ -1461,9 +1462,20 @@ pub enum Reconciled {
/// The task is up (or on its way): re-open the tunnel.
Live(Box<TaskInfo>),
/// The task stopped, or ECS no longer knows it: the project is Stopped.
/// Also an EC2 task with ECS Exec off that could never be attached to
/// (placed, or orphaned): reconcile stopped it.
Gone,
/// An EC2 task with ECS Exec off, still waiting for an instance and
/// recent: a start (perhaps this app's, before a restart) was
/// interrupted mid cold start. The project is Error, its ARN kept so
/// Stop still reaches the task; [`INTERRUPTED`] says what to do.
Interrupted,
}
/// What reconcile says about [`Reconciled::Interrupted`].
pub const INTERRUPTED: &str = "A start of this workspace was interrupted while its EC2 \
instance came up; Start again (or Stop it).";
/// Ask ECS about a project's task: the stored ARN, else (none stored) its
/// live task found by family and tags.
pub async fn reconcile_task<T: TaskApi>(
@@ -1541,7 +1553,33 @@ pub async fn reconcile_project<T: TaskApi>(
find: &FindTask,
) -> Result<Reconciled, String> {
let arn = project.aws.as_ref().and_then(|a| a.task_arn.clone());
let r = reconcile_task(api, arn.as_deref(), find).await?;
let mut r = reconcile_task(api, arn.as_deref(), find).await?;
// A task with ECS Exec off can never be attached to (a cold EC2 start
// the app was closed during): never Running. Placed, or too old for any
// start to still wait for it → stop it (then it's Gone); still waiting
// and recent → Interrupted, left for a Start or Stop.
if let Reconciled::Live(t) = &r {
if !t.exec_enabled {
if waiting_for_instance(t) && !is_orphaned(t, WaitTiming::ec2(), now_unix()) {
if arn.as_deref() != Some(t.arn.as_str()) {
record_task(store, &project.id, Some(&live_record(t)));
}
store.update_status(&project.id, ProjectStatus::Error)?;
return Ok(Reconciled::Interrupted);
}
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
let cluster = cluster_of_task(&t.arn).ok_or_else(|| format!("{} is not a task ARN with a cluster", t.arn))?;
if let Err(e) = bounded_stop(api.stop_task(cluster, &t.arn, REPLACED_REASON)).await {
// It may still run: keep (or record) its ARN so Stop finds it.
if arn.as_deref() != Some(t.arn.as_str()) {
record_task(store, &project.id, Some(&live_record(t)));
}
store.update_status(&project.id, ProjectStatus::Error)?;
return Err(format!("Could not stop the workspace's unreachable task {}: {e}", t.arn));
}
r = Reconciled::Gone;
}
}
match &r {
Reconciled::Gone => {
store.update_status(&project.id, ProjectStatus::Stopped)?;
@@ -1549,25 +1587,28 @@ pub async fn reconcile_project<T: TaskApi>(
}
Reconciled::Live(t) => {
if arn.as_deref() != Some(t.arn.as_str()) {
record_task(
store,
&project.id,
Some(&TaskRecord {
task_arn: t.arn.clone(),
access_point_id: None,
task_definition_arn: Some(t.task_definition_arn.clone()),
workspace_access_point_id: None,
}),
);
record_task(store, &project.id, Some(&live_record(t)));
}
if project.status != ProjectStatus::Running {
store.update_status(&project.id, ProjectStatus::Running)?;
}
}
Reconciled::Interrupted => {}
}
Ok(r)
}
/// The store record of a live task reconcile found (no access points: the
/// stored ones stand).
fn live_record(t: &TaskInfo) -> TaskRecord {
TaskRecord {
task_arn: t.arn.clone(),
access_point_id: None,
task_definition_arn: Some(t.task_definition_arn.clone()),
workspace_access_point_id: None,
}
}
/// `reconcile_project_statuses` for an AWS project: decide, and when the
/// task is live re-open its tunnel in the background (terminals wait for
/// it through [`tunnels::wait_endpoint`]), then — copy-at-connect — copy the
@@ -1630,6 +1671,10 @@ pub async fn reconcile(
Ok(Reconciled::Gone) => {
log::info!("Project '{}' ({}): its task is gone — set to Stopped", project.name, project.id)
}
Ok(Reconciled::Interrupted) => {
log::info!("Project '{}' ({}): {INTERRUPTED}", project.name, project.id);
on_connect_warning(INTERRUPTED.to_string());
}
Err(e) => log::warn!("Project {}: could not check its task ({e}); status left as is", project.id),
}
}
@@ -1679,6 +1724,11 @@ async fn reattach(
let task = wait_ready(&api, &cluster, &task.arn, &logs, timing, |_| {})
.await
.map_err(s)?;
// Reconcile never hands over an exec-off task; never wait on a tunnel
// ECS Exec can't open either.
if !task.exec_enabled {
return Err(format!("The task {} has ECS Exec off; Start the project again", task.arn));
}
let (forward, endpoint) = SsmConnector { sdk: ctx.sdk.clone() }.connect(&task, &token).await?;
let tunnel = LiveTunnel {
task: ecs_task(&task)?,
+96
View File
@@ -107,6 +107,8 @@ struct Fake {
log_reads: Mutex<Vec<String>>,
/// The caller's EC2 record (absent by default).
user_record: Option<crate::aws::environments::Param>,
/// Reading the EC2 record fails (throttled, say).
user_record_fails: bool,
/// RunTask requests, in order.
run_requests: Mutex<Vec<RunTaskInput>>,
/// The clusters ListTasks / StopTask / DescribeTasks were asked about.
@@ -263,6 +265,9 @@ impl RuntimeApi for Fake {
}
async fn user_record(&self, env: &str, _owner_key: &str) -> Result<crate::aws::environments::Param, AwsError> {
self.log(format!("user-record {env}"));
if self.user_record_fails {
return Err(AwsError::Other { message: "Throttling".into() });
}
Ok(self.user_record.clone().unwrap_or(crate::aws::environments::Param::Missing))
}
async fn role_exists(&self, _name: &str) -> Result<Knowable, AwsError> {
@@ -1815,3 +1820,94 @@ fn ec2_capacity_reads_as_host_settings_need_it() {
assert_eq!(v, serde_json::json!({ "state": "unusable", "reason": "schema is not 1" }));
assert_eq!(serde_json::to_value(ec2_capacity_from(None)).unwrap(), serde_json::json!({ "state": "unknown" }));
}
/// Reconcile never marks an exec-off task Running (an app restart during a
/// cold EC2 start): a placed or orphaned one is stopped (its ARN cleared
/// only once the stop succeeded); a recent one still waiting is Error with
/// its ARN kept.
#[tokio::test]
async fn reconcile_never_attaches_to_a_task_with_ecs_exec_off() {
let mut p = ec2_project();
p.status = ProjectStatus::Running;
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("x"));
let execless = |last: &str, placed: bool, created: i64| TaskInfo {
exec_enabled: false,
created_unix: Some(created),
..ec2_task(&p, "x", last, false, placed)
};
let find = FindTask::new(&p, REGION, USER, Some(user_cluster_arn()));
let stopped_with_replaced = |f: &Fake| f.calls().contains(&format!("stop {} {REPLACED_REASON}", ec2_task_arn("x")));
// Placed: stopped, Stopped, ARN cleared.
let fake = Fake::default().with_task(execless("PENDING", true, now_unix()));
let (_d, store) = store_with(&p);
assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Gone);
assert!(stopped_with_replaced(&fake), "{:?}", fake.calls());
let after = store.get(&p.id).unwrap();
assert_eq!(after.status, ProjectStatus::Stopped);
assert_eq!(after.aws.unwrap().task_arn, None);
// Orphaned (still waiting, but older than any start waits): the same.
let fake = Fake::default().with_task(execless("PROVISIONING", false, now_unix() - 3600));
let (_d, store) = store_with(&p);
assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Gone);
assert!(stopped_with_replaced(&fake));
assert_eq!(store.get(&p.id).unwrap().status, ProjectStatus::Stopped);
// The stop fails: Error, the ARN kept.
let fake = Fake { stop_fails: true, ..Default::default() }.with_task(execless("RUNNING", true, now_unix()));
let (_d, store) = store_with(&p);
let e = reconcile_project(&fake, &store, &p, &find).await.unwrap_err();
assert!(e.contains("Could not stop") && e.contains("ExpiredToken"), "{e}");
let after = store.get(&p.id).unwrap();
assert_eq!(after.status, ProjectStatus::Error);
assert_eq!(after.aws.unwrap().task_arn, Some(ec2_task_arn("x")));
// Recent and still waiting: Interrupted (Error), ARN kept, nothing stopped.
let fake = Fake::default().with_task(execless("PROVISIONING", false, now_unix() - 60));
let (_d, store) = store_with(&p);
assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Interrupted);
assert!(!fake.called("stop"));
let after = store.get(&p.id).unwrap();
assert_eq!(after.status, ProjectStatus::Error);
assert_eq!(after.aws.unwrap().task_arn, Some(ec2_task_arn("x")));
assert!(INTERRUPTED.contains("Start again"));
}
#[test]
fn an_orphan_is_older_than_the_ec2_wait_plus_ten_minutes() {
assert_eq!(ORPHAN_MARGIN, Duration::from_secs(10 * 60));
let p = ec2_project();
let at = |age: i64| TaskInfo { created_unix: Some(1_000_000 - age), ..ec2_task(&p, "o", "PROVISIONING", false, false) };
let wait = WaitTiming::ec2();
assert!(!is_orphaned(&at(19 * 60), wait, 1_000_000));
assert!(is_orphaned(&at(21 * 60), wait, 1_000_000));
assert!(!is_orphaned(&TaskInfo { created_unix: None, ..at(0) }, wait, 1_000_000));
}
/// A failed read of the EC2 record does not stop an attach: the stored
/// task's cluster is still searched. Only a new run needs the record.
#[tokio::test]
async fn an_ec2_record_read_error_still_attaches_and_fails_only_a_new_run() {
let mut p = ec2_project();
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("e8"));
let live = ec2_task(&p, "e8", "RUNNING", true, true);
let fake = Fake {
user_record_fails: true,
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![live.arn.clone()])])),
..Default::default()
}
.with_task(live);
let (r, _) = start(&fake, &p).await;
let started = r.unwrap();
assert!(!started.created);
assert_eq!(started.task.arn, ec2_task_arn("e8"));
assert_eq!(*fake.list_clusters.lock().unwrap(), [cluster_arn(), user_cluster_arn()]);
// Nothing live: the start fails with the read error, before any write.
let fake = Fake { user_record_fails: true, ..Default::default() };
let (r, _) = start(&fake, &p).await;
let e = r.err().unwrap();
assert!(e.contains("Could not read your EC2 capacity record") && e.contains("Throttling"), "{e}");
assert!(!fake.called("put-secret") && !fake.called("run"));
}