fix(aws): run a cold EC2 task again once its instance is up

ECS turns enableExecuteCommand off on a RunTask that has to wait for an
instance (PROVISIONING behind a capacity provider), and the task keeps it
off once placed: verified live (CloudTrail shows true in the request,
false in the answer; a RunTask with an instance up keeps it). Such a
task can never be reached, so every cold EC2 start timed out.

A cold start now lets that first task bring the instance up, stops it as
soon as it is placed, waits for it to stop and runs the task again onto
the instance; a second task without ECS Exec is stopped and reported. A
live task found with ECS Exec off is stopped and replaced.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 16:34:38 -08:00
1 parent bf5a810e0d
commit 43fcc520ab
5 files changed
+225 -8

No files matched your search

+1
View File
@@ -277,6 +277,7 @@ fn live(id: &str) -> TaskInfo {
),
runtime_id: None,
container_instance_arn: None,
exec_enabled: true,
exec_agent_running: false,
exit_code: None,
owner: Some(USER.into()),
+5
View File
@@ -67,6 +67,10 @@ pub struct TaskInfo {
pub container_instance_arn: Option<String>,
/// The `workspace` container's ExecuteCommandAgent is RUNNING.
pub exec_agent_running: bool,
/// ECS Exec is on for the task (`enableExecuteCommand`). ECS turns it
/// off on a RunTask that has to wait for an EC2 instance (verified live,
/// phase 7), and such a task can never be reached.
pub exec_enabled: bool,
/// The `workspace` container's exit code, once it has one.
pub exit_code: Option<i32>,
/// Its `triple-c:owner` / `triple-c:project` tags (DescribeTasks with
@@ -116,6 +120,7 @@ pub(crate) fn task_info(t: &Task) -> TaskInfo {
task_definition_arn: t.task_definition_arn().unwrap_or_default().to_string(),
runtime_id: workspace.and_then(|c| c.runtime_id()).map(str::to_string),
container_instance_arn: t.container_instance_arn().map(str::to_string),
exec_enabled: t.enable_execute_command(),
exec_agent_running: workspace.is_some_and(|c| {
c.managed_agents().iter().any(|a| {
a.name() == Some(&ManagedAgentName::ExecuteCommandAgent)
+1
View File
@@ -65,6 +65,7 @@ fn info(id: &str, last: &str) -> TaskInfo {
task_definition_arn: td_arn(3),
runtime_id: None,
container_instance_arn: None,
exec_enabled: true,
exec_agent_running: false,
exit_code: None,
owner: Some(USER.into()),
+122 -7
View File
@@ -532,6 +532,96 @@ async fn wait_task<T: TaskApi>(
}
}
/// What a start says when ECS ran its task with ECS Exec off even with an
/// instance up (it should not happen; the task can't be reached).
pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Triple-C can't \
reach it. Start the project again.";
/// A cold EC2 start's second RunTask. ECS turns `enableExecuteCommand` off
/// on a RunTask that has to wait for an instance (PROVISIONING; verified
/// live, phase 7) and the task keeps it off once placed, so it could never
/// be reached. That first task still makes the capacity provider launch an
/// instance: once it is placed, it is stopped (it runs nothing yet) and the
/// task run again, now straight onto that instance with ECS Exec. Returns
/// the new task; the caller stops `first` itself if this fails before the
/// stop.
#[allow(clippy::too_many_arguments)]
async fn rerun_once_placed<T: TaskApi>(
tasks: &T,
compute: &crate::aws::environments::UserCompute,
first: &str,
task_def_arn: &str,
tags: &[(String, String)],
timing: WaitTiming,
progress: &(dyn Fn(&str) + Send + Sync),
) -> Result<TaskInfo, String> {
let cluster = compute.cluster_arn.as_str();
let deadline = tokio::time::Instant::now() + timing.timeout;
let wanted = [first.to_string()];
progress(PROGRESS_INSTANCE);
// Until an instance has taken the first task (or it stopped).
loop {
let t = tasks
.describe(cluster, &wanted)
.await
.map_err(|e| e.to_string())?
.into_iter()
.find(|t| t.arn == first);
match t {
Some(t) if waiting_for_instance(&t) => {}
_ => break,
}
if tokio::time::Instant::now() + timing.interval > deadline {
let max = compute.max_instances;
return Err(format!(
"No EC2 instance had room for the task within {} minutes. Your capacity may \
already be running its maximum of {max} instance{} (stop another EC2 \
workspace), or its Auto Scaling group could not launch one (an admin can see \
why in the group's activity in the EC2 console).",
timing.timeout.as_secs() / 60,
if max == 1 { "" } else { "s" }
));
}
tokio::time::sleep(timing.interval).await;
}
// Free its reservation before running again, or the second task would
// wait for an instance too.
tasks
.stop_task(cluster, first, STOP_REASON)
.await
.map_err(|e| e.to_string())?;
loop {
let gone = tasks
.describe(cluster, &wanted)
.await
.map_err(|e| e.to_string())?
.into_iter()
.find(|t| t.arn == first)
.is_none_or(|t| t.last_status == "STOPPED");
if gone {
break;
}
if tokio::time::Instant::now() + timing.interval > deadline {
return Err(format!(
"The first task on the new EC2 instance did not stop within {} minutes; \
start the project again.",
timing.timeout.as_secs() / 60
));
}
tokio::time::sleep(timing.interval).await;
}
progress(PROGRESS_WAITING);
let task = tasks
.run_task(crate::aws::tasks::ec2_run_task_request(compute, task_def_arn, tags))
.await
.map_err(|e| crate::aws::tasks::ec2_run_error(e).to_string())?;
if !task.exec_enabled {
let _ = stop_created(tasks, cluster, &task.arn).await;
return Err(EXEC_OFF.into());
}
Ok(task)
}
/// Where a project's task may be: the environment's cluster, the user's EC2
/// cluster from their record, and the per-user cluster a stored task ARN
/// names (the record may be gone while that task runs), without repeats.
@@ -613,6 +703,12 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
break;
}
}
// A task with ECS Exec off (a cold EC2 start that stopped halfway) can
// never be attached to: stop it and start afresh.
if let Some(t) = existing.take_if(|t| !t.exec_enabled) {
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
let _ = stop_created(tasks, &task_cluster(&env, &t.arn), &t.arn).await;
}
let (task, token, access_point_id, workspace_access_point_id, created) = if let Some(task) = existing {
// Attach: never rotate a running task's token (P2).
@@ -691,15 +787,34 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
}));
return Err(START_RACE.into());
}
let arn = start.task.arn.clone();
(inp.on_task)(Some(&TaskRecord {
task_arn: arn.clone(),
access_point_id: prepared.access_point_id.clone(),
task_definition_arn: Some(prepared.task_definition_arn.clone()),
workspace_access_point_id: prepared.workspace_access_point_id.clone(),
}));
let mut arn = start.task.arn.clone();
let record_task = |arn: &str| {
(inp.on_task)(Some(&TaskRecord {
task_arn: arn.to_string(),
access_point_id: prepared.access_point_id.clone(),
task_definition_arn: Some(prepared.task_definition_arn.clone()),
workspace_access_point_id: prepared.workspace_access_point_id.clone(),
}))
};
record_task(&arn);
progress(PROGRESS_WAITING);
let cluster = run_on.cluster(&env).to_string();
if let (RunOn::Ec2(compute), false) = (run_on, start.task.exec_enabled) {
let tags = crate::aws::workspace::workspace_tags(&inp.who.user_id, &project.id, &env.name);
match rerun_once_placed(tasks, compute, &arn, &prepared.task_definition_arn, &tags, inp.ec2_wait, progress).await {
Ok(t) => {
arn = t.arn;
record_task(&arn);
}
Err(e) => {
let msg = with_log_tail(tasks, &logs, &arn, e).await;
if stop_created(tasks, &cluster, &arn).await {
(inp.on_task)(None);
}
return Err(msg);
}
}
}
let ec2 = match run_on {
RunOn::Ec2(c) => Some(c.max_instances),
RunOn::Fargate { .. } => None,
+96 -1
View File
@@ -57,6 +57,7 @@ fn task(p: &Project, id: &str, last: &str, ready: bool) -> TaskInfo {
task_definition_arn: format!("arn:aws:ecs:{REGION}:{ACCOUNT}:task-definition/{}:1", family(p)),
runtime_id: ready.then(|| "rt-1".into()),
container_instance_arn: None,
exec_enabled: true,
exec_agent_running: ready,
exit_code: None,
owner: Some(USER.into()),
@@ -86,6 +87,8 @@ struct Fake {
tasks: Mutex<HashMap<String, TaskInfo>>,
/// What RunTask starts.
run: Option<TaskInfo>,
/// When non-empty, what successive RunTasks start (before `run`).
runs: Mutex<Vec<TaskInfo>>,
logs: Vec<String>,
role: Option<Knowable>,
connect_fails: bool,
@@ -134,7 +137,11 @@ impl TaskApi for Fake {
async fn run_task(&self, req: RunTaskInput) -> Result<TaskInfo, AwsError> {
self.log("run");
self.run_requests.lock().unwrap().push(req);
let t = self.run.clone().expect("a run was expected");
let queued = {
let mut q = self.runs.lock().unwrap();
(!q.is_empty()).then(|| q.remove(0))
};
let t = queued.or_else(|| self.run.clone()).expect("a run was expected");
self.tasks.lock().unwrap().insert(t.arn.clone(), t.clone());
Ok(t)
}
@@ -167,6 +174,10 @@ impl TaskApi for Fake {
if self.stop_fails {
return Err(AwsError::Other { message: "ExpiredToken".into() });
}
// As ECS does: a stopped task is no longer live.
if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) {
t.desired_status = "STOPPED".into();
}
Ok(())
}
async fn log_tail(&self, g: &str, st: &str, l: i32) -> Result<Vec<String>, AwsError> {
@@ -1384,6 +1395,90 @@ async fn an_ec2_start_runs_on_the_users_capacity_and_says_when_it_waits_for_an_i
assert!(fake.describe_clusters.lock().unwrap().iter().all(|c| *c == user_cluster_arn()));
}
/// ECS turns ECS Exec off on a RunTask that has to wait for an instance
/// (verified live): that task only brings the instance up; once placed it
/// is stopped and the task run again, onto the instance, with ECS Exec.
#[tokio::test]
async fn a_cold_ec2_start_runs_the_task_again_once_an_instance_is_up() {
let p = ec2_project();
let execless = |last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, placed) };
let fake = Fake {
user_record: Some(user_record_param()),
runs: Mutex::new(vec![execless("PROVISIONING", false), ec2_task(&p, "e9", "PENDING", false, true)]),
..Default::default()
};
*fake.script.lock().unwrap() = vec![
execless("PROVISIONING", false),
execless("PENDING", true),
TaskInfo { desired_status: "STOPPED".into(), ..execless("PENDING", true) },
execless("STOPPED", true),
ec2_task(&p, "e9", "RUNNING", true, true),
];
let (r, steps) = start(&fake, &p).await;
let started = r.unwrap();
assert_eq!(started.task.arn, ec2_task_arn("e9"));
let calls = fake.calls();
assert_eq!(calls.iter().filter(|c| *c == "run").count(), 2, "{calls:?}");
let stop = format!("stop {} {STOP_REASON}", ec2_task_arn("e1"));
let pos = |c: &str| calls.iter().rposition(|x| x == c).unwrap();
assert!(calls.iter().position(|c| *c == stop).unwrap() < pos("run"), "{calls:?}");
assert_eq!(*fake.stop_clusters.lock().unwrap(), [user_cluster_arn()]);
// Both requests ask for ECS Exec on the user's capacity.
for req in fake.run_requests.lock().unwrap().iter() {
assert_eq!(req.enable_execute_command(), Some(true));
assert_eq!(req.capacity_provider_strategy()[0].capacity_provider(), USER_CLUSTER);
}
// The second task is the one recorded.
assert_eq!(*fake.recorded.lock().unwrap(), vec![Some(ec2_task_arn("e1")), Some(ec2_task_arn("e9"))]);
assert_eq!(
steps,
vec![
PROGRESS_READING,
PROGRESS_PREPARING,
PROGRESS_STARTING,
PROGRESS_WAITING,
PROGRESS_INSTANCE,
PROGRESS_WAITING,
PROGRESS_CONNECTING
]
);
}
#[tokio::test]
async fn a_second_task_without_ecs_exec_is_stopped_and_reported() {
let p = ec2_project();
let execless = |id: &str, last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, id, last, false, placed) };
let fake = Fake {
user_record: Some(user_record_param()),
runs: Mutex::new(vec![execless("e1", "PROVISIONING", false), execless("e9", "PENDING", true)]),
..Default::default()
};
*fake.script.lock().unwrap() = vec![execless("e1", "PENDING", true), execless("e1", "STOPPED", true)];
let (r, _) = start(&fake, &p).await;
assert_eq!(r.err().unwrap().lines().next(), Some(EXEC_OFF));
let calls = fake.calls();
assert!(calls.contains(&format!("stop {} {STOP_REASON}", ec2_task_arn("e9"))), "{calls:?}");
}
#[tokio::test]
async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() {
let mut p = ec2_project();
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old"));
let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) };
let fake = Fake {
user_record: Some(user_record_param()),
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])),
run: Some(ec2_task(&p, "new", "RUNNING", true, true)),
..Default::default()
}
.with_task(old);
let (r, _) = start(&fake, &p).await;
assert_eq!(r.unwrap().task.arn, ec2_task_arn("new"));
let calls = fake.calls();
let stop = calls.iter().position(|c| *c == format!("stop {} {STOP_REASON}", ec2_task_arn("old"))).expect("the old task is stopped");
assert!(stop < calls.iter().position(|c| c == "run").unwrap(), "{calls:?}");
}
#[tokio::test]
async fn a_fargate_start_never_reads_the_ec2_record() {
let p = aws_project(StorageMode::Ephemeral);