fix(aws): never run a second task for a stopped or contested cold EC2 start

The cold-start rerun now runs again only once an instance has taken the
first task and it is still meant to run; a first task that stops by
itself (a user's Stop, a failed placement) ends the start with its
reason. The second launch goes through find_or_run, so a concurrent
start's task is reported as START_RACE instead of doubled. A found task
with ECS Exec off that is still waiting for an instance belongs to a
start in progress (INSTANCE_RACE); a placed one is stopped, the stop is
checked and waited for (its own 3-minute budget) before a new run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 17:59:09 -08:00
1 parent fc3456c44e
commit c25761789e
2 files changed
+154 -41

No files matched your search

+77 -41
View File
@@ -539,21 +539,58 @@ async fn wait_task<T: TaskApi>(
pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Triple-C can't \
reach it. Start the project again.";
/// The most a start waits for a task it stopped to reach STOPPED before
/// running another (its own budget, apart from the placement wait).
const STOP_WAIT: Duration = Duration::from_secs(3 * 60);
/// A start found the project's task with ECS Exec off and still waiting for
/// an instance: another start is bringing one up.
pub const INSTANCE_RACE: &str = "Another start of this workspace is bringing up an EC2 \
instance; try again in a minute (once it is up, Start connects to it)";
/// Wait (at most `min(timeout, STOP_WAIT)`) until `arn` is STOPPED or gone.
async fn wait_stopped<T: TaskApi>(tasks: &T, cluster: &str, arn: &str, timing: WaitTiming) -> Result<(), String> {
let deadline = tokio::time::Instant::now() + timing.timeout.min(STOP_WAIT);
let wanted = [arn.to_string()];
loop {
let gone = tasks
.describe(cluster, &wanted)
.await
.map_err(|e| e.to_string())?
.into_iter()
.find(|t| t.arn == arn)
.is_none_or(|t| t.last_status == "STOPPED");
if gone {
return Ok(());
}
if tokio::time::Instant::now() + timing.interval > deadline {
return Err(format!(
"The task {arn} did not stop in time; start the project again in a minute."
));
}
tokio::time::sleep(timing.interval).await;
}
}
/// A cold EC2 start's second RunTask. ECS turns `enableExecuteCommand` off
/// on a RunTask that has to wait for an instance (PROVISIONING; verified
/// live, phase 7) and the task keeps it off once placed, so it could never
/// be reached. That first task still makes the capacity provider launch an
/// instance: once it is placed, it is stopped (it runs nothing yet) and the
/// task run again, now straight onto that instance with ECS Exec. Returns
/// the new task; the caller stops `first` itself if this fails before the
/// stop.
/// instance: once an instance has taken it (and it is still meant to run),
/// it is stopped and the task run again — through [`find_or_run`], so a
/// concurrent start's task is never doubled ([`START_RACE`]) — now straight
/// onto that instance with ECS Exec. A first task that stops by itself (a
/// user's Stop, a failed placement) ends the start with its reason; nothing
/// is run again. Returns the new task.
#[allow(clippy::too_many_arguments)]
async fn rerun_once_placed<T: TaskApi>(
tasks: &T,
env: &EnvironmentInfo,
owner: &str,
project_id: &str,
compute: &crate::aws::environments::UserCompute,
first: &str,
task_def_arn: &str,
tags: &[(String, String)],
timing: WaitTiming,
progress: &(dyn Fn(&str) + Send + Sync),
) -> Result<TaskInfo, String> {
@@ -561,17 +598,25 @@ async fn rerun_once_placed<T: TaskApi>(
let deadline = tokio::time::Instant::now() + timing.timeout;
let wanted = [first.to_string()];
progress(PROGRESS_INSTANCE);
// Until an instance has taken the first task (or it stopped).
// Until an instance has taken the first task.
loop {
let t = tasks
let Some(t) = tasks
.describe(cluster, &wanted)
.await
.map_err(|e| e.to_string())?
.into_iter()
.find(|t| t.arn == first);
match t {
Some(t) if waiting_for_instance(&t) => {}
_ => break,
.find(|t| t.arn == first)
else {
return Err(format!("The task {first} is no longer known to ECS; start the project again"));
};
if t.desired_status == "STOPPED" || t.last_status == "STOPPED" {
return Err(format!(
"The task stopped before an EC2 instance took it ({}); start the project again.",
t.stopped_reason.as_deref().unwrap_or("no reason given")
));
}
if t.container_instance_arn.is_some() && t.desired_status == "RUNNING" {
break;
}
if tokio::time::Instant::now() + timing.interval > deadline {
let max = compute.max_instances;
@@ -592,36 +637,19 @@ async fn rerun_once_placed<T: TaskApi>(
.stop_task(cluster, first, STOP_REASON)
.await
.map_err(|e| e.to_string())?;
loop {
let gone = tasks
.describe(cluster, &wanted)
.await
.map_err(|e| e.to_string())?
.into_iter()
.find(|t| t.arn == first)
.is_none_or(|t| t.last_status == "STOPPED");
if gone {
break;
}
if tokio::time::Instant::now() + timing.interval > deadline {
return Err(format!(
"The first task on the new EC2 instance did not stop within {} minutes; \
start the project again.",
timing.timeout.as_secs() / 60
));
}
tokio::time::sleep(timing.interval).await;
}
wait_stopped(tasks, cluster, first, timing).await?;
progress(PROGRESS_WAITING);
let task = tasks
.run_task(crate::aws::tasks::ec2_run_task_request(compute, task_def_arn, tags))
let start = find_or_run(tasks, env, owner, project_id, task_def_arn, RunOn::Ec2(compute))
.await
.map_err(|e| crate::aws::tasks::ec2_run_error(e).to_string())?;
if !task.exec_enabled {
let _ = stop_created(tasks, cluster, &task.arn).await;
.map_err(|e| e.to_string())?;
if start.reused {
return Err(START_RACE.into());
}
if !start.task.exec_enabled {
let _ = stop_created(tasks, cluster, &start.task.arn).await;
return Err(EXEC_OFF.into());
}
Ok(task)
Ok(start.task)
}
/// Where a project's task may be: the environment's cluster, the user's EC2
@@ -707,9 +735,18 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
}
// A task with ECS Exec off (a cold EC2 start that stopped halfway) can
// never be attached to: stop it and start afresh.
// One still waiting for an instance belongs to a start in progress.
if let Some(t) = existing.take_if(|t| !t.exec_enabled) {
if waiting_for_instance(&t) {
return Err(INSTANCE_RACE.into());
}
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
let _ = stop_created(tasks, &task_cluster(&env, &t.arn), &t.arn).await;
let cluster = task_cluster(&env, &t.arn);
tasks
.stop_task(&cluster, &t.arn, STOP_REASON)
.await
.map_err(|e| format!("Could not stop the workspace's unreachable task {}: {e}", t.arn))?;
wait_stopped(tasks, &cluster, &t.arn, timing_of(Some(1))).await?;
}
let (task, token, access_point_id, workspace_access_point_id, created) = if let Some(task) = existing {
@@ -802,14 +839,13 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
progress(PROGRESS_WAITING);
let cluster = run_on.cluster(&env).to_string();
if let (RunOn::Ec2(compute), false) = (run_on, start.task.exec_enabled) {
let tags = crate::aws::workspace::workspace_tags(&inp.who.user_id, &project.id, &env.name);
match rerun_once_placed(tasks, compute, &arn, &prepared.task_definition_arn, &tags, inp.ec2_wait, progress).await {
match rerun_once_placed(tasks, &env, &inp.who.user_id, &project.id, compute, &arn, &prepared.task_definition_arn, inp.ec2_wait, progress).await {
Ok(t) => {
arn = t.arn;
record_task(&arn);
}
Err(e) => {
let msg = with_log_tail(tasks, &logs, &arn, e).await;
let msg = if e == START_RACE { e } else { with_log_tail(tasks, &logs, &arn, e).await };
if stop_created(tasks, &cluster, &arn).await {
(inp.on_task)(None);
}
+77
View File
@@ -177,6 +177,7 @@ impl TaskApi for Fake {
// As ECS does: a stopped task is no longer live.
if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) {
t.desired_status = "STOPPED".into();
t.last_status = "STOPPED".into();
}
Ok(())
}
@@ -1461,6 +1462,82 @@ async fn a_second_task_without_ecs_exec_is_stopped_and_reported() {
assert!(calls.contains(&format!("stop {} {STOP_REASON}", ec2_task_arn("e9"))), "{calls:?}");
}
#[tokio::test]
async fn a_cold_ec2_task_stopped_before_placement_ends_the_start_with_its_reason() {
let p = ec2_project();
let execless = |last: &str| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, false) };
let fake = Fake {
user_record: Some(user_record_param()),
runs: Mutex::new(vec![execless("PROVISIONING")]),
..Default::default()
};
*fake.script.lock().unwrap() = vec![
execless("PROVISIONING"),
TaskInfo {
desired_status: "STOPPED".into(),
stopped_reason: Some("Task stopped by user".into()),
..execless("PROVISIONING")
},
];
let (r, _) = start(&fake, &p).await;
let e = r.err().unwrap();
assert!(e.contains("Task stopped by user"), "{e}");
assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "no second RunTask");
}
#[tokio::test]
async fn an_execless_task_still_waiting_for_an_instance_is_another_starts() {
let mut p = ec2_project();
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("other"));
let other = TaskInfo { exec_enabled: false, ..ec2_task(&p, "other", "PROVISIONING", false, false) };
let fake = Fake {
user_record: Some(user_record_param()),
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("other")])])),
..Default::default()
}
.with_task(other);
let (r, _) = start(&fake, &p).await;
assert_eq!(r.err().unwrap(), INSTANCE_RACE);
assert!(!fake.called("run") && !fake.called("stop"), "{:?}", fake.calls());
}
#[tokio::test]
async fn the_cold_reruns_launch_never_doubles_another_starts_task() {
let p = ec2_project();
let execless = |last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, placed) };
let winner = ec2_task(&p, "w1", "PENDING", false, true);
let fake = Fake {
user_record: Some(user_record_param()),
runs: Mutex::new(vec![execless("PROVISIONING", false)]),
// Nothing for the start's own finds; the winner's task by the rerun.
listed: Mutex::new(vec![vec![], vec![], vec![], vec![ec2_task_arn("w1")]]),
..Default::default()
}
.with_task(winner);
*fake.script.lock().unwrap() = vec![execless("PENDING", true), execless("STOPPED", true)];
let (r, _) = start(&fake, &p).await;
assert_eq!(r.err().unwrap(), START_RACE);
assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "{:?}", fake.calls());
}
#[tokio::test]
async fn an_execless_task_that_cannot_be_stopped_fails_the_start() {
let mut p = ec2_project();
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old"));
let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) };
let fake = Fake {
user_record: Some(user_record_param()),
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])),
stop_fails: true,
..Default::default()
}
.with_task(old);
let (r, _) = start(&fake, &p).await;
let e = r.err().unwrap();
assert!(e.contains("Could not stop") && e.contains("ExpiredToken"), "{e}");
assert!(!fake.called("run"));
}
#[tokio::test]
async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() {
let mut p = ec2_project();