fix(aws): never run a second task for a stopped or contested cold EC2 start
The cold-start rerun now runs again only once an instance has taken the first task and it is still meant to run; a first task that stops by itself (a user's Stop, a failed placement) ends the start with its reason. The second launch goes through find_or_run, so a concurrent start's task is reported as START_RACE instead of doubled. A found task with ECS Exec off that is still waiting for an instance belongs to a start in progress (INSTANCE_RACE); a placed one is stopped, the stop is checked and waited for (its own 3-minute budget) before a new run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fc3456c44e
commit
c25761789e
2 files changed
+154
-41
No files matched your search
@@ -539,21 +539,58 @@ async fn wait_task<T: TaskApi>(
|
||||
pub const EXEC_OFF: &str = "ECS started the task with ECS Exec turned off, so Triple-C can't \
|
||||
reach it. Start the project again.";
|
||||
|
||||
/// The most a start waits for a task it stopped to reach STOPPED before
|
||||
/// running another (its own budget, apart from the placement wait).
|
||||
const STOP_WAIT: Duration = Duration::from_secs(3 * 60);
|
||||
|
||||
/// A start found the project's task with ECS Exec off and still waiting for
|
||||
/// an instance: another start is bringing one up.
|
||||
pub const INSTANCE_RACE: &str = "Another start of this workspace is bringing up an EC2 \
|
||||
instance; try again in a minute (once it is up, Start connects to it)";
|
||||
|
||||
/// Wait (at most `min(timeout, STOP_WAIT)`) until `arn` is STOPPED or gone.
|
||||
async fn wait_stopped<T: TaskApi>(tasks: &T, cluster: &str, arn: &str, timing: WaitTiming) -> Result<(), String> {
|
||||
let deadline = tokio::time::Instant::now() + timing.timeout.min(STOP_WAIT);
|
||||
let wanted = [arn.to_string()];
|
||||
loop {
|
||||
let gone = tasks
|
||||
.describe(cluster, &wanted)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.into_iter()
|
||||
.find(|t| t.arn == arn)
|
||||
.is_none_or(|t| t.last_status == "STOPPED");
|
||||
if gone {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() + timing.interval > deadline {
|
||||
return Err(format!(
|
||||
"The task {arn} did not stop in time; start the project again in a minute."
|
||||
));
|
||||
}
|
||||
tokio::time::sleep(timing.interval).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// A cold EC2 start's second RunTask. ECS turns `enableExecuteCommand` off
|
||||
/// on a RunTask that has to wait for an instance (PROVISIONING; verified
|
||||
/// live, phase 7) and the task keeps it off once placed, so it could never
|
||||
/// be reached. That first task still makes the capacity provider launch an
|
||||
/// instance: once it is placed, it is stopped (it runs nothing yet) and the
|
||||
/// task run again, now straight onto that instance with ECS Exec. Returns
|
||||
/// the new task; the caller stops `first` itself if this fails before the
|
||||
/// stop.
|
||||
/// instance: once an instance has taken it (and it is still meant to run),
|
||||
/// it is stopped and the task run again — through [`find_or_run`], so a
|
||||
/// concurrent start's task is never doubled ([`START_RACE`]) — now straight
|
||||
/// onto that instance with ECS Exec. A first task that stops by itself (a
|
||||
/// user's Stop, a failed placement) ends the start with its reason; nothing
|
||||
/// is run again. Returns the new task.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn rerun_once_placed<T: TaskApi>(
|
||||
tasks: &T,
|
||||
env: &EnvironmentInfo,
|
||||
owner: &str,
|
||||
project_id: &str,
|
||||
compute: &crate::aws::environments::UserCompute,
|
||||
first: &str,
|
||||
task_def_arn: &str,
|
||||
tags: &[(String, String)],
|
||||
timing: WaitTiming,
|
||||
progress: &(dyn Fn(&str) + Send + Sync),
|
||||
) -> Result<TaskInfo, String> {
|
||||
@@ -561,17 +598,25 @@ async fn rerun_once_placed<T: TaskApi>(
|
||||
let deadline = tokio::time::Instant::now() + timing.timeout;
|
||||
let wanted = [first.to_string()];
|
||||
progress(PROGRESS_INSTANCE);
|
||||
// Until an instance has taken the first task (or it stopped).
|
||||
// Until an instance has taken the first task.
|
||||
loop {
|
||||
let t = tasks
|
||||
let Some(t) = tasks
|
||||
.describe(cluster, &wanted)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.into_iter()
|
||||
.find(|t| t.arn == first);
|
||||
match t {
|
||||
Some(t) if waiting_for_instance(&t) => {}
|
||||
_ => break,
|
||||
.find(|t| t.arn == first)
|
||||
else {
|
||||
return Err(format!("The task {first} is no longer known to ECS; start the project again"));
|
||||
};
|
||||
if t.desired_status == "STOPPED" || t.last_status == "STOPPED" {
|
||||
return Err(format!(
|
||||
"The task stopped before an EC2 instance took it ({}); start the project again.",
|
||||
t.stopped_reason.as_deref().unwrap_or("no reason given")
|
||||
));
|
||||
}
|
||||
if t.container_instance_arn.is_some() && t.desired_status == "RUNNING" {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() + timing.interval > deadline {
|
||||
let max = compute.max_instances;
|
||||
@@ -592,36 +637,19 @@ async fn rerun_once_placed<T: TaskApi>(
|
||||
.stop_task(cluster, first, STOP_REASON)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
loop {
|
||||
let gone = tasks
|
||||
.describe(cluster, &wanted)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.into_iter()
|
||||
.find(|t| t.arn == first)
|
||||
.is_none_or(|t| t.last_status == "STOPPED");
|
||||
if gone {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() + timing.interval > deadline {
|
||||
return Err(format!(
|
||||
"The first task on the new EC2 instance did not stop within {} minutes; \
|
||||
start the project again.",
|
||||
timing.timeout.as_secs() / 60
|
||||
));
|
||||
}
|
||||
tokio::time::sleep(timing.interval).await;
|
||||
}
|
||||
wait_stopped(tasks, cluster, first, timing).await?;
|
||||
progress(PROGRESS_WAITING);
|
||||
let task = tasks
|
||||
.run_task(crate::aws::tasks::ec2_run_task_request(compute, task_def_arn, tags))
|
||||
let start = find_or_run(tasks, env, owner, project_id, task_def_arn, RunOn::Ec2(compute))
|
||||
.await
|
||||
.map_err(|e| crate::aws::tasks::ec2_run_error(e).to_string())?;
|
||||
if !task.exec_enabled {
|
||||
let _ = stop_created(tasks, cluster, &task.arn).await;
|
||||
.map_err(|e| e.to_string())?;
|
||||
if start.reused {
|
||||
return Err(START_RACE.into());
|
||||
}
|
||||
if !start.task.exec_enabled {
|
||||
let _ = stop_created(tasks, cluster, &start.task.arn).await;
|
||||
return Err(EXEC_OFF.into());
|
||||
}
|
||||
Ok(task)
|
||||
Ok(start.task)
|
||||
}
|
||||
|
||||
/// Where a project's task may be: the environment's cluster, the user's EC2
|
||||
@@ -707,9 +735,18 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
|
||||
}
|
||||
// A task with ECS Exec off (a cold EC2 start that stopped halfway) can
|
||||
// never be attached to: stop it and start afresh.
|
||||
// One still waiting for an instance belongs to a start in progress.
|
||||
if let Some(t) = existing.take_if(|t| !t.exec_enabled) {
|
||||
if waiting_for_instance(&t) {
|
||||
return Err(INSTANCE_RACE.into());
|
||||
}
|
||||
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
|
||||
let _ = stop_created(tasks, &task_cluster(&env, &t.arn), &t.arn).await;
|
||||
let cluster = task_cluster(&env, &t.arn);
|
||||
tasks
|
||||
.stop_task(&cluster, &t.arn, STOP_REASON)
|
||||
.await
|
||||
.map_err(|e| format!("Could not stop the workspace's unreachable task {}: {e}", t.arn))?;
|
||||
wait_stopped(tasks, &cluster, &t.arn, timing_of(Some(1))).await?;
|
||||
}
|
||||
|
||||
let (task, token, access_point_id, workspace_access_point_id, created) = if let Some(task) = existing {
|
||||
@@ -802,14 +839,13 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
|
||||
progress(PROGRESS_WAITING);
|
||||
let cluster = run_on.cluster(&env).to_string();
|
||||
if let (RunOn::Ec2(compute), false) = (run_on, start.task.exec_enabled) {
|
||||
let tags = crate::aws::workspace::workspace_tags(&inp.who.user_id, &project.id, &env.name);
|
||||
match rerun_once_placed(tasks, compute, &arn, &prepared.task_definition_arn, &tags, inp.ec2_wait, progress).await {
|
||||
match rerun_once_placed(tasks, &env, &inp.who.user_id, &project.id, compute, &arn, &prepared.task_definition_arn, inp.ec2_wait, progress).await {
|
||||
Ok(t) => {
|
||||
arn = t.arn;
|
||||
record_task(&arn);
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = with_log_tail(tasks, &logs, &arn, e).await;
|
||||
let msg = if e == START_RACE { e } else { with_log_tail(tasks, &logs, &arn, e).await };
|
||||
if stop_created(tasks, &cluster, &arn).await {
|
||||
(inp.on_task)(None);
|
||||
}
|
||||
|
||||
@@ -177,6 +177,7 @@ impl TaskApi for Fake {
|
||||
// As ECS does: a stopped task is no longer live.
|
||||
if let Some(t) = self.tasks.lock().unwrap().get_mut(arn) {
|
||||
t.desired_status = "STOPPED".into();
|
||||
t.last_status = "STOPPED".into();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1461,6 +1462,82 @@ async fn a_second_task_without_ecs_exec_is_stopped_and_reported() {
|
||||
assert!(calls.contains(&format!("stop {} {STOP_REASON}", ec2_task_arn("e9"))), "{calls:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_cold_ec2_task_stopped_before_placement_ends_the_start_with_its_reason() {
|
||||
let p = ec2_project();
|
||||
let execless = |last: &str| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, false) };
|
||||
let fake = Fake {
|
||||
user_record: Some(user_record_param()),
|
||||
runs: Mutex::new(vec![execless("PROVISIONING")]),
|
||||
..Default::default()
|
||||
};
|
||||
*fake.script.lock().unwrap() = vec![
|
||||
execless("PROVISIONING"),
|
||||
TaskInfo {
|
||||
desired_status: "STOPPED".into(),
|
||||
stopped_reason: Some("Task stopped by user".into()),
|
||||
..execless("PROVISIONING")
|
||||
},
|
||||
];
|
||||
let (r, _) = start(&fake, &p).await;
|
||||
let e = r.err().unwrap();
|
||||
assert!(e.contains("Task stopped by user"), "{e}");
|
||||
assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "no second RunTask");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_execless_task_still_waiting_for_an_instance_is_another_starts() {
|
||||
let mut p = ec2_project();
|
||||
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("other"));
|
||||
let other = TaskInfo { exec_enabled: false, ..ec2_task(&p, "other", "PROVISIONING", false, false) };
|
||||
let fake = Fake {
|
||||
user_record: Some(user_record_param()),
|
||||
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("other")])])),
|
||||
..Default::default()
|
||||
}
|
||||
.with_task(other);
|
||||
let (r, _) = start(&fake, &p).await;
|
||||
assert_eq!(r.err().unwrap(), INSTANCE_RACE);
|
||||
assert!(!fake.called("run") && !fake.called("stop"), "{:?}", fake.calls());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_cold_reruns_launch_never_doubles_another_starts_task() {
|
||||
let p = ec2_project();
|
||||
let execless = |last: &str, placed: bool| TaskInfo { exec_enabled: false, ..ec2_task(&p, "e1", last, false, placed) };
|
||||
let winner = ec2_task(&p, "w1", "PENDING", false, true);
|
||||
let fake = Fake {
|
||||
user_record: Some(user_record_param()),
|
||||
runs: Mutex::new(vec![execless("PROVISIONING", false)]),
|
||||
// Nothing for the start's own finds; the winner's task by the rerun.
|
||||
listed: Mutex::new(vec![vec![], vec![], vec![], vec![ec2_task_arn("w1")]]),
|
||||
..Default::default()
|
||||
}
|
||||
.with_task(winner);
|
||||
*fake.script.lock().unwrap() = vec![execless("PENDING", true), execless("STOPPED", true)];
|
||||
let (r, _) = start(&fake, &p).await;
|
||||
assert_eq!(r.err().unwrap(), START_RACE);
|
||||
assert_eq!(fake.calls().iter().filter(|c| *c == "run").count(), 1, "{:?}", fake.calls());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_execless_task_that_cannot_be_stopped_fails_the_start() {
|
||||
let mut p = ec2_project();
|
||||
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("old"));
|
||||
let old = TaskInfo { exec_enabled: false, ..ec2_task(&p, "old", "RUNNING", false, true) };
|
||||
let fake = Fake {
|
||||
user_record: Some(user_record_param()),
|
||||
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![ec2_task_arn("old")])])),
|
||||
stop_fails: true,
|
||||
..Default::default()
|
||||
}
|
||||
.with_task(old);
|
||||
let (r, _) = start(&fake, &p).await;
|
||||
let e = r.err().unwrap();
|
||||
assert!(e.contains("Could not stop") && e.contains("ExpiredToken"), "{e}");
|
||||
assert!(!fake.called("run"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_live_task_with_ecs_exec_off_is_stopped_and_replaced() {
|
||||
let mut p = ec2_project();
|
||||
|
||||
Reference in new issue
Block a user