fix(aws): never attach to an exec-off EC2 task after a restart; widen the orphan margin
Reconcile stops a placed or orphaned exec-off task (REPLACED_REASON, checked; the ARN is cleared only once the stop succeeded) and marks a recent one still waiting Error with its ARN kept and an 'interrupted' warning. Reattach refuses an exec-off task. ORPHAN_MARGIN is 10 minutes (clock skew). A failed EC2 record read no longer blocks attaching to a live task. One helper builds the task cluster list. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
719a75b46d
commit
8c9ffd4c4c
3 files changed
+223
-61
No files matched your search
@@ -112,6 +112,28 @@ pub fn ec2_cluster_of_task(env: &str, task_arn: &str) -> Option<String> {
|
||||
.and_then(|_| cluster_arn_of_task(task_arn))
|
||||
}
|
||||
|
||||
/// Where a project's task may be: `env_cluster` (the environment's), then
|
||||
/// the user's EC2 cluster from their record, then the per-user cluster
|
||||
/// `stored_task_arn` names (the record may be gone while that task runs),
|
||||
/// without repeats.
|
||||
pub fn project_task_clusters(
|
||||
env: &str,
|
||||
env_cluster: String,
|
||||
ec2_cluster: Option<String>,
|
||||
stored_task_arn: Option<&str>,
|
||||
) -> Vec<String> {
|
||||
let mut out = vec![env_cluster];
|
||||
for c in [ec2_cluster, stored_task_arn.and_then(|a| ec2_cluster_of_task(env, a))]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !out.contains(&c) {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether the caller's task role exists in the environment: the onboarding
|
||||
/// signal a workspace user can read. A denied lookup is not an answer, so
|
||||
/// it is let through (RunTask will say what is wrong).
|
||||
@@ -296,18 +318,12 @@ pub async fn remove_workspace_resources<T: TaskApi, W: WorkspaceApi, R: RuntimeA
|
||||
// 1. The task: in the environment's cluster, the user's EC2 cluster (the
|
||||
// record's, and the stored task's when that is a per-user cluster of
|
||||
// this environment — the record may be gone).
|
||||
let mut clusters = vec![cluster.clone()];
|
||||
for c in [
|
||||
let clusters = project_task_clusters(
|
||||
req.env,
|
||||
cluster.clone(),
|
||||
req.ec2_cluster.map(str::to_string),
|
||||
req.stored_task_arn.and_then(|a| ec2_cluster_of_task(req.env, a)),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !clusters.contains(&c) {
|
||||
clusters.push(c);
|
||||
}
|
||||
}
|
||||
req.stored_task_arn,
|
||||
);
|
||||
let mut to_stop: Vec<(String, String)> = Vec::new();
|
||||
for c in &clusters {
|
||||
if let Some(t) =
|
||||
|
||||
@@ -32,7 +32,7 @@ use super::ContainerTarget;
|
||||
use crate::aws::environments::EnvironmentInfo;
|
||||
use crate::aws::environments::UserRecord;
|
||||
use crate::aws::runtime::{
|
||||
check_onboarded, cluster_arn_of_task, cluster_of_task, ec2_cluster_of_task,
|
||||
check_onboarded, cluster_arn_of_task, cluster_of_task, ec2_cluster_of_task, project_task_clusters,
|
||||
ensure_agent_token, read_agent_token, read_environment, read_user_record,
|
||||
remove_workspace_resources, wait_access_point, PollTiming, RemoveRequest, RuntimeApi,
|
||||
SdkRuntimeApi,
|
||||
@@ -554,7 +554,19 @@ pub const REPLACED_REASON: &str = "Replaced by another Triple-C start";
|
||||
|
||||
/// How long past the EC2 placement wait an exec-off task still waiting for
|
||||
/// an instance counts as another start's, before it is taken as orphaned.
|
||||
const ORPHAN_MARGIN: Duration = Duration::from_secs(2 * 60);
|
||||
/// The task's age is ECS's `createdAt` against this machine's clock, so the
|
||||
/// margin also absorbs clock skew: a local clock more than this far ahead
|
||||
/// could take another start's waiting task for an orphan and replace it (the
|
||||
/// other start then says so: "Another start … replaced this one"); one far
|
||||
/// behind only delays the replacement. NTP-synced clocks are well inside it.
|
||||
const ORPHAN_MARGIN: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
/// Seconds since the Unix epoch on this machine's clock (0 if it is before).
|
||||
fn now_unix() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_secs() as i64)
|
||||
}
|
||||
|
||||
/// Whether an exec-off task still waiting for an instance is older than
|
||||
/// any start could still be waiting for it (`ec2_wait` + a margin): a
|
||||
@@ -691,23 +703,15 @@ async fn rerun_once_placed<T: TaskApi>(
|
||||
Ok(start.task)
|
||||
}
|
||||
|
||||
/// Where a project's task may be: the environment's cluster, the user's EC2
|
||||
/// cluster from their record, and the per-user cluster a stored task ARN
|
||||
/// names (the record may be gone while that task runs), without repeats.
|
||||
/// Where a project's task may be ([`project_task_clusters`]): the
|
||||
/// environment's cluster, the record's EC2 cluster, the stored task's.
|
||||
fn task_clusters(env: &EnvironmentInfo, record: Option<&UserRecord>, stored: Option<&str>) -> Vec<String> {
|
||||
let mut out = vec![env.cluster_arn.clone()];
|
||||
for c in [
|
||||
project_task_clusters(
|
||||
&env.name,
|
||||
env.cluster_arn.clone(),
|
||||
record.and_then(UserRecord::valid).map(|c| c.cluster_arn.clone()),
|
||||
stored.and_then(|a| ec2_cluster_of_task(&env.name, a)),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !out.contains(&c) {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out
|
||||
stored,
|
||||
)
|
||||
}
|
||||
|
||||
/// The cluster (ARN) a task runs on, from its ARN.
|
||||
@@ -744,15 +748,19 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
|
||||
};
|
||||
|
||||
// EC2 (phase 7): the user's record says where new tasks run. Read it
|
||||
// only for EC2 projects; a Fargate start asks exactly what it did.
|
||||
let record = if cfg.launch_type == LaunchType::Ec2 {
|
||||
Some(
|
||||
read_user_record(rt, &env.name, &ok, &inp.who.account, region)
|
||||
.await
|
||||
.map_err(s)?,
|
||||
)
|
||||
// only for EC2 projects; a Fargate start asks exactly what it did. A
|
||||
// failed read still lets a live task (found in the environment's
|
||||
// cluster or the stored task's) be attached; only a new run needs it.
|
||||
let (record, record_err) = if cfg.launch_type == LaunchType::Ec2 {
|
||||
match read_user_record(rt, &env.name, &ok, &inp.who.account, region).await {
|
||||
Ok(r) => (Some(r), None),
|
||||
Err(e) => {
|
||||
log::warn!("Project {}: could not read the EC2 record ({e})", project.id);
|
||||
(None, Some(format!("Could not read your EC2 capacity record: {e}")))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
// The EC2 wait (and its progress) for a task on a per-user cluster.
|
||||
let ec2_of = |task_arn: &str| {
|
||||
@@ -778,10 +786,7 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
|
||||
// An old one no start can still be waiting for is an orphan (a crashed
|
||||
// start): it is replaced like a placed one.
|
||||
if let Some(t) = existing.take_if(|t| !t.exec_enabled) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |d| d.as_secs() as i64);
|
||||
if waiting_for_instance(&t) && !is_orphaned(&t, inp.ec2_wait, now) {
|
||||
if waiting_for_instance(&t) && !is_orphaned(&t, inp.ec2_wait, now_unix()) {
|
||||
return Err(INSTANCE_RACE.into());
|
||||
}
|
||||
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
|
||||
@@ -813,7 +818,11 @@ pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector
|
||||
(task, token, None, None, false)
|
||||
} else {
|
||||
// Where the new task runs; an EC2 project without usable capacity
|
||||
// fails here, before anything is written.
|
||||
// (or whose record could not be read) fails here, before anything
|
||||
// is written.
|
||||
if let Some(e) = record_err {
|
||||
return Err(e);
|
||||
}
|
||||
let run_on = match &record {
|
||||
None => RunOn::Fargate { spot: cfg.spot },
|
||||
Some(UserRecord::Valid(c)) => RunOn::Ec2(c),
|
||||
@@ -1314,16 +1323,8 @@ impl FindTask {
|
||||
.map(|p| p.environment.clone())
|
||||
.unwrap_or_default();
|
||||
let ok = owner_key(region, owner_user_id);
|
||||
let mut clusters = vec![environment_cluster_name(&env)];
|
||||
let stored = project.aws.as_ref().and_then(|a| a.task_arn.as_deref());
|
||||
for c in [ec2_cluster, stored.and_then(|a| ec2_cluster_of_task(&env, a))]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !clusters.contains(&c) {
|
||||
clusters.push(c);
|
||||
}
|
||||
}
|
||||
let clusters = project_task_clusters(&env, environment_cluster_name(&env), ec2_cluster, stored);
|
||||
FindTask {
|
||||
clusters,
|
||||
family: task_family(&env, &ok, &project.id),
|
||||
@@ -1461,9 +1462,20 @@ pub enum Reconciled {
|
||||
/// The task is up (or on its way): re-open the tunnel.
|
||||
Live(Box<TaskInfo>),
|
||||
/// The task stopped, or ECS no longer knows it: the project is Stopped.
|
||||
/// Also an EC2 task with ECS Exec off that could never be attached to
|
||||
/// (placed, or orphaned): reconcile stopped it.
|
||||
Gone,
|
||||
/// An EC2 task with ECS Exec off, still waiting for an instance and
|
||||
/// recent: a start (perhaps this app's, before a restart) was
|
||||
/// interrupted mid cold start. The project is Error, its ARN kept so
|
||||
/// Stop still reaches the task; [`INTERRUPTED`] says what to do.
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
/// What reconcile says about [`Reconciled::Interrupted`].
|
||||
pub const INTERRUPTED: &str = "A start of this workspace was interrupted while its EC2 \
|
||||
instance came up; Start again (or Stop it).";
|
||||
|
||||
/// Ask ECS about a project's task: the stored ARN, else (none stored) its
|
||||
/// live task found by family and tags.
|
||||
pub async fn reconcile_task<T: TaskApi>(
|
||||
@@ -1541,7 +1553,33 @@ pub async fn reconcile_project<T: TaskApi>(
|
||||
find: &FindTask,
|
||||
) -> Result<Reconciled, String> {
|
||||
let arn = project.aws.as_ref().and_then(|a| a.task_arn.clone());
|
||||
let r = reconcile_task(api, arn.as_deref(), find).await?;
|
||||
let mut r = reconcile_task(api, arn.as_deref(), find).await?;
|
||||
// A task with ECS Exec off can never be attached to (a cold EC2 start
|
||||
// the app was closed during): never Running. Placed, or too old for any
|
||||
// start to still wait for it → stop it (then it's Gone); still waiting
|
||||
// and recent → Interrupted, left for a Start or Stop.
|
||||
if let Reconciled::Live(t) = &r {
|
||||
if !t.exec_enabled {
|
||||
if waiting_for_instance(t) && !is_orphaned(t, WaitTiming::ec2(), now_unix()) {
|
||||
if arn.as_deref() != Some(t.arn.as_str()) {
|
||||
record_task(store, &project.id, Some(&live_record(t)));
|
||||
}
|
||||
store.update_status(&project.id, ProjectStatus::Error)?;
|
||||
return Ok(Reconciled::Interrupted);
|
||||
}
|
||||
log::warn!("Project {}: its task {} has ECS Exec off; stopping it", project.id, t.arn);
|
||||
let cluster = cluster_of_task(&t.arn).ok_or_else(|| format!("{} is not a task ARN with a cluster", t.arn))?;
|
||||
if let Err(e) = bounded_stop(api.stop_task(cluster, &t.arn, REPLACED_REASON)).await {
|
||||
// It may still run: keep (or record) its ARN so Stop finds it.
|
||||
if arn.as_deref() != Some(t.arn.as_str()) {
|
||||
record_task(store, &project.id, Some(&live_record(t)));
|
||||
}
|
||||
store.update_status(&project.id, ProjectStatus::Error)?;
|
||||
return Err(format!("Could not stop the workspace's unreachable task {}: {e}", t.arn));
|
||||
}
|
||||
r = Reconciled::Gone;
|
||||
}
|
||||
}
|
||||
match &r {
|
||||
Reconciled::Gone => {
|
||||
store.update_status(&project.id, ProjectStatus::Stopped)?;
|
||||
@@ -1549,25 +1587,28 @@ pub async fn reconcile_project<T: TaskApi>(
|
||||
}
|
||||
Reconciled::Live(t) => {
|
||||
if arn.as_deref() != Some(t.arn.as_str()) {
|
||||
record_task(
|
||||
store,
|
||||
&project.id,
|
||||
Some(&TaskRecord {
|
||||
task_arn: t.arn.clone(),
|
||||
access_point_id: None,
|
||||
task_definition_arn: Some(t.task_definition_arn.clone()),
|
||||
workspace_access_point_id: None,
|
||||
}),
|
||||
);
|
||||
record_task(store, &project.id, Some(&live_record(t)));
|
||||
}
|
||||
if project.status != ProjectStatus::Running {
|
||||
store.update_status(&project.id, ProjectStatus::Running)?;
|
||||
}
|
||||
}
|
||||
Reconciled::Interrupted => {}
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
/// The store record of a live task reconcile found (no access points: the
|
||||
/// stored ones stand).
|
||||
fn live_record(t: &TaskInfo) -> TaskRecord {
|
||||
TaskRecord {
|
||||
task_arn: t.arn.clone(),
|
||||
access_point_id: None,
|
||||
task_definition_arn: Some(t.task_definition_arn.clone()),
|
||||
workspace_access_point_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `reconcile_project_statuses` for an AWS project: decide, and when the
|
||||
/// task is live re-open its tunnel in the background (terminals wait for
|
||||
/// it through [`tunnels::wait_endpoint`]), then — copy-at-connect — copy the
|
||||
@@ -1630,6 +1671,10 @@ pub async fn reconcile(
|
||||
Ok(Reconciled::Gone) => {
|
||||
log::info!("Project '{}' ({}): its task is gone — set to Stopped", project.name, project.id)
|
||||
}
|
||||
Ok(Reconciled::Interrupted) => {
|
||||
log::info!("Project '{}' ({}): {INTERRUPTED}", project.name, project.id);
|
||||
on_connect_warning(INTERRUPTED.to_string());
|
||||
}
|
||||
Err(e) => log::warn!("Project {}: could not check its task ({e}); status left as is", project.id),
|
||||
}
|
||||
}
|
||||
@@ -1679,6 +1724,11 @@ async fn reattach(
|
||||
let task = wait_ready(&api, &cluster, &task.arn, &logs, timing, |_| {})
|
||||
.await
|
||||
.map_err(s)?;
|
||||
// Reconcile never hands over an exec-off task; never wait on a tunnel
|
||||
// ECS Exec can't open either.
|
||||
if !task.exec_enabled {
|
||||
return Err(format!("The task {} has ECS Exec off; Start the project again", task.arn));
|
||||
}
|
||||
let (forward, endpoint) = SsmConnector { sdk: ctx.sdk.clone() }.connect(&task, &token).await?;
|
||||
let tunnel = LiveTunnel {
|
||||
task: ecs_task(&task)?,
|
||||
|
||||
@@ -107,6 +107,8 @@ struct Fake {
|
||||
log_reads: Mutex<Vec<String>>,
|
||||
/// The caller's EC2 record (absent by default).
|
||||
user_record: Option<crate::aws::environments::Param>,
|
||||
/// Reading the EC2 record fails (throttled, say).
|
||||
user_record_fails: bool,
|
||||
/// RunTask requests, in order.
|
||||
run_requests: Mutex<Vec<RunTaskInput>>,
|
||||
/// The clusters ListTasks / StopTask / DescribeTasks were asked about.
|
||||
@@ -263,6 +265,9 @@ impl RuntimeApi for Fake {
|
||||
}
|
||||
async fn user_record(&self, env: &str, _owner_key: &str) -> Result<crate::aws::environments::Param, AwsError> {
|
||||
self.log(format!("user-record {env}"));
|
||||
if self.user_record_fails {
|
||||
return Err(AwsError::Other { message: "Throttling".into() });
|
||||
}
|
||||
Ok(self.user_record.clone().unwrap_or(crate::aws::environments::Param::Missing))
|
||||
}
|
||||
async fn role_exists(&self, _name: &str) -> Result<Knowable, AwsError> {
|
||||
@@ -1815,3 +1820,94 @@ fn ec2_capacity_reads_as_host_settings_need_it() {
|
||||
assert_eq!(v, serde_json::json!({ "state": "unusable", "reason": "schema is not 1" }));
|
||||
assert_eq!(serde_json::to_value(ec2_capacity_from(None)).unwrap(), serde_json::json!({ "state": "unknown" }));
|
||||
}
|
||||
|
||||
/// Reconcile never marks an exec-off task Running (an app restart during a
|
||||
/// cold EC2 start): a placed or orphaned one is stopped (its ARN cleared
|
||||
/// only once the stop succeeded); a recent one still waiting is Error with
|
||||
/// its ARN kept.
|
||||
#[tokio::test]
|
||||
async fn reconcile_never_attaches_to_a_task_with_ecs_exec_off() {
|
||||
let mut p = ec2_project();
|
||||
p.status = ProjectStatus::Running;
|
||||
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("x"));
|
||||
let execless = |last: &str, placed: bool, created: i64| TaskInfo {
|
||||
exec_enabled: false,
|
||||
created_unix: Some(created),
|
||||
..ec2_task(&p, "x", last, false, placed)
|
||||
};
|
||||
let find = FindTask::new(&p, REGION, USER, Some(user_cluster_arn()));
|
||||
let stopped_with_replaced = |f: &Fake| f.calls().contains(&format!("stop {} {REPLACED_REASON}", ec2_task_arn("x")));
|
||||
|
||||
// Placed: stopped, Stopped, ARN cleared.
|
||||
let fake = Fake::default().with_task(execless("PENDING", true, now_unix()));
|
||||
let (_d, store) = store_with(&p);
|
||||
assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Gone);
|
||||
assert!(stopped_with_replaced(&fake), "{:?}", fake.calls());
|
||||
let after = store.get(&p.id).unwrap();
|
||||
assert_eq!(after.status, ProjectStatus::Stopped);
|
||||
assert_eq!(after.aws.unwrap().task_arn, None);
|
||||
|
||||
// Orphaned (still waiting, but older than any start waits): the same.
|
||||
let fake = Fake::default().with_task(execless("PROVISIONING", false, now_unix() - 3600));
|
||||
let (_d, store) = store_with(&p);
|
||||
assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Gone);
|
||||
assert!(stopped_with_replaced(&fake));
|
||||
assert_eq!(store.get(&p.id).unwrap().status, ProjectStatus::Stopped);
|
||||
|
||||
// The stop fails: Error, the ARN kept.
|
||||
let fake = Fake { stop_fails: true, ..Default::default() }.with_task(execless("RUNNING", true, now_unix()));
|
||||
let (_d, store) = store_with(&p);
|
||||
let e = reconcile_project(&fake, &store, &p, &find).await.unwrap_err();
|
||||
assert!(e.contains("Could not stop") && e.contains("ExpiredToken"), "{e}");
|
||||
let after = store.get(&p.id).unwrap();
|
||||
assert_eq!(after.status, ProjectStatus::Error);
|
||||
assert_eq!(after.aws.unwrap().task_arn, Some(ec2_task_arn("x")));
|
||||
|
||||
// Recent and still waiting: Interrupted (Error), ARN kept, nothing stopped.
|
||||
let fake = Fake::default().with_task(execless("PROVISIONING", false, now_unix() - 60));
|
||||
let (_d, store) = store_with(&p);
|
||||
assert_eq!(reconcile_project(&fake, &store, &p, &find).await.unwrap(), Reconciled::Interrupted);
|
||||
assert!(!fake.called("stop"));
|
||||
let after = store.get(&p.id).unwrap();
|
||||
assert_eq!(after.status, ProjectStatus::Error);
|
||||
assert_eq!(after.aws.unwrap().task_arn, Some(ec2_task_arn("x")));
|
||||
assert!(INTERRUPTED.contains("Start again"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_orphan_is_older_than_the_ec2_wait_plus_ten_minutes() {
|
||||
assert_eq!(ORPHAN_MARGIN, Duration::from_secs(10 * 60));
|
||||
let p = ec2_project();
|
||||
let at = |age: i64| TaskInfo { created_unix: Some(1_000_000 - age), ..ec2_task(&p, "o", "PROVISIONING", false, false) };
|
||||
let wait = WaitTiming::ec2();
|
||||
assert!(!is_orphaned(&at(19 * 60), wait, 1_000_000));
|
||||
assert!(is_orphaned(&at(21 * 60), wait, 1_000_000));
|
||||
assert!(!is_orphaned(&TaskInfo { created_unix: None, ..at(0) }, wait, 1_000_000));
|
||||
}
|
||||
|
||||
/// A failed read of the EC2 record does not stop an attach: the stored
|
||||
/// task's cluster is still searched. Only a new run needs the record.
|
||||
#[tokio::test]
|
||||
async fn an_ec2_record_read_error_still_attaches_and_fails_only_a_new_run() {
|
||||
let mut p = ec2_project();
|
||||
p.aws.as_mut().unwrap().task_arn = Some(ec2_task_arn("e8"));
|
||||
let live = ec2_task(&p, "e8", "RUNNING", true, true);
|
||||
let fake = Fake {
|
||||
user_record_fails: true,
|
||||
listed_by_cluster: Some(HashMap::from([(user_cluster_arn(), vec![live.arn.clone()])])),
|
||||
..Default::default()
|
||||
}
|
||||
.with_task(live);
|
||||
let (r, _) = start(&fake, &p).await;
|
||||
let started = r.unwrap();
|
||||
assert!(!started.created);
|
||||
assert_eq!(started.task.arn, ec2_task_arn("e8"));
|
||||
assert_eq!(*fake.list_clusters.lock().unwrap(), [cluster_arn(), user_cluster_arn()]);
|
||||
|
||||
// Nothing live: the start fails with the read error, before any write.
|
||||
let fake = Fake { user_record_fails: true, ..Default::default() };
|
||||
let (r, _) = start(&fake, &p).await;
|
||||
let e = r.err().unwrap();
|
||||
assert!(e.contains("Could not read your EC2 capacity record") && e.contains("Throttling"), "{e}");
|
||||
assert!(!fake.called("put-secret") && !fake.called("run"));
|
||||
}
|
||||
Reference in new issue
Block a user