fix(aws): follow clone and setup in the background, after the start

Ruling R5: the start ends after Connecting (Running, lock released) and a
background follower (cancelled on Stop, Remove, tunnel close, app exit)
reports only a failure it saw or one on a task this start ran. A stale
Starting/Stopping AWS project loads as Error, and reconcile also checks a
Stopped one that still has a task ARN, so a crash never hides a live task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 08:09:51 -08:00
1 parent 6b62a2a241
commit 9c888caf13
11 files changed
+518 -127

No files matched your search

+8 -4
View File
@@ -422,10 +422,14 @@ docker exec stdout → tokio task → emit("terminal-output-{sessionId}") → li
subshell; `status` is `running` from the start until `ok` / `failed <code>` (a chain that dies
early writes `failed`, so the idle stop is never blocked). `triple-c-workspace-status` (python,
no secrets) prints `{clone:{done,repos},setup:{state,code,tail}}`.
The start command follows it after "Connecting…" (`host/aws_host/status.rs`: background exec
every 2 s, a progress line only on change, at most 35 min, quiet stop when the tunnel goes);
a failed clone/setup never fails the start — it arrives as the `workspace-warning` event (warning
toast), and Project Home's Overview polls `aws_workspace_status` every 5 s while visible.
**The start does not wait for them** (it ends after "Connecting…": Running, lock released);
`aws_host::spawn_follow` follows in a background task (`host/aws_host/status.rs`: background exec
every 2 s, at most 35 min, holds no project lock; ended by `tunnels::cancel_follow` on Stop/Remove
and by any tunnel close or app exit). Its only output is a `workspace-warning` event (warning
toast) for a failure it saw happen or one on a task this start ran; progress shows in Project
Home's Overview card, which polls `aws_workspace_status` every 5 s while Running and visible.
At load, a Starting/Stopping AWS project becomes Error (not Stopped — a crash mid-start may have
left a billed task), and `reconcile_aws_projects` also checks a Stopped one still holding a task ARN.
**Lifetime cap** (`TRIPLEC_MAX_LIFETIME_MINUTES`, advisory — sudo can kill it): the idle watcher
(started when idle stop *or* a non-zero cap is set) pushes best effort, records `lifetime` and
sends TERM; the hook exits 76 (`aws::tasks::LIFETIME_EXIT_CODE` → `StopKind::LifetimeCap` →
-23
View File
@@ -286,29 +286,6 @@ fn serialize_payload(m: &BTreeMap<String, String>) -> Result<String, String> {
/// `TRIPLEC_SETUP_SCRIPT` when set, `TRIPLEC_MAX_LIFETIME_MINUTES` when the
/// environment caps it), keys sorted. Fails when it would exceed 65,536 bytes.
#[cfg_attr(not(test), allow(dead_code))]
pub fn project_secret_payload(
env_pairs: &[(String, String)],
agent_token: &str,
files: &[(String, Vec<u8>)],
repos: &[RepoSource],
wip: bool,
idle: Option<u32>,
storage: StorageMode,
) -> Result<String, String> {
project_secret_payload_with(
env_pairs,
agent_token,
files,
repos,
wip,
idle,
storage,
PayloadExtras::default(),
)
}
/// [`project_secret_payload`] with the phase 6 keys.
#[cfg_attr(not(test), allow(dead_code))]
#[allow(clippy::too_many_arguments)]
pub fn project_secret_payload_with(
env_pairs: &[(String, String)],
+92 -10
View File
@@ -469,7 +469,7 @@ fn payload_holds_all_project_data_with_sorted_keys() {
("ca/corp.pem".to_string(), b"PEM".to_vec()),
];
let repos = project(StorageMode::Hybrid).aws.unwrap().repos;
let json = project_secret_payload(
let json = project_secret_payload_with(
&env,
"tok",
&files,
@@ -477,6 +477,7 @@ fn payload_holds_all_project_data_with_sorted_keys() {
true,
Some(30),
StorageMode::Hybrid,
PayloadExtras::default(),
)
.unwrap();
let m = payload_map(&json);
@@ -525,7 +526,16 @@ fn payload_holds_all_project_data_with_sorted_keys() {
#[test]
fn payload_off_values() {
let json =
project_secret_payload(&[], "t", &[], &[], false, None, StorageMode::Ephemeral).unwrap();
project_secret_payload_with(
&[],
"t",
&[],
&[],
false,
None,
StorageMode::Ephemeral,
PayloadExtras::default(),
).unwrap();
let m = payload_map(&json);
assert_eq!(m["TRIPLEC_WIP_PUSH"], "0");
assert_eq!(m["TRIPLEC_STORAGE"], "ephemeral");
@@ -538,8 +548,26 @@ fn payload_off_values() {
fn payload_is_independent_of_env_pair_order() {
let a = pairs(&[("B", "2"), ("A", "1"), ("C", "3")]);
let b = pairs(&[("C", "3"), ("A", "1"), ("B", "2")]);
let pa = project_secret_payload(&a, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap();
let pb = project_secret_payload(&b, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap();
let pa = project_secret_payload_with(
&a,
"t",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
).unwrap();
let pb = project_secret_payload_with(
&b,
"t",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
).unwrap();
assert_eq!(pa, pb);
}
@@ -549,7 +577,16 @@ fn payload_rejects_bad_file_names() {
"ssh/", "ssh/.", "ssh/..", "ssh/a/b", "other/x", "x", "ca/../k", "ssh/a:b",
] {
let files = vec![(name.to_string(), b"x".to_vec())];
let r = project_secret_payload(&[], "t", &files, &[], true, None, StorageMode::Hybrid);
let r = project_secret_payload_with(
&[],
"t",
&files,
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
);
if name == "ssh/a:b" {
// A colon is a fine file name; only the secret's *keys* are restricted.
assert!(r.is_ok());
@@ -563,7 +600,16 @@ fn payload_rejects_bad_file_names() {
fn payload_rejects_env_names_a_secret_reference_cannot_carry() {
for bad in ["A:B", "", "1ABC", "A-B", "A B"] {
let env = pairs(&[(bad, "SENTINEL-VALUE-7f3a")]);
let r = project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid);
let r = project_secret_payload_with(
&env,
"t",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
);
assert!(r.is_err(), "{bad:?} accepted");
// The error never echoes a value.
assert!(!r.unwrap_err().contains("SENTINEL-VALUE-7f3a"));
@@ -581,7 +627,16 @@ fn payload_keeps_its_own_keys_over_custom_env() {
("OK", "1"),
]);
let m = payload_map(
&project_secret_payload(&env, "real", &[], &[], true, None, StorageMode::Hybrid).unwrap(),
&project_secret_payload_with(
&env,
"real",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
).unwrap(),
);
assert_eq!(m["TRIPLEC_AGENT_TOKEN"], "real");
// The task definition's fixed environment owns these two names.
@@ -597,7 +652,16 @@ fn payload_keeps_its_own_keys_over_custom_env() {
fn payload_over_64_kib_fails() {
let env = pairs(&[("BIG", &"x".repeat(65_536))]);
let err =
project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap_err();
project_secret_payload_with(
&env,
"t",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
).unwrap_err();
assert!(err.contains("65,536"), "{err}");
}
@@ -643,7 +707,16 @@ fn payload_omits_setup_script_and_lifetime_when_unset() {
fn payload_setup_script_and_lifetime_cannot_be_forged_by_custom_env() {
let env = pairs(&[("TRIPLEC_SETUP_SCRIPT", "rm -rf /"), ("TRIPLEC_MAX_LIFETIME_MINUTES", "1")]);
let m = payload_map(
&project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap(),
&project_secret_payload_with(
&env,
"t",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
).unwrap(),
);
assert!(!m.contains_key("TRIPLEC_SETUP_SCRIPT"));
assert!(!m.contains_key("TRIPLEC_MAX_LIFETIME_MINUTES"));
@@ -679,7 +752,16 @@ fn payload_size_error_names_the_setup_script_when_it_is_the_large_part() {
fn payload_size_error_names_a_large_custom_variable_by_name_only() {
let env = pairs(&[("BIG_BLOB", &"SENTINEL".repeat(9000))]);
let err =
project_secret_payload(&env, "t", &[], &[], true, None, StorageMode::Hybrid).unwrap_err();
project_secret_payload_with(
&env,
"t",
&[],
&[],
true,
None,
StorageMode::Hybrid,
PayloadExtras::default(),
).unwrap_err();
assert!(err.contains("the environment variable BIG_BLOB (71 KiB)"), "{err}");
assert!(!err.contains("SENTINEL"), "{err}");
}
+67 -22
View File
@@ -1407,9 +1407,6 @@ async fn start_project_container_locked(
// Wrap container operations so that any failure resets status to Stopped.
let host = ProjectHost::for_project(&project);
// An AWS start's clone/setup problems: the start still succeeds, and this
// goes to the user as a `workspace-warning` event once it is Running.
let mut workspace_warning: Option<String> = None;
let result: Result<ContainerTarget, String> = async {
let target = host
.ensure_started(&project, &settings, &state.projects_store, &|msg: &str| {
@@ -1434,14 +1431,6 @@ async fn start_project_container_locked(
target.clone(),
);
// AWS: follow the background clone and setup script until they are
// done (or 35 minutes pass), as progress lines (phase 6, Q4). A
// local project returns at once.
workspace_warning = crate::host::aws_host::follow_start(&project, &|msg: &str| {
emit_progress(&app_handle, &project_id, msg)
})
.await;
Ok(target)
}.await;
@@ -1502,8 +1491,14 @@ async fn start_project_container_locked(
.await;
}
if let Some(detail) = workspace_warning {
emit_workspace_warning(&app_handle, &project_id, &project.name, &detail);
// AWS: the start is done (Running, lock released when this returns);
// the background clone and setup script are followed by a background
// task that only reports a failure (ruling R5). Never for Docker.
if project.is_aws() {
let (handle, id, name) = (app_handle.clone(), project_id.clone(), project.name.clone());
crate::host::aws_host::spawn_follow(&project, move |detail| {
emit_workspace_warning(&handle, &id, &name, &detail)
});
}
project.container_id = Some(container_id);
@@ -1511,11 +1506,10 @@ async fn start_project_container_locked(
Ok(project)
}
/// The start's warning channel (phase 6, Q4): the start command returns the
/// stored `Project`, which has no place for a one-off warning, so a started
/// workspace whose clone or setup script failed is reported as a
/// `workspace-warning` event `{project_id, message, detail}`; the frontend
/// shows it as a warning toast.
/// The follower's warning channel (phase 6, Q4/R5): a started workspace
/// whose clone or setup script failed (or still ran after 35 minutes) is
/// reported as a `workspace-warning` event `{project_id, message, detail}`;
/// the frontend shows it as a warning toast.
fn emit_workspace_warning(app_handle: &tauri::AppHandle, project_id: &str, name: &str, detail: &str) {
let _ = app_handle.emit(
"workspace-warning",
@@ -1569,6 +1563,9 @@ pub async fn stop_project_container(
}
state.projects_store.update_status(&project_id, ProjectStatus::Stopping)?;
// AWS: the background clone/setup follower ends now (it holds no lock;
// closing the tunnel would end it too).
crate::host::tunnels::cancel_follow(&project_id);
// AWS: the terminals and the auth bridge close *before* the stop guard's
// push, so nothing written after its snapshot is silently dropped (the
@@ -2014,6 +2011,17 @@ pub async fn reconcile_project_statuses(
/// How many AWS projects reconcile at once.
const AWS_RECONCILE_CONCURRENCY: usize = 4;
/// Whether an AWS project's status must be checked against ECS at startup:
/// anything but a clean Stopped — and a Stopped one that still has a task
/// ARN (Stop clears it, so the stored ARN means a start or stop never
/// finished and a billed task may be running).
pub(crate) fn needs_aws_reconcile(p: &Project) -> bool {
match p.status {
ProjectStatus::Running | ProjectStatus::Error | ProjectStatus::Starting | ProjectStatus::Stopping => true,
ProjectStatus::Stopped => p.aws.as_ref().is_some_and(|a| a.task_arn.is_some()),
}
}
/// Reconcile every AWS project against ECS (the authority), independently of
/// Docker: an app started without Docker still learns which tasks run and
/// re-opens their tunnels. One bounded DescribeTasks per project, a few at a
@@ -2037,10 +2045,7 @@ pub async fn reconcile_aws_projects(state: State<'_, AppState>) -> Result<Vec<Pr
let settings = state.settings_store.get();
let store = state.projects_store.clone();
futures_util::stream::iter(projects.into_iter().filter(|p| {
matches!(
p.status,
ProjectStatus::Running | ProjectStatus::Error | ProjectStatus::Starting | ProjectStatus::Stopping
) && crate::project_lock::held(&p.id).is_none()
needs_aws_reconcile(p) && crate::project_lock::held(&p.id).is_none()
}))
.for_each_concurrent(AWS_RECONCILE_CONCURRENCY, |project| {
let settings = settings.clone();
@@ -2983,4 +2988,44 @@ mod tests {
local.host_id = None;
assert!(check_aws_resources_removable(&local).is_err());
}
/// The text of `fn name` in this file, up to the next top-level item.
fn fn_body<'a>(src: &'a str, name: &str) -> &'a str {
let start = src.find(&format!("fn {name}(")).unwrap_or_else(|| panic!("{name} missing"));
let rest = &src[start..];
let end = rest.find("\n}\n").unwrap();
&rest[..end]
}
#[test]
fn the_workspace_follower_is_spawned_after_the_start_succeeds_and_only_for_aws() {
// Ruling R5: the start ends after "Connecting…" (Running, lock
// released when the command returns); the clone/setup follow runs
// in the background and never inside the start.
let src = include_str!("project_commands.rs");
let start = fn_body(src, "start_project_container_locked");
let running = start
.find("update_status(&project_id, ProjectStatus::Running)")
.expect("the start sets Running");
let spawn = start.find("aws_host::spawn_follow(").expect("the start spawns the follower");
assert!(spawn > running, "the follower is spawned only after the project is Running");
assert_eq!(start.matches("spawn_follow(").count(), 1);
let guard = start[..spawn].rfind("if project.is_aws() {").expect("AWS-only");
assert!(start[guard..spawn].lines().count() <= 4, "the spawn sits right in the AWS guard");
assert!(!start.contains("follow_workspace_status"), "no follow inside the start");
// Docker: `spawn_follow` itself also refuses a local project
// (`status::tests::a_local_project_has_no_workspace_status_and_no_follower`).
}
#[test]
fn stop_ends_the_follower_before_it_stops_the_task() {
let src = include_str!("project_commands.rs");
let stop = fn_body(src, "stop_project_container");
let cancel = stop.find("tunnels::cancel_follow(&project_id)").expect("stop cancels the follower");
let task = stop.find(".stop_project(&project, &settings)").expect("stop stops the task");
assert!(cancel < task);
// Stop's own claim never waits on the follower: it holds none
// (`status::tests::the_follower_takes_no_project_lock`).
}
}
+6 -1
View File
@@ -82,7 +82,7 @@ const WATCH_CALL_TIMEOUT: Duration = Duration::from_secs(60);
const STOP_TIMEOUT: Duration = Duration::from_secs(45);
pub mod status;
pub use status::{follow_start, workspace_status, WorkspaceStatus};
pub use status::{spawn_follow, workspace_status, WorkspaceStatus};
fn s<E: std::fmt::Display>(e: E) -> String {
e.to_string()
@@ -589,6 +589,7 @@ pub async fn ensure_started(
// Already connected and answering: nothing to do.
if let Some(ep) = tunnels::get_endpoint(&project.id) {
if matches!(agent::is_running(&ep).await, Ok(true)) {
tunnels::mark_attached(&project.id);
return Ok(ContainerTarget::Agent(ep));
}
tunnels::close(&project.id).await;
@@ -653,6 +654,7 @@ pub async fn ensure_started(
access_point_id,
workspace_access_point_id,
task_definition_arn: started.task.task_definition_arn.clone(),
created: started.created,
},
) {
old.forward.close().await;
@@ -834,6 +836,7 @@ async fn task_arn_or_find<T: TaskApi>(
/// the stored task ARN, else whatever live task the project has (a crash
/// between RunTask and the store write). Closes the tunnel.
pub async fn stop_project(project: &Project, settings: &AppSettings) -> Result<(), String> {
tunnels::cancel_follow(&project.id);
if let Some(ep) = tunnels::get_endpoint(&project.id) {
if let Some(res) = stop_endpoint(&ep).await {
return res;
@@ -867,6 +870,7 @@ pub async fn remove_resources(
settings: &AppSettings,
discard_data: bool,
) -> Result<(), String> {
tunnels::cancel_follow(&project.id);
let (_, region, sdk) = project_sdk(project, settings).await?;
let who = caller_identity(&sdk).await.map_err(s)?;
let ok = owner_key(&region, &who.user_id);
@@ -1104,6 +1108,7 @@ async fn reattach(
.as_ref()
.and_then(|a| a.workspace_access_point_id.clone()),
task_definition_arn: task.task_definition_arn.clone(),
created: false,
};
if !reattach_still_wanted(store, &project.id, &task.arn) {
tunnel.forward.close().await;
+90 -33
View File
@@ -3,18 +3,25 @@
//!
//! The entrypoint clones in the background and then runs the setup script;
//! `triple-c-workspace-status` (container) prints both as one JSON line,
//! mirrored by [`WorkspaceStatus`]. The start reads it through the agent every
//! [`FOLLOW_INTERVAL`] as a **background** exec (never activity for idle
//! stop), emits a progress line only when it changes, and stops when the
//! clone is done and no setup is running, after [`FOLLOW_LIMIT`], or quietly
//! when the agent is gone (tunnel dropped, old image without the script).
//! A failed clone or setup does not fail the start: [`follow_warning`] is
//! what the user is told instead.
//! mirrored by [`WorkspaceStatus`].
//!
//! **The start does not wait for them** (ruling R5, amending Q4): it ends
//! after "Connecting…" as in phase 5 — project Running, project lock
//! released, terminals and Stop usable. [`spawn_follow`] then follows in the
//! background: a read through the agent every [`FOLLOW_INTERVAL`] as a
//! **background** exec (never activity for idle stop), a log line only when
//! it changes, until the clone is done and no setup runs, [`FOLLOW_LIMIT`]
//! passes, the agent is gone (tunnel dropped, task died, image without the
//! script) or its token is cancelled (`tunnels::cancel_follow` on Stop and
//! Remove; any tunnel close; app exit). Its only output is the final warning
//! ([`follow_warning`]); Project Home's Overview card polls the status itself
//! while the project is Running, so progress shows there.
use std::future::Future;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio_util::sync::CancellationToken;
use super::super::{tunnels, ContainerTarget};
use crate::models::Project;
@@ -281,16 +288,37 @@ impl Default for FollowTiming {
/// How a follow ended.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FollowEnd {
Settled(WorkspaceStatus),
Settled {
status: WorkspaceStatus,
/// The very first read was already settled: nothing happened in
/// this follow (an attach to a workspace that finished long ago).
first_read: bool,
},
/// [`FollowTiming::limit`] passed; the last status read, if any.
TimedOut(Option<WorkspaceStatus>),
/// The agent went away or kept failing; nothing to say.
/// The agent went away, kept failing, or the follow was cancelled:
/// nothing to say.
Lost,
}
/// Follow `src` until it settles, the limit passes or it is lost, calling
/// `progress` with each new progress line and each new setup log line.
/// Follow `src` until it settles, the limit passes, it is lost or `cancel`
/// fires, calling `progress` with each new progress line and each new setup
/// log line.
pub async fn follow_workspace_status<S: StatusSource>(
src: &S,
total_repos: usize,
timing: FollowTiming,
cancel: &CancellationToken,
progress: &(dyn Fn(&str) + Send + Sync),
) -> FollowEnd {
tokio::select! {
biased;
_ = cancel.cancelled() => FollowEnd::Lost,
end = follow_inner(src, total_repos, timing, progress) => end,
}
}
async fn follow_inner<S: StatusSource>(
src: &S,
total_repos: usize,
timing: FollowTiming,
@@ -301,12 +329,14 @@ pub async fn follow_workspace_status<S: StatusSource>(
let mut last_log: Option<String> = None;
let mut last_status: Option<WorkspaceStatus> = None;
let mut failures = 0u32;
let mut reads = 0u32;
loop {
match src.read().await {
StatusRead::Status(s) => {
failures = 0;
reads += 1;
if s.settled() {
return FollowEnd::Settled(s);
return FollowEnd::Settled { status: s, first_read: reads == 1 };
}
if let Some(line) = s.progress_line(total_repos) {
if last_line.as_deref() != Some(line.as_str()) {
@@ -341,11 +371,20 @@ pub async fn follow_workspace_status<S: StatusSource>(
}
}
/// What the user is told after a follow; `None` when all went well (or the
/// follow was lost — Project Home still shows the status on demand).
pub fn follow_warning(end: &FollowEnd, limit: Duration) -> Option<String> {
/// What the user is told after a follow; `None` when all went well, the
/// follow was lost, or the workspace had settled before this start —
/// a failure is reported only when this follow saw it happen, or the
/// task is one this start ran (`created`). Project Home still shows an old
/// failure on demand.
pub fn follow_warning(end: &FollowEnd, limit: Duration, created: bool) -> Option<String> {
match end {
FollowEnd::Settled(s) => s.problems(),
FollowEnd::Settled { status, first_read } => {
if *first_read && !created {
None
} else {
status.problems()
}
}
FollowEnd::TimedOut(s) => {
let what = match s {
Some(s) if !s.clone.done => "still cloning its repos",
@@ -366,25 +405,43 @@ pub fn follow_warning(end: &FollowEnd, limit: Duration) -> Option<String> {
}
}
/// The start's follow of a just-connected AWS project; the warning to show,
/// if any. A local project returns `None` at once.
pub async fn follow_start(
project: &Project,
progress: &(dyn Fn(&str) + Send + Sync),
) -> Option<String> {
if !project.is_aws() {
return None;
/// The follow of one project's workspace, registered with the tunnel
/// registry for cancellation. `on_warning` gets the warning, if any.
pub async fn run_follow<S: StatusSource>(
src: &S,
project_id: &str,
total_repos: usize,
created: bool,
timing: FollowTiming,
on_warning: impl FnOnce(String),
) {
let (id, cancel) = tunnels::begin_follow(project_id);
let log_line = |m: &str| log::debug!("Project {project_id}: {m}");
let end = follow_workspace_status(src, total_repos, timing, &cancel, &log_line).await;
tunnels::end_follow(project_id, id);
if let Some(w) = follow_warning(&end, timing.limit, created) {
on_warning(w);
}
}
/// After a successful start of an AWS project: follow its clone and setup
/// script in the background (see the module docs). `false` (nothing
/// spawned) for a local project or one with no tunnel.
pub fn spawn_follow(
project: &Project,
on_warning: impl FnOnce(String) + Send + 'static,
) -> bool {
if !project.is_aws() || tunnels::get_endpoint(&project.id).is_none() {
return false;
}
let project_id = project.id.clone();
let total = project.aws.as_ref().map_or(0, |a| a.repos.len());
let timing = FollowTiming::default();
let end = follow_workspace_status(
&TunnelStatusSource { project_id: &project.id },
total,
timing,
progress,
)
.await;
follow_warning(&end, timing.limit)
let created = tunnels::ran_task(&project_id);
tauri::async_runtime::spawn(async move {
let src = TunnelStatusSource { project_id: &project_id };
run_follow(&src, &project_id, total, created, FollowTiming::default(), on_warning).await;
});
true
}
#[cfg(test)]
+80 -10
View File
@@ -1,6 +1,7 @@
use std::sync::Mutex;
use super::*;
use crate::host::tunnels;
/// Returns its scripted reads in order, then repeats the last one.
struct Scripted {
@@ -36,7 +37,8 @@ fn st(json: &str) -> StatusRead {
async fn follow(src: &Scripted, total: usize) -> (FollowEnd, Vec<String>) {
let lines = Mutex::new(Vec::new());
let progress = |m: &str| lines.lock().unwrap().push(m.to_string());
let end = follow_workspace_status(src, total, FollowTiming::default(), &progress).await;
let cancel = CancellationToken::new();
let end = follow_workspace_status(src, total, FollowTiming::default(), &cancel, &progress).await;
(end, lines.into_inner().unwrap())
}
@@ -75,8 +77,9 @@ async fn follow_emits_each_change_once_and_stops_when_settled() {
assert_eq!(src.count(), 8);
// Seven waits of 2 s between eight reads.
assert_eq!(started.elapsed(), FOLLOW_INTERVAL * 7);
let FollowEnd::Settled(s) = &end else { panic!("{end:?}") };
let w = follow_warning(&end, FOLLOW_LIMIT).unwrap();
let FollowEnd::Settled { status: s, first_read: false } = &end else { panic!("{end:?}") };
// Seen failing in this follow: reported even on an attach.
let w = follow_warning(&end, FOLLOW_LIMIT, false).unwrap();
assert!(w.contains("Cloning failed for a repo: b."), "{w}");
assert!(w.contains("The setup script failed (exit 2)."), "{w}");
assert!(w.ends_with("Last lines:\nSetting up jq\nE: oops"), "{w}");
@@ -90,7 +93,20 @@ async fn a_workspace_already_settled_ends_at_once_without_a_warning() {
let (end, lines) = follow(&src, 1).await;
assert!(lines.is_empty());
assert_eq!(started.elapsed(), std::time::Duration::ZERO);
assert_eq!(follow_warning(&end, FOLLOW_LIMIT), None);
assert_eq!(follow_warning(&end, FOLLOW_LIMIT, true), None);
}
#[tokio::test(start_paused = true)]
async fn an_old_failure_warns_only_for_a_task_this_start_ran() {
let src = Scripted::new(vec![st(SETUP_FAILED)]);
let (end, lines) = follow(&src, 2).await;
assert!(lines.is_empty());
assert!(matches!(end, FollowEnd::Settled { first_read: true, .. }), "{end:?}");
// Attached to a workspace that failed long ago: no toast.
assert_eq!(follow_warning(&end, FOLLOW_LIMIT, false), None);
// This start ran the task and it failed before the first read.
let w = follow_warning(&end, FOLLOW_LIMIT, true).unwrap();
assert!(w.contains("The setup script failed (exit 2)."), "{w}");
}
#[tokio::test(start_paused = true)]
@@ -102,7 +118,7 @@ async fn follow_times_out_after_35_minutes() {
assert!(matches!(end, FollowEnd::TimedOut(Some(_))), "{end:?}");
assert!(started.elapsed() <= FOLLOW_LIMIT);
assert!(started.elapsed() > FOLLOW_LIMIT - FOLLOW_INTERVAL * 2);
let w = follow_warning(&end, FOLLOW_LIMIT).unwrap();
let w = follow_warning(&end, FOLLOW_LIMIT, false).unwrap();
assert!(w.contains("still cloning its repos after 35 minutes"), "{w}");
}
@@ -110,7 +126,7 @@ async fn follow_times_out_after_35_minutes() {
async fn a_timed_out_setup_says_so_and_carries_any_clone_failure() {
let src = Scripted::new(vec![st(SETUP_2)]);
let (end, _) = follow(&src, 2).await;
let w = follow_warning(&end, FOLLOW_LIMIT).unwrap();
let w = follow_warning(&end, FOLLOW_LIMIT, false).unwrap();
assert!(w.contains("still running its setup script"), "{w}");
assert!(w.contains("Cloning failed for a repo: b."), "{w}");
}
@@ -121,7 +137,7 @@ async fn a_lost_agent_ends_the_follow_quietly() {
let (end, lines) = follow(&src, 1).await;
assert_eq!(end, FollowEnd::Lost);
assert_eq!(lines, ["Cloning repos… (1/1)"]);
assert_eq!(follow_warning(&end, FOLLOW_LIMIT), None);
assert_eq!(follow_warning(&end, FOLLOW_LIMIT, false), None);
}
#[tokio::test(start_paused = true)]
@@ -129,7 +145,7 @@ async fn failing_reads_are_retried_then_give_up_quietly() {
let fail = || StatusRead::Failed("tunnel hiccup".into());
// Four failures, then a good read: the count resets.
let src = Scripted::new(vec![fail(), fail(), fail(), fail(), st(ALL_OK)]);
assert!(matches!(follow(&src, 1).await.0, FollowEnd::Settled(_)));
assert!(matches!(follow(&src, 1).await.0, FollowEnd::Settled { .. }));
let src = Scripted::new(vec![fail()]);
let (end, _) = follow(&src, 1).await;
assert_eq!(end, FollowEnd::Lost);
@@ -183,8 +199,62 @@ fn the_status_command_reports_a_missing_script_as_127() {
}
#[tokio::test]
async fn a_local_project_has_no_workspace_status() {
async fn a_local_project_has_no_workspace_status_and_no_follower() {
let p = Project::new("p".into(), vec![]);
assert_eq!(workspace_status(&p).await, Ok(None));
assert_eq!(follow_start(&p, &|_| {}).await, None);
assert!(!spawn_follow(&p, |_| panic!("no warning for Docker")));
assert!(!tunnels::is_following(&p.id));
}
/// Never answers: a follow in progress until something ends it.
struct Hang;
impl StatusSource for Hang {
async fn read(&self) -> StatusRead {
std::future::pending().await
}
}
#[tokio::test(start_paused = true)]
async fn stop_ends_a_follow_in_progress_without_a_warning() {
let id = "follow-stop-test";
let task = tokio::spawn(async move {
let warned = std::sync::Arc::new(Mutex::new(false));
let w = warned.clone();
run_follow(&Hang, id, 1, true, FollowTiming::default(), move |_| *w.lock().unwrap() = true)
.await;
let v = *warned.lock().unwrap();
v
});
tokio::time::sleep(FOLLOW_INTERVAL * 3).await;
assert!(tunnels::is_following(id));
// What Stop and Remove call (`stop_project_container`, `stop_project`,
// `remove_resources`); closing the tunnel does the same.
tunnels::cancel_follow(id);
let warned = tokio::time::timeout(std::time::Duration::from_secs(1), task)
.await
.expect("the follow ends at once")
.unwrap();
assert!(!warned);
assert!(!tunnels::is_following(id));
}
#[tokio::test(start_paused = true)]
async fn a_new_follow_replaces_the_old_one() {
let id = "follow-replace-test";
let first = tokio::spawn(run_follow(&Hang, id, 1, true, FollowTiming::default(), |_| {}));
tokio::time::sleep(FOLLOW_INTERVAL).await;
let (_, second) = tunnels::begin_follow(id);
tokio::time::timeout(std::time::Duration::from_secs(1), first).await.unwrap().unwrap();
// The old follower's end does not unregister the new one.
assert!(tunnels::is_following(id));
assert!(!second.is_cancelled());
tunnels::cancel_follow(id);
assert!(second.is_cancelled());
}
#[test]
fn the_follower_takes_no_project_lock() {
// Stop must never wait for it (ruling R5).
let src = include_str!("../status.rs");
assert!(!src.contains("project_lock"));
}
+49
View File
@@ -875,3 +875,52 @@ async fn an_unattached_aws_project_gets_a_target_id_no_tunnel_can_have() {
let local = Project::new("l".into(), Vec::new());
assert_eq!(unattached_container_id(&local), None);
}
#[tokio::test]
async fn a_persisted_starting_aws_project_with_a_task_arn_is_reconciled_against_ecs_on_load() {
use crate::commands::project_commands::needs_aws_reconcile;
use crate::storage::projects_store::settle_stale_statuses;
// The app quit mid-start: Starting, the task's ARN already recorded.
let mut p = aws_project(StorageMode::Hybrid);
p.status = ProjectStatus::Starting;
p.aws.as_mut().unwrap().task_arn = Some(task_arn("mid"));
let mut docker = Project::new("local".into(), vec![]);
docker.status = ProjectStatus::Starting;
let mut loaded = vec![p.clone(), docker];
assert!(settle_stale_statuses(&mut loaded));
// Not Stopped (that would hide a billed task): Error, ARN kept.
assert_eq!(loaded[0].status, ProjectStatus::Error);
assert_eq!(loaded[0].aws.as_ref().unwrap().task_arn, Some(task_arn("mid")));
assert_eq!(loaded[1].status, ProjectStatus::Stopped, "Docker is unchanged");
assert!(needs_aws_reconcile(&loaded[0]));
// ECS says it runs: Running.
let p = loaded[0].clone();
let fake = Fake::default().with_task(task(&p, "mid", "RUNNING", true));
let (_d, store) = store_with(&p);
assert!(matches!(reconcile_project(&fake, &store, &p, &find(&p)).await.unwrap(), Reconciled::Live(_)));
assert_eq!(store.get(&p.id).unwrap().status, ProjectStatus::Running);
// ECS says it is gone: Stopped, ARN cleared.
let (_d, store) = store_with(&p);
assert_eq!(reconcile_project(&Fake::default(), &store, &p, &find(&p)).await.unwrap(), Reconciled::Gone);
let after = store.get(&p.id).unwrap();
assert_eq!(after.status, ProjectStatus::Stopped);
assert_eq!(after.aws.unwrap().task_arn, None);
}
#[test]
fn a_stopped_aws_project_is_reconciled_only_when_it_still_has_a_task_arn() {
use crate::commands::project_commands::needs_aws_reconcile;
let mut p = aws_project(StorageMode::Hybrid);
p.status = ProjectStatus::Stopped;
assert!(!needs_aws_reconcile(&p), "a clean stop clears the ARN");
p.aws.as_mut().unwrap().task_arn = Some(task_arn("left"));
assert!(needs_aws_reconcile(&p));
for s in [ProjectStatus::Running, ProjectStatus::Error, ProjectStatus::Starting, ProjectStatus::Stopping] {
p.status = s;
p.aws.as_mut().unwrap().task_arn = None;
assert!(needs_aws_reconcile(&p));
}
}
+81
View File
@@ -18,6 +18,7 @@ use std::time::Duration;
use aws_config::SdkConfig;
use futures_util::future::BoxFuture;
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
use super::agent::AgentEndpoint;
use crate::aws::tasks::StopKind;
@@ -53,6 +54,9 @@ pub struct LiveTunnel {
pub task_definition_arn: String,
/// EFS storage's `/workspace` access point.
pub workspace_access_point_id: Option<String>,
/// The start that opened this tunnel ran the task (rather than
/// attaching to one already running, or re-attaching after a restart).
pub created: bool,
}
/// Where an endpoint's task is: `(connection, region, cluster, task ARN)`.
@@ -70,6 +74,12 @@ struct Registry {
/// start's own tunnel) and [`cancel_reattach`]: a background re-attach
/// that began under an older generation must never register.
generation: HashMap<String, u64>,
/// The background clone/setup follower per project (`aws_host::status`),
/// with the id [`begin_follow`] gave it. Cancelled by a new follower,
/// [`cancel_follow`] (Stop, Remove), and whenever the tunnel is taken
/// out or every tunnel closed.
followers: HashMap<String, (u64, CancellationToken)>,
next_follower: u64,
}
fn registry() -> &'static Mutex<Registry> {
@@ -209,8 +219,60 @@ pub async fn close_if_task(project_id: &str, task_arn: &str) {
}
}
/// Whether the project's registered tunnel came from a start that ran its
/// task (see [`LiveTunnel::created`]).
pub fn ran_task(project_id: &str) -> bool {
lock().live.get(project_id).is_some_and(|t| t.created)
}
/// A later start found the tunnel already up: it ran no task itself.
pub fn mark_attached(project_id: &str) {
if let Some(t) = lock().live.get_mut(project_id) {
t.created = false;
}
}
/// Register a background follower for `project_id`, cancelling the one
/// before it. Returns its id (for [`end_follow`]) and its token.
pub fn begin_follow(project_id: &str) -> (u64, CancellationToken) {
let mut r = lock();
r.next_follower += 1;
let id = r.next_follower;
let token = CancellationToken::new();
if let Some((_, old)) = r.followers.insert(project_id.to_string(), (id, token.clone())) {
old.cancel();
}
(id, token)
}
/// A follower finished: forget it, unless a newer one has replaced it.
pub fn end_follow(project_id: &str, id: u64) {
let mut r = lock();
if r.followers.get(project_id).is_some_and(|(i, _)| *i == id) {
r.followers.remove(project_id);
}
}
/// End the project's follower, if one runs (Stop, Remove).
pub fn cancel_follow(project_id: &str) {
cancel_follow_locked(&mut lock(), project_id);
}
fn cancel_follow_locked(r: &mut Registry, project_id: &str) {
if let Some((_, t)) = r.followers.remove(project_id) {
t.cancel();
}
}
/// Whether a follower runs for the project.
#[cfg(test)]
pub fn is_following(project_id: &str) -> bool {
lock().followers.contains_key(project_id)
}
/// Under the lock: invalidate a re-attach in flight and take the tunnel out.
fn take(r: &mut Registry, project_id: &str) -> Option<LiveTunnel> {
cancel_follow_locked(r, project_id);
r.pending.remove(project_id);
*r.generation.entry(project_id.to_string()).or_default() += 1;
let t = r.live.remove(project_id);
@@ -259,6 +321,9 @@ pub async fn close_all() {
let mut r = lock();
r.pending.clear();
r.stopped.clear();
for (_, (_, t)) in r.followers.drain() {
t.cancel();
}
for g in r.generation.values_mut() {
*g += 1;
}
@@ -339,6 +404,7 @@ pub(crate) mod tests {
access_point_id: Some("fsap-1".into()),
task_definition_arn: "arn:td:1".into(),
workspace_access_point_id: None,
created: true,
}
}
@@ -364,6 +430,21 @@ pub(crate) mod tests {
assert!(get_endpoint(&p).is_none() && task_of(&ep).is_none());
}
#[tokio::test]
async fn closing_the_tunnel_ends_its_follower_and_an_attach_clears_created() {
let closes = Arc::new(AtomicUsize::new(0));
let p = pid("follow");
register(&p, fake_tunnel(61009, closes.clone()));
assert!(ran_task(&p));
mark_attached(&p);
assert!(!ran_task(&p));
let (_, token) = begin_follow(&p);
assert!(is_following(&p) && !token.is_cancelled());
close(&p).await;
assert!(token.is_cancelled());
assert!(!is_following(&p));
}
#[tokio::test]
async fn re_registering_hands_back_the_old_tunnel_and_clears_a_stale_stop() {
let closes = Arc::new(AtomicUsize::new(0));
+41 -21
View File
@@ -93,6 +93,46 @@ fn record_corrupt_load(file_path: &Path, now: &chrono::DateTime<chrono::Utc>) {
}
}
/// The load's rewrite of stale transient statuses; `true` when it changed
/// any.
pub(crate) fn settle_stale_statuses(projects: &mut [Project]) -> bool {
// Reconcile stale transient statuses: on a cold app start no Docker
// operations can be in flight, so Starting/Stopping are always stale.
// Running/Error are left as-is and reconciled against Docker later
// via the reconcile_project_statuses command.
//
// An AWS project goes to **Error**, not Stopped: an app that quit
// mid-start (or mid-stop) may have left a billed task running, with
// or without its ARN recorded yet. Error is what a failed start that
// left a task already uses; `reconcile_aws_projects` then asks ECS
// (by ARN, else by the project's tags) and settles it Running or
// Stopped. Stopped would hide the live task.
let mut changed = false;
for p in projects.iter_mut() {
match p.status {
crate::models::ProjectStatus::Starting | crate::models::ProjectStatus::Stopping => {
let to = if p.is_aws() {
crate::models::ProjectStatus::Error
} else {
crate::models::ProjectStatus::Stopped
};
log::warn!(
"Reconciling stale '{}' status for project '{}' ({}) → {:?}",
serde_json::to_string(&p.status).unwrap_or_default().trim_matches('"'),
p.name,
p.id,
to
);
p.status = to;
p.updated_at = chrono::Utc::now().to_rfc3339();
changed = true;
}
_ => {}
}
}
changed
}
pub struct ProjectsStore {
projects: Mutex<Vec<Project>>,
file_path: PathBuf,
@@ -155,28 +195,8 @@ impl ProjectsStore {
(Vec::new(), false)
};
// Reconcile stale transient statuses: on a cold app start no Docker
// operations can be in flight, so Starting/Stopping are always stale.
// Running/Error are left as-is and reconciled against Docker later
// via the reconcile_project_statuses command.
let mut projects = projects;
let mut needs_save = needs_save;
for p in projects.iter_mut() {
match p.status {
crate::models::ProjectStatus::Starting | crate::models::ProjectStatus::Stopping => {
log::warn!(
"Reconciling stale '{}' status for project '{}' ({}) → Stopped",
serde_json::to_string(&p.status).unwrap_or_default().trim_matches('"'),
p.name,
p.id
);
p.status = crate::models::ProjectStatus::Stopped;
p.updated_at = chrono::Utc::now().to_rfc3339();
needs_save = true;
}
_ => {}
}
}
let needs_save = settle_stale_statuses(&mut projects) || needs_save;
let store = Self {
projects: Mutex::new(projects),
@@ -13,9 +13,10 @@ const REPO_STATE: Record<string, string> = {
};
/**
* An AWS workspace's background clone and setup script (phase 6, Q4): the
* same status the start followed, re-read every 5 s while the project is
* Running and the window is visible. Each read is a background exec, so the
* An AWS workspace's background clone and setup script (phase 6, Q4/R5):
* the start ends at "Connecting…" and this card is where their progress
* shows, re-read every 5 s while the project is Running and the window is
* visible. Each read is a background exec, so the
* card never keeps an idle workspace up. Renders nothing for a local project,
* a workspace that cannot be asked, or one with no repos and no setup script.
*/