Files
Triple-C/app/src-tauri/src/host/aws_host.rs
T
shadowdaoandClaude Opus 5.5 9e1e72d2a4
Build App (Preview) / compute-version (pull_request) Successful in 6s
Secret Scan / scan (push) Successful in 7s
Secret Scan / scan (pull_request) Successful in 14s
Build App (Preview) / create-release (pull_request) Successful in 6s
Build App (Preview) / build-macos (pull_request) Successful in 6m3s
Build App (Preview) / test (pull_request) Successful in 8m39s
Build App (Preview) / build-windows (pull_request) Successful in 11m38s
Build Container / build-container (pull_request) Successful in 17m24s
Build App (Preview) / build-linux (pull_request) Failing after 21m19s
Build App (Preview) / prune-previews (pull_request) Skipped
feat(aws): setup script editor
HostSection gets a monospace "Setup script" textarea (saved on blur, blank
as none) with the 32 KiB UTF-8 limit checked using the backend's message,
and notes on when it runs (as claude, bash -e, after the clone, every start,
30 minutes, output in the Overview card) and what it gets under
copy-at-connect.

Also in Host settings: the environment's advisory lifetime cap (Q8), read
by a new aws_environment_max_hours command, and the copy-at-connect limits
(no CA before the first connect, removed credentials linger in a running
task, no setup re-run after a re-clone), documented in CLAUDE.md too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 10:59:37 -08:00

1352 lines
53 KiB
Rust

//! `ProjectHost::Aws`: a project's container is an ECS Fargate task in one
//! of the account's environments, reached through an SSM port-forward to
//! the in-task agent (`ssm.rs`, `agent.rs`).
//!
//! Start: connection → `SdkConfig` → caller identity → environment record →
//! onboarding → the project's live task, if there is one (its token is read
//! back from the project secret, P2 — **a running task's token is never
//! rotated**); otherwise `prepare_workspace` (which rotates the token) →
//! access point available → RunTask → ready → tunnel → agent health →
//! registry ([`super::tunnels`]). A task this start ran is stopped again on
//! any later failure, so nothing billed is left behind.
//!
//! The decisions are made by [`start_core`] over the `aws/` traits and a
//! [`Connector`], so fakes exercise them; AWS SDK calls stay in `aws/`.
use std::future::Future;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use aws_config::SdkConfig;
use futures_util::future::BoxFuture;
use super::agent::{self, AgentEndpoint};
use super::ssm::{EcsTask, PortForward, AGENT_PORT};
use super::tunnels::{self, LiveTunnel, TunnelHandle};
use super::watch::{AwsTaskLookup, TaskGroup, WatchedTask};
use super::ContainerTarget;
use crate::aws::environments::EnvironmentInfo;
use crate::aws::runtime::{
check_onboarded, cluster_arn_of_task, cluster_of_task, ensure_agent_token, read_agent_token,
read_environment,
remove_workspace_resources, wait_access_point, PollTiming, RemoveRequest, RuntimeApi,
SdkRuntimeApi,
};
use crate::aws::tasks::{
find_or_run, find_project_task, stopped_among, task_log_stream, wait_ready, SdkTaskApi,
StopKind, TaskApi, TaskInfo, TaskLogs, WaitTiming, LOG_TAIL_LINES, STOP_REASON,
};
use crate::aws::tofu_modules::{
environment_cluster_name, log_stream_prefix, owner_key, task_family,
};
use crate::aws::workspace::{
gateway_sidecar_problem, new_agent_token, prepare_workspace, GatewayPayload, SdkWorkspaceApi,
WorkspaceApi,
};
use crate::aws::{caller_identity, sdk_config, CallerIdentity};
use crate::docker::ca_certs::{container_cert_name, ResolvedCa};
use crate::models::app_settings::ImageSource;
use crate::models::container_config::REGISTRY_IMAGE;
use crate::models::{AppSettings, Project, ProjectStatus};
use crate::storage::projects_store::ProjectsStore;
use crate::storage::settings_store::SettingsStore;
// Progress strings the start view shows, in order (Task 8 matches on none
// of them; they are for the user).
pub const PROGRESS_READING: &str = "Reading environment…";
pub const PROGRESS_PREPARING: &str = "Preparing workspace…";
pub const PROGRESS_STARTING: &str = "Starting task…";
pub const PROGRESS_WAITING: &str = "Waiting for the task (about 2 minutes)…";
pub const PROGRESS_CONNECTING: &str = "Connecting…";
/// A concurrent start won the RunTask race: the secret now holds a token
/// that task never saw.
pub const START_RACE: &str =
"Another start of this workspace is in progress; try again in a minute (once it is up, \
Start connects to it)";
/// The agent refused the token from the secret (a start elsewhere rotated
/// it, and the repair did not happen).
pub const TOKEN_MISMATCH: &str =
"The workspace's access token no longer matches; stop and start the workspace";
/// Per-file cap on what is shipped in the project secret (R1).
pub const MAX_FILE_BYTES: u64 = 64 * 1024;
/// How long the agent gets to answer through a fresh tunnel.
const HEALTH_TIMEOUT: Duration = Duration::from_secs(30);
const HEALTH_INTERVAL: Duration = Duration::from_secs(1);
/// How long a terminal waits for a background re-attach.
pub const REATTACH_WAIT: Duration = Duration::from_secs(90);
/// The most one AWS death-watch poll of a group may take.
const WATCH_CALL_TIMEOUT: Duration = Duration::from_secs(60);
/// The most a StopTask (credentials included) may take before the stop is
/// reported as failed; the SDK's own bound is per call.
const STOP_TIMEOUT: Duration = Duration::from_secs(45);
pub mod connect;
pub mod status;
pub use connect::{spawn_connect_push, terminal_env};
pub use status::{spawn_follow, workspace_status, WorkspaceStatus};
fn s<E: std::fmt::Display>(e: E) -> String {
e.to_string()
}
// ─────────────────────────────────────────────────────────────── inputs
/// The image an AWS task runs (P5): registry images only.
pub fn aws_image(settings: &AppSettings) -> Result<String, String> {
match settings.image_source {
ImageSource::Registry => Ok(REGISTRY_IMAGE.to_string()),
ImageSource::LocalBuild => {
log::info!("AWS projects run the registry image; the local build exists only on this machine");
Ok(REGISTRY_IMAGE.to_string())
}
ImageSource::Custom => {
let name = settings.custom_image_name.as_deref().unwrap_or("").trim();
if is_registry_reference(name) {
Ok(name.to_string())
} else {
Err(format!(
"The custom image \"{}\" is not a registry reference, so AWS can't pull it. \
Use a name like registry.example.com/team/image:tag (Settings → Docker).",
name.escape_debug()
))
}
}
}
}
/// `host/path[:tag]` or `user/repo[:tag]`: something Fargate can pull. A
/// bare name (a local tag) or a loopback registry is not.
fn is_registry_reference(name: &str) -> bool {
if name.is_empty() || name.len() > 512 || name.chars().any(|c| c.is_whitespace() || c.is_control()) {
return false;
}
let Some((first, rest)) = name.split_once('/') else {
return false;
};
if first.is_empty() || rest.is_empty() || rest.starts_with('/') {
return false;
}
let host = first.split(':').next().unwrap_or("");
!(host == "localhost" || host.starts_with("127.") || host == "[::1]")
}
/// [`aws_env`]'s answer: the pairs, and the resolved CA whose files ship too.
pub type AwsEnv = (Vec<(String, String)>, Option<ResolvedCa>);
/// The environment pairs an AWS task gets: [`workspace_env`] with the same
/// inputs Docker resolves, and Claude Code's sandbox forced off (R4: Fargate
/// support for bubblewrap is unverified). Returns the resolved CA too.
///
/// [`workspace_env`]: crate::docker::workspace_env::workspace_env
pub fn aws_env(project: &Project, settings: &AppSettings) -> Result<AwsEnv, String> {
let mut p = project.clone();
if p.sandbox_mode_enabled {
log::info!(
"Project {}: Claude Code's sandbox is turned off on AWS (not yet supported on Fargate)",
project.id
);
p.sandbox_mode_enabled = false;
}
crate::docker::workspace_env::workspace_env_from_settings(&p, settings)
}
/// The gateway sidecar's payload (Q7), owned: its key-free config, the
/// provider key and a fresh master key.
pub struct SidecarSecrets {
config: String,
api_key: zeroize::Zeroizing<String>,
master_key: zeroize::Zeroizing<String>,
}
impl SidecarSecrets {
pub fn payload(&self) -> GatewayPayload<'_> {
GatewayPayload {
config: &self.config,
api_key: &self.api_key,
master_key: &self.master_key,
}
}
}
/// What the gateway sidecar needs for this start, or `None` when the project
/// doesn't run it. Refuses a project the sidecar can't serve
/// ([`gateway_sidecar_problem`]) and a missing provider key (`read_api_key`:
/// the keychain's, Settings → Model Gateway). The master key is new every
/// task start — only this task's sidecar and workspace know it — so the
/// Docker gateway's own master key never leaves this machine.
pub fn gateway_sidecar(
project: &Project,
settings: &AppSettings,
read_api_key: impl FnOnce() -> Result<Option<String>, String>,
) -> Result<Option<SidecarSecrets>, String> {
if !project.aws.as_ref().is_some_and(|a| a.gateway_sidecar) {
return Ok(None);
}
if let Some(problem) = gateway_sidecar_problem(project, &settings.gateway) {
return Err(problem);
}
let api_key = read_api_key()?
.filter(|k| !k.trim().is_empty())
.ok_or_else(|| {
"The LLM gateway sidecar needs the provider API key: set it in Settings → Model \
Gateway"
.to_string()
})?;
Ok(Some(SidecarSecrets {
config: crate::docker::gateway::sidecar_config(&settings.gateway),
api_key: zeroize::Zeroizing::new(api_key),
master_key: zeroize::Zeroizing::new(new_agent_token()),
}))
}
/// One file read for the secret, refusing anything over [`MAX_FILE_BYTES`]
/// (enforced on the bytes read, not on a size the file could change after).
fn read_capped(path: &Path, shown: &str) -> Result<Vec<u8>, String> {
use std::io::Read;
let file = std::fs::File::open(path).map_err(|e| format!("Could not read {shown}: {e}"))?;
let mut bytes = Vec::new();
file.take(MAX_FILE_BYTES + 1)
.read_to_end(&mut bytes)
.map_err(|e| format!("Could not read {shown}: {e}"))?;
if bytes.len() as u64 > MAX_FILE_BYTES {
return Err(format!(
"{shown} is larger than {} KiB, the limit for each file sent to an AWS workspace",
MAX_FILE_BYTES / 1024
));
}
Ok(bytes)
}
/// The regular files directly in `dir`: subdirectories skipped, and a
/// symlink only when it resolves to a file inside `dir`.
fn files_in(dir: &Path) -> Result<Vec<(String, PathBuf)>, String> {
let root = std::fs::canonicalize(dir)
.map_err(|e| format!("Could not read {}: {e}", dir.display()))?;
let mut out = Vec::new();
let entries =
std::fs::read_dir(&root).map_err(|e| format!("Could not read {}: {e}", dir.display()))?;
for entry in entries {
let entry = entry.map_err(|e| format!("Could not read {}: {e}", dir.display()))?;
let path = entry.path();
let name = entry.file_name().to_str().map(str::to_string).ok_or_else(|| {
format!(
"{} has a name that is not UTF-8, so it can't be sent to an AWS workspace; \
rename or move it",
path.display()
)
})?;
let ft = entry
.file_type()
.map_err(|e| format!("Could not read {}: {e}", path.display()))?;
let real = if ft.is_file() {
path.clone()
} else if ft.is_symlink() {
match std::fs::canonicalize(&path) {
Ok(t) if t.starts_with(&root) && t.is_file() => t,
_ => {
log::warn!("Skipping {} (a link out of the folder, or not to a file)", path.display());
continue;
}
}
} else {
continue;
};
out.push((name, real));
}
out.sort();
Ok(out)
}
/// The files the entrypoint writes to `/tmp/.host-ssh` and `/tmp/.host-ca`
/// (R1), as `ssh/<file>` / `ca/<file>` pairs: the project's effective SSH
/// key folder (or file) and the resolved CA, the same sources Docker mounts.
pub fn workspace_files(
ssh_path: Option<&str>,
ca: Option<&ResolvedCa>,
) -> Result<Vec<(String, Vec<u8>)>, String> {
let mut out = Vec::new();
if let Some(ssh) = ssh_path.map(str::trim).filter(|p| !p.is_empty()) {
crate::commands::project_commands::validate_mounted_host_path(
"the SSH key folder",
None,
Some(ssh),
)?;
let path = Path::new(ssh);
if path.is_dir() {
for (name, real) in files_in(path)? {
let bytes = read_capped(&real, &format!("SSH file {name}"))?;
out.push((format!("ssh/{name}"), bytes));
}
} else if path.is_file() {
let name = path
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| format!("The SSH key path {ssh} has no usable file name"))?;
out.push((format!("ssh/{name}"), read_capped(path, &format!("SSH file {name}"))?));
} else {
return Err(format!("The SSH key path {ssh} does not exist"));
}
}
if let Some(ca) = ca {
crate::commands::project_commands::validate_mounted_host_path(
"the CA certificate path",
None,
Some(&ca.host_path),
)?;
if ca.is_dir {
let allowed: Vec<PathBuf> = ca.cert_files.clone();
for (name, real) in files_in(Path::new(&ca.host_path))? {
// Only what `collect_cert_files` picked (by extension).
let picked = allowed.iter().any(|f| {
f.file_name().and_then(|n| n.to_str()) == Some(name.as_str())
});
if picked {
out.push((format!("ca/{name}"), read_capped(&real, &format!("certificate {name}"))?));
}
}
} else {
let path = Path::new(&ca.host_path);
let name = container_cert_name(
&path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(),
);
out.push((format!("ca/{name}"), read_capped(path, &format!("certificate {name}"))?));
}
}
Ok(out)
}
// ──────────────────────────────────────────────────────────── connector
/// Opens the tunnel to a ready task and checks its agent answers.
pub trait Connector: Send + Sync {
fn connect(
&self,
task: &TaskInfo,
token: &str,
) -> impl Future<Output = Result<(Box<dyn TunnelHandle>, AgentEndpoint), String>> + Send;
}
/// The real one: an SSM port-forward, then `/v1/health` through it.
pub struct SsmConnector {
pub sdk: SdkConfig,
}
/// The SSM target of a ready task's `workspace` container.
pub fn ecs_task(task: &TaskInfo) -> Result<EcsTask, String> {
let cluster = cluster_of_task(&task.arn)
.ok_or_else(|| format!("{} is not a task ARN with a cluster", task.arn))?;
let task_id = task.arn.rsplit('/').next().unwrap_or_default().to_string();
let runtime_id = task
.runtime_id
.clone()
.ok_or_else(|| "The task's container has no runtime id yet".to_string())?;
Ok(EcsTask { cluster: cluster.to_string(), task_id, runtime_id })
}
/// What the user is told when the agent answers with an error status.
fn refusal_message(e: &str) -> String {
if e.contains("(401)") {
TOKEN_MISMATCH.into()
} else {
format!("The workspace's agent refused the connection: {e}")
}
}
/// Wait for the agent behind `ep` to answer. A refusal (wrong token) is
/// final; silence is retried until [`HEALTH_TIMEOUT`].
async fn wait_healthy(ep: &AgentEndpoint) -> Result<(), String> {
let deadline = tokio::time::Instant::now() + HEALTH_TIMEOUT;
loop {
match agent::is_running(ep).await {
Ok(true) => return Ok(()),
Ok(false) => {}
Err(e) => return Err(refusal_message(&e)),
}
if tokio::time::Instant::now() + HEALTH_INTERVAL > deadline {
return Err(
"The workspace's agent did not answer through the tunnel. Check that the image \
includes triple-c-agent, then start the project again."
.into(),
);
}
tokio::time::sleep(HEALTH_INTERVAL).await;
}
}
impl Connector for SsmConnector {
async fn connect(
&self,
task: &TaskInfo,
token: &str,
) -> Result<(Box<dyn TunnelHandle>, AgentEndpoint), String> {
let target = ecs_task(task)?;
let forward = PortForward::open(&self.sdk, &target, AGENT_PORT).await?;
let ep = forward.endpoint(token);
if let Err(e) = wait_healthy(&ep).await {
forward.close().await;
return Err(e);
}
Ok((Box::new(forward), ep))
}
}
// ─────────────────────────────────────────────────────────────── start
/// What [`start_core`] needs besides the APIs.
pub struct StartInputs<'a> {
pub project: &'a Project,
pub who: &'a CallerIdentity,
pub env_pairs: &'a [(String, String)],
pub files: &'a [(String, Vec<u8>)],
/// The custom variables' names: credentials under copy-at-connect.
pub custom_env_keys: &'a [String],
pub image: &'a str,
/// The gateway sidecar's payload, when the project runs it
/// ([`gateway_sidecar`]). Called only when the start prepares a new task:
/// attaching to a running one never needs the provider key or models.
pub gateway: &'a (dyn Fn() -> Result<Option<SidecarSecrets>, String> + Send + Sync),
pub wait: WaitTiming,
pub access_point_wait: PollTiming,
/// Called as soon as the project has a task (found, or RunTask
/// returned) so the store holds its ARN before the long wait — a crash
/// then still leaves the task findable — and with `None` when a task
/// this start ran was stopped again.
pub on_task: &'a (dyn Fn(Option<&TaskRecord>) + Send + Sync),
}
/// A project's task handles, for the store's `set_aws_runtime`. `None`
/// fields keep what the store has.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TaskRecord {
pub task_arn: String,
pub access_point_id: Option<String>,
pub task_definition_arn: Option<String>,
/// EFS storage's `/workspace` access point.
pub workspace_access_point_id: Option<String>,
}
/// A connected task.
pub struct Started {
pub env: EnvironmentInfo,
pub task: TaskInfo,
pub forward: Box<dyn TunnelHandle>,
pub endpoint: AgentEndpoint,
/// The access point the task mounts (Hybrid). `None` when an existing
/// task was attached to: the store's value stands.
pub access_point_id: Option<String>,
/// The `/workspace` access point (EFS), like `access_point_id`.
pub workspace_access_point_id: Option<String>,
/// This start ran the task (rather than attaching to a running one).
pub created: bool,
}
/// Append the task's last log lines to a wait failure that lacks them.
async fn with_log_tail<T: TaskApi>(api: &T, logs: &TaskLogs, arn: &str, err: String) -> String {
if err.contains("Last log lines") || err.contains("wrote no log lines") || err.contains("log could not be read") {
return err;
}
match api.log_tail(&logs.group, &task_log_stream(&logs.stream_prefix, arn), LOG_TAIL_LINES).await {
Ok(lines) if !lines.is_empty() => format!("{err}\nLast log lines:\n{}", lines.join("\n")),
_ => err,
}
}
/// Stop a task this start ran, best-effort, so nothing billed is orphaned.
/// `true` when StopTask succeeded; otherwise the task may still run and its
/// ARN must stay recorded.
async fn stop_created<T: TaskApi>(api: &T, cluster: &str, arn: &str) -> bool {
match api.stop_task(cluster, arn, STOP_REASON).await {
Ok(()) => true,
Err(e) => {
log::warn!("Could not stop task {arn} after a failed start: {e}");
false
}
}
}
/// The start sequence; see the module docs.
pub async fn start_core<T: TaskApi, W: WorkspaceApi, R: RuntimeApi, C: Connector>(
tasks: &T,
work: &W,
rt: &R,
connector: &C,
inp: &StartInputs<'_>,
progress: &(dyn Fn(&str) + Send + Sync),
) -> Result<Started, String> {
let project = inp.project;
let (placement, cfg) = match (&project.aws_placement, &project.aws) {
(Some(p), Some(c)) if project.is_aws() => (p, c),
_ => return Err("The project is not set up to run on AWS".into()),
};
let region = placement.region.as_str();
let ok = owner_key(region, &inp.who.user_id);
progress(PROGRESS_READING);
let env = read_environment(rt, &placement.environment, &inp.who.account, region)
.await
.map_err(s)?;
check_onboarded(rt, &env.name, &ok).await.map_err(s)?;
let family = task_family(&env.name, &ok, &project.id);
let logs = TaskLogs {
group: env.log_group.clone(),
stream_prefix: log_stream_prefix(&ok, &project.id),
};
let existing = find_project_task(tasks, &env.cluster_arn, &family, &inp.who.user_id, &project.id)
.await
.map_err(s)?;
let (task, token, access_point_id, workspace_access_point_id, created) = if let Some(task) = existing {
// Attach: never rotate a running task's token (P2).
log::info!("Project {}: attaching to its running task {}", project.id, task.arn);
(inp.on_task)(Some(&TaskRecord {
task_arn: task.arn.clone(),
access_point_id: None,
task_definition_arn: Some(task.task_definition_arn.clone()),
workspace_access_point_id: None,
}));
let token = read_agent_token(rt, &env.name, &ok, &project.id).await.map_err(s)?;
progress(PROGRESS_WAITING);
let task = match wait_ready(tasks, &env.cluster_arn, &task.arn, &logs, inp.wait, |_| {}).await {
Ok(t) => t,
Err(e) => return Err(with_log_tail(tasks, &logs, &task.arn, e.to_string()).await),
};
(task, token, None, None, false)
} else {
progress(PROGRESS_PREPARING);
let sidecar = (inp.gateway)()?;
let prepared = prepare_workspace(
work,
inp.who,
&env,
project,
inp.env_pairs,
inp.files,
inp.custom_env_keys,
inp.image,
sidecar.as_ref().map(SidecarSecrets::payload),
)
.await
.map_err(s)?;
for ap in [&prepared.access_point_id, &prepared.workspace_access_point_id]
.into_iter()
.flatten()
{
wait_access_point(rt, ap, inp.access_point_wait).await.map_err(s)?;
}
progress(PROGRESS_STARTING);
let start = find_or_run(
tasks,
&env,
&inp.who.user_id,
&project.id,
&prepared.task_definition_arn,
cfg.spot,
)
.await
.map_err(s)?;
if start.reused {
// The winner repairs the token this start just rotated
// (`ensure_agent_token`); nothing to undo here.
(inp.on_task)(Some(&TaskRecord {
task_arn: start.task.arn.clone(),
access_point_id: prepared.access_point_id.clone(),
task_definition_arn: Some(start.task.task_definition_arn.clone()),
workspace_access_point_id: prepared.workspace_access_point_id.clone(),
}));
return Err(START_RACE.into());
}
let arn = start.task.arn.clone();
(inp.on_task)(Some(&TaskRecord {
task_arn: arn.clone(),
access_point_id: prepared.access_point_id.clone(),
task_definition_arn: Some(prepared.task_definition_arn.clone()),
workspace_access_point_id: prepared.workspace_access_point_id.clone(),
}));
progress(PROGRESS_WAITING);
let task = match wait_ready(tasks, &env.cluster_arn, &arn, &logs, inp.wait, |_| {}).await {
Ok(t) => t,
Err(e) => {
let msg = with_log_tail(tasks, &logs, &arn, e.to_string()).await;
if stop_created(tasks, &env.cluster_arn, &arn).await {
(inp.on_task)(None);
}
return Err(msg);
}
};
(
task,
prepared.agent_token,
prepared.access_point_id,
prepared.workspace_access_point_id,
true,
)
};
progress(PROGRESS_CONNECTING);
match connector.connect(&task, &token).await {
Ok((forward, endpoint)) => {
if created {
// A start that lost the RunTask race rotated the secret's
// token after this one wrote it; put ours back so the next
// attach (another machine, a restart) reads the right one.
match ensure_agent_token(work, rt, &env, &inp.who.user_id, &ok, &project.id, &token).await {
Ok(true) => log::warn!(
"Project {}: the secret's agent token had been rotated by another start; restored",
project.id
),
Ok(false) => {}
Err(e) => log::warn!(
"Project {}: could not check the secret's agent token ({e}); a later attach may be refused",
project.id
),
}
}
Ok(Started {
env,
task,
forward,
endpoint,
access_point_id,
workspace_access_point_id,
created,
})
}
Err(e) => {
if created && stop_created(tasks, &env.cluster_arn, &task.arn).await {
(inp.on_task)(None);
}
Err(e)
}
}
}
/// The connection, region and SDK config of an AWS project.
async fn project_sdk(project: &Project, settings: &AppSettings) -> Result<(String, String, SdkConfig), String> {
let host_id = project.host_id.as_deref().ok_or("The project has no host")?;
let region = project
.aws_placement
.as_ref()
.map(|p| p.region.clone())
.ok_or("The project has no AWS placement")?;
let conn = crate::commands::host_commands::aws_conn(host_id, settings)?;
let sdk = sdk_config(&conn, &region).await.map_err(s)?;
Ok((host_id.to_string(), region, sdk))
}
/// The lifetime cap, in hours (0 = none), of the environment `project` runs
/// in, from its record (Q8: Host settings show it). `None` when it can't be
/// read — offline, denied, slow, or the environment is gone.
pub async fn environment_max_hours(project: &Project, settings: &AppSettings) -> Option<u32> {
let placement = project.aws_placement.as_ref()?;
let read = async {
let (_, region, sdk) = project_sdk(project, settings).await.ok()?;
let who = caller_identity(&sdk).await.ok()?;
read_environment(&SdkRuntimeApi::new(&sdk), &placement.environment, &who.account, &region)
.await
.ok()
};
let info = tokio::time::timeout(std::time::Duration::from_secs(10), read).await.ok()??;
Some(info.max_workspace_hours)
}
/// `ProjectHost::Aws::ensure_started`.
pub async fn ensure_started(
project: &Project,
settings: &AppSettings,
store: &ProjectsStore,
progress: &(dyn Fn(&str) + Send + Sync),
) -> Result<ContainerTarget, String> {
ensure_started_with(
project,
settings,
store,
progress,
crate::storage::secure::get_gateway_api_key,
)
.await
}
/// [`ensure_started`] with the gateway sidecar's provider key read by
/// `read_gateway_key` instead of the keychain (the live tests have none).
pub async fn ensure_started_with(
project: &Project,
settings: &AppSettings,
store: &ProjectsStore,
progress: &(dyn Fn(&str) + Send + Sync),
read_gateway_key: impl FnOnce() -> Result<Option<String>, String> + Send,
) -> Result<ContainerTarget, String> {
// A background re-attach (after a restart) may be opening a tunnel:
// cancel it, this start attaches itself. Its tunnel, if it gets one, is
// refused by the registry and closed.
tunnels::cancel_reattach(&project.id);
// Already connected and answering: nothing to do.
if let Some(ep) = tunnels::get_endpoint(&project.id) {
if matches!(agent::is_running(&ep).await, Ok(true)) {
tunnels::mark_attached(&project.id);
return Ok(ContainerTarget::Agent(ep));
}
tunnels::close(&project.id).await;
}
// Everything local first: a bad image or file fails before any AWS call.
let image = aws_image(settings)?;
let (env_pairs, ca) = aws_env(project, settings)?;
let ssh = project
.ssh_key_path
.as_deref()
.or(settings.default_ssh_key_path.as_deref());
let files = workspace_files(ssh, ca.as_ref())?;
let custom_env_keys =
crate::docker::workspace_env::custom_env_keys(project, &settings.global_custom_env_vars);
// Read lazily: only a start that prepares a new task needs it (an attach
// must work with the provider key gone from the keychain).
let read_gateway_key = std::sync::Mutex::new(Some(read_gateway_key));
let sidecar = || {
gateway_sidecar(project, settings, || match read_gateway_key.lock().unwrap().take() {
Some(read) => read(),
None => Ok(None),
})
};
let (connection_id, region, sdk) = project_sdk(project, settings).await?;
let who = caller_identity(&sdk).await.map_err(s)?;
let on_task = |rec: Option<&TaskRecord>| record_task(store, &project.id, rec);
let started = start_core(
&SdkTaskApi::new(&sdk),
&SdkWorkspaceApi::new(&sdk),
&SdkRuntimeApi::new(&sdk),
&SsmConnector { sdk: sdk.clone() },
&StartInputs {
project,
who: &who,
env_pairs: &env_pairs,
files: &files,
custom_env_keys: &custom_env_keys,
image: &image,
gateway: &sidecar,
wait: WaitTiming::default(),
access_point_wait: PollTiming::default(),
on_task: &on_task,
},
progress,
)
.await?;
log::info!(
"Project {}: {} task {}",
project.id,
if started.created { "started" } else { "attached to" },
started.task.arn
);
let endpoint = started.endpoint.clone();
let access_point_id = started
.access_point_id
.or_else(|| project.aws.as_ref().and_then(|a| a.access_point_id.clone()));
let workspace_access_point_id = started
.workspace_access_point_id
.or_else(|| project.aws.as_ref().and_then(|a| a.workspace_access_point_id.clone()));
let ecs = ecs_task(&started.task)?;
if let Some(old) = tunnels::register(
&project.id,
LiveTunnel {
task: ecs,
task_arn: started.task.arn.clone(),
cluster_arn: started.env.cluster_arn.clone(),
connection_id,
region,
sdk,
forward: started.forward,
endpoint: endpoint.clone(),
access_point_id,
workspace_access_point_id,
task_definition_arn: started.task.task_definition_arn.clone(),
created: started.created,
},
) {
old.forward.close().await;
}
Ok(ContainerTarget::Agent(endpoint))
}
/// Write a start's task handles to the store as soon as they exist
/// ([`StartInputs::on_task`]); `None` clears the task ARN.
pub fn record_task(store: &ProjectsStore, project_id: &str, rec: Option<&TaskRecord>) {
let Some(stored) = store.get(project_id).and_then(|p| p.aws) else {
return;
};
if let Some(r) = rec {
remember_task(project_id, &r.task_arn);
}
let res = match rec {
Some(r) => store.set_aws_runtime(
project_id,
Some(r.task_arn.clone()),
r.access_point_id.clone().or(stored.access_point_id),
r.task_definition_arn.clone().or(stored.task_definition_arn),
r.workspace_access_point_id.clone().or(stored.workspace_access_point_id),
),
None => store.set_aws_runtime(
project_id,
None,
stored.access_point_id,
stored.task_definition_arn,
stored.workspace_access_point_id,
),
};
if let Err(e) = res {
log::warn!("Project {project_id}: could not record its task: {e}");
}
}
// ──────────────────────────────────────────────────────────────── logs
/// The last task each project's starts ran or found, kept for this run of
/// the app: a start that failed and stopped its own task clears the stored
/// ARN, and its log is what the user then wants to read.
static LAST_TASK: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<String, String>>> =
std::sync::OnceLock::new();
fn last_tasks() -> std::sync::MutexGuard<'static, std::collections::HashMap<String, String>> {
LAST_TASK
.get_or_init(Default::default)
.lock()
.unwrap_or_else(|e| e.into_inner())
}
pub fn remember_task(project_id: &str, task_arn: &str) {
last_tasks().insert(project_id.to_string(), task_arn.to_string());
}
pub fn last_task_arn(project_id: &str) -> Option<String> {
last_tasks().get(project_id).cloned()
}
/// The task whose log `aws_task_log` reads: the stored (current) one, else
/// the tunnel's, else the last one a start of this run ran or found.
pub fn log_task_arn(project: &Project) -> Option<String> {
project
.aws
.as_ref()
.and_then(|a| a.task_arn.clone())
.or_else(|| tunnels::runtime(&project.id).map(|r| r.0))
.or_else(|| last_task_arn(&project.id))
}
/// The last `limit` lines of `task_arn`'s log stream, in the environment
/// record's log group.
pub async fn task_log_lines<T: TaskApi, R: RuntimeApi>(
api: &T,
rt: &R,
project: &Project,
who: &CallerIdentity,
region: &str,
task_arn: &str,
limit: i32,
) -> Result<Vec<String>, String> {
let placement = project.aws_placement.as_ref().ok_or("The project has no AWS placement")?;
let env = read_environment(rt, &placement.environment, &who.account, region)
.await
.map_err(s)?;
let prefix = log_stream_prefix(&owner_key(region, &who.user_id), &project.id);
api.log_tail(&env.log_group, &task_log_stream(&prefix, task_arn), limit)
.await
.map_err(s)
}
/// `aws_task_log`: the current or last task's log; empty with no task.
pub async fn task_log(project: &Project, settings: &AppSettings, limit: i32) -> Result<Vec<String>, String> {
let Some(arn) = log_task_arn(project) else {
return Ok(Vec::new());
};
let (_, region, sdk) = project_sdk(project, settings).await?;
let who = caller_identity(&sdk).await.map_err(s)?;
task_log_lines(&SdkTaskApi::new(&sdk), &SdkRuntimeApi::new(&sdk), project, &who, &region, &arn, limit).await
}
// ──────────────────────────────────────────────────────────────── stop
/// StopTask the task in `task_arn`'s cluster.
pub async fn stop_task_arn<T: TaskApi>(api: &T, task_arn: &str) -> Result<(), String> {
let cluster = cluster_of_task(task_arn)
.ok_or_else(|| format!("{task_arn} is not a task ARN with a cluster"))?;
bounded_stop(api.stop_task(cluster, task_arn, STOP_REASON)).await
}
/// A StopTask, bounded by [`STOP_TIMEOUT`]: a stop that cannot reach AWS
/// fails (and the project keeps its task ARN) instead of hanging the UI.
async fn bounded_stop(
call: impl std::future::Future<Output = Result<(), crate::aws::AwsError>>,
) -> Result<(), String> {
tokio::time::timeout(STOP_TIMEOUT, call)
.await
.map_err(|_| format!("StopTask did not answer within {STOP_TIMEOUT:?}; try again"))?
.map_err(s)
}
/// `ProjectHost::stop` for an agent target: the task behind a registered
/// tunnel. `None` when `ep` is not one of ours.
pub async fn stop_endpoint(ep: &AgentEndpoint) -> Option<Result<(), String>> {
let (_connection, _region, cluster, arn) = tunnels::task_of(ep)?;
let (project_id, sdk) = tunnels::owner_of(ep)?;
let res = bounded_stop(SdkTaskApi::new(&sdk).stop_task(&cluster, &arn, STOP_REASON)).await;
if res.is_ok() {
tunnels::close(&project_id).await;
}
Some(res)
}
/// Where to look for a project's live task when no ARN is stored (a crash
/// before the store write, or a store from an older build).
pub struct FindTask {
pub cluster: String,
pub family: String,
pub owner_user_id: String,
pub project_id: String,
}
impl FindTask {
fn new(project: &Project, region: &str, owner_user_id: &str) -> Self {
let env = project
.aws_placement
.as_ref()
.map(|p| p.environment.clone())
.unwrap_or_default();
let ok = owner_key(region, owner_user_id);
FindTask {
cluster: environment_cluster_name(&env),
family: task_family(&env, &ok, &project.id),
owner_user_id: owner_user_id.to_string(),
project_id: project.id.clone(),
}
}
}
/// The project's task ARN: the stored one, else its live task's.
async fn task_arn_or_find<T: TaskApi>(
api: &T,
stored: Option<&str>,
find: &FindTask,
) -> Result<Option<String>, String> {
if let Some(a) = stored {
return Ok(Some(a.to_string()));
}
Ok(
find_project_task(api, &find.cluster, &find.family, &find.owner_user_id, &find.project_id)
.await
.map_err(s)?
.map(|t| t.arn),
)
}
/// Stop an AWS project's task: through its tunnel when there is one, else
/// the stored task ARN, else whatever live task the project has (a crash
/// between RunTask and the store write). Closes the tunnel.
pub async fn stop_project(project: &Project, settings: &AppSettings) -> Result<(), String> {
tunnels::cancel_follow(&project.id);
if let Some(ep) = tunnels::get_endpoint(&project.id) {
if let Some(res) = stop_endpoint(&ep).await {
return res;
}
}
let (_, region, sdk) = project_sdk(project, settings).await?;
let api = SdkTaskApi::new(&sdk);
let stored = project.aws.as_ref().and_then(|a| a.task_arn.clone());
let arn = match stored {
Some(a) => Some(a),
None => {
let who = caller_identity(&sdk).await.map_err(s)?;
task_arn_or_find(&api, None, &FindTask::new(project, &region, &who.user_id)).await?
}
};
if let Some(arn) = arn {
stop_task_arn(&api, &arn).await?;
}
tunnels::close(&project.id).await;
Ok(())
}
// ────────────────────────────────────────────────────────────── remove
/// Stop the task and remove the project's AWS resources (task definition
/// revisions, the secret at once, and with `discard_data` the
/// access point). The tunnel is closed once that succeeded; on failure the
/// project keeps it.
pub async fn remove_resources(
project: &Project,
settings: &AppSettings,
discard_data: bool,
) -> Result<(), String> {
tunnels::cancel_follow(&project.id);
let (_, region, sdk) = project_sdk(project, settings).await?;
let who = caller_identity(&sdk).await.map_err(s)?;
let ok = owner_key(&region, &who.user_id);
let env = project.aws_placement.as_ref().map(|p| p.environment.clone()).unwrap_or_default();
let aws = project.aws.as_ref();
remove_workspace_resources(
&SdkTaskApi::new(&sdk),
&SdkWorkspaceApi::new(&sdk),
&SdkRuntimeApi::new(&sdk),
&RemoveRequest {
env: &env,
region: &region,
owner_user_id: &who.user_id,
owner_key: &ok,
project_id: &project.id,
stored_task_arn: aws.and_then(|a| a.task_arn.as_deref()),
stored_access_point_id: aws.and_then(|a| a.access_point_id.as_deref()),
stored_workspace_access_point_id: aws
.and_then(|a| a.workspace_access_point_id.as_deref()),
discard_data,
},
)
.await
.map_err(s)?;
tunnels::close(&project.id).await;
Ok(())
}
// ─────────────────────────────────────────────────────────── reconcile
/// What reconcile found for a Running AWS project.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Reconciled {
/// The task is up (or on its way): re-open the tunnel.
Live(Box<TaskInfo>),
/// The task stopped, or ECS no longer knows it: the project is Stopped.
Gone,
}
/// Ask ECS about a project's task: the stored ARN, else (none stored) its
/// live task found by family and tags.
pub async fn reconcile_task<T: TaskApi>(
api: &T,
task_arn: Option<&str>,
find: &FindTask,
) -> Result<Reconciled, String> {
let Some(arn) = task_arn_or_find(api, task_arn, find).await? else {
return Ok(Reconciled::Gone);
};
if let Some(cluster) = cluster_of_task(&arn) {
let found = api.describe(cluster, std::slice::from_ref(&arn)).await.map_err(s)?;
if let Some(t) = found.into_iter().find(|t| t.arn == arn).filter(is_live) {
return Ok(Reconciled::Live(Box::new(t)));
}
}
// The stored task is gone or stopping. Another start (another machine,
// or a crash before the store write) may have left the project a newer
// one: look before calling it Gone (m3).
if task_arn.is_some() {
if let Some(t) =
find_project_task(api, &find.cluster, &find.family, &find.owner_user_id, &find.project_id)
.await
.map_err(s)?
.filter(|t| t.arn != arn && is_live(t))
{
return Ok(Reconciled::Live(Box::new(t)));
}
}
Ok(Reconciled::Gone)
}
fn is_live(t: &TaskInfo) -> bool {
t.desired_status == "RUNNING"
&& matches!(t.last_status.as_str(), "PROVISIONING" | "PENDING" | "ACTIVATING" | "RUNNING")
}
/// The target id an AWS project gets while it has no registered tunnel, or
/// `None` when its stored one is fine (a registered tunnel's, already this
/// placeholder, or none). Never a loopback URL, so a port a new tunnel
/// reuses can't make the death watch match the wrong project (m9). A valid
/// agent URL, so the watch can still report the project by it.
pub fn unattached_container_id(p: &Project) -> Option<String> {
if !p.is_aws() {
return None;
}
let stored = p.container_id.as_deref()?;
if tunnels::get_endpoint(&p.id).is_some_and(|ep| ep.base_url.as_str() == stored) {
return None;
}
let placeholder = tunnels::unattached_id(&p.id);
(stored != placeholder).then_some(placeholder)
}
/// Before reconcile asks ECS about `p`: a project with no registered tunnel
/// is (or is about to be) known by its [`tunnels::unattached_id`], which a
/// past unattached spell may have left marked stopped. Reconcile is the
/// authority about the task now, so that mark goes; the watch re-marks it if
/// the task really is stopped. Returns the placeholder to store, if any.
pub fn prepare_unattached(p: &Project) -> Option<String> {
if !p.is_aws() || tunnels::get_endpoint(&p.id).is_some() {
return None;
}
tunnels::forget_stopped(&tunnels::unattached_id(&p.id));
unattached_container_id(p)
}
/// Reconcile one AWS project the store has as Running (or Starting/
/// Stopping/Error left by a crash): a gone task makes it Stopped (its task
/// ARN cleared); a live one keeps it Running and its ARN stored. An ECS
/// that can't be asked changes nothing — marking Stopped would hide a
/// billed task.
pub async fn reconcile_project<T: TaskApi>(
api: &T,
store: &ProjectsStore,
project: &Project,
find: &FindTask,
) -> Result<Reconciled, String> {
let arn = project.aws.as_ref().and_then(|a| a.task_arn.clone());
let r = reconcile_task(api, arn.as_deref(), find).await?;
match &r {
Reconciled::Gone => {
store.update_status(&project.id, ProjectStatus::Stopped)?;
record_task(store, &project.id, None);
}
Reconciled::Live(t) => {
if arn.as_deref() != Some(t.arn.as_str()) {
record_task(
store,
&project.id,
Some(&TaskRecord {
task_arn: t.arn.clone(),
access_point_id: None,
task_definition_arn: Some(t.task_definition_arn.clone()),
workspace_access_point_id: None,
}),
);
}
if project.status != ProjectStatus::Running {
store.update_status(&project.id, ProjectStatus::Running)?;
}
}
}
Ok(r)
}
/// `reconcile_project_statuses` for an AWS project: decide, and when the
/// task is live re-open its tunnel in the background (terminals wait for
/// it through [`tunnels::wait_endpoint`]), then — copy-at-connect — copy the
/// credentials in, calling `on_connect_warning` if that fails — after
/// opening its port mappings (`on_ports_warning` for any that fail). Never
/// blocks on the tunnel.
pub async fn reconcile(
project: &Project,
settings: &AppSettings,
store: Arc<ProjectsStore>,
on_connect_warning: impl FnOnce(String) + Send + 'static,
on_ports_warning: impl FnOnce(String) + Send + 'static,
) {
let (connection_id, region, sdk) = match project_sdk(project, settings).await {
Ok(v) => v,
Err(e) => {
log::warn!("Project {}: could not reach AWS to reconcile ({e}); status left as is", project.id);
return;
}
};
let who = match caller_identity(&sdk).await {
Ok(w) => w,
Err(e) => {
log::warn!("Project {}: could not reach AWS to reconcile ({e}); status left as is", project.id);
return;
}
};
let api = SdkTaskApi::new(&sdk);
let find = FindTask::new(project, &region, &who.user_id);
match reconcile_project(&api, &store, project, &find).await {
Ok(Reconciled::Live(task)) => {
let generation = tunnels::begin_reattach(&project.id);
let (project, settings) = (project.clone(), settings.clone());
tauri::async_runtime::spawn(async move {
let ctx = Reattach { generation, connection_id, region, sdk, who };
match reattach(&project, *task, ctx, &store).await {
// Re-attached: a connect like any other (Q2).
Ok(true) => {
// Q6: the port mappings' forwards, before the
// (slower) credential copy.
if let Some(detail) = super::port_forwards::open_for_project(&store, &project.id).await {
on_ports_warning(detail);
}
if let Err(e) = connect::apply_connect(&project, &settings).await {
log::warn!("Project {}: could not copy the credentials in: {e}", project.id);
if tunnels::get_endpoint(&project.id).is_some() {
on_connect_warning(e);
}
}
}
Ok(false) => {}
Err(e) => {
log::warn!("Project {}: could not re-open the tunnel to its task: {e}", project.id);
tunnels::end_reattach(&project.id, generation);
}
}
});
}
Ok(Reconciled::Gone) => {
log::info!("Project '{}' ({}): its task is gone — set to Stopped", project.name, project.id)
}
Err(e) => log::warn!("Project {}: could not check its task ({e}); status left as is", project.id),
}
}
struct Reattach {
generation: u64,
connection_id: String,
region: String,
sdk: SdkConfig,
who: CallerIdentity,
}
/// Whether a re-attach may still register: the project exists, is Running
/// and still has this task. A Stop or Remove meanwhile also bumps the
/// registry's generation, which `register_if_current` checks.
pub fn reattach_still_wanted(store: &ProjectsStore, project_id: &str, task_arn: &str) -> bool {
store.get(project_id).is_some_and(|p| {
p.status == ProjectStatus::Running
&& p.aws.as_ref().and_then(|a| a.task_arn.as_deref()) == Some(task_arn)
})
}
/// Re-open the tunnel to a live task found at startup. `Ok(true)` once it
/// is registered; `Ok(false)` when it was no longer wanted.
async fn reattach(
project: &Project,
task: TaskInfo,
ctx: Reattach,
store: &ProjectsStore,
) -> Result<bool, String> {
let placement = project.aws_placement.as_ref().ok_or("no placement")?;
let ok = owner_key(&ctx.region, &ctx.who.user_id);
let rt = SdkRuntimeApi::new(&ctx.sdk);
let api = SdkTaskApi::new(&ctx.sdk);
let env = read_environment(&rt, &placement.environment, &ctx.who.account, &ctx.region)
.await
.map_err(s)?;
let token = read_agent_token(&rt, &env.name, &ok, &project.id).await.map_err(s)?;
let logs = TaskLogs { group: env.log_group.clone(), stream_prefix: log_stream_prefix(&ok, &project.id) };
let task = wait_ready(&api, &env.cluster_arn, &task.arn, &logs, WaitTiming::default(), |_| {})
.await
.map_err(s)?;
let (forward, endpoint) = SsmConnector { sdk: ctx.sdk.clone() }.connect(&task, &token).await?;
let tunnel = LiveTunnel {
task: ecs_task(&task)?,
task_arn: task.arn.clone(),
cluster_arn: env.cluster_arn.clone(),
connection_id: ctx.connection_id,
region: ctx.region,
sdk: ctx.sdk,
forward,
endpoint: endpoint.clone(),
access_point_id: project.aws.as_ref().and_then(|a| a.access_point_id.clone()),
workspace_access_point_id: project
.aws
.as_ref()
.and_then(|a| a.workspace_access_point_id.clone()),
task_definition_arn: task.task_definition_arn.clone(),
created: false,
};
if !reattach_still_wanted(store, &project.id, &task.arn) {
tunnel.forward.close().await;
tunnels::end_reattach(&project.id, ctx.generation);
log::info!("Project {}: stopped or removed while re-attaching; tunnel closed", project.id);
return Ok(false);
}
match tunnels::register_if_current(&project.id, ctx.generation, tunnel) {
Ok(old) => {
if let Some(old) = old {
old.forward.close().await;
}
}
Err(tunnel) => {
tunnel.forward.close().await;
log::info!("Project {}: stopped or removed while re-attaching; tunnel closed", project.id);
return Ok(false);
}
}
// The local port is new, so the stored target id is too.
store.set_container_id(&project.id, Some(endpoint.base_url.to_string()))?;
log::info!("Project {}: tunnel to task {} re-opened", project.id, task.arn);
Ok(true)
}
// ─────────────────────────────────────────────────────────────── watch
/// The AWS death watch's lookup: Running AWS projects from the store, their
/// targets from the registry (or, while unattached, an endpoint with the
/// stored id so `container_watch::settle` still matches), and
/// `stopped_among` per connection/region/cluster.
pub struct StoreTaskLookup {
pub projects: Arc<ProjectsStore>,
pub settings: Arc<SettingsStore>,
}
/// A Running AWS project's watched task, if it has one.
pub fn watched_task(p: &Project) -> Option<WatchedTask> {
if p.status != ProjectStatus::Running || !p.is_aws() {
return None;
}
let arn = p.aws.as_ref()?.task_arn.clone()?;
let target = match tunnels::get_endpoint(&p.id) {
Some(ep) => ep,
None => AgentEndpoint::new(url::Url::parse(p.container_id.as_deref()?).ok()?, "unattached").ok()?,
};
if Some(target.base_url.as_str()) != p.container_id.as_deref() {
return None;
}
Some(WatchedTask {
target: ContainerTarget::Agent(target),
group: TaskGroup {
connection_id: p.host_id.clone()?,
region: p.aws_placement.as_ref()?.region.clone(),
cluster_arn: cluster_arn_of_task(&arn)?,
},
task_arn: arn,
})
}
impl AwsTaskLookup for StoreTaskLookup {
fn running_tasks(&self) -> Vec<WatchedTask> {
self.projects.list().iter().filter_map(watched_task).collect()
}
fn stopped<'a>(
&'a self,
group: &'a TaskGroup,
arns: &'a [String],
) -> BoxFuture<'a, Result<Vec<String>, String>> {
Box::pin(async move {
let settings = self.settings.get();
let conn = crate::commands::host_commands::aws_conn(&group.connection_id, &settings)?;
// The SDK bounds each call; this bounds the whole poll of a group
// (credentials included, which an SSO refresh can stall), so one
// stuck group never stalls the watch of every other.
let stopped = tokio::time::timeout(WATCH_CALL_TIMEOUT, async {
let sdk = sdk_config(&conn, &group.region).await.map_err(s)?;
stopped_among(&SdkTaskApi::new(&sdk), &group.cluster_arn, arns)
.await
.map_err(s)
})
.await
.map_err(|_| format!("ECS did not answer within {WATCH_CALL_TIMEOUT:?}"))??;
record_stopped(&self.projects.list(), &stopped);
Ok(stopped.into_iter().map(|(arn, _)| arn).collect())
})
}
}
/// Tell the registry which targets' tasks ECS reports stopped, and why, so
/// `has_exited` confirms them (R3) and the death report can say why (P8).
pub fn record_stopped(projects: &[Project], stopped: &[(String, StopKind)]) {
for w in projects.iter().filter_map(watched_task) {
if let Some((_, why)) = stopped.iter().find(|(arn, _)| *arn == w.task_arn) {
tunnels::mark_stopped(w.target.id(), why.clone());
}
}
}
#[cfg(test)]
mod tests;