begin_follow runs in spawn_follow and the token is passed in, so a quick Stop→Start can't leave the old follower registered last. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
456 lines
16 KiB
Rust
456 lines
16 KiB
Rust
//! An AWS workspace's clone and setup script, as the start follows them and
|
|
//! Project Home shows them (phase 6, Q4).
|
|
//!
|
|
//! The entrypoint clones in the background and then runs the setup script;
|
|
//! `triple-c-workspace-status` (container) prints both as one JSON line,
|
|
//! mirrored by [`WorkspaceStatus`].
|
|
//!
|
|
//! **The start does not wait for them** (ruling R5, amending Q4): it ends
|
|
//! after "Connecting…" as in phase 5 — project Running, project lock
|
|
//! released, terminals and Stop usable. [`spawn_follow`] then follows in the
|
|
//! background: a read through the agent every [`FOLLOW_INTERVAL`] as a
|
|
//! **background** exec (never activity for idle stop), a log line only when
|
|
//! it changes, until the clone is done and no setup runs, [`FOLLOW_LIMIT`]
|
|
//! passes, the agent is gone (tunnel dropped, task died, image without the
|
|
//! script) or its token is cancelled (`tunnels::cancel_follow` on Stop and
|
|
//! Remove; any tunnel close; app exit). Its only output is the final warning
|
|
//! ([`follow_warning`]); Project Home's Overview card polls the status itself
|
|
//! while the project is Running, so progress shows there.
|
|
|
|
use std::future::Future;
|
|
use std::time::Duration;
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
use tokio_util::sync::CancellationToken;
|
|
|
|
use super::super::{tunnels, ContainerTarget};
|
|
use crate::models::Project;
|
|
|
|
/// The in-container status reader (Task 4).
|
|
pub const STATUS_SCRIPT: &str = "/usr/local/bin/triple-c-workspace-status";
|
|
pub const FOLLOW_INTERVAL: Duration = Duration::from_secs(2);
|
|
/// The setup script's own timeout is 30 minutes, after the clone.
|
|
pub const FOLLOW_LIMIT: Duration = Duration::from_secs(35 * 60);
|
|
/// One status read through the agent.
|
|
const READ_TIMEOUT: Duration = Duration::from_secs(20);
|
|
/// Reads failing in a row before the follow gives up (quietly).
|
|
const MAX_READ_FAILURES: u32 = 5;
|
|
/// The script prints at most ~20 KB (20 tail lines of ≤ 1000 characters).
|
|
const STATUS_OUTPUT_LIMIT: usize = 256 * 1024;
|
|
/// A setup log line shown as progress is cut to this many characters.
|
|
const MAX_PROGRESS_LINE: usize = 200;
|
|
/// Setup log lines a warning carries.
|
|
const WARNING_TAIL_LINES: usize = 10;
|
|
|
|
pub const PROGRESS_CLONING: &str = "Cloning repos…";
|
|
pub const PROGRESS_SETUP: &str = "Running setup script…";
|
|
|
|
// ──────────────────────────────────────────────────────────────── model
|
|
|
|
/// `triple-c-workspace-status`'s JSON. Unknown fields are ignored and
|
|
/// missing ones default, so an older or newer image never breaks a read.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(default)]
|
|
pub struct WorkspaceStatus {
|
|
pub clone: CloneStatus,
|
|
pub setup: SetupStatus,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(default)]
|
|
pub struct CloneStatus {
|
|
/// Every repo has been tried.
|
|
pub done: bool,
|
|
/// The repos tried so far, in order.
|
|
pub repos: Vec<RepoProgress>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(default)]
|
|
pub struct RepoProgress {
|
|
pub folder: String,
|
|
/// `cloning`, `cloned`, `failed` or `exists`.
|
|
pub state: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(default)]
|
|
pub struct SetupStatus {
|
|
pub state: SetupState,
|
|
/// The script's exit code: 0 when `ok`, the code when `failed` (if known).
|
|
pub code: Option<i64>,
|
|
/// The setup log's last lines.
|
|
pub tail: Vec<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub enum SetupState {
|
|
/// No setup script.
|
|
#[default]
|
|
None,
|
|
Running,
|
|
Ok,
|
|
/// Also any state this app does not know.
|
|
#[serde(other)]
|
|
Failed,
|
|
}
|
|
|
|
impl RepoProgress {
|
|
fn finished(&self) -> bool {
|
|
matches!(self.state.as_str(), "cloned" | "failed" | "exists")
|
|
}
|
|
}
|
|
|
|
impl WorkspaceStatus {
|
|
/// Nothing more will change: the clone is done and no setup is running.
|
|
pub fn settled(&self) -> bool {
|
|
self.clone.done && self.setup.state != SetupState::Running
|
|
}
|
|
|
|
pub fn failed_repos(&self) -> Vec<&str> {
|
|
self.clone
|
|
.repos
|
|
.iter()
|
|
.filter(|r| r.state == "failed")
|
|
.map(|r| r.folder.as_str())
|
|
.collect()
|
|
}
|
|
|
|
/// The start's progress line for this state (`None` once settled).
|
|
/// `total` is how many repos the project configures; the status only
|
|
/// lists the ones started so far.
|
|
pub fn progress_line(&self, total: usize) -> Option<String> {
|
|
if !self.clone.done {
|
|
let n = total.max(self.clone.repos.len());
|
|
if n == 0 {
|
|
return Some(PROGRESS_CLONING.into());
|
|
}
|
|
let finished = self.clone.repos.iter().filter(|r| r.finished()).count();
|
|
return Some(format!("{PROGRESS_CLONING} ({}/{n})", (finished + 1).min(n)));
|
|
}
|
|
(self.setup.state == SetupState::Running).then(|| PROGRESS_SETUP.into())
|
|
}
|
|
|
|
/// The setup log's last non-blank line, cleaned for a progress line.
|
|
fn last_log_line(&self) -> Option<String> {
|
|
self.setup
|
|
.tail
|
|
.iter()
|
|
.rev()
|
|
.map(|l| clean_line(l))
|
|
.find(|l| !l.is_empty())
|
|
}
|
|
|
|
/// What went wrong, for the user; `None` when nothing did.
|
|
pub fn problems(&self) -> Option<String> {
|
|
let mut parts = Vec::new();
|
|
let failed = self.failed_repos();
|
|
if !failed.is_empty() {
|
|
parts.push(format!(
|
|
"Cloning failed for {}: {}. The workspace started without {}.",
|
|
if failed.len() == 1 { "a repo" } else { "some repos" },
|
|
failed.join(", "),
|
|
if failed.len() == 1 { "it" } else { "them" },
|
|
));
|
|
}
|
|
if self.setup.state == SetupState::Failed {
|
|
let code = self.setup.code.map(|c| format!(" (exit {c})")).unwrap_or_default();
|
|
let mut s = format!("The setup script failed{code}.");
|
|
let tail: Vec<&String> = self.setup.tail.iter().rev().take(WARNING_TAIL_LINES).collect();
|
|
if !tail.is_empty() {
|
|
s.push_str("\nLast lines:\n");
|
|
s.push_str(
|
|
&tail.into_iter().rev().map(String::as_str).collect::<Vec<_>>().join("\n"),
|
|
);
|
|
}
|
|
parts.push(s);
|
|
}
|
|
(!parts.is_empty()).then(|| parts.join("\n\n"))
|
|
}
|
|
}
|
|
|
|
/// One line of the setup log as a progress message: control characters
|
|
/// dropped, trimmed, at most [`MAX_PROGRESS_LINE`] characters.
|
|
fn clean_line(line: &str) -> String {
|
|
let cleaned: String = line
|
|
.chars()
|
|
.map(|c| if c == '\t' { ' ' } else { c })
|
|
.filter(|c| !c.is_control())
|
|
.collect();
|
|
let t = cleaned.trim();
|
|
if t.chars().count() > MAX_PROGRESS_LINE {
|
|
let mut s: String = t.chars().take(MAX_PROGRESS_LINE - 1).collect();
|
|
s.push('…');
|
|
s
|
|
} else {
|
|
t.to_string()
|
|
}
|
|
}
|
|
|
|
/// Parse the script's output: its last non-blank line is the JSON (anything
|
|
/// before it — a stray warning — is ignored).
|
|
pub fn parse_status(output: &str) -> Result<WorkspaceStatus, String> {
|
|
let line = output
|
|
.lines()
|
|
.rev()
|
|
.find(|l| !l.trim().is_empty())
|
|
.ok_or("The workspace status was empty")?;
|
|
serde_json::from_str(line.trim()).map_err(|e| format!("The workspace status could not be read: {e}"))
|
|
}
|
|
|
|
// ───────────────────────────────────────────────────────────── reading
|
|
|
|
/// One read of the status.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum StatusRead {
|
|
Status(WorkspaceStatus),
|
|
/// No agent to ask (tunnel gone), or an image without the script.
|
|
Gone,
|
|
/// A read that failed; may be retried.
|
|
Failed(String),
|
|
}
|
|
|
|
pub trait StatusSource: Send + Sync {
|
|
fn read(&self) -> impl Future<Output = StatusRead> + Send;
|
|
}
|
|
|
|
/// The command that runs the script; exit 127 when the image predates it.
|
|
fn status_command() -> Vec<String> {
|
|
vec![
|
|
"sh".into(),
|
|
"-c".into(),
|
|
format!("[ -x {STATUS_SCRIPT} ] || exit 127; exec {STATUS_SCRIPT}"),
|
|
]
|
|
}
|
|
|
|
/// Read the status through `target`, as a background exec.
|
|
pub async fn read_status(target: &ContainerTarget) -> StatusRead {
|
|
let res = tokio::time::timeout(
|
|
READ_TIMEOUT,
|
|
target.exec_oneshot_background("claude", status_command(), STATUS_OUTPUT_LIMIT),
|
|
)
|
|
.await;
|
|
match res {
|
|
Err(_) => StatusRead::Failed("The workspace status read timed out".into()),
|
|
Ok(Err(e)) => StatusRead::Failed(e),
|
|
Ok(Ok((out, 0))) => match parse_status(&out) {
|
|
Ok(s) => StatusRead::Status(s),
|
|
Err(e) => StatusRead::Failed(e),
|
|
},
|
|
Ok(Ok((_, 126 | 127))) => StatusRead::Gone,
|
|
Ok(Ok((_, code))) => StatusRead::Failed(format!("The workspace status exited with {code}")),
|
|
}
|
|
}
|
|
|
|
/// The project's registered tunnel, looked up on every read: a stop or a
|
|
/// dropped tunnel unregisters it, and the follow then ends.
|
|
struct TunnelStatusSource<'a> {
|
|
project_id: &'a str,
|
|
}
|
|
|
|
impl StatusSource for TunnelStatusSource<'_> {
|
|
async fn read(&self) -> StatusRead {
|
|
match tunnels::get_endpoint(self.project_id) {
|
|
Some(ep) => read_status(&ContainerTarget::Agent(ep)).await,
|
|
None => StatusRead::Gone,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The status of an AWS project's workspace for Project Home: `None` for a
|
|
/// local project, one with no tunnel, or an image without the script.
|
|
pub async fn workspace_status(project: &Project) -> Result<Option<WorkspaceStatus>, String> {
|
|
if !project.is_aws() {
|
|
return Ok(None);
|
|
}
|
|
match (TunnelStatusSource { project_id: &project.id }).read().await {
|
|
StatusRead::Status(s) => Ok(Some(s)),
|
|
StatusRead::Gone => Ok(None),
|
|
StatusRead::Failed(e) => Err(e),
|
|
}
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────── following
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub struct FollowTiming {
|
|
pub interval: Duration,
|
|
pub limit: Duration,
|
|
}
|
|
|
|
impl Default for FollowTiming {
|
|
fn default() -> Self {
|
|
Self { interval: FOLLOW_INTERVAL, limit: FOLLOW_LIMIT }
|
|
}
|
|
}
|
|
|
|
/// How a follow ended.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum FollowEnd {
|
|
Settled {
|
|
status: WorkspaceStatus,
|
|
/// The very first read was already settled: nothing happened in
|
|
/// this follow (an attach to a workspace that finished long ago).
|
|
first_read: bool,
|
|
},
|
|
/// [`FollowTiming::limit`] passed; the last status read, if any.
|
|
TimedOut(Option<WorkspaceStatus>),
|
|
/// The agent went away, kept failing, or the follow was cancelled:
|
|
/// nothing to say.
|
|
Lost,
|
|
}
|
|
|
|
/// Follow `src` until it settles, the limit passes, it is lost or `cancel`
|
|
/// fires, calling `progress` with each new progress line and each new setup
|
|
/// log line.
|
|
pub async fn follow_workspace_status<S: StatusSource>(
|
|
src: &S,
|
|
total_repos: usize,
|
|
timing: FollowTiming,
|
|
cancel: &CancellationToken,
|
|
progress: &(dyn Fn(&str) + Send + Sync),
|
|
) -> FollowEnd {
|
|
tokio::select! {
|
|
biased;
|
|
_ = cancel.cancelled() => FollowEnd::Lost,
|
|
end = follow_inner(src, total_repos, timing, progress) => end,
|
|
}
|
|
}
|
|
|
|
async fn follow_inner<S: StatusSource>(
|
|
src: &S,
|
|
total_repos: usize,
|
|
timing: FollowTiming,
|
|
progress: &(dyn Fn(&str) + Send + Sync),
|
|
) -> FollowEnd {
|
|
let deadline = tokio::time::Instant::now() + timing.limit;
|
|
let mut last_line: Option<String> = None;
|
|
let mut last_log: Option<String> = None;
|
|
let mut last_status: Option<WorkspaceStatus> = None;
|
|
let mut failures = 0u32;
|
|
let mut reads = 0u32;
|
|
loop {
|
|
match src.read().await {
|
|
StatusRead::Status(s) => {
|
|
failures = 0;
|
|
reads += 1;
|
|
if s.settled() {
|
|
return FollowEnd::Settled { status: s, first_read: reads == 1 };
|
|
}
|
|
if let Some(line) = s.progress_line(total_repos) {
|
|
if last_line.as_deref() != Some(line.as_str()) {
|
|
progress(&line);
|
|
last_line = Some(line);
|
|
}
|
|
}
|
|
if s.setup.state == SetupState::Running {
|
|
if let Some(log) = s.last_log_line() {
|
|
if last_log.as_deref() != Some(log.as_str()) {
|
|
progress(&log);
|
|
last_log = Some(log);
|
|
}
|
|
}
|
|
}
|
|
last_status = Some(s);
|
|
}
|
|
StatusRead::Gone => return FollowEnd::Lost,
|
|
StatusRead::Failed(e) => {
|
|
failures += 1;
|
|
log::debug!("Workspace status read failed ({failures}): {e}");
|
|
if failures >= MAX_READ_FAILURES {
|
|
log::info!("Stopped following the workspace's start: {e}");
|
|
return FollowEnd::Lost;
|
|
}
|
|
}
|
|
}
|
|
if tokio::time::Instant::now() + timing.interval > deadline {
|
|
return FollowEnd::TimedOut(last_status);
|
|
}
|
|
tokio::time::sleep(timing.interval).await;
|
|
}
|
|
}
|
|
|
|
/// What the user is told after a follow; `None` when all went well, the
|
|
/// follow was lost, or the workspace had settled before this start —
|
|
/// a failure is reported only when this follow saw it happen, or the
|
|
/// task is one this start ran (`created`). Project Home still shows an old
|
|
/// failure on demand.
|
|
pub fn follow_warning(end: &FollowEnd, limit: Duration, created: bool) -> Option<String> {
|
|
match end {
|
|
FollowEnd::Settled { status, first_read } => {
|
|
if *first_read && !created {
|
|
None
|
|
} else {
|
|
status.problems()
|
|
}
|
|
}
|
|
FollowEnd::TimedOut(s) => {
|
|
let what = match s {
|
|
Some(s) if !s.clone.done => "still cloning its repos",
|
|
_ => "still running its setup script",
|
|
};
|
|
let mut msg = format!(
|
|
"The workspace is {what} after {} minutes. It is running; Project Home shows the \
|
|
progress.",
|
|
limit.as_secs() / 60
|
|
);
|
|
if let Some(p) = s.as_ref().and_then(WorkspaceStatus::problems) {
|
|
msg.push_str("\n\n");
|
|
msg.push_str(&p);
|
|
}
|
|
Some(msg)
|
|
}
|
|
FollowEnd::Lost => None,
|
|
}
|
|
}
|
|
|
|
/// The follow of one project's workspace. `follower` is its registration
|
|
/// with the tunnel registry (`tunnels::begin_follow`), made by the caller
|
|
/// before the follow is spawned so registrations keep the order of the starts
|
|
/// (a quick Stop→Start can't leave the old follower registered last).
|
|
/// `on_warning` gets the warning, if any.
|
|
pub async fn run_follow<S: StatusSource>(
|
|
src: &S,
|
|
project_id: &str,
|
|
follower: (u64, CancellationToken),
|
|
total_repos: usize,
|
|
created: bool,
|
|
timing: FollowTiming,
|
|
on_warning: impl FnOnce(String),
|
|
) {
|
|
let (id, cancel) = follower;
|
|
let log_line = |m: &str| log::debug!("Project {project_id}: {m}");
|
|
let end = follow_workspace_status(src, total_repos, timing, &cancel, &log_line).await;
|
|
tunnels::end_follow(project_id, id);
|
|
if let Some(w) = follow_warning(&end, timing.limit, created) {
|
|
on_warning(w);
|
|
}
|
|
}
|
|
|
|
/// After a successful start of an AWS project: follow its clone and setup
|
|
/// script in the background (see the module docs). `false` (nothing
|
|
/// spawned) for a local project or one with no tunnel.
|
|
pub fn spawn_follow(
|
|
project: &Project,
|
|
on_warning: impl FnOnce(String) + Send + 'static,
|
|
) -> bool {
|
|
if !project.is_aws() || tunnels::get_endpoint(&project.id).is_none() {
|
|
return false;
|
|
}
|
|
let project_id = project.id.clone();
|
|
let total = project.aws.as_ref().map_or(0, |a| a.repos.len());
|
|
let created = tunnels::ran_task(&project_id);
|
|
// Registered now, not in the spawned task: see `run_follow`.
|
|
let follower = tunnels::begin_follow(&project_id);
|
|
tauri::async_runtime::spawn(async move {
|
|
let src = TunnelStatusSource { project_id: &project_id };
|
|
run_follow(&src, &project_id, follower, total, created, FollowTiming::default(), on_warning)
|
|
.await;
|
|
});
|
|
true
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests;
|