test(aws): guard the live workspace test against hangs and leaks

Review round 1 of Task 9:
- the phase-5 body runs under a 45-minute timeout and each exec under
  180 s, so a hang becomes a panic the guard catches; the body future is
  boxed (moved into the timeout it overflowed the test thread's stack,
  which aborts without any guard);
- scripts/aws-e2e-workspace.sh builds and pushes the image to a throwaway
  ECR repository, runs the test and deletes the repository in an exit
  trap; the test's guard deletes it too (TRIPLEC_E2E_ECR_REPO);
- the guard survives a poisoned lock, closes the tunnel and restores
  AWS_CONFIG_FILE;
- new live probe: a bare elasticfilesystem:TagResource on the user's own
  file system is denied; README records it and the final run (R8 removal
  end state), and the admin command uses xargs -r.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/src-tauri/src/aws/live_tests.rs

@@ -1731,6 +1731,24 @@ mod environment {
         /// The user's file system's creation token.
         fs_token: Option<String>,
         secrets: Vec<String>,
+        /// The project whose tunnel to close.
+        project_id: Option<String>,
+        /// `AWS_CONFIG_FILE` before the test pointed it at its own file
+        /// (outer `None`: not changed).
+        config_file_before: Option<Option<std::ffi::OsString>>,
+        /// `TRIPLEC_E2E_ECR_REPO`: the throwaway repository to force-delete.
+        ecr_repo: Option<String>,
+    }
+
+    /// The whole phase-5 body: a hang anywhere becomes a panic the guard
+    /// catches, so nothing billed outlives it.
+    const PHASE5_TIMEOUT: Duration = Duration::from_secs(45 * 60);
+    /// One exec through the agent.
+    const SH_TIMEOUT: Duration = Duration::from_secs(180);
+
+    /// The phase-5 guard's state, even after a panic poisoned the lock.
+    fn p5_lock(m: &std::sync::Mutex<Phase5>) -> std::sync::MutexGuard<'_, Phase5> {
+        m.lock().unwrap_or_else(|e| e.into_inner())
     }
 
     const ECR_POLICY: &str = "triple-c-e2e-ecr-pull";
@@ -1741,6 +1759,14 @@ mod environment {
     /// test-only ECR policy off the execution role, so the modules'
     /// `tofu destroy` can remove the rest. Logs, never fails.
     async fn phase5_cleanup(e: &E2e, p: &Phase5) {
+        if let Some(id) = &p.project_id {
+            crate::host::tunnels::close(id).await;
+        }
+        match &p.config_file_before {
+            Some(Some(v)) => std::env::set_var("AWS_CONFIG_FILE", v),
+            Some(None) => std::env::remove_var("AWS_CONFIG_FILE"),
+            None => {}
+        }
         let ecs = aws_sdk_ecs::Client::new(&e.cfg);
         if let Some(cluster) = &p.cluster {
             let mut arns = Vec::new();
@@ -1840,6 +1866,19 @@ mod environment {
                 }
             }
         }
+            if let Some(repo) = &p.ecr_repo {
+            // No ECR crate in the app: the AWS CLI, as the e2e profile.
+            let out = tokio::process::Command::new("aws")
+                .args(["--profile", &super::var("TRIPLEC_AWS_E2E_PROFILE"), "--region", &e.region])
+                .args(["ecr", "delete-repository", "--repository-name", repo, "--force"])
+                .output()
+                .await;
+            match out {
+                Ok(o) if o.status.success() => eprintln!("cleanup: deleted ECR repository {repo}"),
+                Ok(o) if String::from_utf8_lossy(&o.stderr).contains("RepositoryNotFound") => {}
+                other => eprintln!("cleanup: delete ECR repository {repo}: {other:?}"),
+            }
+        }
     }
 
     /// Poll `f` every `every` until it returns `Some`, or `None` after `within`.
@@ -1862,9 +1901,13 @@ mod environment {
 
     /// `sh -c script` as `user`: (output, exit code).
     async fn sh(t: &crate::host::ContainerTarget, user: &str, script: &str) -> (String, i64) {
-        t.exec_oneshot_as(user, vec!["sh".into(), "-c".into(), script.into()], Vec::new())
-            .await
-            .unwrap_or_else(|err| panic!("exec `{script}`: {err}"))
+        tokio::time::timeout(
+            SH_TIMEOUT,
+            t.exec_oneshot_as(user, vec!["sh".into(), "-c".into(), script.into()], Vec::new()),
+        )
+        .await
+        .unwrap_or_else(|_| panic!("exec `{script}` did not finish in {SH_TIMEOUT:?}"))
+        .unwrap_or_else(|err| panic!("exec `{script}`: {err}"))
     }
 
     /// Phase 5: a Fargate workspace's whole life through `ProjectHost::Aws`,
@@ -1887,8 +1930,14 @@ mod environment {
     /// onboarded execution role a **test-only** inline pull policy
     /// (`triple-c-e2e-ecr-pull`; the boundary allows ECR pulls, the
     /// onboarding module grants none) and removes it before the onboarding
-    /// is destroyed. Creating and deleting that repository is the runner's
-    /// job (`aws/README.md`, "Live verification").
+    /// is destroyed. `scripts/aws-e2e-workspace.sh` does it all: builds the
+    /// image, creates the throwaway repository `triple-c-e2e-sandbox`,
+    /// pushes, runs this test with `TRIPLEC_E2E_ECR_REPO` set (this test's
+    /// guard then force-deletes that repository too) and deletes it in a
+    /// trap on exit.
+    ///
+    /// The body runs under a 45-minute timeout and every exec under its own,
+    /// so a hang becomes a panic and the guard still runs.
     ///
     /// The workspace user signs in the way a saved profile connection does:
     /// a profile (`role_arn` + `source_profile` = the e2e profile, session
@@ -1911,9 +1960,12 @@ mod environment {
         use crate::storage::projects_store::ProjectsStore;
 
         let image = super::var("TRIPLEC_E2E_IMAGE");
+        let ecr_repo = std::env::var("TRIPLEC_E2E_ECR_REPO").ok().filter(|r| !r.is_empty());
         env_e2e(|t| async move {
-            let p5 = std::sync::Mutex::new(Phase5::default());
-            let body = async {
+            let p5 = std::sync::Mutex::new(Phase5 { ecr_repo, ..Default::default() });
+            // Boxed: the body's state machine is far larger than a test
+            // thread's 2 MiB stack once it is moved into the timeout.
+            let inner = Box::pin(async {
                 let region = t.e.region.clone();
                 let admin = t.admin().await;
                 let (keys, cfg, who) = t.session(&admin).await;
@@ -1922,8 +1974,12 @@ mod environment {
 
                 // ── the environment, deployed by the environment admin ──
                 let name = format!("e2e{}", &uuid::Uuid::new_v4().simple().to_string()[..6]);
-                p5.lock().unwrap().cluster = Some(environment_cluster_name(&name));
-                p5.lock().unwrap().families.push(format!("triple-c-{name}-"));
+                let (cluster, families) = (environment_cluster_name(&name), format!("triple-c-{name}-"));
+                {
+                    let mut g = p5_lock(&p5);
+                    g.cluster = Some(cluster);
+                    g.families.push(families);
+                }
                 let vars = environment_vars(&name, &region, &NetworkChoice::Public).unwrap();
                 let (_, outcome, denied) = t
                     .run(ModuleKind::Environment, Deployment::Environment(&name), &vars, TofuOp::Apply, t.keys_env(&keys))
@@ -1934,7 +1990,8 @@ mod environment {
                 let user = t.e.tier_role("wsuser", &[Tier::WorkspaceUser]).await;
                 let owner = validate_owner_id(&user.session_user_id()).expect("owner id");
                 let okey = owner_key(&region, &owner);
-                p5.lock().unwrap().fs_token = Some(user_file_system_token(&name, &okey));
+                let fs_token = user_file_system_token(&name, &okey);
+                p5_lock(&p5).fs_token = Some(fs_token.clone());
 
                 // Onboarded by the admin (an onboarding from an admin bundle).
                 let (keys, _, _) = t.session(&admin).await;
@@ -1958,7 +2015,7 @@ mod environment {
                             r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Action":"ecr:GetAuthorizationToken","Resource":"*"}},{{"Effect":"Allow","Action":["ecr:BatchGetImage","ecr:GetDownloadUrlForLayer","ecr:BatchCheckLayerAvailability"],"Resource":"arn:aws:ecr:{region}:{}:repository/{repo}"}}]}}"#,
                             t.e.account
                         );
-                        p5.lock().unwrap().exec_role = Some(exec_role.clone());
+                        p5_lock(&p5).exec_role = Some(exec_role.clone());
                         ok(
                             "put the test-only ECR policy",
                             t.e.iam.put_role_policy().role_name(&exec_role).policy_name(ECR_POLICY).policy_document(doc).send().await,
@@ -1980,6 +2037,8 @@ mod environment {
                     ),
                 )
                 .unwrap();
+                let before = std::env::var_os("AWS_CONFIG_FILE");
+                p5_lock(&p5).config_file_before = Some(before);
                 std::env::set_var("AWS_CONFIG_FILE", &cfg_file);
                 let conn = HostConnection {
                     id: "e2e-wsuser".into(),
@@ -2019,7 +2078,12 @@ mod environment {
                     ..Default::default()
                 });
                 let id = project.id.clone();
-                p5.lock().unwrap().secrets.push(project_secret_name(&name, &okey, &id));
+                let secret_name = project_secret_name(&name, &okey, &id);
+                {
+                    let mut g = p5_lock(&p5);
+                    g.project_id = Some(id.clone());
+                    g.secrets.push(secret_name);
+                }
                 let store_dir = tempfile::tempdir().unwrap();
                 let store = ProjectsStore::over_file_for_test(store_dir.path().join("projects.json"), vec![project.clone()]);
                 let host = ProjectHost::for_project(&project);
@@ -2157,6 +2221,26 @@ mod environment {
                 );
                 eprintln!("RESULT GetSecretValue on another owner's secret: denied");
 
+                // A bare TagResource on the user's own file system (no
+                // CreateAction) is refused: tagging is only at creation.
+                let efs_admin = aws_sdk_efs::Client::new(&t.e.cfg);
+                let fs = ok(
+                    "find the user's file system",
+                    efs_admin.describe_file_systems().creation_token(&fs_token).send().await,
+                );
+                let fs_id = fs.file_systems().first().expect("the user's file system").file_system_id().to_string();
+                let ws_efs = aws_sdk_efs::Client::new(&ws_cfg);
+                super::denied(
+                    "bare TagResource on own file system",
+                    ws_efs
+                        .tag_resource()
+                        .resource_id(&fs_id)
+                        .tags(aws_sdk_efs::types::Tag::builder().key("triple-c:owner").value(&owner).build().unwrap())
+                        .send()
+                        .await,
+                );
+                eprintln!("RESULT bare elasticfilesystem:TagResource on the user's own file system {fs_id}: denied");
+
                 // RunTask on a revision the user does not own is denied.
                 let foreign_family = format!("triple-c-{name}-e2e-foreign");
                 let reg = ok(
@@ -2300,9 +2384,14 @@ mod environment {
                 // R8: the revisions stay registered for an admin to clear
                 // (the guard does, as the account admin).
                 assert_eq!(defs.task_definition_arns(), [revision1.clone()], "the project's revision is left ACTIVE");
+            });
+            let body = async {
+                if tokio::time::timeout(PHASE5_TIMEOUT, inner).await.is_err() {
+                    panic!("the phase-5 body did not finish in {PHASE5_TIMEOUT:?}");
+                }
             };
             let result = std::panic::AssertUnwindSafe(body).catch_unwind().await;
-            let guard = std::mem::take(&mut *p5.lock().unwrap());
+            let guard = std::mem::take(&mut *p5_lock(&p5));
             phase5_cleanup(&t.e, &guard).await;
             if let Err(panic) = result {
                 std::panic::resume_unwind(panic);

aws/README.md

@@ -134,7 +134,7 @@ revision (`ecs:DeregisterTaskDefinition` on `*`), so an admin clears an environm
 
 ```sh
 aws ecs list-task-definitions --family-prefix triple-c-<env>- --query 'taskDefinitionArns[]' --output text \
-  | xargs -n1 aws ecs deregister-task-definition --task-definition >/dev/null
+  | xargs -r -n1 aws ecs deregister-task-definition --task-definition >/dev/null
 ```
 
 Deregistered revisions become INACTIVE; ECS keeps them until they are deleted with
@@ -551,15 +551,14 @@ Still open:
 ### Phase 5: a Fargate workspace (2026-11-15)
 
 ```sh
-cd app/src-tauri
-cargo build
-TRIPLEC_AWS_E2E_PROFILE=<admin profile> TRIPLEC_AWS_E2E_REGION=us-west-2 \
-TRIPLEC_E2E_IMAGE=<account>.dkr.ecr.<region>.amazonaws.com/triple-c-e2e-sandbox:<tag> \
-  cargo test aws::live_tests::environment::a_fargate_workspace_round_trips -- --ignored --test-threads=1 --nocapture
+TRIPLEC_AWS_E2E_PROFILE=<admin profile> TRIPLEC_AWS_E2E_REGION=us-west-2 scripts/aws-e2e-workspace.sh
 ```
 
-The image is this branch's `container/` (`docker build --platform linux/amd64`), pushed to a
-throwaway private ECR repository `triple-c-e2e-sandbox` that the runner creates and deletes.
+The script builds this checkout's `container/` (`--platform linux/amd64`), creates a throwaway
+private ECR repository `triple-c-e2e-sandbox`, pushes, runs the test
+(`aws::live_tests::environment::a_fargate_workspace_round_trips`, with `TRIPLEC_E2E_IMAGE` and
+`TRIPLEC_E2E_ECR_REPO`) and deletes the repository in an exit trap; the test's guard deletes it
+as well. The test body runs under a 45-minute timeout, every exec under its own.
 **Test only:** the onboarding grants the execution role no ECR permissions (the boundary allows
 pulls), so the test puts an inline `triple-c-e2e-ecr-pull` policy on it and removes it before the
 onboarding is destroyed. The test deploys a `public` environment as the environment admin, onboards
@@ -575,9 +574,10 @@ and the secret and removes the ECR policy; then the fixture destroys the modules
 | `DeregisterTaskDefinition` | ✗→decided (R8): ECS authorizes it against `*` only. `task-definition/triple-c-*`, and `*` with `aws:ResourceTag/triple-c:owner`, are both denied; only an unconditional `*` works (probe). The workspace tier no longer has it (`RetireOwnTaskDefinitions` dropped); removing a project leaves its revisions registered for an admin to clear |
 | `DeleteTaskDefinitions` | (probe) resource-level: authorized against the revision's ARN with `aws:ResourceTag` (the owner's tag passes, another owner's is denied), but it only deletes INACTIVE revisions, so without deregistration it is useless to a user. Dropped from the workspace tier with it (R8) |
 | `CreateAccessPoint`'s `TagResource` | ✗→fixed: authorized against the **file system**, not the access point (probe: `access-point/*` denied; `file-system/*` allowed; `CreateAction`, `aws:RequestTag` and `aws:TagKeys` all present) |
+| Bare `TagResource` on the user's own file system | ✓ denied (no `elasticfilesystem:CreateAction`): `TagOwnAccessPointsAtCreate` tags only at access-point creation |
 | `GetSecretValue` | ✓ own secret (an attach reads the token back); another owner's `triple-c/*` secret denied |
 | EFS access point with the task role (IAM + TLS) | ✓ mounted at `/mnt/triple-c-home` (`127.0.0.1:/ … nfs4`, the TLS tunnel), `claude:claude 700`; `~/.claude`, `~/.claude.json` and `~/.ssh` link into it |
-| Times | first start (register, access point, RunTask, pull of ~1.2 GB, agent, tunnel) 104 s; ECS `HEALTHY` 135 s after the start began; attach to a running task 2 s; StopTask to `STOPPED` 42 s; restart 86–92 s |
+| Times | first start (register, access point, RunTask, pull of ~1.2 GB, agent, tunnel) 94–104 s; ECS `HEALTHY` 135 s after the start began; attach to a running task 2 s; StopTask to `STOPPED` 42 s; restart 86–92 s |
 | Attach | ✓ a second start while running reuses the task (one task in the family, same ARN, no RunTask) |
 | Restart | ✓ new task, same revision; the `~/.claude` marker survived |
 | Repo clone | ✓ the background clone finished (`/run/triple-c-clone/repos`: `Hello-World cloned`) |
@@ -586,4 +586,5 @@ and the secret and removes the ECR policy; then the fixture destroys the modules
 | `~/.claude.json` after Claude Code runs | ✓ still a link (Claude Code started without a login: "Not logged in") |
 | Bubblewrap (P9) | ✗ `bwrap --unshare-user` fails: "No permissions to create new namespace" (kernel 6.1, `max_user_namespaces` 16384, no `CAP_SYS_ADMIN` on Fargate). The sandbox stays off on AWS |
 | A revision per configuration | turning idle stop on adds a secret key, so the next start registers a new revision (by design) |
+| Removal (R8) | ✓ final run: `remove_aws_resources` succeeds; the secret is scheduled for deletion, the access point deleted, the revision left ACTIVE |
 | Cleanup | ✓ nothing `triple-c` left in us-west-2; environment KMS keys pending deletion |

scripts/aws-e2e-workspace.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Run the phase-5 live test (`a_fargate_workspace_round_trips`) end to end:
+# build the sandbox image from this checkout, push it to a throwaway private
+# ECR repository, run the test, and delete the repository again — on success,
+# failure or Ctrl-C (trap). The test deletes the repository in its own guard
+# as well (TRIPLEC_E2E_ECR_REPO), so a crash of either side still removes it.
+#
+# Needs: docker, the AWS CLI, cargo, session-manager-plugin, and the admin
+# profile of a DISPOSABLE account. Usage, from anywhere in the repo:
+#
+#   TRIPLEC_AWS_E2E_PROFILE=<profile> TRIPLEC_AWS_E2E_REGION=us-west-2 \
+#     scripts/aws-e2e-workspace.sh
+#
+# Costs a few cents (about 40 Fargate task-minutes at 1 vCPU / 2 GB).
+set -euo pipefail
+
+: "${TRIPLEC_AWS_E2E_PROFILE:?set TRIPLEC_AWS_E2E_PROFILE (see aws/README.md, Live verification)}"
+: "${TRIPLEC_AWS_E2E_REGION:?set TRIPLEC_AWS_E2E_REGION}"
+root=$(cd "$(dirname "$0")/.." && pwd)
+repo=triple-c-e2e-sandbox
+tag="e2e-$(date -u +%Y%m%d%H%M%S)"
+aws_() { aws --profile "$TRIPLEC_AWS_E2E_PROFILE" --region "$TRIPLEC_AWS_E2E_REGION" "$@"; }
+
+docker_config=$(mktemp -d)
+cleanup() {
+    status=$?
+    if aws_ ecr describe-repositories --repository-names "$repo" >/dev/null 2>&1; then
+        aws_ ecr delete-repository --repository-name "$repo" --force >/dev/null \
+            && echo "aws-e2e-workspace: deleted ECR repository $repo" \
+            || echo "aws-e2e-workspace: WARNING: could not delete ECR repository $repo" >&2
+    fi
+    rm -rf "$docker_config"
+    exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+account=$(aws_ sts get-caller-identity --query Account --output text)
+registry="$account.dkr.ecr.$TRIPLEC_AWS_E2E_REGION.amazonaws.com"
+image="$registry/$repo:$tag"
+
+docker build --platform linux/amd64 -t "$image" "$root/container"
+aws_ ecr create-repository --repository-name "$repo" \
+    --tags Key=triple-c-test,Value=phase5 >/dev/null
+aws_ ecr get-login-password \
+    | DOCKER_CONFIG="$docker_config" docker login --username AWS --password-stdin "$registry" >/dev/null
+DOCKER_CONFIG="$docker_config" docker push "$image"
+docker rmi "$image" >/dev/null || true
+
+cd "$root/app/src-tauri"
+mkdir -p ../dist
+cargo build --locked
+# A stack overflow aborts without running any guard: give the test thread room.
+RUST_MIN_STACK=16777216 TRIPLEC_E2E_IMAGE="$image" TRIPLEC_E2E_ECR_REPO="$repo" \
+    cargo test --locked aws::live_tests::environment::a_fargate_workspace_round_trips \
+    -- --ignored --test-threads=1 --nocapture