From 084fa6efe444bc8c24bc8dd452654d03e7d8bc49 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Sun, 15 Nov 2026 11:56:40 -0800 Subject: [PATCH] test(aws): guard the live workspace test against hangs and leaks Review round 1 of Task 9: - the phase-5 body runs under a 45-minute timeout and each exec under 180 s, so a hang becomes a panic the guard catches; the body future is boxed (moved into the timeout it overflowed the test thread's stack, which aborts without any guard); - scripts/aws-e2e-workspace.sh builds and pushes the image to a throwaway ECR repository, runs the test and deletes the repository in an exit trap; the test's guard deletes it too (TRIPLEC_E2E_ECR_REPO); - the guard survives a poisoned lock, closes the tunnel and restores AWS_CONFIG_FILE; - new live probe: a bare elasticfilesystem:TagResource on the user's own file system is denied; README records it and the final run (R8 removal end state), and the admin command uses xargs -r. Co-Authored-By: Claude Opus 5.5 --- app/src-tauri/src/aws/live_tests.rs | 115 ++++++++++++++++++++++++---- aws/README.md | 19 ++--- scripts/aws-e2e-workspace.sh | 55 +++++++++++++ 3 files changed, 167 insertions(+), 22 deletions(-) create mode 100755 scripts/aws-e2e-workspace.sh diff --git a/app/src-tauri/src/aws/live_tests.rs b/app/src-tauri/src/aws/live_tests.rs index a04b23b..f653e4b 100644 --- a/app/src-tauri/src/aws/live_tests.rs +++ b/app/src-tauri/src/aws/live_tests.rs @@ -1731,6 +1731,24 @@ mod environment { /// The user's file system's creation token. fs_token: Option, secrets: Vec, + /// The project whose tunnel to close. + project_id: Option, + /// `AWS_CONFIG_FILE` before the test pointed it at its own file + /// (outer `None`: not changed). + config_file_before: Option>, + /// `TRIPLEC_E2E_ECR_REPO`: the throwaway repository to force-delete. + ecr_repo: Option, + } + + /// The whole phase-5 body: a hang anywhere becomes a panic the guard + /// catches, so nothing billed outlives it. + const PHASE5_TIMEOUT: Duration = Duration::from_secs(45 * 60); + /// One exec through the agent. + const SH_TIMEOUT: Duration = Duration::from_secs(180); + + /// The phase-5 guard's state, even after a panic poisoned the lock. + fn p5_lock(m: &std::sync::Mutex) -> std::sync::MutexGuard<'_, Phase5> { + m.lock().unwrap_or_else(|e| e.into_inner()) } const ECR_POLICY: &str = "triple-c-e2e-ecr-pull"; @@ -1741,6 +1759,14 @@ mod environment { /// test-only ECR policy off the execution role, so the modules' /// `tofu destroy` can remove the rest. Logs, never fails. async fn phase5_cleanup(e: &E2e, p: &Phase5) { + if let Some(id) = &p.project_id { + crate::host::tunnels::close(id).await; + } + match &p.config_file_before { + Some(Some(v)) => std::env::set_var("AWS_CONFIG_FILE", v), + Some(None) => std::env::remove_var("AWS_CONFIG_FILE"), + None => {} + } let ecs = aws_sdk_ecs::Client::new(&e.cfg); if let Some(cluster) = &p.cluster { let mut arns = Vec::new(); @@ -1840,6 +1866,19 @@ mod environment { } } } + if let Some(repo) = &p.ecr_repo { + // No ECR crate in the app: the AWS CLI, as the e2e profile. + let out = tokio::process::Command::new("aws") + .args(["--profile", &super::var("TRIPLEC_AWS_E2E_PROFILE"), "--region", &e.region]) + .args(["ecr", "delete-repository", "--repository-name", repo, "--force"]) + .output() + .await; + match out { + Ok(o) if o.status.success() => eprintln!("cleanup: deleted ECR repository {repo}"), + Ok(o) if String::from_utf8_lossy(&o.stderr).contains("RepositoryNotFound") => {} + other => eprintln!("cleanup: delete ECR repository {repo}: {other:?}"), + } + } } /// Poll `f` every `every` until it returns `Some`, or `None` after `within`. @@ -1862,9 +1901,13 @@ mod environment { /// `sh -c script` as `user`: (output, exit code). async fn sh(t: &crate::host::ContainerTarget, user: &str, script: &str) -> (String, i64) { - t.exec_oneshot_as(user, vec!["sh".into(), "-c".into(), script.into()], Vec::new()) - .await - .unwrap_or_else(|err| panic!("exec `{script}`: {err}")) + tokio::time::timeout( + SH_TIMEOUT, + t.exec_oneshot_as(user, vec!["sh".into(), "-c".into(), script.into()], Vec::new()), + ) + .await + .unwrap_or_else(|_| panic!("exec `{script}` did not finish in {SH_TIMEOUT:?}")) + .unwrap_or_else(|err| panic!("exec `{script}`: {err}")) } /// Phase 5: a Fargate workspace's whole life through `ProjectHost::Aws`, @@ -1887,8 +1930,14 @@ mod environment { /// onboarded execution role a **test-only** inline pull policy /// (`triple-c-e2e-ecr-pull`; the boundary allows ECR pulls, the /// onboarding module grants none) and removes it before the onboarding - /// is destroyed. Creating and deleting that repository is the runner's - /// job (`aws/README.md`, "Live verification"). + /// is destroyed. `scripts/aws-e2e-workspace.sh` does it all: builds the + /// image, creates the throwaway repository `triple-c-e2e-sandbox`, + /// pushes, runs this test with `TRIPLEC_E2E_ECR_REPO` set (this test's + /// guard then force-deletes that repository too) and deletes it in a + /// trap on exit. + /// + /// The body runs under a 45-minute timeout and every exec under its own, + /// so a hang becomes a panic and the guard still runs. /// /// The workspace user signs in the way a saved profile connection does: /// a profile (`role_arn` + `source_profile` = the e2e profile, session @@ -1911,9 +1960,12 @@ mod environment { use crate::storage::projects_store::ProjectsStore; let image = super::var("TRIPLEC_E2E_IMAGE"); + let ecr_repo = std::env::var("TRIPLEC_E2E_ECR_REPO").ok().filter(|r| !r.is_empty()); env_e2e(|t| async move { - let p5 = std::sync::Mutex::new(Phase5::default()); - let body = async { + let p5 = std::sync::Mutex::new(Phase5 { ecr_repo, ..Default::default() }); + // Boxed: the body's state machine is far larger than a test + // thread's 2 MiB stack once it is moved into the timeout. + let inner = Box::pin(async { let region = t.e.region.clone(); let admin = t.admin().await; let (keys, cfg, who) = t.session(&admin).await; @@ -1922,8 +1974,12 @@ mod environment { // ── the environment, deployed by the environment admin ── let name = format!("e2e{}", &uuid::Uuid::new_v4().simple().to_string()[..6]); - p5.lock().unwrap().cluster = Some(environment_cluster_name(&name)); - p5.lock().unwrap().families.push(format!("triple-c-{name}-")); + let (cluster, families) = (environment_cluster_name(&name), format!("triple-c-{name}-")); + { + let mut g = p5_lock(&p5); + g.cluster = Some(cluster); + g.families.push(families); + } let vars = environment_vars(&name, ®ion, &NetworkChoice::Public).unwrap(); let (_, outcome, denied) = t .run(ModuleKind::Environment, Deployment::Environment(&name), &vars, TofuOp::Apply, t.keys_env(&keys)) @@ -1934,7 +1990,8 @@ mod environment { let user = t.e.tier_role("wsuser", &[Tier::WorkspaceUser]).await; let owner = validate_owner_id(&user.session_user_id()).expect("owner id"); let okey = owner_key(®ion, &owner); - p5.lock().unwrap().fs_token = Some(user_file_system_token(&name, &okey)); + let fs_token = user_file_system_token(&name, &okey); + p5_lock(&p5).fs_token = Some(fs_token.clone()); // Onboarded by the admin (an onboarding from an admin bundle). let (keys, _, _) = t.session(&admin).await; @@ -1958,7 +2015,7 @@ mod environment { r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Action":"ecr:GetAuthorizationToken","Resource":"*"}},{{"Effect":"Allow","Action":["ecr:BatchGetImage","ecr:GetDownloadUrlForLayer","ecr:BatchCheckLayerAvailability"],"Resource":"arn:aws:ecr:{region}:{}:repository/{repo}"}}]}}"#, t.e.account ); - p5.lock().unwrap().exec_role = Some(exec_role.clone()); + p5_lock(&p5).exec_role = Some(exec_role.clone()); ok( "put the test-only ECR policy", t.e.iam.put_role_policy().role_name(&exec_role).policy_name(ECR_POLICY).policy_document(doc).send().await, @@ -1980,6 +2037,8 @@ mod environment { ), ) .unwrap(); + let before = std::env::var_os("AWS_CONFIG_FILE"); + p5_lock(&p5).config_file_before = Some(before); std::env::set_var("AWS_CONFIG_FILE", &cfg_file); let conn = HostConnection { id: "e2e-wsuser".into(), @@ -2019,7 +2078,12 @@ mod environment { ..Default::default() }); let id = project.id.clone(); - p5.lock().unwrap().secrets.push(project_secret_name(&name, &okey, &id)); + let secret_name = project_secret_name(&name, &okey, &id); + { + let mut g = p5_lock(&p5); + g.project_id = Some(id.clone()); + g.secrets.push(secret_name); + } let store_dir = tempfile::tempdir().unwrap(); let store = ProjectsStore::over_file_for_test(store_dir.path().join("projects.json"), vec![project.clone()]); let host = ProjectHost::for_project(&project); @@ -2157,6 +2221,26 @@ mod environment { ); eprintln!("RESULT GetSecretValue on another owner's secret: denied"); + // A bare TagResource on the user's own file system (no + // CreateAction) is refused: tagging is only at creation. + let efs_admin = aws_sdk_efs::Client::new(&t.e.cfg); + let fs = ok( + "find the user's file system", + efs_admin.describe_file_systems().creation_token(&fs_token).send().await, + ); + let fs_id = fs.file_systems().first().expect("the user's file system").file_system_id().to_string(); + let ws_efs = aws_sdk_efs::Client::new(&ws_cfg); + super::denied( + "bare TagResource on own file system", + ws_efs + .tag_resource() + .resource_id(&fs_id) + .tags(aws_sdk_efs::types::Tag::builder().key("triple-c:owner").value(&owner).build().unwrap()) + .send() + .await, + ); + eprintln!("RESULT bare elasticfilesystem:TagResource on the user's own file system {fs_id}: denied"); + // RunTask on a revision the user does not own is denied. let foreign_family = format!("triple-c-{name}-e2e-foreign"); let reg = ok( @@ -2300,9 +2384,14 @@ mod environment { // R8: the revisions stay registered for an admin to clear // (the guard does, as the account admin). assert_eq!(defs.task_definition_arns(), [revision1.clone()], "the project's revision is left ACTIVE"); + }); + let body = async { + if tokio::time::timeout(PHASE5_TIMEOUT, inner).await.is_err() { + panic!("the phase-5 body did not finish in {PHASE5_TIMEOUT:?}"); + } }; let result = std::panic::AssertUnwindSafe(body).catch_unwind().await; - let guard = std::mem::take(&mut *p5.lock().unwrap()); + let guard = std::mem::take(&mut *p5_lock(&p5)); phase5_cleanup(&t.e, &guard).await; if let Err(panic) = result { std::panic::resume_unwind(panic); diff --git a/aws/README.md b/aws/README.md index 89acbd4..8abf8b1 100644 --- a/aws/README.md +++ b/aws/README.md @@ -134,7 +134,7 @@ revision (`ecs:DeregisterTaskDefinition` on `*`), so an admin clears an environm ```sh aws ecs list-task-definitions --family-prefix triple-c-- --query 'taskDefinitionArns[]' --output text \ - | xargs -n1 aws ecs deregister-task-definition --task-definition >/dev/null + | xargs -r -n1 aws ecs deregister-task-definition --task-definition >/dev/null ``` Deregistered revisions become INACTIVE; ECS keeps them until they are deleted with @@ -551,15 +551,14 @@ Still open: ### Phase 5: a Fargate workspace (2026-11-15) ```sh -cd app/src-tauri -cargo build -TRIPLEC_AWS_E2E_PROFILE= TRIPLEC_AWS_E2E_REGION=us-west-2 \ -TRIPLEC_E2E_IMAGE=.dkr.ecr..amazonaws.com/triple-c-e2e-sandbox: \ - cargo test aws::live_tests::environment::a_fargate_workspace_round_trips -- --ignored --test-threads=1 --nocapture +TRIPLEC_AWS_E2E_PROFILE= TRIPLEC_AWS_E2E_REGION=us-west-2 scripts/aws-e2e-workspace.sh ``` -The image is this branch's `container/` (`docker build --platform linux/amd64`), pushed to a -throwaway private ECR repository `triple-c-e2e-sandbox` that the runner creates and deletes. +The script builds this checkout's `container/` (`--platform linux/amd64`), creates a throwaway +private ECR repository `triple-c-e2e-sandbox`, pushes, runs the test +(`aws::live_tests::environment::a_fargate_workspace_round_trips`, with `TRIPLEC_E2E_IMAGE` and +`TRIPLEC_E2E_ECR_REPO`) and deletes the repository in an exit trap; the test's guard deletes it +as well. The test body runs under a 45-minute timeout, every exec under its own. **Test only:** the onboarding grants the execution role no ECR permissions (the boundary allows pulls), so the test puts an inline `triple-c-e2e-ecr-pull` policy on it and removes it before the onboarding is destroyed. The test deploys a `public` environment as the environment admin, onboards @@ -575,9 +574,10 @@ and the secret and removes the ECR policy; then the fixture destroys the modules | `DeregisterTaskDefinition` | ✗→decided (R8): ECS authorizes it against `*` only. `task-definition/triple-c-*`, and `*` with `aws:ResourceTag/triple-c:owner`, are both denied; only an unconditional `*` works (probe). The workspace tier no longer has it (`RetireOwnTaskDefinitions` dropped); removing a project leaves its revisions registered for an admin to clear | | `DeleteTaskDefinitions` | (probe) resource-level: authorized against the revision's ARN with `aws:ResourceTag` (the owner's tag passes, another owner's is denied), but it only deletes INACTIVE revisions, so without deregistration it is useless to a user. Dropped from the workspace tier with it (R8) | | `CreateAccessPoint`'s `TagResource` | ✗→fixed: authorized against the **file system**, not the access point (probe: `access-point/*` denied; `file-system/*` allowed; `CreateAction`, `aws:RequestTag` and `aws:TagKeys` all present) | +| Bare `TagResource` on the user's own file system | ✓ denied (no `elasticfilesystem:CreateAction`): `TagOwnAccessPointsAtCreate` tags only at access-point creation | | `GetSecretValue` | ✓ own secret (an attach reads the token back); another owner's `triple-c/*` secret denied | | EFS access point with the task role (IAM + TLS) | ✓ mounted at `/mnt/triple-c-home` (`127.0.0.1:/ … nfs4`, the TLS tunnel), `claude:claude 700`; `~/.claude`, `~/.claude.json` and `~/.ssh` link into it | -| Times | first start (register, access point, RunTask, pull of ~1.2 GB, agent, tunnel) 104 s; ECS `HEALTHY` 135 s after the start began; attach to a running task 2 s; StopTask to `STOPPED` 42 s; restart 86–92 s | +| Times | first start (register, access point, RunTask, pull of ~1.2 GB, agent, tunnel) 94–104 s; ECS `HEALTHY` 135 s after the start began; attach to a running task 2 s; StopTask to `STOPPED` 42 s; restart 86–92 s | | Attach | ✓ a second start while running reuses the task (one task in the family, same ARN, no RunTask) | | Restart | ✓ new task, same revision; the `~/.claude` marker survived | | Repo clone | ✓ the background clone finished (`/run/triple-c-clone/repos`: `Hello-World cloned`) | @@ -586,4 +586,5 @@ and the secret and removes the ECR policy; then the fixture destroys the modules | `~/.claude.json` after Claude Code runs | ✓ still a link (Claude Code started without a login: "Not logged in") | | Bubblewrap (P9) | ✗ `bwrap --unshare-user` fails: "No permissions to create new namespace" (kernel 6.1, `max_user_namespaces` 16384, no `CAP_SYS_ADMIN` on Fargate). The sandbox stays off on AWS | | A revision per configuration | turning idle stop on adds a secret key, so the next start registers a new revision (by design) | +| Removal (R8) | ✓ final run: `remove_aws_resources` succeeds; the secret is scheduled for deletion, the access point deleted, the revision left ACTIVE | | Cleanup | ✓ nothing `triple-c` left in us-west-2; environment KMS keys pending deletion | diff --git a/scripts/aws-e2e-workspace.sh b/scripts/aws-e2e-workspace.sh new file mode 100755 index 0000000..2376fc4 --- /dev/null +++ b/scripts/aws-e2e-workspace.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Run the phase-5 live test (`a_fargate_workspace_round_trips`) end to end: +# build the sandbox image from this checkout, push it to a throwaway private +# ECR repository, run the test, and delete the repository again — on success, +# failure or Ctrl-C (trap). The test deletes the repository in its own guard +# as well (TRIPLEC_E2E_ECR_REPO), so a crash of either side still removes it. +# +# Needs: docker, the AWS CLI, cargo, session-manager-plugin, and the admin +# profile of a DISPOSABLE account. Usage, from anywhere in the repo: +# +# TRIPLEC_AWS_E2E_PROFILE= TRIPLEC_AWS_E2E_REGION=us-west-2 \ +# scripts/aws-e2e-workspace.sh +# +# Costs a few cents (about 40 Fargate task-minutes at 1 vCPU / 2 GB). +set -euo pipefail + +: "${TRIPLEC_AWS_E2E_PROFILE:?set TRIPLEC_AWS_E2E_PROFILE (see aws/README.md, Live verification)}" +: "${TRIPLEC_AWS_E2E_REGION:?set TRIPLEC_AWS_E2E_REGION}" +root=$(cd "$(dirname "$0")/.." && pwd) +repo=triple-c-e2e-sandbox +tag="e2e-$(date -u +%Y%m%d%H%M%S)" +aws_() { aws --profile "$TRIPLEC_AWS_E2E_PROFILE" --region "$TRIPLEC_AWS_E2E_REGION" "$@"; } + +docker_config=$(mktemp -d) +cleanup() { + status=$? + if aws_ ecr describe-repositories --repository-names "$repo" >/dev/null 2>&1; then + aws_ ecr delete-repository --repository-name "$repo" --force >/dev/null \ + && echo "aws-e2e-workspace: deleted ECR repository $repo" \ + || echo "aws-e2e-workspace: WARNING: could not delete ECR repository $repo" >&2 + fi + rm -rf "$docker_config" + exit "$status" +} +trap cleanup EXIT INT TERM + +account=$(aws_ sts get-caller-identity --query Account --output text) +registry="$account.dkr.ecr.$TRIPLEC_AWS_E2E_REGION.amazonaws.com" +image="$registry/$repo:$tag" + +docker build --platform linux/amd64 -t "$image" "$root/container" +aws_ ecr create-repository --repository-name "$repo" \ + --tags Key=triple-c-test,Value=phase5 >/dev/null +aws_ ecr get-login-password \ + | DOCKER_CONFIG="$docker_config" docker login --username AWS --password-stdin "$registry" >/dev/null +DOCKER_CONFIG="$docker_config" docker push "$image" +docker rmi "$image" >/dev/null || true + +cd "$root/app/src-tauri" +mkdir -p ../dist +cargo build --locked +# A stack overflow aborts without running any guard: give the test thread room. +RUST_MIN_STACK=16777216 TRIPLEC_E2E_IMAGE="$image" TRIPLEC_E2E_ECR_REPO="$repo" \ + cargo test --locked aws::live_tests::environment::a_fargate_workspace_round_trips \ + -- --ignored --test-threads=1 --nocapture