fix(aws): create the secret before reading it; one staging dir per connect

- Copy-at-connect tries CreateSecret first and reads the current value only
  when the secret exists, so a first start never reads a missing secret
  (which the owner-tag condition may refuse).
- triple-c-apply-connect: --prepare makes and prints a per-connect nonce
  dir; the app uploads into it and applies exactly that dir, so concurrent
  connects never mix files. CA files move to a fixed path so an unchanged
  CA is not reinstalled.
- connect.env parsing kept a trailing '=' (base64 padding) — it was dropped.
- Pending-deletion errors from GetSecretValue are reported as such; message
  and doc-comment fixes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 08:51:14 -08:00
1 parent 820bfab25c
commit 262ea041c8
8 files changed
+299 -112

No files matched your search

+5 -3
View File
@@ -449,8 +449,9 @@ docker exec stdout → tokio task → emit("terminal-output-{sessionId}") → li
the agent token but no `TRIPLEC_FILES` and no credential pairs (`SECRET_ENV_KEYS` + every custom
variable; `TRIPLEC_CREDENTIALS=connect` instead — the gateway sidecar's `TRIPLEC_GATEWAY_*` keys
stay, R1). After every connect (start, attach, background re-attach) `aws_host::connect`
uploads the files and `connect.env` (`KEY=<base64>` lines) as root into `/dev/shm/triple-c`
(root 0700 tmpfs) and runs `triple-c-apply-connect`: CA install as root, SSH merge and git
uploads the files and `connect.env` (`KEY=<base64>` lines) as root into this connect's own
`/dev/shm/triple-c/connect/<nonce>/` (root 0700 tmpfs; `--prepare` prints the nonce, so two
connects at once never mix) and runs `triple-c-apply-connect <nonce>`: CA install as root, SSH merge and git
credential helper **as `claude`** (root never walks a live `~/.ssh`), `/run/triple-c/connected`,
then a background re-clone of `failed` repos only (GIT_TOKEN in that clone's env only). The
first clone waits up to 5 min for `connected` (`aws_wait_connect`). Terminals the app opens add
@@ -458,7 +459,8 @@ docker exec stdout → tokio task → emit("terminal-output-{sessionId}") → li
nothing reaches the agent's snapshot, so cron/scheduled jobs and the setup script never get them.
A failed push is a `workspace-warning`. Hybrid/EFS do **not** link `~/.ssh` to EFS under
copy-at-connect (R6: keys vanish with the task). A secret that still holds credentials when a
copy-at-connect start writes it (`holds_credentials`) is force-deleted and created again (R7:
copy-at-connect start finds on CreateSecret's ResourceExists (`holds_credentials`; a new
project's secret is never read first) is force-deleted and created again (R7:
`PutSecretValue` would keep them as AWSPREVIOUS); `TRIPLE_C_CUSTOM_ENV` is left out too.
The re-clone closes the apply lock's fd (a later connect never waits for it); `--prepare` takes it.
The functions both scripts use live in `container/entrypoint-lib.sh`
+10
View File
@@ -115,6 +115,16 @@ impl WorkspaceApi for Fake {
async fn project_secret_value(&self, _n: &str) -> Result<Option<String>, AwsError> {
unreachable!()
}
async fn create_project_secret(
&self,
n: &str,
k: &str,
j: &str,
t: &[(String, String)],
) -> Result<Option<String>, AwsError> {
let _ = (n, k, j, t);
unreachable!()
}
async fn delete_project_secret(&self, _n: &str) -> Result<(), AwsError> {
unreachable!()
}
+78 -35
View File
@@ -650,6 +650,15 @@ pub trait WorkspaceApi: Send + Sync {
root: &str,
tags: &[(String, String)],
) -> impl Future<Output = Result<String, AwsError>> + Send;
/// Create the secret with `json` (encrypted with `kms_key_arn`, tagged).
/// `Some(arn)` when created, `None` when it already exists.
fn create_project_secret(
&self,
name: &str,
kms_key_arn: &str,
json: &str,
tags: &[(String, String)],
) -> impl Future<Output = Result<Option<String>, AwsError>> + Send;
/// The secret's current `SecretString`; `None` when there is no secret.
fn project_secret_value(
&self,
@@ -868,7 +877,17 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
// Secrets Manager before, or a custom variable has gone): a new value
// would leave them in the secret's version history (AWSPREVIOUS), so
// the secret is deleted outright and made again (ruling R7).
Some(custom) => {
//
// Created first, as for Secrets Manager: a new project has no secret,
// and reading one that doesn't exist may be refused outright (the
// read is allowed only on secrets carrying the caller's owner tag).
// Only an existing secret is read.
Some(custom) => match api
.create_project_secret(&secret_name, &env.kms_key_arn, &json, &tags)
.await?
{
Some(arn) => arn,
None => {
let old = api.project_secret_value(&secret_name).await?;
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
log::info!(
@@ -887,7 +906,8 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
} else {
api.put_project_secret(&secret_name, &env.kms_key_arn, &json, &tags).await?
}
}
}
},
None => api.put_project_secret(&secret_name, &env.kms_key_arn, &json, &tags).await?,
};
@@ -952,8 +972,6 @@ const MAX_LISTED_REVISIONS: i32 = 100;
/// registered (what crowding a family out can cost the caller).
const MAX_REVISION_PAGES: usize = 20;
/// Secrets Manager's `InvalidRequestException` for a name whose secret is
/// scheduled for deletion ("... already scheduled for deletion").
/// Copy-at-connect's custom variable names, or `None` for Secrets Manager.
fn extras_custom<'a>(
cfg: &crate::models::project::AwsWorkspaceConfig,
@@ -1012,7 +1030,9 @@ pub async fn replace_project_secret<W: WorkspaceApi>(
Err(e) if is_pending_deletion_error(&e) => {
if tokio::time::Instant::now() + timing.interval > deadline {
return Err(other(format!(
"The project's secret ({name}) was deleted to remove the credentials it held, but AWS has not finished deleting it after {} s. Start the project again in a minute.",
"The project's secret ({name}) was deleted to remove the credentials it \
held, but AWS has not finished deleting it after {} s. Start the project \
again in a minute.",
timing.limit.as_secs()
)));
}
@@ -1023,8 +1043,11 @@ pub async fn replace_project_secret<W: WorkspaceApi>(
}
}
/// Secrets Manager's `InvalidRequestException` for a name whose secret is
/// scheduled for deletion ("... already scheduled for deletion").
fn is_pending_deletion(message: Option<&str>) -> bool {
message.is_some_and(|m| m.contains("scheduled for deletion"))
// CreateSecret/PutSecretValue say "scheduled", GetSecretValue "marked".
message.is_some_and(|m| m.contains("scheduled for deletion") || m.contains("marked for deletion"))
}
fn pending_deletion_error(name: &str) -> AwsError {
@@ -1177,6 +1200,13 @@ impl WorkspaceApi for SdkWorkspaceApi {
{
Ok(None)
}
Err(e)
if e.as_service_error().is_some_and(|s| {
s.is_invalid_request_exception() && is_pending_deletion(s.message())
}) =>
{
Err(pending_deletion_error(name))
}
Err(e) => Err(classify_sdk_error(ACTION, &e)),
}
}
@@ -1202,16 +1232,15 @@ impl WorkspaceApi for SdkWorkspaceApi {
}
}
async fn put_project_secret(
async fn create_project_secret(
&self,
name: &str,
kms_key_arn: &str,
json: &str,
tags: &[(String, String)],
) -> Result<String, AwsError> {
) -> Result<Option<String>, AwsError> {
use aws_sdk_secretsmanager::types::Tag as SmTag;
const CREATE: &str = "secretsmanager:CreateSecret";
const PUT: &str = "secretsmanager:PutSecretValue";
let created = self
.secrets
.create_secret()
@@ -1225,43 +1254,57 @@ impl WorkspaceApi for SdkWorkspaceApi {
))
.send()
.await;
let arn = match created {
Ok(out) => out.arn().map(str::to_string),
match created {
Ok(out) => out
.arn()
.map(|a| Some(a.to_string()))
.ok_or_else(|| other(format!("{CREATE} returned no ARN"))),
Err(e)
if e.as_service_error()
.is_some_and(|s| s.is_resource_exists_exception()) =>
{
// Only a secret tagged with the caller as owner accepts this
// (the policy), so another user's same-named secret fails
// here rather than receiving the payload.
self.secrets
.put_secret_value()
.secret_id(name)
.secret_string(json)
.send()
.await
.map_err(|e| match e.as_service_error() {
Some(s)
if s.is_invalid_request_exception()
&& is_pending_deletion(s.message()) =>
{
pending_deletion_error(name)
}
_ => classify_sdk_error(PUT, &e),
})?
.arn()
.map(str::to_string)
Ok(None)
}
Err(e)
if e.as_service_error().is_some_and(|s| {
s.is_invalid_request_exception() && is_pending_deletion(s.message())
}) =>
{
return Err(pending_deletion_error(name));
Err(pending_deletion_error(name))
}
Err(e) => return Err(classify_sdk_error(CREATE, &e)),
};
arn.ok_or_else(|| other(format!("{CREATE} returned no ARN")))
Err(e) => Err(classify_sdk_error(CREATE, &e)),
}
}
async fn put_project_secret(
&self,
name: &str,
kms_key_arn: &str,
json: &str,
tags: &[(String, String)],
) -> Result<String, AwsError> {
const PUT: &str = "secretsmanager:PutSecretValue";
if let Some(arn) = self.create_project_secret(name, kms_key_arn, json, tags).await? {
return Ok(arn);
}
// Only a secret tagged with the caller as owner accepts this (the
// policy), so another user's same-named secret fails here rather
// than receiving the payload.
self.secrets
.put_secret_value()
.secret_id(name)
.secret_string(json)
.send()
.await
.map_err(|e| match e.as_service_error() {
Some(s) if s.is_invalid_request_exception() && is_pending_deletion(s.message()) => {
pending_deletion_error(name)
}
_ => classify_sdk_error(PUT, &e),
})?
.arn()
.map(str::to_string)
.ok_or_else(|| other(format!("{PUT} returned no ARN")))
}
async fn task_definition_revisions(&self, family: &str) -> Result<Vec<String>, AwsError> {
+22 -1
View File
@@ -799,6 +799,8 @@ struct Fake {
secret_reads: Mutex<usize>,
/// Puts that fail as "scheduled for deletion" before one succeeds.
pending_puts: Mutex<usize>,
/// Calls `create_project_secret` got.
creates: Mutex<usize>,
}
impl Fake {
@@ -850,8 +852,22 @@ impl WorkspaceApi for Fake {
}
async fn delete_project_secret(&self, name: &str) -> Result<(), AwsError> {
self.deleted_secrets.lock().unwrap().push(name.into());
*self.existing_secret.lock().unwrap() = None;
Ok(())
}
async fn create_project_secret(
&self,
name: &str,
kms_key_arn: &str,
json: &str,
tags: &[(String, String)],
) -> Result<Option<String>, AwsError> {
*self.creates.lock().unwrap() += 1;
if self.existing_secret.lock().unwrap().is_some() {
return Ok(None);
}
self.put_project_secret(name, kms_key_arn, json, tags).await.map(Some)
}
async fn put_project_secret(
&self,
name: &str,
@@ -1638,6 +1654,7 @@ async fn switching_to_copy_at_connect_replaces_a_secret_that_held_credentials()
prepare_workspace(&fake, &who(), &env_info(), &copy_at_connect_project(), &env, &[], &[], IMAGE)
.await
.unwrap();
assert_eq!(*fake.secret_reads.lock().unwrap(), 1, "read only once it exists");
assert_eq!(fake.deleted_secrets.lock().unwrap().len(), 1);
let m = payload_map(&fake.secrets.lock().unwrap()[0].2);
assert!(!m.contains_key("GIT_TOKEN"));
@@ -1653,12 +1670,16 @@ async fn switching_to_copy_at_connect_replaces_a_secret_that_held_credentials()
assert!(fake.deleted_secrets.lock().unwrap().is_empty());
assert_eq!(fake.secrets.lock().unwrap().len(), 1);
// No secret yet: created, nothing deleted.
// No secret yet (a first start): created, never read — a read of a
// secret that doesn't exist may be refused — nothing deleted.
let fake = Fake::onboarded();
prepare_workspace(&fake, &who(), &env_info(), &copy_at_connect_project(), &env, &[], &[], IMAGE)
.await
.unwrap();
assert_eq!(*fake.creates.lock().unwrap(), 1);
assert_eq!(*fake.secret_reads.lock().unwrap(), 0);
assert!(fake.deleted_secrets.lock().unwrap().is_empty());
assert!(!payload_map(&fake.secrets.lock().unwrap()[0].2).contains_key("GIT_TOKEN"));
// Secrets Manager never reads or deletes.
let fake = Fake::onboarded();
+78 -29
View File
@@ -2226,7 +2226,7 @@ impl Connect {
.replace("lib=/usr/local/lib/triple-c/entrypoint-lib.sh\n", &format!("lib='{}'\n", p("lib.sh").display()))
.replace("shm=/dev/shm/triple-c\n", &format!("shm='{}'\n", p("shm").display()))
.replace("stage=/dev/shm/triple-c/connect\n", &format!("stage='{}'\n", p("shm/connect").display()))
.replace("env_file=/dev/shm/triple-c/connect.env\n", &format!("env_file='{}'\n", p("shm/connect.env").display()))
.replace("ca_dir=/dev/shm/triple-c/connect-ca\n", &format!("ca_dir='{}'\n", p("shm/connect-ca").display()))
.replace("clone_status=/run/triple-c-clone\n", &format!("clone_status='{}'\n", p("clone").display()))
.replace("workspace=/workspace\n", &format!("workspace='{}'\n", p("ws").display()))
.replace("retry_wait=2400\n", "retry_wait=20\n")
@@ -2252,6 +2252,31 @@ impl Connect {
self.root.join(s)
}
/// `--prepare`: this connect's nonce (its dir is `shm/connect/<nonce>`).
fn prepare(&self) -> String {
let out = self.run(&["--prepare"]);
assert!(out.status.success(), "{}", text(&out));
let nonce = String::from_utf8_lossy(&out.stdout).trim().to_string();
assert!(nonce.len() == 16 && nonce.chars().all(|c| c.is_ascii_hexdigit()), "{nonce:?}");
nonce
}
/// What the app uploads for one connect: `files` (`ssh/x`, `ca/y`) and
/// `connect.env`.
fn stage(&self, nonce: &str, files: &[(&str, &str)], env: &str) {
for (k, v) in files {
fs::write(self.p(&format!("shm/connect/{nonce}/{k}")), v).unwrap();
}
fs::write(self.p(&format!("shm/connect/{nonce}/connect.env")), env).unwrap();
}
/// A whole connect: prepare, stage, apply.
fn connect(&self, files: &[(&str, &str)], env: &str) -> Output {
let nonce = self.prepare();
self.stage(&nonce, files, env);
self.run(&[&nonce])
}
/// Waits for the background re-clone to finish (`done` back).
fn wait_done(&self) {
for _ in 0..200 {
@@ -2272,26 +2297,22 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
let c = Connect::new(tmp.path());
let remote = bare_remote(tmp.path(), "remote");
// A stale staging dir and env file from an earlier connect are replaced.
fs::create_dir_all(c.p("shm/connect/ssh")).unwrap();
fs::write(c.p("shm/connect/ssh/old"), "old").unwrap();
fs::write(c.p("shm/connect.env"), "OLD=x\n").unwrap();
let out = c.run(&["--prepare"]);
assert!(out.status.success(), "{}", text(&out));
for d in ["shm/connect", "shm/connect/ssh", "shm/connect/ca"] {
assert_eq!(mode(&c.p(d)), 0o700, "{d}");
// A connect's own staging dir; one an abandoned connect left long ago is
// cleared, a recent one (a connect in flight) is not.
fs::create_dir_all(c.p("shm/connect/0000000000000old")).unwrap();
fs::create_dir_all(c.p("shm/connect/00000000000recent")).unwrap();
assert!(Command::new("touch").args(["-d", "1 hour ago"]).arg(c.p("shm/connect/0000000000000old")).status().unwrap().success());
let nonce = c.prepare();
for d in ["shm/connect".to_string(), format!("shm/connect/{nonce}"), format!("shm/connect/{nonce}/ssh"), format!("shm/connect/{nonce}/ca")] {
assert_eq!(mode(&c.p(&d)), 0o700, "{d}");
}
assert!(!c.p("shm/connect/ssh/old").exists() && !c.p("shm/connect.env").exists());
assert!(!c.p("shm/connect/0000000000000old").exists());
assert!(c.p("shm/connect/00000000000recent").exists());
// What the app uploads.
fs::write(c.p("shm/connect/ssh/id_ed25519"), "KEY").unwrap();
fs::write(c.p("shm/connect/ca/corp.pem"), PEM).unwrap();
let token = "ghp_secret-value";
fs::write(
c.p("shm/connect.env"),
format!("ANTHROPIC_API_KEY={}\nGIT_TOKEN={}\n", b64("sk-other"), b64(token)),
)
.unwrap();
let env = format!("ANTHROPIC_API_KEY={}\nGIT_TOKEN={}\n", b64("sk-other"), b64(token));
c.stage(&nonce, &[("ssh/id_ed25519", "KEY"), ("ca/corp.pem", PEM)], &env);
// The first clone is done: `ok` cloned, `bad` failed (no credentials then).
git(tmp.path(), &["clone", "-q", remote.to_str().unwrap(), c.p("ws/ok").to_str().unwrap()]);
fs::write(c.p("clone/repos"), "ok\tcloning\nok\tcloned\nbad\tcloning\nbad\tfailed\n").unwrap();
@@ -2305,10 +2326,11 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
)
.unwrap();
let out = c.run(&[]);
let out = c.run(&[&nonce]);
let log = text(&out);
assert!(out.status.success(), "{log}");
assert!(log.contains("credentials applied"), "{log}");
assert!(!c.p(&format!("shm/connect/{nonce}")).exists(), "the apply removes its staging");
assert!(!log.contains(token) && !log.contains("sk-other"), "no value is printed: {log}");
// CA, as the entrypoint installs it.
@@ -2337,8 +2359,9 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
assert!(clone_log.contains("cloning again with the project's credentials: bad"), "{clone_log}");
assert!(!clone_log.contains(token), "{clone_log}");
// Idempotent: again, nothing failed, nothing re-cloned, same state.
let out = c.run(&[]);
// A new connect with the same credentials: nothing failed, nothing
// re-cloned, same state, the CA not reinstalled.
let out = c.connect(&[("ssh/id_ed25519", "KEY"), ("ca/corp.pem", PEM)], &env);
assert!(out.status.success(), "{}", text(&out));
assert_eq!(fs::read_to_string(c.p("clone/repos")).unwrap(), repos);
assert!(c.p("clone/done").exists());
@@ -2351,11 +2374,11 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
fn apply_connect_with_nothing_configured_changes_nothing_and_leaves_a_waiting_clone_alone() {
let tmp = tempfile::tempdir().unwrap();
let c = Connect::new(tmp.path());
assert!(c.run(&["--prepare"]).status.success());
let nonce = c.prepare();
// The first clone is still waiting for this connect: no `done`, no
// `no-credentials`.
fs::write(c.p("run/repos.json"), "[]").unwrap();
let out = c.run(&[]);
let out = c.run(&[&nonce]);
let log = text(&out);
assert!(out.status.success(), "{log}");
assert!(!log.contains("warning"), "{log}");
@@ -2371,7 +2394,7 @@ fn apply_connect_retries_a_clone_that_started_without_credentials_once_it_is_don
let tmp = tempfile::tempdir().unwrap();
let c = Connect::new(tmp.path());
let remote = bare_remote(tmp.path(), "remote");
assert!(c.run(&["--prepare"]).status.success());
let nonce = c.prepare();
fs::write(
c.p("run/repos.json"),
format!(r#"[{{"url":"{}","branch":null,"folder":"late"}}]"#, remote.display()),
@@ -2380,7 +2403,7 @@ fn apply_connect_retries_a_clone_that_started_without_credentials_once_it_is_don
// Still cloning without credentials (the wait timed out).
fs::write(c.p("clone/no-credentials"), "").unwrap();
fs::write(c.p("clone/repos"), "late\tcloning\n").unwrap();
let out = c.run(&[]);
let out = c.run(&[&nonce]);
assert!(out.status.success(), "{}", text(&out));
std::thread::sleep(std::time::Duration::from_millis(500));
assert!(!c.p("ws/late").exists(), "nothing until the running clone is done");
@@ -2410,16 +2433,15 @@ fn a_running_re_clone_never_holds_the_apply_lock_and_prepare_waits_for_an_apply(
let mut lib = fs::read_to_string(c.p("lib.sh")).unwrap();
lib.push_str("\nclone_repos() { sleep 6; }\n");
fs::write(c.p("lib.sh"), lib).unwrap();
assert!(c.run(&["--prepare"]).status.success());
let nonce = c.prepare();
fs::write(c.p("run/repos.json"), r#"[{"url":"x","branch":null,"folder":"bad"}]"#).unwrap();
fs::write(c.p("clone/repos"), "bad\tcloning\nbad\tfailed\n").unwrap();
fs::write(c.p("clone/done"), "").unwrap();
assert!(c.run(&[]).status.success());
assert!(c.run(&[&nonce]).status.success());
assert!(!c.p("clone/done").exists(), "the re-clone is running");
// The next connect goes straight through.
let started = std::time::Instant::now();
assert!(c.run(&["--prepare"]).status.success());
let out = c.run(&[]);
let out = c.connect(&[], "");
assert!(out.status.success(), "{}", text(&out));
assert!(!text(&out).contains("another apply"), "{}", text(&out));
assert!(started.elapsed() < std::time::Duration::from_secs(4), "{:?}", started.elapsed());
@@ -2429,7 +2451,6 @@ fn a_running_re_clone_never_holds_the_apply_lock_and_prepare_waits_for_an_apply(
let lock = c.p("shm/apply-connect.lock");
let mut holder = Command::new("flock").arg(&lock).args(["sleep", "2"]).spawn().unwrap();
std::thread::sleep(std::time::Duration::from_millis(300));
fs::write(c.p("shm/connect/marker"), "").ok();
let started = std::time::Instant::now();
assert!(c.run(&["--prepare"]).status.success());
assert!(started.elapsed() >= std::time::Duration::from_millis(1200), "{:?}", started.elapsed());
@@ -2461,3 +2482,31 @@ fn under_copy_at_connect_hybrid_keeps_ssh_off_efs() {
assert_eq!(fs::read_to_string(home.join(".ssh/id_ed25519")).unwrap(), "KEY");
assert!(!mnt.join(".ssh").exists(), "nothing written to EFS");
}
#[test]
fn two_connects_at_once_never_mix_their_files() {
// Two machines connecting at the same moment: both prepare, then both
// upload, then both apply. Each applies its own set; the later wins.
let tmp = tempfile::tempdir().unwrap();
let c = Connect::new(tmp.path());
let a = c.prepare();
let b = c.prepare();
assert_ne!(a, b);
c.stage(&a, &[("ssh/id_a", "A"), ("ssh/config", "Host a\n")], &format!("GIT_TOKEN={}\n", b64("tok-a")));
c.stage(&b, &[("ssh/id_b", "B"), ("ssh/config", "Host b\n")], &format!("GIT_TOKEN={}\n", b64("tok-b")));
for n in [&a, &b] {
let out = c.run(&[n]);
assert!(out.status.success(), "{}", text(&out));
}
assert_eq!(fs::read_to_string(c.p("home/.ssh/config")).unwrap(), "Host b\n");
let cred = fs::read_to_string(c.p("home/.git-credentials")).unwrap_or_else(|e| panic!("{e}: {:?}", fs::read_dir(c.p("home")).unwrap().map(|e| e.unwrap().file_name()).collect::<Vec<_>>()));
assert!(cred.contains("tok-b"), "{cred}");
// B's set as a whole: A's key, merged first, is dropped again by the
// merge's own bookkeeping (B no longer provides it).
assert_eq!(fs::read_to_string(c.p("home/.ssh/id_b")).unwrap(), "B");
assert!(!c.p("home/.ssh/id_a").exists());
assert_eq!(fs::read_dir(c.p("shm/connect")).unwrap().count(), 0, "both staging dirs gone");
// A nonce that is not one, or not prepared, is refused.
assert_eq!(c.run(&["../../etc"]).status.code(), Some(2));
assert_eq!(c.run(&[&a]).status.code(), Some(1));
}
+24 -11
View File
@@ -8,8 +8,9 @@
//! [`workspace_files`](super::workspace_files) the secret would carry) and
//! the credential environment pairs (`SECRET_ENV_KEYS` and every custom
//! variable, `aws::workspace::credential_pairs`). [`push_connect_bundle`]
//! uploads them through the agent, as root, into root-only staging on the
//! task's `/dev/shm` tmpfs and runs `triple-c-apply-connect` (container), which
//! uploads them through the agent, as root, into this connect's own root-only
//! staging dir on the task's `/dev/shm` tmpfs and runs `triple-c-apply-connect
//! <nonce>` (container), which
//! installs the CA, merges the SSH files, writes the git credential helper
//! and re-clones what failed. The terminals the app opens get the pairs in
//! their own `ExecSpec.env` ([`terminal_env`]); nothing puts them into the
@@ -28,9 +29,8 @@ use crate::models::{AppSettings, Project};
/// The in-container half (container/triple-c-apply-connect).
pub const APPLY_CONNECT: &str = "/usr/local/bin/triple-c-apply-connect";
/// Root 0700 on the tmpfs (made by the entrypoint for the agent token).
pub const CONNECT_SHM: &str = "/dev/shm/triple-c";
/// Staging for the files: `ssh/` and `ca/` under it.
/// Staging (root 0700, on the tmpfs the entrypoint made for the agent
/// token): each connect gets `<nonce>/` with `ssh/`, `ca/` and `connect.env`.
pub const CONNECT_STAGE: &str = "/dev/shm/triple-c/connect";
pub const CONNECT_ENV_FILE: &str = "connect.env";
/// Every staged file is root 0600.
@@ -174,21 +174,34 @@ async fn run_apply<T: ConnectTarget>(t: &T, args: &[&str]) -> Result<String, Str
}
}
/// Upload `bundle` and apply it (see the module docs): `--prepare`, the
/// files and `connect.env` (root 0600), then the apply.
/// `--prepare`'s answer: this connect's nonce, 16 hex digits on the last line.
pub(crate) fn parse_nonce(out: &str) -> Result<String, String> {
let last = out.lines().rev().map(str::trim).find(|l| !l.is_empty()).unwrap_or_default();
if last.len() == 16 && last.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) {
Ok(last.to_string())
} else {
Err(format!("triple-c-apply-connect --prepare gave no connect id:\n{}", tail(out)))
}
}
/// Upload `bundle` and apply it (see the module docs): `--prepare` (which
/// names this connect's own staging dir), the files and `connect.env` (root
/// 0600) into that dir, then the apply of exactly that dir — so two
/// connects at once (two machines) never mix their files.
pub async fn push_connect_bundle<T: ConnectTarget>(t: &T, bundle: &ConnectBundle) -> Result<(), String> {
run_apply(t, &["--prepare"]).await?;
let nonce = parse_nonce(&run_apply(t, &["--prepare"]).await?)?;
let dir = format!("{CONNECT_STAGE}/{nonce}");
for (key, data) in &bundle.files {
validate_file_key(key)?;
let (kind, name) = key.split_once('/').expect("validated");
t.upload_root(&format!("{CONNECT_STAGE}/{kind}"), name, data, STAGED_MODE)
t.upload_root(&format!("{dir}/{kind}"), name, data, STAGED_MODE)
.await
.map_err(|e| format!("Could not copy {key} into the workspace: {e}"))?;
}
t.upload_root(CONNECT_SHM, CONNECT_ENV_FILE, &connect_env_file(&bundle.env), STAGED_MODE)
t.upload_root(&dir, CONNECT_ENV_FILE, &connect_env_file(&bundle.env), STAGED_MODE)
.await
.map_err(|e| format!("Could not copy the credential variables into the workspace: {e}"))?;
let out = run_apply(t, &[]).await?;
let out = run_apply(t, &[&nonce]).await?;
for line in out.lines().filter(|l| l.contains("warning")) {
log::warn!("{line}");
}
+26 -8
View File
@@ -164,6 +164,15 @@ impl WorkspaceApi for Fake {
async fn project_secret_value(&self, _name: &str) -> Result<Option<String>, AwsError> {
Ok(None)
}
async fn create_project_secret(
&self,
n: &str,
k: &str,
j: &str,
t: &[(String, String)],
) -> Result<Option<String>, AwsError> {
self.put_project_secret(n, k, j, t).await.map(Some)
}
async fn delete_project_secret(&self, _name: &str) -> Result<(), AwsError> {
Ok(())
}
@@ -959,10 +968,11 @@ mod copy_at_connect {
impl ConnectTarget for FakeTarget {
async fn exec_root(&self, cmd: Vec<String>) -> Result<(String, i64), String> {
self.calls.lock().unwrap().push(format!("exec {}", cmd.join(" ")));
if cmd.len() == 1 {
if let Some(r) = self.apply_result.lock().unwrap().clone() {
return Ok(r);
}
if cmd.get(1).map(String::as_str) == Some("--prepare") {
return Ok(("0123456789abcdef\n".into(), 0));
}
if let Some(r) = self.apply_result.lock().unwrap().clone() {
return Ok(r);
}
Ok(("triple-c-apply-connect: credentials applied\n".into(), 0))
}
@@ -994,10 +1004,10 @@ mod copy_at_connect {
*t.calls.lock().unwrap(),
vec![
"exec /usr/local/bin/triple-c-apply-connect --prepare",
"upload /dev/shm/triple-c/connect/ssh/id_ed25519",
"upload /dev/shm/triple-c/connect/ca/corp.crt",
"upload /dev/shm/triple-c/connect.env",
"exec /usr/local/bin/triple-c-apply-connect",
"upload /dev/shm/triple-c/connect/0123456789abcdef/ssh/id_ed25519",
"upload /dev/shm/triple-c/connect/0123456789abcdef/ca/corp.crt",
"upload /dev/shm/triple-c/connect/0123456789abcdef/connect.env",
"exec /usr/local/bin/triple-c-apply-connect 0123456789abcdef",
]
);
let uploads = t.uploads.lock().unwrap();
@@ -1030,6 +1040,14 @@ mod copy_at_connect {
assert!(t.uploads.lock().unwrap().is_empty());
}
#[test]
fn only_a_well_formed_connect_id_is_used_as_a_path() {
assert_eq!(parse_nonce("noise\n0123456789abcdef\n").unwrap(), "0123456789abcdef");
for bad in ["", "../../etc/passwd", "0123456789ABCDEF", "0123456789abcde", "0123456789abcdef0"] {
assert!(parse_nonce(bad).is_err(), "{bad}");
}
}
#[test]
fn a_bundle_never_shows_its_values() {
let shown = format!("{:?}", bundle());
+56 -25
View File
@@ -7,13 +7,14 @@
# custom variables. Instead, after every connect (a start, an attach, the
# background re-attach when the app starts) the app, through the agent and as
# root:
# 1. runs `triple-c-apply-connect --prepare` (fresh, empty, root-only
# staging dirs on the /dev/shm tmpfs — never the disk);
# 2. uploads the SSH files to /dev/shm/triple-c/connect/ssh/, the CA files to
# /dev/shm/triple-c/connect/ca/ (each root 0600) and the credential
# environment to /dev/shm/triple-c/connect.env (root 0600, one
# `KEY=<base64 of the value>` line per variable);
# 3. runs `triple-c-apply-connect`, which
# 1. runs `triple-c-apply-connect --prepare`, which makes this connect's
# own root-only staging dir on the /dev/shm tmpfs (never the disk),
# /dev/shm/triple-c/connect/<nonce>/{ssh,ca}, and prints <nonce>
# (16 hex digits);
# 2. uploads the SSH files to <nonce>/ssh/, the CA files to <nonce>/ca/ and
# the credential environment to <nonce>/connect.env (each root 0600;
# connect.env holds one `KEY=<base64 of the value>` line per variable);
# 3. runs `triple-c-apply-connect <nonce>`, which, under the apply lock,
# - installs the CA (install_corporate_ca, as root: its source and the
# system store are root-only paths),
# - merges the SSH files into ~/.ssh (merge_host_ssh, **as the user**:
@@ -31,7 +32,12 @@
# connect.env is never sourced into, or exported to, anything but the clone,
# and the agent's environment (every terminal's) is untouched — the app adds
# the credential variables to the terminals it opens itself. Nothing here
# prints a value. Idempotent: run it again and it converges on the same state.
# prints a value. Each connect stages and applies its own set, so two connects
# at once (two machines, a start racing a re-attach) never mix their files:
# their applies run one after the other, the later one's set wins. The apply
# removes its staging dir; the CA files move to /dev/shm/triple-c/connect-ca
# (a fixed path, so an unchanged CA is not reinstalled). Idempotent: a new
# connect with the same credentials converges on the same state.
# Re-cloning does not re-run the setup script.
set -u
@@ -40,7 +46,7 @@ set -u
lib=/usr/local/lib/triple-c/entrypoint-lib.sh
shm=/dev/shm/triple-c
stage=/dev/shm/triple-c/connect
env_file=/dev/shm/triple-c/connect.env
ca_dir=/dev/shm/triple-c/connect-ca
clone_status=/run/triple-c-clone
workspace=/workspace
# How long a retry waits for a clone that started without credentials.
@@ -58,33 +64,47 @@ if ! . "$lib"; then
fi
triplec_aws=1
# The apply lock, also taken by --prepare so a prepare never empties the
# staging dir under a running apply.
# The apply lock: applies run one at a time; --prepare takes it too, so its
# clean-up of stale staging dirs never races an apply.
take_lock() {
exec 9>"$shm/apply-connect.lock" && flock -w 120 9 \
|| say "warning — another apply is still running; going ahead"
}
prepare() {
local nonce
umask 077
if [ ! -d "$shm" ] || [ -L "$shm" ]; then
say "$shm is missing (the workspace has no agent token?); nothing prepared"
exit 1
fi
take_lock
rm -rf "$stage" "$env_file"
mkdir -m 700 "$stage" "$stage/ssh" "$stage/ca" || exit 1
if [ -L "$stage" ] || { [ -e "$stage" ] && [ ! -d "$stage" ]; }; then rm -f "$stage"; fi
mkdir -p -m 700 "$stage" && chmod 700 "$stage" || exit 1
# What a connect that never applied (the app gone mid-push) left.
find "$stage" -mindepth 1 -maxdepth 1 -mmin +10 -exec rm -rf {} + 2>/dev/null
nonce=$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')
case "$nonce" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) say "no random nonce"; exit 1 ;; esac
mkdir -m 700 "$stage/$nonce" "$stage/$nonce/ssh" "$stage/$nonce/ca" || exit 1
printf '%s\n' "$nonce"
}
# This connect's staging dir (set by apply).
dir=
# GIT_TOKEN, decoded from connect.env into this shell only.
connect_git_token=
read_git_token() {
local key value
[ -f "$env_file" ] || return 0
while IFS='=' read -r key value; do
local line key value
[ -f "$dir/connect.env" ] || return 0
# Split by hand at the first `=`: `read` with IFS='=' would drop a
# trailing `=`, which is base64 padding here.
while IFS= read -r line; do
key=${line%%=*}
value=${line#*=}
[ "$key" = GIT_TOKEN ] || continue
connect_git_token=$(printf '%s' "$value" | base64 -d 2>/dev/null) || connect_git_token=
done < "$env_file"
done < "$dir/connect.env"
}
# The staged SSH files merged as the user: copied to a fresh root-made dir
@@ -94,7 +114,7 @@ merge_ssh_as_user() {
local tmp rc=0
tmp=$(mktemp -d "${ssh_tmp_dir}/triple-c-connect.XXXXXXXX") \
|| { say "warning — no temporary directory; SSH files not merged"; return 0; }
if cp -R "$stage/ssh" "$tmp/ssh"; then
if cp -R "$dir/ssh" "$tmp/ssh"; then
if [ -n "${triplec_run_as:-}" ] && [ "$(id -u)" = 0 ]; then
chown -R "$triplec_run_as:" "$tmp"
fi
@@ -183,19 +203,29 @@ retry_failed_clones() {
}
apply() {
local nonce="$1"
umask 077
if [ ! -d "$stage" ] || [ -L "$stage" ]; then
say "$stage is missing; run --prepare and upload first"
case "$nonce" in
*[!0-9a-f]*|'') say "not a connect nonce: $nonce"; exit 2 ;;
esac
[ "${#nonce}" = 16 ] || { say "not a connect nonce: $nonce"; exit 2; }
dir="$stage/$nonce"
if [ ! -d "$dir" ] || [ -L "$dir" ] || [ -L "$stage" ]; then
say "$dir is missing; run --prepare and upload first"
exit 1
fi
take_lock
# Empty staging dirs: nothing of that kind is configured.
rmdir "$stage/ca" "$stage/ssh" 2>/dev/null
rmdir "$dir/ca" "$dir/ssh" 2>/dev/null
CA_SRC="$stage/ca"
rm -rf "$ca_dir"
if [ -d "$dir/ca" ]; then
mv "$dir/ca" "$ca_dir" || say "warning — could not move the CA files into place"
fi
CA_SRC="$ca_dir"
install_corporate_ca
if [ -d "$stage/ssh" ]; then
if [ -d "$dir/ssh" ]; then
merge_ssh_as_user
fi
@@ -210,11 +240,12 @@ apply() {
say "warning — could not record the connect in $triplec_run_dir"
fi
retry_failed_clones
rm -rf "$dir"
say "credentials applied"
}
case "${1:-}" in
--prepare) prepare ;;
'') apply ;;
*) say "usage: triple-c-apply-connect [--prepare]"; exit 2 ;;
''|-*) say "usage: triple-c-apply-connect --prepare | <nonce>"; exit 2 ;;
*) apply "$1" ;;
esac