fix(aws): create the secret before reading it; one staging dir per connect
- Copy-at-connect tries CreateSecret first and reads the current value only when the secret exists, so a first start never reads a missing secret (which the owner-tag condition may refuse). - triple-c-apply-connect: --prepare makes and prints a per-connect nonce dir; the app uploads into it and applies exactly that dir, so concurrent connects never mix files. CA files move to a fixed path so an unchanged CA is not reinstalled. - connect.env parsing kept a trailing '=' (base64 padding) — it was dropped. - Pending-deletion errors from GetSecretValue are reported as such; message and doc-comment fixes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
820bfab25c
commit
262ea041c8
8 files changed
+299
-112
No files matched your search
@@ -449,8 +449,9 @@ docker exec stdout → tokio task → emit("terminal-output-{sessionId}") → li
|
||||
the agent token but no `TRIPLEC_FILES` and no credential pairs (`SECRET_ENV_KEYS` + every custom
|
||||
variable; `TRIPLEC_CREDENTIALS=connect` instead — the gateway sidecar's `TRIPLEC_GATEWAY_*` keys
|
||||
stay, R1). After every connect (start, attach, background re-attach) `aws_host::connect`
|
||||
uploads the files and `connect.env` (`KEY=<base64>` lines) as root into `/dev/shm/triple-c`
|
||||
(root 0700 tmpfs) and runs `triple-c-apply-connect`: CA install as root, SSH merge and git
|
||||
uploads the files and `connect.env` (`KEY=<base64>` lines) as root into this connect's own
|
||||
`/dev/shm/triple-c/connect/<nonce>/` (root 0700 tmpfs; `--prepare` prints the nonce, so two
|
||||
connects at once never mix) and runs `triple-c-apply-connect <nonce>`: CA install as root, SSH merge and git
|
||||
credential helper **as `claude`** (root never walks a live `~/.ssh`), `/run/triple-c/connected`,
|
||||
then a background re-clone of `failed` repos only (GIT_TOKEN in that clone's env only). The
|
||||
first clone waits up to 5 min for `connected` (`aws_wait_connect`). Terminals the app opens add
|
||||
@@ -458,7 +459,8 @@ docker exec stdout → tokio task → emit("terminal-output-{sessionId}") → li
|
||||
nothing reaches the agent's snapshot, so cron/scheduled jobs and the setup script never get them.
|
||||
A failed push is a `workspace-warning`. Hybrid/EFS do **not** link `~/.ssh` to EFS under
|
||||
copy-at-connect (R6: keys vanish with the task). A secret that still holds credentials when a
|
||||
copy-at-connect start writes it (`holds_credentials`) is force-deleted and created again (R7:
|
||||
copy-at-connect start finds on CreateSecret's ResourceExists (`holds_credentials`; a new
|
||||
project's secret is never read first) is force-deleted and created again (R7:
|
||||
`PutSecretValue` would keep them as AWSPREVIOUS); `TRIPLE_C_CUSTOM_ENV` is left out too.
|
||||
The re-clone closes the apply lock's fd (a later connect never waits for it); `--prepare` takes it.
|
||||
The functions both scripts use live in `container/entrypoint-lib.sh`
|
||||
|
||||
@@ -115,6 +115,16 @@ impl WorkspaceApi for Fake {
|
||||
async fn project_secret_value(&self, _n: &str) -> Result<Option<String>, AwsError> {
|
||||
unreachable!()
|
||||
}
|
||||
async fn create_project_secret(
|
||||
&self,
|
||||
n: &str,
|
||||
k: &str,
|
||||
j: &str,
|
||||
t: &[(String, String)],
|
||||
) -> Result<Option<String>, AwsError> {
|
||||
let _ = (n, k, j, t);
|
||||
unreachable!()
|
||||
}
|
||||
async fn delete_project_secret(&self, _n: &str) -> Result<(), AwsError> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
@@ -650,6 +650,15 @@ pub trait WorkspaceApi: Send + Sync {
|
||||
root: &str,
|
||||
tags: &[(String, String)],
|
||||
) -> impl Future<Output = Result<String, AwsError>> + Send;
|
||||
/// Create the secret with `json` (encrypted with `kms_key_arn`, tagged).
|
||||
/// `Some(arn)` when created, `None` when it already exists.
|
||||
fn create_project_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
kms_key_arn: &str,
|
||||
json: &str,
|
||||
tags: &[(String, String)],
|
||||
) -> impl Future<Output = Result<Option<String>, AwsError>> + Send;
|
||||
/// The secret's current `SecretString`; `None` when there is no secret.
|
||||
fn project_secret_value(
|
||||
&self,
|
||||
@@ -868,7 +877,17 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
|
||||
// Secrets Manager before, or a custom variable has gone): a new value
|
||||
// would leave them in the secret's version history (AWSPREVIOUS), so
|
||||
// the secret is deleted outright and made again (ruling R7).
|
||||
Some(custom) => {
|
||||
//
|
||||
// Created first, as for Secrets Manager: a new project has no secret,
|
||||
// and reading one that doesn't exist may be refused outright (the
|
||||
// read is allowed only on secrets carrying the caller's owner tag).
|
||||
// Only an existing secret is read.
|
||||
Some(custom) => match api
|
||||
.create_project_secret(&secret_name, &env.kms_key_arn, &json, &tags)
|
||||
.await?
|
||||
{
|
||||
Some(arn) => arn,
|
||||
None => {
|
||||
let old = api.project_secret_value(&secret_name).await?;
|
||||
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
|
||||
log::info!(
|
||||
@@ -887,7 +906,8 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
|
||||
} else {
|
||||
api.put_project_secret(&secret_name, &env.kms_key_arn, &json, &tags).await?
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
None => api.put_project_secret(&secret_name, &env.kms_key_arn, &json, &tags).await?,
|
||||
};
|
||||
|
||||
@@ -952,8 +972,6 @@ const MAX_LISTED_REVISIONS: i32 = 100;
|
||||
/// registered (what crowding a family out can cost the caller).
|
||||
const MAX_REVISION_PAGES: usize = 20;
|
||||
|
||||
/// Secrets Manager's `InvalidRequestException` for a name whose secret is
|
||||
/// scheduled for deletion ("... already scheduled for deletion").
|
||||
/// Copy-at-connect's custom variable names, or `None` for Secrets Manager.
|
||||
fn extras_custom<'a>(
|
||||
cfg: &crate::models::project::AwsWorkspaceConfig,
|
||||
@@ -1012,7 +1030,9 @@ pub async fn replace_project_secret<W: WorkspaceApi>(
|
||||
Err(e) if is_pending_deletion_error(&e) => {
|
||||
if tokio::time::Instant::now() + timing.interval > deadline {
|
||||
return Err(other(format!(
|
||||
"The project's secret ({name}) was deleted to remove the credentials it held, but AWS has not finished deleting it after {} s. Start the project again in a minute.",
|
||||
"The project's secret ({name}) was deleted to remove the credentials it \
|
||||
held, but AWS has not finished deleting it after {} s. Start the project \
|
||||
again in a minute.",
|
||||
timing.limit.as_secs()
|
||||
)));
|
||||
}
|
||||
@@ -1023,8 +1043,11 @@ pub async fn replace_project_secret<W: WorkspaceApi>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Secrets Manager's `InvalidRequestException` for a name whose secret is
|
||||
/// scheduled for deletion ("... already scheduled for deletion").
|
||||
fn is_pending_deletion(message: Option<&str>) -> bool {
|
||||
message.is_some_and(|m| m.contains("scheduled for deletion"))
|
||||
// CreateSecret/PutSecretValue say "scheduled", GetSecretValue "marked".
|
||||
message.is_some_and(|m| m.contains("scheduled for deletion") || m.contains("marked for deletion"))
|
||||
}
|
||||
|
||||
fn pending_deletion_error(name: &str) -> AwsError {
|
||||
@@ -1177,6 +1200,13 @@ impl WorkspaceApi for SdkWorkspaceApi {
|
||||
{
|
||||
Ok(None)
|
||||
}
|
||||
Err(e)
|
||||
if e.as_service_error().is_some_and(|s| {
|
||||
s.is_invalid_request_exception() && is_pending_deletion(s.message())
|
||||
}) =>
|
||||
{
|
||||
Err(pending_deletion_error(name))
|
||||
}
|
||||
Err(e) => Err(classify_sdk_error(ACTION, &e)),
|
||||
}
|
||||
}
|
||||
@@ -1202,16 +1232,15 @@ impl WorkspaceApi for SdkWorkspaceApi {
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_project_secret(
|
||||
async fn create_project_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
kms_key_arn: &str,
|
||||
json: &str,
|
||||
tags: &[(String, String)],
|
||||
) -> Result<String, AwsError> {
|
||||
) -> Result<Option<String>, AwsError> {
|
||||
use aws_sdk_secretsmanager::types::Tag as SmTag;
|
||||
const CREATE: &str = "secretsmanager:CreateSecret";
|
||||
const PUT: &str = "secretsmanager:PutSecretValue";
|
||||
let created = self
|
||||
.secrets
|
||||
.create_secret()
|
||||
@@ -1225,43 +1254,57 @@ impl WorkspaceApi for SdkWorkspaceApi {
|
||||
))
|
||||
.send()
|
||||
.await;
|
||||
let arn = match created {
|
||||
Ok(out) => out.arn().map(str::to_string),
|
||||
match created {
|
||||
Ok(out) => out
|
||||
.arn()
|
||||
.map(|a| Some(a.to_string()))
|
||||
.ok_or_else(|| other(format!("{CREATE} returned no ARN"))),
|
||||
Err(e)
|
||||
if e.as_service_error()
|
||||
.is_some_and(|s| s.is_resource_exists_exception()) =>
|
||||
{
|
||||
// Only a secret tagged with the caller as owner accepts this
|
||||
// (the policy), so another user's same-named secret fails
|
||||
// here rather than receiving the payload.
|
||||
self.secrets
|
||||
.put_secret_value()
|
||||
.secret_id(name)
|
||||
.secret_string(json)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| match e.as_service_error() {
|
||||
Some(s)
|
||||
if s.is_invalid_request_exception()
|
||||
&& is_pending_deletion(s.message()) =>
|
||||
{
|
||||
pending_deletion_error(name)
|
||||
}
|
||||
_ => classify_sdk_error(PUT, &e),
|
||||
})?
|
||||
.arn()
|
||||
.map(str::to_string)
|
||||
Ok(None)
|
||||
}
|
||||
Err(e)
|
||||
if e.as_service_error().is_some_and(|s| {
|
||||
s.is_invalid_request_exception() && is_pending_deletion(s.message())
|
||||
}) =>
|
||||
{
|
||||
return Err(pending_deletion_error(name));
|
||||
Err(pending_deletion_error(name))
|
||||
}
|
||||
Err(e) => return Err(classify_sdk_error(CREATE, &e)),
|
||||
};
|
||||
arn.ok_or_else(|| other(format!("{CREATE} returned no ARN")))
|
||||
Err(e) => Err(classify_sdk_error(CREATE, &e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_project_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
kms_key_arn: &str,
|
||||
json: &str,
|
||||
tags: &[(String, String)],
|
||||
) -> Result<String, AwsError> {
|
||||
const PUT: &str = "secretsmanager:PutSecretValue";
|
||||
if let Some(arn) = self.create_project_secret(name, kms_key_arn, json, tags).await? {
|
||||
return Ok(arn);
|
||||
}
|
||||
// Only a secret tagged with the caller as owner accepts this (the
|
||||
// policy), so another user's same-named secret fails here rather
|
||||
// than receiving the payload.
|
||||
self.secrets
|
||||
.put_secret_value()
|
||||
.secret_id(name)
|
||||
.secret_string(json)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| match e.as_service_error() {
|
||||
Some(s) if s.is_invalid_request_exception() && is_pending_deletion(s.message()) => {
|
||||
pending_deletion_error(name)
|
||||
}
|
||||
_ => classify_sdk_error(PUT, &e),
|
||||
})?
|
||||
.arn()
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| other(format!("{PUT} returned no ARN")))
|
||||
}
|
||||
|
||||
async fn task_definition_revisions(&self, family: &str) -> Result<Vec<String>, AwsError> {
|
||||
|
||||
@@ -799,6 +799,8 @@ struct Fake {
|
||||
secret_reads: Mutex<usize>,
|
||||
/// Puts that fail as "scheduled for deletion" before one succeeds.
|
||||
pending_puts: Mutex<usize>,
|
||||
/// Calls `create_project_secret` got.
|
||||
creates: Mutex<usize>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
@@ -850,8 +852,22 @@ impl WorkspaceApi for Fake {
|
||||
}
|
||||
async fn delete_project_secret(&self, name: &str) -> Result<(), AwsError> {
|
||||
self.deleted_secrets.lock().unwrap().push(name.into());
|
||||
*self.existing_secret.lock().unwrap() = None;
|
||||
Ok(())
|
||||
}
|
||||
async fn create_project_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
kms_key_arn: &str,
|
||||
json: &str,
|
||||
tags: &[(String, String)],
|
||||
) -> Result<Option<String>, AwsError> {
|
||||
*self.creates.lock().unwrap() += 1;
|
||||
if self.existing_secret.lock().unwrap().is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
self.put_project_secret(name, kms_key_arn, json, tags).await.map(Some)
|
||||
}
|
||||
async fn put_project_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
@@ -1638,6 +1654,7 @@ async fn switching_to_copy_at_connect_replaces_a_secret_that_held_credentials()
|
||||
prepare_workspace(&fake, &who(), &env_info(), ©_at_connect_project(), &env, &[], &[], IMAGE)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(*fake.secret_reads.lock().unwrap(), 1, "read only once it exists");
|
||||
assert_eq!(fake.deleted_secrets.lock().unwrap().len(), 1);
|
||||
let m = payload_map(&fake.secrets.lock().unwrap()[0].2);
|
||||
assert!(!m.contains_key("GIT_TOKEN"));
|
||||
@@ -1653,12 +1670,16 @@ async fn switching_to_copy_at_connect_replaces_a_secret_that_held_credentials()
|
||||
assert!(fake.deleted_secrets.lock().unwrap().is_empty());
|
||||
assert_eq!(fake.secrets.lock().unwrap().len(), 1);
|
||||
|
||||
// No secret yet: created, nothing deleted.
|
||||
// No secret yet (a first start): created, never read — a read of a
|
||||
// secret that doesn't exist may be refused — nothing deleted.
|
||||
let fake = Fake::onboarded();
|
||||
prepare_workspace(&fake, &who(), &env_info(), ©_at_connect_project(), &env, &[], &[], IMAGE)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(*fake.creates.lock().unwrap(), 1);
|
||||
assert_eq!(*fake.secret_reads.lock().unwrap(), 0);
|
||||
assert!(fake.deleted_secrets.lock().unwrap().is_empty());
|
||||
assert!(!payload_map(&fake.secrets.lock().unwrap()[0].2).contains_key("GIT_TOKEN"));
|
||||
|
||||
// Secrets Manager never reads or deletes.
|
||||
let fake = Fake::onboarded();
|
||||
|
||||
@@ -2226,7 +2226,7 @@ impl Connect {
|
||||
.replace("lib=/usr/local/lib/triple-c/entrypoint-lib.sh\n", &format!("lib='{}'\n", p("lib.sh").display()))
|
||||
.replace("shm=/dev/shm/triple-c\n", &format!("shm='{}'\n", p("shm").display()))
|
||||
.replace("stage=/dev/shm/triple-c/connect\n", &format!("stage='{}'\n", p("shm/connect").display()))
|
||||
.replace("env_file=/dev/shm/triple-c/connect.env\n", &format!("env_file='{}'\n", p("shm/connect.env").display()))
|
||||
.replace("ca_dir=/dev/shm/triple-c/connect-ca\n", &format!("ca_dir='{}'\n", p("shm/connect-ca").display()))
|
||||
.replace("clone_status=/run/triple-c-clone\n", &format!("clone_status='{}'\n", p("clone").display()))
|
||||
.replace("workspace=/workspace\n", &format!("workspace='{}'\n", p("ws").display()))
|
||||
.replace("retry_wait=2400\n", "retry_wait=20\n")
|
||||
@@ -2252,6 +2252,31 @@ impl Connect {
|
||||
self.root.join(s)
|
||||
}
|
||||
|
||||
/// `--prepare`: this connect's nonce (its dir is `shm/connect/<nonce>`).
|
||||
fn prepare(&self) -> String {
|
||||
let out = self.run(&["--prepare"]);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
let nonce = String::from_utf8_lossy(&out.stdout).trim().to_string();
|
||||
assert!(nonce.len() == 16 && nonce.chars().all(|c| c.is_ascii_hexdigit()), "{nonce:?}");
|
||||
nonce
|
||||
}
|
||||
|
||||
/// What the app uploads for one connect: `files` (`ssh/x`, `ca/y`) and
|
||||
/// `connect.env`.
|
||||
fn stage(&self, nonce: &str, files: &[(&str, &str)], env: &str) {
|
||||
for (k, v) in files {
|
||||
fs::write(self.p(&format!("shm/connect/{nonce}/{k}")), v).unwrap();
|
||||
}
|
||||
fs::write(self.p(&format!("shm/connect/{nonce}/connect.env")), env).unwrap();
|
||||
}
|
||||
|
||||
/// A whole connect: prepare, stage, apply.
|
||||
fn connect(&self, files: &[(&str, &str)], env: &str) -> Output {
|
||||
let nonce = self.prepare();
|
||||
self.stage(&nonce, files, env);
|
||||
self.run(&[&nonce])
|
||||
}
|
||||
|
||||
/// Waits for the background re-clone to finish (`done` back).
|
||||
fn wait_done(&self) {
|
||||
for _ in 0..200 {
|
||||
@@ -2272,26 +2297,22 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
|
||||
let c = Connect::new(tmp.path());
|
||||
let remote = bare_remote(tmp.path(), "remote");
|
||||
|
||||
// A stale staging dir and env file from an earlier connect are replaced.
|
||||
fs::create_dir_all(c.p("shm/connect/ssh")).unwrap();
|
||||
fs::write(c.p("shm/connect/ssh/old"), "old").unwrap();
|
||||
fs::write(c.p("shm/connect.env"), "OLD=x\n").unwrap();
|
||||
let out = c.run(&["--prepare"]);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
for d in ["shm/connect", "shm/connect/ssh", "shm/connect/ca"] {
|
||||
assert_eq!(mode(&c.p(d)), 0o700, "{d}");
|
||||
// A connect's own staging dir; one an abandoned connect left long ago is
|
||||
// cleared, a recent one (a connect in flight) is not.
|
||||
fs::create_dir_all(c.p("shm/connect/0000000000000old")).unwrap();
|
||||
fs::create_dir_all(c.p("shm/connect/00000000000recent")).unwrap();
|
||||
assert!(Command::new("touch").args(["-d", "1 hour ago"]).arg(c.p("shm/connect/0000000000000old")).status().unwrap().success());
|
||||
let nonce = c.prepare();
|
||||
for d in ["shm/connect".to_string(), format!("shm/connect/{nonce}"), format!("shm/connect/{nonce}/ssh"), format!("shm/connect/{nonce}/ca")] {
|
||||
assert_eq!(mode(&c.p(&d)), 0o700, "{d}");
|
||||
}
|
||||
assert!(!c.p("shm/connect/ssh/old").exists() && !c.p("shm/connect.env").exists());
|
||||
assert!(!c.p("shm/connect/0000000000000old").exists());
|
||||
assert!(c.p("shm/connect/00000000000recent").exists());
|
||||
|
||||
// What the app uploads.
|
||||
fs::write(c.p("shm/connect/ssh/id_ed25519"), "KEY").unwrap();
|
||||
fs::write(c.p("shm/connect/ca/corp.pem"), PEM).unwrap();
|
||||
let token = "ghp_secret-value";
|
||||
fs::write(
|
||||
c.p("shm/connect.env"),
|
||||
format!("ANTHROPIC_API_KEY={}\nGIT_TOKEN={}\n", b64("sk-other"), b64(token)),
|
||||
)
|
||||
.unwrap();
|
||||
let env = format!("ANTHROPIC_API_KEY={}\nGIT_TOKEN={}\n", b64("sk-other"), b64(token));
|
||||
c.stage(&nonce, &[("ssh/id_ed25519", "KEY"), ("ca/corp.pem", PEM)], &env);
|
||||
// The first clone is done: `ok` cloned, `bad` failed (no credentials then).
|
||||
git(tmp.path(), &["clone", "-q", remote.to_str().unwrap(), c.p("ws/ok").to_str().unwrap()]);
|
||||
fs::write(c.p("clone/repos"), "ok\tcloning\nok\tcloned\nbad\tcloning\nbad\tfailed\n").unwrap();
|
||||
@@ -2305,10 +2326,11 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let out = c.run(&[]);
|
||||
let out = c.run(&[&nonce]);
|
||||
let log = text(&out);
|
||||
assert!(out.status.success(), "{log}");
|
||||
assert!(log.contains("credentials applied"), "{log}");
|
||||
assert!(!c.p(&format!("shm/connect/{nonce}")).exists(), "the apply removes its staging");
|
||||
assert!(!log.contains(token) && !log.contains("sk-other"), "no value is printed: {log}");
|
||||
|
||||
// CA, as the entrypoint installs it.
|
||||
@@ -2337,8 +2359,9 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
|
||||
assert!(clone_log.contains("cloning again with the project's credentials: bad"), "{clone_log}");
|
||||
assert!(!clone_log.contains(token), "{clone_log}");
|
||||
|
||||
// Idempotent: again, nothing failed, nothing re-cloned, same state.
|
||||
let out = c.run(&[]);
|
||||
// A new connect with the same credentials: nothing failed, nothing
|
||||
// re-cloned, same state, the CA not reinstalled.
|
||||
let out = c.connect(&[("ssh/id_ed25519", "KEY"), ("ca/corp.pem", PEM)], &env);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
assert_eq!(fs::read_to_string(c.p("clone/repos")).unwrap(), repos);
|
||||
assert!(c.p("clone/done").exists());
|
||||
@@ -2351,11 +2374,11 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
|
||||
fn apply_connect_with_nothing_configured_changes_nothing_and_leaves_a_waiting_clone_alone() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let c = Connect::new(tmp.path());
|
||||
assert!(c.run(&["--prepare"]).status.success());
|
||||
let nonce = c.prepare();
|
||||
// The first clone is still waiting for this connect: no `done`, no
|
||||
// `no-credentials`.
|
||||
fs::write(c.p("run/repos.json"), "[]").unwrap();
|
||||
let out = c.run(&[]);
|
||||
let out = c.run(&[&nonce]);
|
||||
let log = text(&out);
|
||||
assert!(out.status.success(), "{log}");
|
||||
assert!(!log.contains("warning"), "{log}");
|
||||
@@ -2371,7 +2394,7 @@ fn apply_connect_retries_a_clone_that_started_without_credentials_once_it_is_don
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let c = Connect::new(tmp.path());
|
||||
let remote = bare_remote(tmp.path(), "remote");
|
||||
assert!(c.run(&["--prepare"]).status.success());
|
||||
let nonce = c.prepare();
|
||||
fs::write(
|
||||
c.p("run/repos.json"),
|
||||
format!(r#"[{{"url":"{}","branch":null,"folder":"late"}}]"#, remote.display()),
|
||||
@@ -2380,7 +2403,7 @@ fn apply_connect_retries_a_clone_that_started_without_credentials_once_it_is_don
|
||||
// Still cloning without credentials (the wait timed out).
|
||||
fs::write(c.p("clone/no-credentials"), "").unwrap();
|
||||
fs::write(c.p("clone/repos"), "late\tcloning\n").unwrap();
|
||||
let out = c.run(&[]);
|
||||
let out = c.run(&[&nonce]);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
assert!(!c.p("ws/late").exists(), "nothing until the running clone is done");
|
||||
@@ -2410,16 +2433,15 @@ fn a_running_re_clone_never_holds_the_apply_lock_and_prepare_waits_for_an_apply(
|
||||
let mut lib = fs::read_to_string(c.p("lib.sh")).unwrap();
|
||||
lib.push_str("\nclone_repos() { sleep 6; }\n");
|
||||
fs::write(c.p("lib.sh"), lib).unwrap();
|
||||
assert!(c.run(&["--prepare"]).status.success());
|
||||
let nonce = c.prepare();
|
||||
fs::write(c.p("run/repos.json"), r#"[{"url":"x","branch":null,"folder":"bad"}]"#).unwrap();
|
||||
fs::write(c.p("clone/repos"), "bad\tcloning\nbad\tfailed\n").unwrap();
|
||||
fs::write(c.p("clone/done"), "").unwrap();
|
||||
assert!(c.run(&[]).status.success());
|
||||
assert!(c.run(&[&nonce]).status.success());
|
||||
assert!(!c.p("clone/done").exists(), "the re-clone is running");
|
||||
// The next connect goes straight through.
|
||||
let started = std::time::Instant::now();
|
||||
assert!(c.run(&["--prepare"]).status.success());
|
||||
let out = c.run(&[]);
|
||||
let out = c.connect(&[], "");
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
assert!(!text(&out).contains("another apply"), "{}", text(&out));
|
||||
assert!(started.elapsed() < std::time::Duration::from_secs(4), "{:?}", started.elapsed());
|
||||
@@ -2429,7 +2451,6 @@ fn a_running_re_clone_never_holds_the_apply_lock_and_prepare_waits_for_an_apply(
|
||||
let lock = c.p("shm/apply-connect.lock");
|
||||
let mut holder = Command::new("flock").arg(&lock).args(["sleep", "2"]).spawn().unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_millis(300));
|
||||
fs::write(c.p("shm/connect/marker"), "").ok();
|
||||
let started = std::time::Instant::now();
|
||||
assert!(c.run(&["--prepare"]).status.success());
|
||||
assert!(started.elapsed() >= std::time::Duration::from_millis(1200), "{:?}", started.elapsed());
|
||||
@@ -2461,3 +2482,31 @@ fn under_copy_at_connect_hybrid_keeps_ssh_off_efs() {
|
||||
assert_eq!(fs::read_to_string(home.join(".ssh/id_ed25519")).unwrap(), "KEY");
|
||||
assert!(!mnt.join(".ssh").exists(), "nothing written to EFS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_connects_at_once_never_mix_their_files() {
|
||||
// Two machines connecting at the same moment: both prepare, then both
|
||||
// upload, then both apply. Each applies its own set; the later wins.
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let c = Connect::new(tmp.path());
|
||||
let a = c.prepare();
|
||||
let b = c.prepare();
|
||||
assert_ne!(a, b);
|
||||
c.stage(&a, &[("ssh/id_a", "A"), ("ssh/config", "Host a\n")], &format!("GIT_TOKEN={}\n", b64("tok-a")));
|
||||
c.stage(&b, &[("ssh/id_b", "B"), ("ssh/config", "Host b\n")], &format!("GIT_TOKEN={}\n", b64("tok-b")));
|
||||
for n in [&a, &b] {
|
||||
let out = c.run(&[n]);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
}
|
||||
assert_eq!(fs::read_to_string(c.p("home/.ssh/config")).unwrap(), "Host b\n");
|
||||
let cred = fs::read_to_string(c.p("home/.git-credentials")).unwrap_or_else(|e| panic!("{e}: {:?}", fs::read_dir(c.p("home")).unwrap().map(|e| e.unwrap().file_name()).collect::<Vec<_>>()));
|
||||
assert!(cred.contains("tok-b"), "{cred}");
|
||||
// B's set as a whole: A's key, merged first, is dropped again by the
|
||||
// merge's own bookkeeping (B no longer provides it).
|
||||
assert_eq!(fs::read_to_string(c.p("home/.ssh/id_b")).unwrap(), "B");
|
||||
assert!(!c.p("home/.ssh/id_a").exists());
|
||||
assert_eq!(fs::read_dir(c.p("shm/connect")).unwrap().count(), 0, "both staging dirs gone");
|
||||
// A nonce that is not one, or not prepared, is refused.
|
||||
assert_eq!(c.run(&["../../etc"]).status.code(), Some(2));
|
||||
assert_eq!(c.run(&[&a]).status.code(), Some(1));
|
||||
}
|
||||
@@ -8,8 +8,9 @@
|
||||
//! [`workspace_files`](super::workspace_files) the secret would carry) and
|
||||
//! the credential environment pairs (`SECRET_ENV_KEYS` and every custom
|
||||
//! variable, `aws::workspace::credential_pairs`). [`push_connect_bundle`]
|
||||
//! uploads them through the agent, as root, into root-only staging on the
|
||||
//! task's `/dev/shm` tmpfs and runs `triple-c-apply-connect` (container), which
|
||||
//! uploads them through the agent, as root, into this connect's own root-only
|
||||
//! staging dir on the task's `/dev/shm` tmpfs and runs `triple-c-apply-connect
|
||||
//! <nonce>` (container), which
|
||||
//! installs the CA, merges the SSH files, writes the git credential helper
|
||||
//! and re-clones what failed. The terminals the app opens get the pairs in
|
||||
//! their own `ExecSpec.env` ([`terminal_env`]); nothing puts them into the
|
||||
@@ -28,9 +29,8 @@ use crate::models::{AppSettings, Project};
|
||||
|
||||
/// The in-container half (container/triple-c-apply-connect).
|
||||
pub const APPLY_CONNECT: &str = "/usr/local/bin/triple-c-apply-connect";
|
||||
/// Root 0700 on the tmpfs (made by the entrypoint for the agent token).
|
||||
pub const CONNECT_SHM: &str = "/dev/shm/triple-c";
|
||||
/// Staging for the files: `ssh/` and `ca/` under it.
|
||||
/// Staging (root 0700, on the tmpfs the entrypoint made for the agent
|
||||
/// token): each connect gets `<nonce>/` with `ssh/`, `ca/` and `connect.env`.
|
||||
pub const CONNECT_STAGE: &str = "/dev/shm/triple-c/connect";
|
||||
pub const CONNECT_ENV_FILE: &str = "connect.env";
|
||||
/// Every staged file is root 0600.
|
||||
@@ -174,21 +174,34 @@ async fn run_apply<T: ConnectTarget>(t: &T, args: &[&str]) -> Result<String, Str
|
||||
}
|
||||
}
|
||||
|
||||
/// Upload `bundle` and apply it (see the module docs): `--prepare`, the
|
||||
/// files and `connect.env` (root 0600), then the apply.
|
||||
/// `--prepare`'s answer: this connect's nonce, 16 hex digits on the last line.
|
||||
pub(crate) fn parse_nonce(out: &str) -> Result<String, String> {
|
||||
let last = out.lines().rev().map(str::trim).find(|l| !l.is_empty()).unwrap_or_default();
|
||||
if last.len() == 16 && last.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) {
|
||||
Ok(last.to_string())
|
||||
} else {
|
||||
Err(format!("triple-c-apply-connect --prepare gave no connect id:\n{}", tail(out)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Upload `bundle` and apply it (see the module docs): `--prepare` (which
|
||||
/// names this connect's own staging dir), the files and `connect.env` (root
|
||||
/// 0600) into that dir, then the apply of exactly that dir — so two
|
||||
/// connects at once (two machines) never mix their files.
|
||||
pub async fn push_connect_bundle<T: ConnectTarget>(t: &T, bundle: &ConnectBundle) -> Result<(), String> {
|
||||
run_apply(t, &["--prepare"]).await?;
|
||||
let nonce = parse_nonce(&run_apply(t, &["--prepare"]).await?)?;
|
||||
let dir = format!("{CONNECT_STAGE}/{nonce}");
|
||||
for (key, data) in &bundle.files {
|
||||
validate_file_key(key)?;
|
||||
let (kind, name) = key.split_once('/').expect("validated");
|
||||
t.upload_root(&format!("{CONNECT_STAGE}/{kind}"), name, data, STAGED_MODE)
|
||||
t.upload_root(&format!("{dir}/{kind}"), name, data, STAGED_MODE)
|
||||
.await
|
||||
.map_err(|e| format!("Could not copy {key} into the workspace: {e}"))?;
|
||||
}
|
||||
t.upload_root(CONNECT_SHM, CONNECT_ENV_FILE, &connect_env_file(&bundle.env), STAGED_MODE)
|
||||
t.upload_root(&dir, CONNECT_ENV_FILE, &connect_env_file(&bundle.env), STAGED_MODE)
|
||||
.await
|
||||
.map_err(|e| format!("Could not copy the credential variables into the workspace: {e}"))?;
|
||||
let out = run_apply(t, &[]).await?;
|
||||
let out = run_apply(t, &[&nonce]).await?;
|
||||
for line in out.lines().filter(|l| l.contains("warning")) {
|
||||
log::warn!("{line}");
|
||||
}
|
||||
|
||||
@@ -164,6 +164,15 @@ impl WorkspaceApi for Fake {
|
||||
async fn project_secret_value(&self, _name: &str) -> Result<Option<String>, AwsError> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn create_project_secret(
|
||||
&self,
|
||||
n: &str,
|
||||
k: &str,
|
||||
j: &str,
|
||||
t: &[(String, String)],
|
||||
) -> Result<Option<String>, AwsError> {
|
||||
self.put_project_secret(n, k, j, t).await.map(Some)
|
||||
}
|
||||
async fn delete_project_secret(&self, _name: &str) -> Result<(), AwsError> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -959,10 +968,11 @@ mod copy_at_connect {
|
||||
impl ConnectTarget for FakeTarget {
|
||||
async fn exec_root(&self, cmd: Vec<String>) -> Result<(String, i64), String> {
|
||||
self.calls.lock().unwrap().push(format!("exec {}", cmd.join(" ")));
|
||||
if cmd.len() == 1 {
|
||||
if let Some(r) = self.apply_result.lock().unwrap().clone() {
|
||||
return Ok(r);
|
||||
}
|
||||
if cmd.get(1).map(String::as_str) == Some("--prepare") {
|
||||
return Ok(("0123456789abcdef\n".into(), 0));
|
||||
}
|
||||
if let Some(r) = self.apply_result.lock().unwrap().clone() {
|
||||
return Ok(r);
|
||||
}
|
||||
Ok(("triple-c-apply-connect: credentials applied\n".into(), 0))
|
||||
}
|
||||
@@ -994,10 +1004,10 @@ mod copy_at_connect {
|
||||
*t.calls.lock().unwrap(),
|
||||
vec![
|
||||
"exec /usr/local/bin/triple-c-apply-connect --prepare",
|
||||
"upload /dev/shm/triple-c/connect/ssh/id_ed25519",
|
||||
"upload /dev/shm/triple-c/connect/ca/corp.crt",
|
||||
"upload /dev/shm/triple-c/connect.env",
|
||||
"exec /usr/local/bin/triple-c-apply-connect",
|
||||
"upload /dev/shm/triple-c/connect/0123456789abcdef/ssh/id_ed25519",
|
||||
"upload /dev/shm/triple-c/connect/0123456789abcdef/ca/corp.crt",
|
||||
"upload /dev/shm/triple-c/connect/0123456789abcdef/connect.env",
|
||||
"exec /usr/local/bin/triple-c-apply-connect 0123456789abcdef",
|
||||
]
|
||||
);
|
||||
let uploads = t.uploads.lock().unwrap();
|
||||
@@ -1030,6 +1040,14 @@ mod copy_at_connect {
|
||||
assert!(t.uploads.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_well_formed_connect_id_is_used_as_a_path() {
|
||||
assert_eq!(parse_nonce("noise\n0123456789abcdef\n").unwrap(), "0123456789abcdef");
|
||||
for bad in ["", "../../etc/passwd", "0123456789ABCDEF", "0123456789abcde", "0123456789abcdef0"] {
|
||||
assert!(parse_nonce(bad).is_err(), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bundle_never_shows_its_values() {
|
||||
let shown = format!("{:?}", bundle());
|
||||
|
||||
@@ -7,13 +7,14 @@
|
||||
# custom variables. Instead, after every connect (a start, an attach, the
|
||||
# background re-attach when the app starts) the app, through the agent and as
|
||||
# root:
|
||||
# 1. runs `triple-c-apply-connect --prepare` (fresh, empty, root-only
|
||||
# staging dirs on the /dev/shm tmpfs — never the disk);
|
||||
# 2. uploads the SSH files to /dev/shm/triple-c/connect/ssh/, the CA files to
|
||||
# /dev/shm/triple-c/connect/ca/ (each root 0600) and the credential
|
||||
# environment to /dev/shm/triple-c/connect.env (root 0600, one
|
||||
# `KEY=<base64 of the value>` line per variable);
|
||||
# 3. runs `triple-c-apply-connect`, which
|
||||
# 1. runs `triple-c-apply-connect --prepare`, which makes this connect's
|
||||
# own root-only staging dir on the /dev/shm tmpfs (never the disk),
|
||||
# /dev/shm/triple-c/connect/<nonce>/{ssh,ca}, and prints <nonce>
|
||||
# (16 hex digits);
|
||||
# 2. uploads the SSH files to <nonce>/ssh/, the CA files to <nonce>/ca/ and
|
||||
# the credential environment to <nonce>/connect.env (each root 0600;
|
||||
# connect.env holds one `KEY=<base64 of the value>` line per variable);
|
||||
# 3. runs `triple-c-apply-connect <nonce>`, which, under the apply lock,
|
||||
# - installs the CA (install_corporate_ca, as root: its source and the
|
||||
# system store are root-only paths),
|
||||
# - merges the SSH files into ~/.ssh (merge_host_ssh, **as the user**:
|
||||
@@ -31,7 +32,12 @@
|
||||
# connect.env is never sourced into, or exported to, anything but the clone,
|
||||
# and the agent's environment (every terminal's) is untouched — the app adds
|
||||
# the credential variables to the terminals it opens itself. Nothing here
|
||||
# prints a value. Idempotent: run it again and it converges on the same state.
|
||||
# prints a value. Each connect stages and applies its own set, so two connects
|
||||
# at once (two machines, a start racing a re-attach) never mix their files:
|
||||
# their applies run one after the other, the later one's set wins. The apply
|
||||
# removes its staging dir; the CA files move to /dev/shm/triple-c/connect-ca
|
||||
# (a fixed path, so an unchanged CA is not reinstalled). Idempotent: a new
|
||||
# connect with the same credentials converges on the same state.
|
||||
# Re-cloning does not re-run the setup script.
|
||||
set -u
|
||||
|
||||
@@ -40,7 +46,7 @@ set -u
|
||||
lib=/usr/local/lib/triple-c/entrypoint-lib.sh
|
||||
shm=/dev/shm/triple-c
|
||||
stage=/dev/shm/triple-c/connect
|
||||
env_file=/dev/shm/triple-c/connect.env
|
||||
ca_dir=/dev/shm/triple-c/connect-ca
|
||||
clone_status=/run/triple-c-clone
|
||||
workspace=/workspace
|
||||
# How long a retry waits for a clone that started without credentials.
|
||||
@@ -58,33 +64,47 @@ if ! . "$lib"; then
|
||||
fi
|
||||
triplec_aws=1
|
||||
|
||||
# The apply lock, also taken by --prepare so a prepare never empties the
|
||||
# staging dir under a running apply.
|
||||
# The apply lock: applies run one at a time; --prepare takes it too, so its
|
||||
# clean-up of stale staging dirs never races an apply.
|
||||
take_lock() {
|
||||
exec 9>"$shm/apply-connect.lock" && flock -w 120 9 \
|
||||
|| say "warning — another apply is still running; going ahead"
|
||||
}
|
||||
|
||||
prepare() {
|
||||
local nonce
|
||||
umask 077
|
||||
if [ ! -d "$shm" ] || [ -L "$shm" ]; then
|
||||
say "$shm is missing (the workspace has no agent token?); nothing prepared"
|
||||
exit 1
|
||||
fi
|
||||
take_lock
|
||||
rm -rf "$stage" "$env_file"
|
||||
mkdir -m 700 "$stage" "$stage/ssh" "$stage/ca" || exit 1
|
||||
if [ -L "$stage" ] || { [ -e "$stage" ] && [ ! -d "$stage" ]; }; then rm -f "$stage"; fi
|
||||
mkdir -p -m 700 "$stage" && chmod 700 "$stage" || exit 1
|
||||
# What a connect that never applied (the app gone mid-push) left.
|
||||
find "$stage" -mindepth 1 -maxdepth 1 -mmin +10 -exec rm -rf {} + 2>/dev/null
|
||||
nonce=$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')
|
||||
case "$nonce" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) say "no random nonce"; exit 1 ;; esac
|
||||
mkdir -m 700 "$stage/$nonce" "$stage/$nonce/ssh" "$stage/$nonce/ca" || exit 1
|
||||
printf '%s\n' "$nonce"
|
||||
}
|
||||
|
||||
# This connect's staging dir (set by apply).
|
||||
dir=
|
||||
|
||||
# GIT_TOKEN, decoded from connect.env into this shell only.
|
||||
connect_git_token=
|
||||
read_git_token() {
|
||||
local key value
|
||||
[ -f "$env_file" ] || return 0
|
||||
while IFS='=' read -r key value; do
|
||||
local line key value
|
||||
[ -f "$dir/connect.env" ] || return 0
|
||||
# Split by hand at the first `=`: `read` with IFS='=' would drop a
|
||||
# trailing `=`, which is base64 padding here.
|
||||
while IFS= read -r line; do
|
||||
key=${line%%=*}
|
||||
value=${line#*=}
|
||||
[ "$key" = GIT_TOKEN ] || continue
|
||||
connect_git_token=$(printf '%s' "$value" | base64 -d 2>/dev/null) || connect_git_token=
|
||||
done < "$env_file"
|
||||
done < "$dir/connect.env"
|
||||
}
|
||||
|
||||
# The staged SSH files merged as the user: copied to a fresh root-made dir
|
||||
@@ -94,7 +114,7 @@ merge_ssh_as_user() {
|
||||
local tmp rc=0
|
||||
tmp=$(mktemp -d "${ssh_tmp_dir}/triple-c-connect.XXXXXXXX") \
|
||||
|| { say "warning — no temporary directory; SSH files not merged"; return 0; }
|
||||
if cp -R "$stage/ssh" "$tmp/ssh"; then
|
||||
if cp -R "$dir/ssh" "$tmp/ssh"; then
|
||||
if [ -n "${triplec_run_as:-}" ] && [ "$(id -u)" = 0 ]; then
|
||||
chown -R "$triplec_run_as:" "$tmp"
|
||||
fi
|
||||
@@ -183,19 +203,29 @@ retry_failed_clones() {
|
||||
}
|
||||
|
||||
apply() {
|
||||
local nonce="$1"
|
||||
umask 077
|
||||
if [ ! -d "$stage" ] || [ -L "$stage" ]; then
|
||||
say "$stage is missing; run --prepare and upload first"
|
||||
case "$nonce" in
|
||||
*[!0-9a-f]*|'') say "not a connect nonce: $nonce"; exit 2 ;;
|
||||
esac
|
||||
[ "${#nonce}" = 16 ] || { say "not a connect nonce: $nonce"; exit 2; }
|
||||
dir="$stage/$nonce"
|
||||
if [ ! -d "$dir" ] || [ -L "$dir" ] || [ -L "$stage" ]; then
|
||||
say "$dir is missing; run --prepare and upload first"
|
||||
exit 1
|
||||
fi
|
||||
take_lock
|
||||
# Empty staging dirs: nothing of that kind is configured.
|
||||
rmdir "$stage/ca" "$stage/ssh" 2>/dev/null
|
||||
rmdir "$dir/ca" "$dir/ssh" 2>/dev/null
|
||||
|
||||
CA_SRC="$stage/ca"
|
||||
rm -rf "$ca_dir"
|
||||
if [ -d "$dir/ca" ]; then
|
||||
mv "$dir/ca" "$ca_dir" || say "warning — could not move the CA files into place"
|
||||
fi
|
||||
CA_SRC="$ca_dir"
|
||||
install_corporate_ca
|
||||
|
||||
if [ -d "$stage/ssh" ]; then
|
||||
if [ -d "$dir/ssh" ]; then
|
||||
merge_ssh_as_user
|
||||
fi
|
||||
|
||||
@@ -210,11 +240,12 @@ apply() {
|
||||
say "warning — could not record the connect in $triplec_run_dir"
|
||||
fi
|
||||
retry_failed_clones
|
||||
rm -rf "$dir"
|
||||
say "credentials applied"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--prepare) prepare ;;
|
||||
'') apply ;;
|
||||
*) say "usage: triple-c-apply-connect [--prepare]"; exit 2 ;;
|
||||
''|-*) say "usage: triple-c-apply-connect --prepare | <nonce>"; exit 2 ;;
|
||||
*) apply "$1" ;;
|
||||
esac
|
||||
Reference in new issue
Block a user