fix(aws): retry the copy-at-connect secret read while the secret is pending deletion

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 12:08:46 -08:00
1 parent 544409c554
commit ea37f8434f
2 files changed
+63 -19

No files matched your search

+25 -18
View File
@@ -1048,24 +1048,31 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
{
Some(arn) => arn,
None => {
let old = api.project_secret_value(&secret_name).await?;
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
log::info!(
"Project {}: its secret still held credentials; replacing it so no version keeps them",
project.id
);
replace_project_secret(
api,
&secret_name,
&env.kms_key_arn,
&json,
&tags,
RECREATE_TIMING,
)
.await?
} else {
put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?
}
// The secret existed at the create but may be marked for
// deletion by now (another start replacing it): wait that out
// like the writes do. Once it is gone the read says `None`
// and the put below makes it again.
let old = retry_while_pending_deletion(&secret_name, RECREATE_TIMING, || {
api.project_secret_value(&secret_name)
})
.await?;
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
log::info!(
"Project {}: its secret still held credentials; replacing it so no version keeps them",
project.id
);
replace_project_secret(
api,
&secret_name,
&env.kms_key_arn,
&json,
&tags,
RECREATE_TIMING,
)
.await?
} else {
put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?
}
}
},
None => put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?,
+38 -1
View File
@@ -800,6 +800,8 @@ struct Fake {
secret_reads: Mutex<usize>,
/// Puts that fail as "scheduled for deletion" before one succeeds.
pending_puts: Mutex<usize>,
/// Reads that fail as "marked for deletion" before one succeeds.
pending_reads: Mutex<usize>,
/// Calls `create_project_secret` got.
creates: Mutex<usize>,
}
@@ -847,8 +849,15 @@ impl WorkspaceApi for Fake {
c.push((fs_id.into(), root.into(), tags.to_vec()));
Ok(format!("fsap-new{}", c.len()))
}
async fn project_secret_value(&self, _name: &str) -> Result<Option<String>, AwsError> {
async fn project_secret_value(&self, name: &str) -> Result<Option<String>, AwsError> {
*self.secret_reads.lock().unwrap() += 1;
{
let mut pending = self.pending_reads.lock().unwrap();
if *pending > 0 {
*pending -= 1;
return Err(pending_deletion_error(name));
}
}
Ok(self.existing_secret.lock().unwrap().clone())
}
async fn delete_project_secret(&self, name: &str) -> Result<(), AwsError> {
@@ -1751,6 +1760,34 @@ async fn the_start_after_a_timed_out_replace_retries_the_create() {
assert!(!e.contains("Restore"), "{e}");
}
#[tokio::test(start_paused = true)]
async fn a_copy_at_connect_read_of_a_secret_being_deleted_is_retried() {
// CreateSecret said it exists, then the read meets a deletion AWS is
// still finishing (another start replacing it): retried like the writes.
let env = pairs(&[("TZ", "UTC")]);
let fake = Fake::onboarded();
*fake.existing_secret.lock().unwrap() = Some(
r#"{"TRIPLEC_AGENT_TOKEN":"t","TRIPLEC_CREDENTIALS":"connect","TZ":"UTC","TRIPLEC_REPOS":"[]"}"#.into(),
);
*fake.pending_reads.lock().unwrap() = 3;
prepare_workspace(&fake, &who(), &env_info(), &copy_at_connect_project(), &env, &[], &[], IMAGE, None)
.await
.unwrap();
assert_eq!(*fake.secret_reads.lock().unwrap(), 4);
assert_eq!(fake.secrets.lock().unwrap().len(), 1);
// Still being deleted past the limit: try again in a minute.
let fake = Fake::onboarded();
*fake.existing_secret.lock().unwrap() = Some(r#"{"TZ":"UTC"}"#.into());
*fake.pending_reads.lock().unwrap() = 1000;
let e = prepare_workspace(&fake, &who(), &env_info(), &copy_at_connect_project(), &env, &[], &[], IMAGE, None)
.await
.unwrap_err()
.to_string();
assert!(e.contains("Start the project again in a minute"), "{e}");
assert!(fake.secrets.lock().unwrap().is_empty());
}
#[test]
fn a_copy_at_connect_payload_drops_the_custom_variables_fingerprint() {
let env = pairs(&[("TRIPLE_C_CUSTOM_ENV", "63ce86"), ("TZ", "UTC")]);