fix(aws): retry the copy-at-connect secret read while the secret is pending deletion
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
544409c554
commit
ea37f8434f
2 files changed
+63
-19
No files matched your search
@@ -1048,24 +1048,31 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
|
||||
{
|
||||
Some(arn) => arn,
|
||||
None => {
|
||||
let old = api.project_secret_value(&secret_name).await?;
|
||||
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
|
||||
log::info!(
|
||||
"Project {}: its secret still held credentials; replacing it so no version keeps them",
|
||||
project.id
|
||||
);
|
||||
replace_project_secret(
|
||||
api,
|
||||
&secret_name,
|
||||
&env.kms_key_arn,
|
||||
&json,
|
||||
&tags,
|
||||
RECREATE_TIMING,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?
|
||||
}
|
||||
// The secret existed at the create but may be marked for
|
||||
// deletion by now (another start replacing it): wait that out
|
||||
// like the writes do. Once it is gone the read says `None`
|
||||
// and the put below makes it again.
|
||||
let old = retry_while_pending_deletion(&secret_name, RECREATE_TIMING, || {
|
||||
api.project_secret_value(&secret_name)
|
||||
})
|
||||
.await?;
|
||||
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
|
||||
log::info!(
|
||||
"Project {}: its secret still held credentials; replacing it so no version keeps them",
|
||||
project.id
|
||||
);
|
||||
replace_project_secret(
|
||||
api,
|
||||
&secret_name,
|
||||
&env.kms_key_arn,
|
||||
&json,
|
||||
&tags,
|
||||
RECREATE_TIMING,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?
|
||||
}
|
||||
}
|
||||
},
|
||||
None => put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?,
|
||||
|
||||
@@ -800,6 +800,8 @@ struct Fake {
|
||||
secret_reads: Mutex<usize>,
|
||||
/// Puts that fail as "scheduled for deletion" before one succeeds.
|
||||
pending_puts: Mutex<usize>,
|
||||
/// Reads that fail as "marked for deletion" before one succeeds.
|
||||
pending_reads: Mutex<usize>,
|
||||
/// Calls `create_project_secret` got.
|
||||
creates: Mutex<usize>,
|
||||
}
|
||||
@@ -847,8 +849,15 @@ impl WorkspaceApi for Fake {
|
||||
c.push((fs_id.into(), root.into(), tags.to_vec()));
|
||||
Ok(format!("fsap-new{}", c.len()))
|
||||
}
|
||||
async fn project_secret_value(&self, _name: &str) -> Result<Option<String>, AwsError> {
|
||||
async fn project_secret_value(&self, name: &str) -> Result<Option<String>, AwsError> {
|
||||
*self.secret_reads.lock().unwrap() += 1;
|
||||
{
|
||||
let mut pending = self.pending_reads.lock().unwrap();
|
||||
if *pending > 0 {
|
||||
*pending -= 1;
|
||||
return Err(pending_deletion_error(name));
|
||||
}
|
||||
}
|
||||
Ok(self.existing_secret.lock().unwrap().clone())
|
||||
}
|
||||
async fn delete_project_secret(&self, name: &str) -> Result<(), AwsError> {
|
||||
@@ -1751,6 +1760,34 @@ async fn the_start_after_a_timed_out_replace_retries_the_create() {
|
||||
assert!(!e.contains("Restore"), "{e}");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_copy_at_connect_read_of_a_secret_being_deleted_is_retried() {
|
||||
// CreateSecret said it exists, then the read meets a deletion AWS is
|
||||
// still finishing (another start replacing it): retried like the writes.
|
||||
let env = pairs(&[("TZ", "UTC")]);
|
||||
let fake = Fake::onboarded();
|
||||
*fake.existing_secret.lock().unwrap() = Some(
|
||||
r#"{"TRIPLEC_AGENT_TOKEN":"t","TRIPLEC_CREDENTIALS":"connect","TZ":"UTC","TRIPLEC_REPOS":"[]"}"#.into(),
|
||||
);
|
||||
*fake.pending_reads.lock().unwrap() = 3;
|
||||
prepare_workspace(&fake, &who(), &env_info(), ©_at_connect_project(), &env, &[], &[], IMAGE, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(*fake.secret_reads.lock().unwrap(), 4);
|
||||
assert_eq!(fake.secrets.lock().unwrap().len(), 1);
|
||||
|
||||
// Still being deleted past the limit: try again in a minute.
|
||||
let fake = Fake::onboarded();
|
||||
*fake.existing_secret.lock().unwrap() = Some(r#"{"TZ":"UTC"}"#.into());
|
||||
*fake.pending_reads.lock().unwrap() = 1000;
|
||||
let e = prepare_workspace(&fake, &who(), &env_info(), ©_at_connect_project(), &env, &[], &[], IMAGE, None)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(e.contains("Start the project again in a minute"), "{e}");
|
||||
assert!(fake.secrets.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_copy_at_connect_payload_drops_the_custom_variables_fingerprint() {
|
||||
let env = pairs(&[("TRIPLE_C_CUSTOM_ENV", "63ce86"), ("TZ", "UTC")]);
|
||||
|
||||
Reference in new issue
Block a user