diff --git a/app/src-tauri/src/aws/workspace.rs b/app/src-tauri/src/aws/workspace.rs index 700db2f..f41fa67 100644 --- a/app/src-tauri/src/aws/workspace.rs +++ b/app/src-tauri/src/aws/workspace.rs @@ -1048,24 +1048,31 @@ pub async fn prepare_workspace( { Some(arn) => arn, None => { - let old = api.project_secret_value(&secret_name).await?; - if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) { - log::info!( - "Project {}: its secret still held credentials; replacing it so no version keeps them", - project.id - ); - replace_project_secret( - api, - &secret_name, - &env.kms_key_arn, - &json, - &tags, - RECREATE_TIMING, - ) - .await? - } else { - put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await? - } + // The secret existed at the create but may be marked for + // deletion by now (another start replacing it): wait that out + // like the writes do. Once it is gone the read says `None` + // and the put below makes it again. + let old = retry_while_pending_deletion(&secret_name, RECREATE_TIMING, || { + api.project_secret_value(&secret_name) + }) + .await?; + if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) { + log::info!( + "Project {}: its secret still held credentials; replacing it so no version keeps them", + project.id + ); + replace_project_secret( + api, + &secret_name, + &env.kms_key_arn, + &json, + &tags, + RECREATE_TIMING, + ) + .await? + } else { + put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await? + } } }, None => put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?, diff --git a/app/src-tauri/src/aws/workspace/tests.rs b/app/src-tauri/src/aws/workspace/tests.rs index 2774ec0..46cd135 100644 --- a/app/src-tauri/src/aws/workspace/tests.rs +++ b/app/src-tauri/src/aws/workspace/tests.rs @@ -800,6 +800,8 @@ struct Fake { secret_reads: Mutex, /// Puts that fail as "scheduled for deletion" before one succeeds. pending_puts: Mutex, + /// Reads that fail as "marked for deletion" before one succeeds. + pending_reads: Mutex, /// Calls `create_project_secret` got. creates: Mutex, } @@ -847,8 +849,15 @@ impl WorkspaceApi for Fake { c.push((fs_id.into(), root.into(), tags.to_vec())); Ok(format!("fsap-new{}", c.len())) } - async fn project_secret_value(&self, _name: &str) -> Result, AwsError> { + async fn project_secret_value(&self, name: &str) -> Result, AwsError> { *self.secret_reads.lock().unwrap() += 1; + { + let mut pending = self.pending_reads.lock().unwrap(); + if *pending > 0 { + *pending -= 1; + return Err(pending_deletion_error(name)); + } + } Ok(self.existing_secret.lock().unwrap().clone()) } async fn delete_project_secret(&self, name: &str) -> Result<(), AwsError> { @@ -1751,6 +1760,34 @@ async fn the_start_after_a_timed_out_replace_retries_the_create() { assert!(!e.contains("Restore"), "{e}"); } +#[tokio::test(start_paused = true)] +async fn a_copy_at_connect_read_of_a_secret_being_deleted_is_retried() { + // CreateSecret said it exists, then the read meets a deletion AWS is + // still finishing (another start replacing it): retried like the writes. + let env = pairs(&[("TZ", "UTC")]); + let fake = Fake::onboarded(); + *fake.existing_secret.lock().unwrap() = Some( + r#"{"TRIPLEC_AGENT_TOKEN":"t","TRIPLEC_CREDENTIALS":"connect","TZ":"UTC","TRIPLEC_REPOS":"[]"}"#.into(), + ); + *fake.pending_reads.lock().unwrap() = 3; + prepare_workspace(&fake, &who(), &env_info(), ©_at_connect_project(), &env, &[], &[], IMAGE, None) + .await + .unwrap(); + assert_eq!(*fake.secret_reads.lock().unwrap(), 4); + assert_eq!(fake.secrets.lock().unwrap().len(), 1); + + // Still being deleted past the limit: try again in a minute. + let fake = Fake::onboarded(); + *fake.existing_secret.lock().unwrap() = Some(r#"{"TZ":"UTC"}"#.into()); + *fake.pending_reads.lock().unwrap() = 1000; + let e = prepare_workspace(&fake, &who(), &env_info(), ©_at_connect_project(), &env, &[], &[], IMAGE, None) + .await + .unwrap_err() + .to_string(); + assert!(e.contains("Start the project again in a minute"), "{e}"); + assert!(fake.secrets.lock().unwrap().is_empty()); +} + #[test] fn a_copy_at_connect_payload_drops_the_custom_variables_fingerprint() { let env = pairs(&[("TRIPLE_C_CUSTOM_ENV", "63ce86"), ("TZ", "UTC")]);