feat: enhance service account key rotation with validation callback (#24)

* feat: enhance service account key rotation with validation callback

- add validation callback will enable the host service to validate the newly created key before the deleting the old key

- wrapped both the call to validate permissions and validate callback in a retry because sometimes it takes some time for the newly created key to be active on GCP services. This throws an error and prevents host catalog creation

plugin/service/host/plugin.go

@@ -92,7 +92,17 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 	}
 
 	if credState.CredentialsConfig.IsRotatable() && !catalogAttributes.DisableCredentialRotation {
-		if err := credState.RotateCreds(ctx, permissions, p.testGCPClientOpts...); err != nil {
+		state, err := newGCPCatalogPersistedState(
+			append([]gcpCatalogPersistedStateOption{
+				withCredentials(credState),
+			}, p.testCatalogStateOpts...)...,
+		)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "error setting up persisted state: %s", err)
+		}
+		validateCredsCallback := state.ValidateCredsCallbackFn(ctx, p.testGCPClientOpts...)
+
+		if err := credState.RotateCreds(ctx, permissions, validateCredsCallback, p.testGCPClientOpts...); err != nil {
 			return nil, err
 		}
 	}
@@ -219,7 +229,17 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		// If we're enabling rotation now but didn't before, or have
 		// freshly replaced credentials, we can rotate here.
 		if !newCatalogAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
-			if err := credState.RotateCreds(ctx, permissions, p.testGCPClientOpts...); err != nil {
+			state, err := newGCPCatalogPersistedState(
+				append([]gcpCatalogPersistedStateOption{
+					withCredentials(credState),
+				}, p.testCatalogStateOpts...)...,
+			)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "error setting up persisted state: %s", err)
+			}
+			validateCredsCallback := state.ValidateCredsCallbackFn(ctx, p.testGCPClientOpts...)
+
+			if err := credState.RotateCreds(ctx, permissions, validateCredsCallback, p.testGCPClientOpts...); err != nil {
 				return nil, err
 			}
 		}