feat: enhance service account key rotation with validation callback (#24)

* feat: enhance service account key rotation with validation callback

- add validation callback will enable the host service to validate the newly created key before the deleting the old key

- wrapped both the call to validate permissions and validate callback in a retry because sometimes it takes some time for the newly created key to be active on GCP services. This throws an error and prevents host catalog creation

plugin/service/host/plugin.go

@@ -92,7 +92,17 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 	}
 
 	if credState.CredentialsConfig.IsRotatable() && !catalogAttributes.DisableCredentialRotation {
-		if err := credState.RotateCreds(ctx, permissions, p.testGCPClientOpts...); err != nil {
+		state, err := newGCPCatalogPersistedState(
+			append([]gcpCatalogPersistedStateOption{
+				withCredentials(credState),
+			}, p.testCatalogStateOpts...)...,
+		)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "error setting up persisted state: %s", err)
+		}
+		validateCredsCallback := state.ValidateCredsCallbackFn(ctx, p.testGCPClientOpts...)
+
+		if err := credState.RotateCreds(ctx, permissions, validateCredsCallback, p.testGCPClientOpts...); err != nil {
 			return nil, err
 		}
 	}
@@ -219,7 +229,17 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		// If we're enabling rotation now but didn't before, or have
 		// freshly replaced credentials, we can rotate here.
 		if !newCatalogAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
-			if err := credState.RotateCreds(ctx, permissions, p.testGCPClientOpts...); err != nil {
+			state, err := newGCPCatalogPersistedState(
+				append([]gcpCatalogPersistedStateOption{
+					withCredentials(credState),
+				}, p.testCatalogStateOpts...)...,
+			)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "error setting up persisted state: %s", err)
+			}
+			validateCredsCallback := state.ValidateCredsCallbackFn(ctx, p.testGCPClientOpts...)
+
+			if err := credState.RotateCreds(ctx, permissions, validateCredsCallback, p.testGCPClientOpts...); err != nil {
 				return nil, err
 			}
 		}

plugin/service/host/state.go

@@ -13,6 +13,7 @@ import (
 	"github.com/googleapis/gax-go/v2"
 	"github.com/hashicorp/boundary-plugin-gcp/internal/credential"
 	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	"google.golang.org/protobuf/types/known/structpb"
 )
@@ -99,3 +100,30 @@ func (s *gcpCatalogPersistedState) InstancesClient(
 
 	return client, nil
 }
+
+// validateCredsCallbackFn returns a ValidateCredsCallback function that
+// validates the credentials by listing instances in the project and zone.
+func (s *gcpCatalogPersistedState) ValidateCredsCallbackFn(
+	ctx context.Context,
+	opts ...option.ClientOption,
+) credential.ValidateCredsCallback {
+	return func(config *credential.Config, opts ...option.ClientOption) error {
+		s.CredentialsConfig = config
+		instancesClient, err := s.InstancesClient(ctx, opts...)
+		if err != nil {
+			return fmt.Errorf("error creating instances client: %w", err)
+		}
+
+		it := instancesClient.List(ctx, &computepb.ListInstancesRequest{
+			Project: s.CredentialsConfig.ProjectId,
+			Zone:    s.CredentialsConfig.Zone,
+		})
+
+		_, err = it.Next()
+		if err != nil && err != iterator.Done {
+			return fmt.Errorf("error listing instances: %w", err)
+		}
+
+		return nil
+	}
+}

plugin/service/host/state_test.go

@@ -5,9 +5,11 @@ package host
 
 import (
 	"context"
+	"errors"
 	"testing"
 	"time"
 
+	computepb "cloud.google.com/go/compute/apiv1/computepb"
 	"github.com/hashicorp/boundary-plugin-gcp/internal/credential"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -87,3 +89,44 @@ func TestInstancesClient(t *testing.T) {
 		assert.Contains(t, err.Error(), "error creating instances client")
 	})
 }
+func TestValidateCredsCallbackFn(t *testing.T) {
+	ctx := context.Background()
+	cred := &credential.PersistedState{
+		CredentialsConfig: &credential.Config{
+			ProjectId: "test-project-id",
+			Zone:      "test-zone",
+		},
+	}
+	state := &gcpCatalogPersistedState{PersistedState: cred}
+
+	t.Run("List instances error", func(t *testing.T) {
+		state.testInstancesAPIFunc = newTestMockInstances(ctx,
+			nil,
+			testMockInstancesWithListInstancesOutput(nil),
+			testMockInstancesWithListInstancesError(errors.New("error listing instances")),
+		)
+
+		callback := state.ValidateCredsCallbackFn(ctx)
+		err := callback(&credential.Config{
+			ProjectId: "test-project-id",
+			Zone:      "test-zone",
+		})
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "error listing instances")
+	})
+
+	t.Run("Success", func(t *testing.T) {
+		state.testInstancesAPIFunc = newTestMockInstances(ctx,
+			nil,
+			testMockInstancesWithListInstancesOutput(&computepb.InstanceList{}),
+			testMockInstancesWithListInstancesError(nil),
+		)
+
+		callback := state.ValidateCredsCallbackFn(ctx)
+		err := callback(&credential.Config{
+			ProjectId: "test-project-id",
+			Zone:      "test-zone",
+		})
+		assert.NoError(t, err)
+	})
+}