fixup! update timeout for tests to avoid ctx deadline errors

- update validation for service account validation call to use `select` instead of `sleep`
- add `validateServiceAccountKeyTimeout` to config. This makes it easier to control the timeout from tests so tests do not have to run for long with the default timeout

CODEOWNERS

@@ -1,4 +1,4 @@
 # These owners will be the default owners for everything in
 # the repo, unless a later match takes precedence.
 
-@hashicorp/boundary
+* @hashicorp/boundary

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/googleapis/gax-go/v2 v2.13.0
 	github.com/hashicorp/boundary/sdk v0.0.47
+	github.com/hashicorp/terraform-json v0.23.0
 	golang.org/x/oauth2 v0.22.0
 	google.golang.org/api v0.196.0
 )
@@ -18,12 +19,15 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	cloud.google.com/go/longrunning v0.6.0 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/zclconf/go-cty v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect

go.sum

@@ -22,6 +22,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
+github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
@@ -125,8 +127,12 @@ github.com/hashicorp/go-sockaddr v1.0.5/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3ly
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI=
+github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c=
 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
 github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
 github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
@@ -187,6 +193,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ=
+github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=

internal/credential/attributes.go

@@ -115,13 +115,16 @@ func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes)
 	}
 
 	switch {
-	// dynamic credentials requires the target service account id, cl and private key
+	// service account impersonation authentication requires the target service account id and client_email
-	case attrs.TargetServiceAccountId != "" && (privateKey == "" || attrs.ClientEmail == ""):
+	case attrs.TargetServiceAccountId != "" && attrs.ClientEmail == "":
 		badFields[fmt.Sprintf("secrets.%s", ConstPrivateKey)] = "must not be empty when target service account id is set"
+	// service account impersonation authentication requires the target service account id and private key
+	case attrs.TargetServiceAccountId != "" && privateKey == "":
 		badFields[fmt.Sprintf("attributes.%s", ConstClientEmail)] = "must not be empty when target service account id is set"
-	// static credentials requires the private key and client email
+	// service account key credentials requires the private key and client email
 	case privateKey != "" && attrs.ClientEmail == "":
 		badFields[fmt.Sprintf("attributes.%s", ConstClientEmail)] = "must not be empty when private key is set"
+	// service account key credentials requires the client_email
 	case attrs.ClientEmail != "" && privateKey == "":
 		badFields[fmt.Sprintf("secrets.%s", ConstPrivateKey)] = "must not be empty when client email is set"
 	}

internal/credential/attributes_test.go

@@ -119,15 +119,17 @@ func TestGetCredentialsConfig(t *testing.T) {
 				Zone: "us-central-1a",
 			},
 			expected: &Config{
-				Zone:   "us-central-1a",
+				Zone:                             "us-central-1a",
-				Scopes: []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 		},
 		{
 			name:  "nil attributes",
 			attrs: nil,
 			expected: &Config{
-				Scopes: []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 		},
 		{
@@ -154,12 +156,13 @@ func TestGetCredentialsConfig(t *testing.T) {
 				ClientEmail: "test@test.com",
 			},
 			expected: &Config{
-				ProjectId:    "test-project",
+				ProjectId:                        "test-project",
-				Zone:         "us-central-1a",
+				Zone:                             "us-central-1a",
-				ClientEmail:  "test@test.com",
+				ClientEmail:                      "test@test.com",
-				PrivateKey:   "test-private-key",
+				PrivateKey:                       "test-private-key",
-				PrivateKeyId: "test-private-key-id",
+				PrivateKeyId:                     "test-private-key-id",
-				Scopes:       []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 		},
 		{
@@ -220,12 +223,13 @@ func TestGetCredentialsConfig(t *testing.T) {
 				Zone:        "us-central-1a",
 			},
 			expected: &Config{
-				ProjectId:    "test-project",
+				ProjectId:                        "test-project",
-				Zone:         "us-central-1a",
+				Zone:                             "us-central-1a",
-				PrivateKey:   "test-private-key",
+				PrivateKey:                       "test-private-key",
-				PrivateKeyId: "test-private-key-id",
+				PrivateKeyId:                     "test-private-key-id",
-				ClientEmail:  "test@test.com",
+				ClientEmail:                      "test@test.com",
-				Scopes:       []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 		},
 		{
@@ -241,13 +245,14 @@ func TestGetCredentialsConfig(t *testing.T) {
 				TargetServiceAccountId: "test-target-service-account",
 			},
 			expected: &Config{
-				ProjectId:              "test-project",
+				ProjectId:                        "test-project",
-				Zone:                   "us-central-1a",
+				Zone:                             "us-central-1a",
-				PrivateKey:             "test-private-key",
+				PrivateKey:                       "test-private-key",
-				PrivateKeyId:           "test-private-key-id",
+				PrivateKeyId:                     "test-private-key-id",
-				ClientEmail:            "test@test.com",
+				ClientEmail:                      "test@test.com",
-				TargetServiceAccountId: "test-target-service-account",
+				TargetServiceAccountId:           "test-target-service-account",
-				Scopes:                 []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 		},
 	}

internal/credential/config.go

@@ -41,17 +41,21 @@ const (
 	// and then controlling the service account's access by granting it IAM roles.
 	// https://cloud.google.com/compute/docs/access/service-accounts#scopes_best_practice
 	defaultGCPScope = "https://www.googleapis.com/auth/cloud-platform"
+	// defaultValidateServiceAccountKeyTimeout is the timeout for validating
+	// a service account key.
+	defaultValidateServiceAccountKeyTimeout = 5 * time.Second
 )
 
 // Config is the configuration for the GCP credential.
 type Config struct {
-	ProjectId              string
+	ProjectId                        string
-	PrivateKey             string
+	PrivateKey                       string
-	PrivateKeyId           string
+	PrivateKeyId                     string
-	Zone                   string
+	Zone                             string
-	ClientEmail            string
+	ClientEmail                      string
-	TargetServiceAccountId string
+	TargetServiceAccountId           string
-	Scopes                 []string
+	Scopes                           []string
+	validateServiceAccountKeyTimeout time.Duration
 }
 
 // credentials represents a simplified version of the GCP credentials file format.
@@ -72,13 +76,14 @@ func NewConfig(opt ...Option) (*Config, error) {
 		return nil, err
 	}
 	c := &Config{
-		ProjectId:              opts.WithProjectId,
+		ProjectId:                        opts.WithProjectId,
-		PrivateKey:             opts.WithPrivateKey,
+		PrivateKey:                       opts.WithPrivateKey,
-		PrivateKeyId:           opts.WithPrivateKeyId,
+		PrivateKeyId:                     opts.WithPrivateKeyId,
-		ClientEmail:            opts.WithClientEmail,
+		ClientEmail:                      opts.WithClientEmail,
-		TargetServiceAccountId: opts.WithTargetServiceAccountId,
+		TargetServiceAccountId:           opts.WithTargetServiceAccountId,
-		Zone:                   opts.WithZone,
+		Zone:                             opts.WithZone,
-		Scopes:                 opts.WithScopes,
+		Scopes:                           opts.WithScopes,
+		validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 	}
 	return c, nil
 }
@@ -97,12 +102,14 @@ func (c *Config) toCredentials() *credentials {
 // clone returns a copy of the configuration.
 func (c *Config) clone() *Config {
 	return &Config{
-		ProjectId:              c.ProjectId,
+		ProjectId:                        c.ProjectId,
-		PrivateKey:             c.PrivateKey,
+		Zone:                             c.Zone,
-		PrivateKeyId:           c.PrivateKeyId,
+		PrivateKey:                       c.PrivateKey,
-		ClientEmail:            c.ClientEmail,
+		PrivateKeyId:                     c.PrivateKeyId,
-		TargetServiceAccountId: c.TargetServiceAccountId,
+		ClientEmail:                      c.ClientEmail,
-		Scopes:                 c.Scopes,
+		TargetServiceAccountId:           c.TargetServiceAccountId,
+		Scopes:                           c.Scopes,
+		validateServiceAccountKeyTimeout: c.validateServiceAccountKeyTimeout,
 	}
 }
 

internal/credential/config_test.go

@@ -403,13 +403,14 @@ func TestNewConfig(t *testing.T) {
 				WithScopes([]string{"scope1", "scope2"}),
 			},
 			expected: &Config{
-				ProjectId:              "test-project-id",
+				ProjectId:                        "test-project-id",
-				PrivateKey:             "test-private-key",
+				PrivateKey:                       "test-private-key",
-				PrivateKeyId:           "test-private-key-id",
+				PrivateKeyId:                     "test-private-key-id",
-				ClientEmail:            "test-email@example.com",
+				ClientEmail:                      "test-email@example.com",
-				TargetServiceAccountId: "target-service-account@example.com",
+				TargetServiceAccountId:           "target-service-account@example.com",
-				Zone:                   "test-zone",
+				Zone:                             "test-zone",
-				Scopes:                 []string{"scope1", "scope2"},
+				Scopes:                           []string{"scope1", "scope2"},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 			expectError: false,
 		},
@@ -417,7 +418,8 @@ func TestNewConfig(t *testing.T) {
 			name:    "No Options",
 			options: []Option{},
 			expected: &Config{
-				Scopes: []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 			expectError: false,
 		},
@@ -442,13 +444,14 @@ func TestNewConfig(t *testing.T) {
 				WithZone("test-zone"),
 			},
 			expected: &Config{
-				ProjectId:              "test-project-id",
+				ProjectId:                        "test-project-id",
-				PrivateKey:             "test-private-key",
+				PrivateKey:                       "test-private-key",
-				PrivateKeyId:           "test-private-key-id",
+				PrivateKeyId:                     "test-private-key-id",
-				ClientEmail:            "test-email@example.com",
+				ClientEmail:                      "test-email@example.com",
-				TargetServiceAccountId: "target-service-account@example.com",
+				TargetServiceAccountId:           "target-service-account@example.com",
-				Zone:                   "test-zone",
+				Zone:                             "test-zone",
-				Scopes:                 []string{defaultGCPScope},
+				Scopes:                           []string{defaultGCPScope},
+				validateServiceAccountKeyTimeout: defaultValidateServiceAccountKeyTimeout,
 			},
 			expectError: false,
 		},

internal/credential/rotate.go

@@ -5,9 +5,11 @@ package credential
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"slices"
-	"strings"
+	"time"
 
 	admin "cloud.google.com/go/iam/admin/apiv1"
 	"cloud.google.com/go/iam/admin/apiv1/adminpb"
@@ -19,12 +21,39 @@ import (
 )
 
 const (
-	ComputeInstancesListPermission        = "compute.instances.list"
+	// ComputeInstancesListPermission is the IAM permission required
+	// to list compute instances.
+	ComputeInstancesListPermission = "compute.instances.list"
+	// IAMServiceAccountKeysCreatePermission is the IAM permission
+	// required to create a service account key.
+	// This permission is required to rotate service account keys.
 	IAMServiceAccountKeysCreatePermission = "iam.serviceAccountKeys.create"
+	// IAMServiceAccountKeysDeletePermission is the IAM permission
+	// required to delete a service account key.
+	// This permission is required to rotate service account keys.
 	IAMServiceAccountKeysDeletePermission = "iam.serviceAccountKeys.delete"
-	CreateServiceAccountKeyResourceName   = "projects/-/serviceAccounts"
 )
 
+// ServiceAccountPrivateKey represents a decoded PrivateKeyData
+// from a Service Account Key.
+// https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts.keys#ServiceAccountKey
+type ServiceAccountPrivateKey struct {
+	Type                    string `json:"type"`
+	ProjectID               string `json:"project_id"`
+	PrivateKeyID            string `json:"private_key_id"`
+	PrivateKey              string `json:"private_key"`
+	ClientEmail             string `json:"client_email"`
+	ClientID                string `json:"client_id"`
+	AuthURI                 string `json:"auth_uri"`
+	TokenURI                string `json:"token_uri"`
+	AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`
+	ClientX509CertURL       string `json:"client_x509_cert_url"`
+}
+
+// ValidateCredsCallback is called to validate the credentials after rotating the service account key.
+// The callback should return an error if the credentials are invalid.
+type ValidateCredsCallback func(*Config, ...option.ClientOption) error
+
 // RotateServiceAccountKey takes the private key from this credentials config
 // and first creates a new private key and private key id, then deletes the
 // old private key.
@@ -32,7 +61,12 @@ const (
 // If deletion of the old private key is successful, the new private key and
 // private key id are written into the credentials config and nil is returned.
 // On any error, the old credentials are not overwritten.
-func (c *Config) RotateServiceAccountKey(ctx context.Context, permissions []string, opts ...option.ClientOption) error {
+func (c *Config) RotateServiceAccountKey(
+	ctx context.Context,
+	permissions []string,
+	validateCredsCallback ValidateCredsCallback,
+	opts ...option.ClientOption,
+) error {
 	if c.PrivateKey == "" {
 		return status.Error(codes.InvalidArgument, "cannot rotate credentials when private key is not set")
 	}
@@ -60,7 +94,7 @@ func (c *Config) RotateServiceAccountKey(ctx context.Context, permissions []stri
 	}
 
 	createServiceAccountKeyRes, err := iamClient.CreateServiceAccountKey(ctx, &adminpb.CreateServiceAccountKeyRequest{
-		Name:           fmt.Sprintf("%s/%s", CreateServiceAccountKeyResourceName, c.ClientEmail),
+		Name:           fmt.Sprintf("projects/%s/serviceAccounts/%s", c.ProjectId, c.ClientEmail),
 		PrivateKeyType: adminpb.ServiceAccountPrivateKeyType_TYPE_GOOGLE_CREDENTIALS_FILE,
 		KeyAlgorithm:   adminpb.ServiceAccountKeyAlgorithm_KEY_ALG_RSA_2048,
 	})
@@ -68,15 +102,16 @@ func (c *Config) RotateServiceAccountKey(ctx context.Context, permissions []stri
 		return status.Errorf(codes.Internal, "error creating service account key: %v", err)
 	}
 
-	privateKeyId, err := getPrivateKeyIdFromName(createServiceAccountKeyRes.Name)
+	var serviceAccountKey ServiceAccountPrivateKey
+	err = json.Unmarshal(createServiceAccountKeyRes.PrivateKeyData, &serviceAccountKey)
 	if err != nil {
-		return status.Errorf(codes.Internal, "error parsing private key ID: %v", err)
+		return status.Errorf(codes.Internal, "error unmarshalling service account key: %v", err)
 	}
 
 	// Clone the config, update the private key and private key ID with the new key.
 	newConfig := c.clone()
-	newConfig.PrivateKey = string(createServiceAccountKeyRes.PrivateKeyData)
+	newConfig.PrivateKey = serviceAccountKey.PrivateKey
-	newConfig.PrivateKeyId = privateKeyId
+	newConfig.PrivateKeyId = serviceAccountKey.PrivateKeyID
 
 	newCreds, err := newConfig.GenerateCredentials(ctx)
 	if err != nil {
@@ -88,10 +123,10 @@ func (c *Config) RotateServiceAccountKey(ctx context.Context, permissions []stri
 	clientOptions = []option.ClientOption{option.WithTokenSource(newCreds.TokenSource)}
 	clientOptions = append(clientOptions, opts...)
 
-	// Validate that the new credentials have the necessary permissions.
+	// Validate the new service account key
-	_, err = newConfig.ValidateIamPermissions(ctx, permissions, clientOptions...)
+	err = newConfig.ValidateServiceAccountKey(ctx, permissions, validateCredsCallback, clientOptions...)
 	if err != nil {
-		return status.Errorf(codes.PermissionDenied, "error testing IAM permissions with rotated service account key: %v", err)
+		return status.Errorf(codes.PermissionDenied, "error validating rotated service account key: %v", err)
 	}
 
 	iamClient, err = admin.NewIamClient(ctx, clientOptions...)
@@ -100,7 +135,7 @@ func (c *Config) RotateServiceAccountKey(ctx context.Context, permissions []stri
 	}
 
 	err = iamClient.DeleteServiceAccountKey(ctx, &adminpb.DeleteServiceAccountKeyRequest{
-		Name: fmt.Sprintf("%s/%s/keys/%s", CreateServiceAccountKeyResourceName, c.ClientEmail, c.PrivateKeyId),
+		Name: fmt.Sprintf("projects/%s/serviceAccounts/%s/keys/%s", c.ProjectId, c.ClientEmail, c.PrivateKeyId),
 	})
 	if err != nil {
 		return status.Errorf(codes.Internal, "error deleting service account key: %v", err)
@@ -112,6 +147,44 @@ func (c *Config) RotateServiceAccountKey(ctx context.Context, permissions []stri
 	return nil
 }
 
+// ValidateServiceAccountKey validates the service account key by checking the IAM permissions
+// and calling the validation callback.
+// The function will retry validation until the timeout is reached.
+func (c *Config) ValidateServiceAccountKey(
+	ctx context.Context,
+	permissions []string,
+	validateCredsCallback ValidateCredsCallback,
+	opts ...option.ClientOption) error {
+	var validatePermissionsErr error
+	var validationCallbackErr error
+
+	ticker := time.NewTicker(200 * time.Millisecond)
+	defer ticker.Stop()
+
+	ctx, cancel := context.WithTimeout(ctx, c.validateServiceAccountKeyTimeout)
+	defer cancel()
+
+	for {
+		select {
+		case <-ctx.Done():
+			if ctx.Err() == context.DeadlineExceeded {
+				return fmt.Errorf("failed to validate service account key before timeout: %w", errors.Join(validatePermissionsErr, validationCallbackErr))
+			}
+			return ctx.Err()
+		case <-ticker.C:
+			_, validatePermissionsErr = c.ValidateIamPermissions(ctx, permissions, opts...)
+
+			if validateCredsCallback != nil {
+				validationCallbackErr = validateCredsCallback(c, opts...)
+			}
+
+			if validatePermissionsErr == nil && validationCallbackErr == nil {
+				return nil
+			}
+		}
+	}
+}
+
 func (c *Config) DeletePrivateKey(ctx context.Context, opts ...option.ClientOption) error {
 	if c.PrivateKey == "" {
 		return status.Error(codes.InvalidArgument, "cannot delete credentials when private key is not set")
@@ -140,7 +213,7 @@ func (c *Config) DeletePrivateKey(ctx context.Context, opts ...option.ClientOpti
 	}
 
 	err = iamClient.DeleteServiceAccountKey(ctx, &adminpb.DeleteServiceAccountKeyRequest{
-		Name: fmt.Sprintf("projects/-/serviceAccounts/%s/keys/%s", c.ClientEmail, c.PrivateKeyId),
+		Name: fmt.Sprintf("projects/%s/serviceAccounts/%s/keys/%s", c.ProjectId, c.ClientEmail, c.PrivateKeyId),
 	})
 	if err != nil {
 		return status.Errorf(codes.Unknown, "error deleting service account key: %v", err)
@@ -156,18 +229,7 @@ func (c *Config) ValidateIamPermissions(ctx context.Context, permissions []strin
 		return nil, status.Error(codes.InvalidArgument, "permissions are required")
 	}
 
-	creds, err := c.GenerateCredentials(ctx)
+	rmClient, err := resourcemanager.NewProjectsClient(ctx, opts...)
-	if err != nil {
-		return nil, status.Errorf(codes.Unauthenticated, "error generating credentials: %v", err)
-	}
-	if creds.TokenSource == nil {
-		return nil, status.Error(codes.Unauthenticated, "error generating credentials: token source is nil")
-	}
-
-	clientOptions := []option.ClientOption{option.WithTokenSource(creds.TokenSource)}
-	clientOptions = append(clientOptions, opts...)
-
-	rmClient, err := resourcemanager.NewProjectsClient(ctx, clientOptions...)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to create Resource Manager client: %v", err)
 	}
@@ -197,12 +259,3 @@ func (c *Config) ValidateIamPermissions(ctx context.Context, permissions []strin
 
 	return resp.Permissions, nil
 }
-
-// getPrivateKeyIdFromName extracts the private key ID from the name of a service account key.
-func getPrivateKeyIdFromName(input string) (string, error) {
-	lastSlashIndex := strings.LastIndex(input, "/")
-	if lastSlashIndex != -1 {
-		return input[lastSlashIndex+1:], nil
-	}
-	return "", fmt.Errorf("could not find private key ID in %s", input)
-}

internal/credential/rotate_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"cloud.google.com/go/iam/admin/apiv1/adminpb"
 	"cloud.google.com/go/iam/apiv1/iampb"
@@ -30,6 +31,7 @@ func TestRotateServiceAccountKey(t *testing.T) {
 			},
 		},
 	}
+	testValidateTimeout := 1 * time.Second
 
 	tests := []struct {
 		name          string
@@ -76,9 +78,10 @@ func TestRotateServiceAccountKey(t *testing.T) {
 		{
 			name: "DeleteServiceAccountKey returns error",
 			config: &Config{
-				PrivateKey:   "some-private-key",
+				PrivateKey:                       "some-private-key",
-				PrivateKeyId: "some-private-key-id",
+				PrivateKeyId:                     "some-private-key-id",
-				ClientEmail:  "client@example.com",
+				ClientEmail:                      "client@example.com",
+				validateServiceAccountKeyTimeout: testValidateTimeout,
 			},
 			setup: func() {
 				testIAMAdminServer.testCreateServiceAccountKeyError = nil
@@ -89,9 +92,10 @@ func TestRotateServiceAccountKey(t *testing.T) {
 		{
 			name: "Successful rotation",
 			config: &Config{
-				PrivateKey:   "some-private-key",
+				PrivateKey:                       "some-private-key",
-				PrivateKeyId: "some-private-key-id",
+				PrivateKeyId:                     "some-private-key-id",
-				ClientEmail:  "client@example.com",
+				ClientEmail:                      "client@example.com",
+				validateServiceAccountKeyTimeout: testValidateTimeout,
 			},
 			setup: func() {
 				testIAMAdminServer.testCreateServiceAccountKeyError = nil
@@ -128,8 +132,14 @@ func TestRotateServiceAccountKey(t *testing.T) {
 				IAMServiceAccountKeysCreatePermission,
 				IAMServiceAccountKeysDeletePermission,
 			}
+			validateCredsCallbackCalled := false
 
-			err = tt.config.RotateServiceAccountKey(ctx, permissions, testOptions...)
+			validateCredsCallBack := func(c *Config, opts ...googleOption.ClientOption) error {
+				validateCredsCallbackCalled = true
+				return nil
+			}
+
+			err = tt.config.RotateServiceAccountKey(ctx, permissions, validateCredsCallBack, testOptions...)
 			if tt.expectedError != nil {
 				require.ErrorContains(t, err, tt.expectedError.Error())
 				require.Equal(t, initialClientEmail, tt.config.ClientEmail)
@@ -143,6 +153,7 @@ func TestRotateServiceAccountKey(t *testing.T) {
 			require.NotEmpty(t, tt.config.PrivateKeyId)
 			require.NotEqual(t, initialPrivateKey, tt.config.PrivateKey)
 			require.NotEqual(t, initialPrivateKeyId, tt.config.PrivateKeyId)
+			require.True(t, validateCredsCallbackCalled)
 		})
 	}
 }
@@ -183,16 +194,6 @@ func TestValidateIamPermissions(t *testing.T) {
 			wantErr:     true,
 			expectedErr: "permissions are required",
 		},
-		{
-			name: "Failed to Generate Credentials",
-			config: &Config{
-				ProjectId:  "test-project-id",
-				PrivateKey: "test-private-key",
-			},
-			permissions: []string{"permission1", "permission2"},
-			wantErr:     true,
-			expectedErr: "failed to generate credentials",
-		},
 		{
 			name: "Failed to Test IAM Permissions",
 			config: &Config{
@@ -264,6 +265,104 @@ func TestValidateIamPermissions(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateServiceAccountKey(t *testing.T) {
+	ctx := context.Background()
+	testResourceServer := &testResourceServer{}
+	testValidateTimeout := 1 * time.Second
+
+	tests := []struct {
+		name                    string
+		config                  *Config
+		permissions             []string
+		validateCredsCallback   ValidateCredsCallback
+		testIamPermissionsError error
+		expectedError           error
+	}{
+		{
+			name: "Successful validation",
+			config: &Config{
+				ProjectId:                        "test-project-id",
+				PrivateKey:                       "test-private-key",
+				ClientEmail:                      "test@test.com",
+				validateServiceAccountKeyTimeout: testValidateTimeout,
+			},
+			permissions: []string{
+				ComputeInstancesListPermission,
+				IAMServiceAccountKeysCreatePermission,
+				IAMServiceAccountKeysDeletePermission,
+			},
+			validateCredsCallback: func(c *Config, opts ...googleOption.ClientOption) error {
+				return nil
+			},
+			expectedError: nil,
+		},
+		{
+			name: "Validation callback returns error",
+			config: &Config{
+				ProjectId:                        "test-project-id",
+				PrivateKey:                       "test-private-key",
+				ClientEmail:                      "test@test.com",
+				validateServiceAccountKeyTimeout: testValidateTimeout,
+			},
+			permissions: []string{
+				ComputeInstancesListPermission,
+				IAMServiceAccountKeysCreatePermission,
+				IAMServiceAccountKeysDeletePermission,
+			},
+			validateCredsCallback: func(c *Config, opts ...googleOption.ClientOption) error {
+				return errors.New("validation callback error")
+			},
+			expectedError: errors.New("validation callback error"),
+		},
+		{
+			name: "IAM permissions validation returns error",
+			config: &Config{
+				ProjectId:                        "test-project-id",
+				PrivateKey:                       "test-private-key",
+				ClientEmail:                      "test@test.com",
+				validateServiceAccountKeyTimeout: testValidateTimeout,
+			},
+			permissions: []string{
+				ComputeInstancesListPermission,
+				IAMServiceAccountKeysCreatePermission,
+				IAMServiceAccountKeysDeletePermission,
+			},
+			testIamPermissionsError: errors.New("IAM permissions error"),
+			expectedError:           errors.New("failed to validate IAM permissions"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testResourceServer.testIamPermissionsResponse = &iampb.TestIamPermissionsResponse{
+				Permissions: tt.permissions,
+			}
+			testResourceServer.testIamPermissionsError = tt.testIamPermissionsError
+
+			gsrv := NewGRPCServer()
+			resourcemanagerpb.RegisterProjectsServer(gsrv.Server, testResourceServer)
+			addr, err := gsrv.Start()
+			require.NoError(t, err)
+
+			testOptions := []googleOption.ClientOption{
+				googleOption.WithEndpoint(addr),
+				googleOption.WithoutAuthentication(),
+				googleOption.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())),
+				googleOption.WithTokenSource(nil),
+			}
+
+			err = tt.config.ValidateServiceAccountKey(ctx, tt.permissions, tt.validateCredsCallback, testOptions...)
+			if tt.expectedError != nil {
+				require.ErrorContains(t, err, tt.expectedError.Error())
+				return
+			}
+
+			require.NoError(t, err)
+		})
+	}
+}
+
 func TestDeletePrivateKey(t *testing.T) {
 	testIAMAdminServer := &testIAMAdminServer{}
 

internal/credential/state.go

@@ -37,6 +37,28 @@ func NewPersistedState(opt ...Option) (*PersistedState, error) {
 	return s, nil
 }
 
+// RotateCreds rotates the credentials for the GCP catalog and updates the last rotated time.
+func (s *PersistedState) RotateCreds(
+	ctx context.Context,
+	permissions []string,
+	validateCredsCallback ValidateCredsCallback,
+	opts ...option.ClientOption) error {
+	if s.CredentialsConfig == nil {
+		return status.Error(codes.InvalidArgument, "missing credentials config")
+	}
+	if !s.CredentialsConfig.IsRotatable() {
+		return status.Error(codes.InvalidArgument, "cannot rotate non-rotatable credentials")
+	}
+
+	err := s.CredentialsConfig.RotateServiceAccountKey(ctx, permissions, validateCredsCallback, opts...)
+	if err != nil {
+		return status.Errorf(codes.Internal, "failed to rotate credentials: %v", err)
+	}
+	s.CredsLastRotatedTime = time.Now()
+
+	return nil
+}
+
 // ReplaceCreds replaces the private key in the state with a new key.
 // If the existing key was rotated at any point in time, it is
 // deleted first, otherwise it's left alone. This method returns a

internal/credential/testing.go

@@ -5,6 +5,7 @@ package credential
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 
@@ -29,9 +30,17 @@ func NewTestIAMAdminServer(createServiceAccountKeyError error, deleteServiceAcco
 }
 
 func (f *testIAMAdminServer) CreateServiceAccountKey(ctx context.Context, req *adminpb.CreateServiceAccountKeyRequest) (*adminpb.ServiceAccountKey, error) {
+	serviceAccountKey := ServiceAccountPrivateKey{
+		PrivateKey:   "updated-private-key",
+		PrivateKeyID: "updated-private-key-id",
+	}
+	serviceAccountKeyData, err := json.Marshal(serviceAccountKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal service account key: %v", err)
+	}
 	resp := &adminpb.ServiceAccountKey{
 		Name:           "projects/-/serviceAccounts/1234/keys/updated-private-key-id",
-		PrivateKeyData: []byte("updated-private-key"),
+		PrivateKeyData: serviceAccountKeyData,
 	}
 	return resp, f.testCreateServiceAccountKeyError
 }

plugin/service/host/attributes.go

@@ -108,7 +108,7 @@ func buildListInstancesRequest(attributes *SetAttributes, catalog *CatalogAttrib
 		return nil, fmt.Errorf("error building filters: %w", err)
 	}
 
-	if len(filters) > 1 {
+	if len(filters) > 0 {
 		filters := strings.Join(filters, " AND ")
 		request.Filter = &filters
 	}

plugin/service/host/attributes_test.go

@@ -183,6 +183,23 @@ func TestBuildListInstancesRequest(t *testing.T) {
 			},
 			expectedErrContains: "error building filters",
 		},
+		{
+			name: "valid request with no filters",
+			attributes: &SetAttributes{
+				Filters: []string{},
+			},
+			catalog: &CatalogAttributes{
+				CredentialAttributes: &cred.CredentialAttributes{
+					ProjectId: "test-12345",
+					Zone:      "us-central-1a",
+				},
+			},
+			expected: &computepb.ListInstancesRequest{
+				Project: "test-12345",
+				Zone:    "us-central-1a",
+				Filter:  stringPtr("status = running"),
+			},
+		},
 	}
 
 	for _, tc := range cases {

plugin/service/host/plugin.go

@@ -87,8 +87,22 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 		credential.IAMServiceAccountKeysDeletePermission,
 	}
 
+	if !credState.CredentialsConfig.IsRotatable() && !catalogAttributes.DisableCredentialRotation {
+		return nil, status.Error(codes.InvalidArgument, "cannot rotate credentials for non-rotatable credentials")
+	}
+
 	if credState.CredentialsConfig.IsRotatable() && !catalogAttributes.DisableCredentialRotation {
-		if err := credState.CredentialsConfig.RotateServiceAccountKey(ctx, permissions, p.testGCPClientOpts...); err != nil {
+		state, err := newGCPCatalogPersistedState(
+			append([]gcpCatalogPersistedStateOption{
+				withCredentials(credState),
+			}, p.testCatalogStateOpts...)...,
+		)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "error setting up persisted state: %s", err)
+		}
+		validateCredsCallback := state.ValidateCredsCallbackFn(ctx, p.testGCPClientOpts...)
+
+		if err := credState.RotateCreds(ctx, permissions, validateCredsCallback, p.testGCPClientOpts...); err != nil {
 			return nil, err
 		}
 	}
@@ -152,12 +166,15 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		return nil, status.Errorf(codes.InvalidArgument, "error getting persisted state from proto: %s", err)
 	}
 
-	updatedCredentials, err := credential.GetCredentialsConfig(newCatalog.GetSecrets(), newCatalogAttributes.CredentialAttributes)
+	// We will be updating credentials when one of two changes occur:
-	if err != nil {
+	// 1. The new catalog has a service account key, in which case we need to delete the old service account key credentials
-		return nil, err
+	// 2. The target service account ID has changed. This indicates a change of credential type to use service account impersonation
-	}
+	if newCatalog.GetSecrets() != nil || newCatalogAttributes.TargetServiceAccountId != oldCatalogAttributes.TargetServiceAccountId {
+		updatedCredentials, err := credential.GetCredentialsConfig(newCatalog.GetSecrets(), newCatalogAttributes.CredentialAttributes)
+		if err != nil {
+			return nil, err
+		}
 
-	if newCatalog.GetSecrets() != nil {
 		newCredState, err := credential.NewPersistedState([]credential.Option{
 			credential.WithCredentialsConfig(updatedCredentials),
 		}...,
@@ -186,11 +203,14 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		}
 	}
 
-	if credState.CredentialsConfig.IsRotatable() {
+	// This is a validate check to make sure that we aren't trying to
-		if !updatedCredentials.IsRotatable() && !newCatalogAttributes.DisableCredentialRotation {
+	// rotate credentials that are not rotatable. For example,
-			return nil, status.Error(codes.InvalidArgument, "cannot rotate credentials for non-rotatable credentials")
+	// Application Default Credentials (ADC) are not rotatable.
-		}
+	if !credState.CredentialsConfig.IsRotatable() && !newCatalogAttributes.DisableCredentialRotation {
+		return nil, status.Error(codes.InvalidArgument, "cannot rotate credentials for non-rotatable credentials")
+	}
 
+	if credState.CredentialsConfig.IsRotatable() {
 		// This is a validate check to make sure that we aren't disabling
 		// rotation for credentials currently being managed by rotation.
 		// This is not allowed.
@@ -209,7 +229,17 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		// If we're enabling rotation now but didn't before, or have
 		// freshly replaced credentials, we can rotate here.
 		if !newCatalogAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
-			if err := credState.CredentialsConfig.RotateServiceAccountKey(ctx, permissions, p.testGCPClientOpts...); err != nil {
+			state, err := newGCPCatalogPersistedState(
+				append([]gcpCatalogPersistedStateOption{
+					withCredentials(credState),
+				}, p.testCatalogStateOpts...)...,
+			)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "error setting up persisted state: %s", err)
+			}
+			validateCredsCallback := state.ValidateCredsCallbackFn(ctx, p.testGCPClientOpts...)
+
+			if err := credState.RotateCreds(ctx, permissions, validateCredsCallback, p.testGCPClientOpts...); err != nil {
 				return nil, err
 			}
 		}

plugin/service/host/plugin_test.go

@@ -878,6 +878,53 @@ func TestCreateCatalog(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "create with application default credentials and credential rotation disabled",
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								cred.ConstProjectId:                 structpb.NewStringValue("test-project"),
+								cred.ConstZone:                      structpb.NewStringValue("us-central1-a"),
+								cred.ConstDisableCredentialRotation: structpb.NewBoolValue(true),
+							},
+						},
+					},
+				},
+			},
+			catalogOpts: []gcpCatalogPersistedStateOption{
+				withTestInstancesAPIFunc(newTestMockInstances(ctx,
+					nil,
+					testMockInstancesWithListInstancesOutput(&computepb.InstanceList{}),
+					testMockInstancesWithListInstancesError(nil),
+				)),
+			},
+			expectedRsp: &pb.OnCreateCatalogResponse{
+				Persisted: &pb.HostCatalogPersisted{
+					Secrets: &structpb.Struct{
+						Fields: map[string]*structpb.Value{},
+					},
+				},
+			},
+		},
+		{
+			name: "create with application default credentials and credential rotation enabled",
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								cred.ConstProjectId:                 structpb.NewStringValue("test-project"),
+								cred.ConstZone:                      structpb.NewStringValue("us-central1-a"),
+								cred.ConstDisableCredentialRotation: structpb.NewBoolValue(false),
+							},
+						},
+					},
+				},
+			},
+			expectedErr: "cannot rotate credentials for non-rotatable credentials",
+		},
 	}
 
 	for _, tc := range cases {
@@ -1172,6 +1219,12 @@ func TestUpdateCatalog(t *testing.T) {
 							},
 						},
 					},
+					Secrets: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							cred.ConstPrivateKeyId: structpb.NewStringValue(""),
+							cred.ConstPrivateKey:   structpb.NewStringValue(""),
+						},
+					},
 				},
 				Persisted: &pb.HostCatalogPersisted{
 					Secrets: &structpb.Struct{
@@ -1183,8 +1236,55 @@ func TestUpdateCatalog(t *testing.T) {
 					},
 				},
 			},
+			catalogOpts: []gcpCatalogPersistedStateOption{
+				withTestInstancesAPIFunc(newTestMockInstances(ctx,
+					nil,
+					testMockInstancesWithListInstancesOutput(&computepb.InstanceList{}),
+					testMockInstancesWithListInstancesError(nil),
+				)),
+			},
 			expectedErr: "cannot rotate credentials for non-rotatable credentials",
 		},
+		{
+			name: "update application default credentials",
+			req: &pb.OnUpdateCatalogRequest{
+				CurrentCatalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								cred.ConstProjectId:                 structpb.NewStringValue("test-project"),
+								cred.ConstClientEmail:               structpb.NewStringValue("test@example.com"),
+								cred.ConstZone:                      structpb.NewStringValue("us-central1-a"),
+								cred.ConstDisableCredentialRotation: structpb.NewBoolValue(true),
+							},
+						},
+					},
+				},
+				NewCatalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								cred.ConstProjectId:                 structpb.NewStringValue("test-project-1"),
+								cred.ConstZone:                      structpb.NewStringValue("us-central1"),
+								cred.ConstDisableCredentialRotation: structpb.NewBoolValue(true),
+							},
+						},
+					},
+				},
+			},
+			catalogOpts: []gcpCatalogPersistedStateOption{
+				withTestInstancesAPIFunc(newTestMockInstances(ctx,
+					nil,
+					testMockInstancesWithListInstancesOutput(&computepb.InstanceList{}),
+					testMockInstancesWithListInstancesError(nil),
+				)),
+			},
+			expectedRsp: &pb.OnUpdateCatalogResponse{
+				Persisted: &pb.HostCatalogPersisted{
+					Secrets: &structpb.Struct{},
+				},
+			},
+		},
 	}
 
 	for _, tc := range cases {

plugin/service/host/state.go

@@ -13,6 +13,7 @@ import (
 	"github.com/googleapis/gax-go/v2"
 	"github.com/hashicorp/boundary-plugin-gcp/internal/credential"
 	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	"google.golang.org/protobuf/types/known/structpb"
 )
@@ -99,3 +100,30 @@ func (s *gcpCatalogPersistedState) InstancesClient(
 
 	return client, nil
 }
+
+// validateCredsCallbackFn returns a ValidateCredsCallback function that
+// validates the credentials by listing instances in the project and zone.
+func (s *gcpCatalogPersistedState) ValidateCredsCallbackFn(
+	ctx context.Context,
+	opts ...option.ClientOption,
+) credential.ValidateCredsCallback {
+	return func(config *credential.Config, opts ...option.ClientOption) error {
+		s.CredentialsConfig = config
+		instancesClient, err := s.InstancesClient(ctx, opts...)
+		if err != nil {
+			return fmt.Errorf("error creating instances client: %w", err)
+		}
+
+		it := instancesClient.List(ctx, &computepb.ListInstancesRequest{
+			Project: s.CredentialsConfig.ProjectId,
+			Zone:    s.CredentialsConfig.Zone,
+		})
+
+		_, err = it.Next()
+		if err != nil && err != iterator.Done {
+			return fmt.Errorf("error listing instances: %w", err)
+		}
+
+		return nil
+	}
+}

plugin/service/host/state_test.go

@@ -5,9 +5,11 @@ package host
 
 import (
 	"context"
+	"errors"
 	"testing"
 	"time"
 
+	computepb "cloud.google.com/go/compute/apiv1/computepb"
 	"github.com/hashicorp/boundary-plugin-gcp/internal/credential"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -87,3 +89,44 @@ func TestInstancesClient(t *testing.T) {
 		assert.Contains(t, err.Error(), "error creating instances client")
 	})
 }
+func TestValidateCredsCallbackFn(t *testing.T) {
+	ctx := context.Background()
+	cred := &credential.PersistedState{
+		CredentialsConfig: &credential.Config{
+			ProjectId: "test-project-id",
+			Zone:      "test-zone",
+		},
+	}
+	state := &gcpCatalogPersistedState{PersistedState: cred}
+
+	t.Run("List instances error", func(t *testing.T) {
+		state.testInstancesAPIFunc = newTestMockInstances(ctx,
+			nil,
+			testMockInstancesWithListInstancesOutput(nil),
+			testMockInstancesWithListInstancesError(errors.New("error listing instances")),
+		)
+
+		callback := state.ValidateCredsCallbackFn(ctx)
+		err := callback(&credential.Config{
+			ProjectId: "test-project-id",
+			Zone:      "test-zone",
+		})
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "error listing instances")
+	})
+
+	t.Run("Success", func(t *testing.T) {
+		state.testInstancesAPIFunc = newTestMockInstances(ctx,
+			nil,
+			testMockInstancesWithListInstancesOutput(&computepb.InstanceList{}),
+			testMockInstancesWithListInstancesError(nil),
+		)
+
+		callback := state.ValidateCredsCallbackFn(ctx)
+		err := callback(&credential.Config{
+			ProjectId: "test-project-id",
+			Zone:      "test-zone",
+		})
+		assert.NoError(t, err)
+	})
+}

testing/e2e_host_test.go

@@ -0,0 +1,589 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package testing
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary-plugin-gcp/internal/credential"
+	"github.com/hashicorp/boundary-plugin-gcp/plugin/service/host"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
+	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+const (
+	expectedInstanceCount = 3
+	expectedTagsCount     = 3
+	expectedLabelsCount   = 3
+)
+
+// All the tests in this test depend on each other
+// to run in order. This is because the tests are
+// testing the same plugin instance and the same
+// persisted state. The tests are designed to run
+// in order to test the plugin's ability to rotate
+// credentials and manage the state of the plugin
+// instance.
+// To run the tests you need the following
+// environment variables set:
+// GOOGLE_APPLICATION_CREDENTIALS: This should be
+// the path to the service account key. This can also
+// be automatically set after authenticating with gcloud
+// using `gcloud auth application-default login`.
+//
+// TF_VAR_project_id: The project_id of the project.
+//
+// TF_VAR_region: The region the resources should be
+// deployed in.
+//
+// TF_VAR_zone: The zone the resources should be
+// deployed in.
+func TestHostPlugin(t *testing.T) {
+	creds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+	if creds == "" {
+		t.Skip("set GOOGLE_APPLICATION_CREDENTIALS to use this test")
+	}
+	projectId := os.Getenv("TF_VAR_project_id")
+	if projectId == "" {
+		t.Skip("set TF_VAR_project_id to use this test")
+	}
+	region := os.Getenv("TF_VAR_region")
+	if region == "" {
+		t.Skip("set TF_VAR_region to use this test")
+	}
+	zone := os.Getenv("TF_VAR_zone")
+	if zone == "" {
+		t.Skip("set TF_VAR_zone to use this test")
+	}
+
+	require := require.New(t)
+	tf, err := NewTestTerraformer("testdata/host")
+	require.NoError(err)
+	require.NotNil(tf)
+
+	t.Log("===== deploying test Terraform workspace =====")
+	err = tf.Deploy()
+	require.NoError(err)
+
+	defer func() {
+		t.Log("===== destroying test Terraform workspace =====")
+		if err := tf.Destroy(); err != nil {
+			t.Logf("WARNING: could not run Terraform destroy: %s", err)
+		}
+	}()
+
+	clientEmail, err := tf.GetOutput("client_email")
+	require.NoError(err)
+	require.NotNil(clientEmail)
+
+	instanceIds, err := tf.GetOutputSlice("instance_ids")
+	require.NoError(err)
+	require.Len(instanceIds, expectedInstanceCount)
+
+	instanceTags, err := tf.GetOutputMap("instance_tags")
+	require.NoError(err)
+	require.Len(instanceTags, expectedInstanceCount)
+
+	instanceLabels, err := tf.GetOutputMap("instance_labels")
+	require.NoError(err)
+	require.Len(instanceLabels, expectedLabelsCount)
+
+	privateKeys, err := tf.GetOutputSlice("private_keys")
+	require.NoError(err)
+	require.Len(privateKeys, expectedServiceAccountKeys)
+
+	privateKeyIds, err := tf.GetOutputSlice("private_key_ids")
+	require.NoError(err)
+	require.Len(privateKeyIds, expectedServiceAccountKeys)
+
+	p := new(host.HostPlugin)
+	ctx := context.Background()
+
+	var keyId, secret string
+
+	t.Run("OnCreateCatalog", func(t *testing.T) {
+		t.Run("Test non-rotation", func(t *testing.T) {
+			keyId, secret = testPluginOnCreateCatalog(
+				ctx,
+				t,
+				p,
+				zone,
+				projectId,
+				clientEmail.(string),
+				privateKeyIds[0].(string),
+				privateKeys[0].(string),
+				false,
+			)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Test rotation", func(t *testing.T) {
+			keyId, secret = testPluginOnCreateCatalog(
+				ctx,
+				t,
+				p,
+				zone,
+				projectId,
+				clientEmail.(string),
+				keyId,
+				secret,
+				true,
+			)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+	})
+
+	t.Run("OnUpdateCatalog", func(t *testing.T) {
+		t.Run("Test no-op non-rotation", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, "", "", false, false)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Switch to rotation", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, "", "", false, true)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Test no-op with rotation", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, "", "", true, true)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Switch credentials to next service account key. Don't rotate", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, privateKeyIds[1].(string), privateKeys[1].(string), true, false)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Switch credentials to next service account key. Add rotation", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, privateKeyIds[2].(string), privateKeys[2].(string), false, true)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Switch to next service account key, with rotation disabled", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, privateKeyIds[3].(string), privateKeys[3].(string), true, false)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+
+		t.Run("Switch to another service account key and keep rotation off", func(t *testing.T) {
+			keyId, secret = testPluginOnUpdateCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, privateKeyIds[4].(string), privateKeys[4].(string), false, false)
+			require.NotNil(keyId)
+			require.NotNil(secret)
+		})
+	})
+
+	t.Run("OnDeleteCatalog", func(t *testing.T) {
+		t.Run("Test non-rotated", func(t *testing.T) {
+			testPluginOnDeleteCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, false)
+		})
+
+		t.Run("Test as if we had rotated these credentials.", func(t *testing.T) {
+			testPluginOnDeleteCatalog(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, true)
+		})
+	})
+
+	t.Run("HostSet", func(t *testing.T) {
+		keyId, secret := privateKeyIds[5].(string), privateKeys[5].(string)
+		expectedLabelInstancesMap := make(map[string][]string)
+		for instanceId, instanceLabel := range instanceLabels {
+			for tagKey := range instanceLabel.(map[string]any) {
+				for _, expectedTag := range instanceLabels {
+					if tagKey == expectedTag {
+						expectedLabelInstancesMap[tagKey] = append(expectedLabelInstancesMap[tagKey], instanceId)
+					}
+				}
+			}
+		}
+
+		cases := map[string][]string{
+			"label-1": {"foo"},
+			"label-2": {"bar"},
+			"label-3": {"baz"},
+			"label-4": {"foo", "bar"},
+			"label-5": {"foo", "baz"},
+			"label-6": {"bar", "baz"},
+			"label-7": {"foo", "bar", "baz"},
+		}
+
+		for key, tc := range cases {
+			t.Run(fmt.Sprintf("Label=%s", key), func(t *testing.T) {
+				labels := make([]string, 0, len(tc))
+				for _, label := range tc {
+					labels = append(labels, fmt.Sprintf("labels.%s=%s", key, label))
+				}
+				testPluginOnCreateUpdateSet(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, labels)
+				testPluginListHosts(ctx, t, p, zone, projectId, clientEmail.(string), keyId, secret, labels, expectedLabelInstancesMap)
+			})
+		}
+	})
+}
+
+func testPluginOnCreateCatalog(
+	ctx context.Context,
+	t *testing.T,
+	p *host.HostPlugin,
+	zone string,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string,
+	rotate bool) (string, string) {
+	t.Helper()
+	t.Logf("testing OnCreateCatalog (zone=%s, rotate=%t)", zone, rotate)
+	require := require.New(t)
+
+	reqAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstClientEmail:               clientEmail,
+		credential.ConstProjectId:                 projectId,
+		credential.ConstZone:                      zone,
+		credential.ConstDisableCredentialRotation: !rotate,
+	})
+	require.NoError(err)
+	reqSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstPrivateKeyId: privateKeyId,
+		credential.ConstPrivateKey:   privateKey,
+	})
+	require.NoError(err)
+	request := &pb.OnCreateCatalogRequest{
+		Catalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: reqAttrs,
+			},
+			Secrets: reqSecrets,
+		},
+	}
+	response, err := p.OnCreateCatalog(ctx, request)
+	require.NoError(err)
+	require.NotNil(response)
+	persisted := response.GetPersisted()
+	require.NotNil(persisted)
+	return validatePersistedSecrets(t, persisted.GetSecrets(), projectId, clientEmail, privateKeyId, privateKey, rotate)
+}
+
+func testPluginOnUpdateCatalog(
+	ctx context.Context,
+	t *testing.T,
+	p *host.HostPlugin,
+	zone string,
+	projectId string,
+	clientEmail string,
+	currentPrivateKeyId string,
+	currentPrivateKey string,
+	newPrivateKeyId string,
+	newPrivateKey string,
+	rotated bool,
+	rotate bool,
+) (string, string) {
+	t.Helper()
+	t.Logf("testing OnUpdateCatalog (zone=%s, newcreds=%t, rotated=%t, rotate=%t)", zone, newPrivateKeyId != "" && newPrivateKey != "", rotated, rotate)
+	require := require.New(t)
+
+	// Take a timestamp of the current time to get a point in time to
+	// reference, ensuring that we are updating credential rotation
+	// timestamps.
+	currentCredsLastRotatedTime := time.Now()
+
+	reqCurrentAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstProjectId:                 projectId,
+		credential.ConstClientEmail:               clientEmail,
+		credential.ConstZone:                      zone,
+		credential.ConstDisableCredentialRotation: !rotated,
+	})
+	require.NoError(err)
+	reqNewAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstProjectId:                 projectId,
+		credential.ConstClientEmail:               clientEmail,
+		credential.ConstZone:                      zone,
+		credential.ConstDisableCredentialRotation: !rotate,
+	})
+	require.NoError(err)
+	var reqSecrets *structpb.Struct
+	if newPrivateKeyId != "" && newPrivateKey != "" {
+		reqSecrets, err = structpb.NewStruct(map[string]any{
+			credential.ConstPrivateKeyId: newPrivateKeyId,
+			credential.ConstPrivateKey:   newPrivateKey,
+		})
+		require.NoError(err)
+	}
+	reqPersistedSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstPrivateKeyId: currentPrivateKeyId,
+		credential.ConstPrivateKey:   currentPrivateKey,
+		credential.ConstCredsLastRotatedTime: func() string {
+			if rotated {
+				return currentCredsLastRotatedTime.Format(time.RFC3339Nano)
+			}
+
+			return (time.Time{}).Format(time.RFC3339Nano)
+		}(),
+	})
+	require.NoError(err)
+	require.NotNil(reqPersistedSecrets)
+	request := &pb.OnUpdateCatalogRequest{
+		CurrentCatalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: reqCurrentAttrs,
+			},
+		},
+		NewCatalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: reqNewAttrs,
+			},
+			Secrets: reqSecrets,
+		},
+		Persisted: &pb.HostCatalogPersisted{
+			Secrets: reqPersistedSecrets,
+		},
+	}
+	response, err := p.OnUpdateCatalog(ctx, request)
+	require.NoError(err)
+	require.NotNil(response)
+	persisted := response.GetPersisted()
+	require.NotNil(persisted)
+	return validateUpdateSecrets(
+		t,
+		projectId,
+		clientEmail,
+		persisted.GetSecrets(),
+		currentCredsLastRotatedTime,
+		currentPrivateKeyId,
+		currentPrivateKey,
+		newPrivateKeyId,
+		newPrivateKey,
+		rotated,
+		rotate,
+	)
+}
+
+func testPluginOnDeleteCatalog(
+	ctx context.Context,
+	t *testing.T,
+	p *host.HostPlugin,
+	zone string,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string,
+	rotated bool) {
+	t.Helper()
+	t.Logf("testing OnDeleteCatalog (zone=%s, rotated=%t)", zone, rotated)
+	require := require.New(t)
+
+	reqAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstProjectId:                 projectId,
+		credential.ConstClientEmail:               clientEmail,
+		credential.ConstZone:                      zone,
+		credential.ConstDisableCredentialRotation: !rotated,
+	})
+	require.NoError(err)
+	reqSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstPrivateKeyId: privateKeyId,
+		credential.ConstPrivateKey:   privateKey,
+	})
+	require.NoError(err)
+	reqPersistedSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstPrivateKeyId: privateKeyId,
+		credential.ConstPrivateKey:   privateKey,
+		credential.ConstCredsLastRotatedTime: func() string {
+			if rotated {
+				return time.Now().Format(time.RFC3339Nano)
+			}
+
+			return (time.Time{}).Format(time.RFC3339Nano)
+		}(),
+	})
+	require.NoError(err)
+	request := &pb.OnDeleteCatalogRequest{
+		Catalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: reqAttrs,
+			},
+			Secrets: reqSecrets,
+		},
+		Persisted: &pb.HostCatalogPersisted{
+			Secrets: reqPersistedSecrets,
+		},
+	}
+	response, err := p.OnDeleteCatalog(ctx, request)
+	require.NoError(err)
+	require.NotNil(response)
+
+	// We want to test the validity of the credentials post-deletion.
+	if rotated {
+		// The credentials should no longer be valid.
+		requireCredentialsInvalid(t, projectId, clientEmail, privateKeyId, privateKey)
+	} else {
+		// The credentials should still be valid. Sleep 10s first just to
+		// be sure, since we're not rotating.
+		time.Sleep(time.Second * 10)
+		requireCredentialsValid(t, projectId, clientEmail, privateKeyId, privateKey)
+	}
+}
+
+func testPluginOnCreateUpdateSet(
+	ctx context.Context,
+	t *testing.T,
+	p *host.HostPlugin,
+	zone string,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string,
+	labels []string) {
+	t.Helper()
+	t.Logf("testing OnCreateSet (zone=%s, labels=%v)", zone, labels)
+	require := require.New(t)
+	catalogAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstProjectId:                 projectId,
+		credential.ConstClientEmail:               clientEmail,
+		credential.ConstZone:                      zone,
+		credential.ConstDisableCredentialRotation: true, // Note that this does nothing in sets, but just noting for tests
+	})
+	require.NoError(err)
+	setAttrs, err := structpb.NewStruct(map[string]any{
+		host.ConstListInstancesFilter: []any{strings.Join(labels, " AND ")},
+	})
+	require.NoError(err)
+	reqPersistedSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstPrivateKeyId:         privateKeyId,
+		credential.ConstPrivateKey:           privateKey,
+		credential.ConstCredsLastRotatedTime: (time.Time{}).Format(time.RFC3339Nano),
+	})
+	require.NoError(err)
+	createRequest := &pb.OnCreateSetRequest{
+		Catalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: catalogAttrs,
+			},
+		},
+		Set: &hostsets.HostSet{
+			Attrs: &hostsets.HostSet_Attributes{
+				Attributes: setAttrs,
+			},
+		},
+		Persisted: &pb.HostCatalogPersisted{
+			Secrets: reqPersistedSecrets,
+		},
+	}
+	createResponse, err := p.OnCreateSet(ctx, createRequest)
+	require.NoError(err)
+	require.NotNil(createResponse)
+
+	// Do an update test in the same function, as it's pretty much the
+	// same function right now.
+	t.Logf("testing OnUpdateSet (zone=%s, labels=%v)", zone, labels)
+	updateRequest := &pb.OnUpdateSetRequest{
+		Catalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: catalogAttrs,
+			},
+		},
+		CurrentSet: &hostsets.HostSet{
+			Attrs: &hostsets.HostSet_Attributes{
+				Attributes: setAttrs,
+			},
+		},
+		NewSet: &hostsets.HostSet{
+			Attrs: &hostsets.HostSet_Attributes{
+				Attributes: setAttrs,
+			},
+		},
+		Persisted: &pb.HostCatalogPersisted{
+			Secrets: reqPersistedSecrets,
+		},
+	}
+	updateResponse, err := p.OnUpdateSet(ctx, updateRequest)
+	require.NoError(err)
+	require.NotNil(updateResponse)
+}
+
+func testPluginListHosts(
+	ctx context.Context,
+	t *testing.T,
+	p *host.HostPlugin,
+	zone string,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string,
+	labels []string,
+	expected map[string][]string) {
+	t.Helper()
+	t.Logf("testing ListHosts (zone=%s, labels=%v)", zone, labels)
+	require := require.New(t)
+	catalogAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstProjectId:                 projectId,
+		credential.ConstClientEmail:               clientEmail,
+		credential.ConstZone:                      zone,
+		credential.ConstDisableCredentialRotation: true, // Note that this does nothing in sets, but just noting for tests
+	})
+	require.NoError(err)
+	sets := make([]*hostsets.HostSet, len(labels))
+	for i, label := range labels {
+		setAttrs, err := structpb.NewStruct(map[string]any{
+			host.ConstListInstancesFilter: []any{label},
+		})
+		require.NoError(err)
+		sets[i] = &hostsets.HostSet{
+			Id: fmt.Sprintf("hostset-%d", i),
+			Attrs: &hostsets.HostSet_Attributes{
+				Attributes: setAttrs,
+			},
+		}
+	}
+	reqPersistedSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstPrivateKeyId:         privateKeyId,
+		credential.ConstPrivateKey:           privateKey,
+		credential.ConstCredsLastRotatedTime: (time.Time{}).Format(time.RFC3339Nano),
+	})
+	require.NoError(err)
+	request := &pb.ListHostsRequest{
+		Catalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: catalogAttrs,
+			},
+		},
+		Sets: sets,
+		Persisted: &pb.HostCatalogPersisted{
+			Secrets: reqPersistedSecrets,
+		},
+	}
+	response, err := p.ListHosts(ctx, request)
+	require.NoError(err)
+	require.NotNil(response)
+
+	// Validate the returned instances by ID. Assemble the instance
+	// details from the expected set.
+	expectedInstances := make(map[string][]string)
+	for i, label := range labels {
+		for _, instanceId := range expected[label] {
+			expectedInstances[instanceId] = append(expectedInstances[instanceId], fmt.Sprintf("hostset-%d", i))
+		}
+	}
+
+	// Take the returned hosts by ID and create the same kind of map.
+	actualInstances := make(map[string][]string)
+	for _, host := range response.GetHosts() {
+		actualInstances[host.ExternalId] = host.SetIds
+	}
+
+	// Compare
+	require.Equal(expectedInstances, actualInstances)
+	// Success
+	t.Logf("testing ListHosts: success (region=%s, tags=%v, expected/actual=(len=%d, ids=%s))", zone, labels, len(actualInstances), actualInstances)
+}

testing/terraform/main.tf

@@ -1,66 +0,0 @@
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: MPL-2.0
-
-data "google_compute_zones" "available" {}
-
-resource "google_compute_instance" "default" {
-  count        = var.num_instances
-  name         = "boundary-${count.index}"
-  machine_type = "n2-standard-2"
-  zone         = data.google_compute_zones.available.names[0]
-
-  labels = var.labels
-
-  boot_disk {
-    initialize_params {
-      image  = "ubuntu-os-cloud/ubuntu-2204-lts"
-      labels = var.labels
-    }
-  }
-
-  network_interface {
-    network = "default"
-
-    access_config {
-      // Ephemeral public IP
-    }
-  }
-
-  metadata = {
-    ssh-keys = "ubuntu:${tls_private_key.ssh.public_key_openssh}"
-  }
-
-  dynamic "service_account" {
-    for_each = var.service_account_email != null ? [var.service_account_email] : []
-    content {
-      email  = service_account.value
-      scopes = ["cloud-platform"]
-    }
-  }
-}
-
-resource "google_compute_instance_group" "webservers" {
-  name        = "boundary-servers"
-  description = "Boundary server group"
-
-  instances = google_compute_instance.default.*.id
-
-  zone = data.google_compute_zones.available.names[0]
-}
-
-# RSA key of size 4096 bits
-resource "tls_private_key" "ssh" {
-  algorithm = "RSA"
-  rsa_bits  = 4096
-}
-
-resource "google_compute_firewall" "rules" {
-  name    = "allow-ssh"
-  network = "default"
-
-  allow {
-    protocol = "tcp"
-    ports    = ["22"]
-  }
-  source_ranges = [var.client_cidr_block]
-}

testing/terraform/output.tf

@@ -1,8 +0,0 @@
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: MPL-2.0
-
-resource "local_file" "ssh_key" {
-  content         = tls_private_key.ssh.private_key_openssh
-  filename        = "${path.module}/secrets/id_rsa.pem"
-  file_permission = "0400"
-}

testing/terraform/variables.tf

@@ -1,29 +0,0 @@
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: MPL-2.0
-
-variable "labels" {
-  type = map(string)
-  default = {
-    repository = "boundary-plugin-gcp"
-    purpose    = "testing"
-  }
-}
-
-variable "region" {
-  type    = string
-  default = "us-central1"
-}
-
-variable "num_instances" {
-  type    = number
-  default = 3
-}
-
-variable "service_account_email" {
-  type    = string
-  default = null
-}
-
-variable "client_cidr_block" {
-  type = string
-}

testing/testdata/host/main.tf

@@ -0,0 +1,77 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resource "random_id" "service_account_client_email" {
+  prefix      = "test-${var.user_prefix}"
+  byte_length = 2
+}
+
+resource "google_service_account" "test_service_account" {
+  project      = var.project_id
+  account_id   = random_id.service_account_client_email.dec
+  display_name = "test-${var.user_prefix}"
+}
+
+resource "google_project_iam_binding" "test_service_account_user" {
+  project = var.project_id
+  count = length(var.rolesList)
+  role =  var.rolesList[count.index]
+  members = [
+    "serviceAccount:${google_service_account.test_service_account.email}"
+  ]
+}
+
+resource "google_service_account_key" "test_service_account_key" {
+  service_account_id  = google_service_account.test_service_account.name
+  public_key_type     = "TYPE_X509_PEM_FILE"
+  private_key_type    = "TYPE_GOOGLE_CREDENTIALS_FILE"
+  key_algorithm       = "KEY_ALG_RSA_2048"
+}
+
+resource "google_service_account_key" "test_service_account_keys" {
+  count               = var.num_service_account_keys
+  service_account_id  = google_service_account.test_service_account.name
+  public_key_type     = "TYPE_X509_PEM_FILE"
+  private_key_type    = "TYPE_GOOGLE_CREDENTIALS_FILE"
+  key_algorithm       = "KEY_ALG_RSA_2048"
+}
+
+resource "google_compute_instance" "instances" {
+  project      = var.project_id
+  count        = var.num_instances
+  name         = "boundary-${count.index}"
+  machine_type = "n2-standard-2"
+  zone         = var.zone
+  labels       = var.labels
+  tags         = ["test-instance-${count.index}"]
+
+  boot_disk {
+    initialize_params {
+      image  = "ubuntu-os-cloud/ubuntu-2204-lts"
+      labels = var.labels
+    }
+  }
+
+  network_interface {
+    network = "default"
+
+    access_config {
+      // Ephemeral public IP
+    }
+  }
+
+  metadata = {
+    ssh-keys = "ubuntu:${tls_private_key.ssh.public_key_openssh}"
+  }
+
+  service_account {
+    email  = google_service_account.test_service_account.email
+    scopes = ["cloud-platform"]
+  }
+}
+
+# RSA key of size 4096 bits
+resource "tls_private_key" "ssh" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}

testing/testdata/host/output.tf

@@ -0,0 +1,32 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+output "client_email" {
+  value = google_service_account.test_service_account.email
+}
+
+output "private_keys" {
+  value = [for key in google_service_account_key.test_service_account_keys : jsondecode(base64decode(key.private_key)).private_key]
+  sensitive = true
+}
+
+output "private_key_ids" {
+  value = [for key in google_service_account_key.test_service_account_keys : chomp(reverse(split("/", key.id))[0])]
+  sensitive = true
+}
+
+output "instance_ids" {
+  value = google_compute_instance.instances.*.id
+}
+
+output "instance_tags" {
+  value = {
+    for instance in google_compute_instance.instances : instance.id => instance.tags
+  }
+}
+
+output "instance_labels" {
+  value = {
+    for instance in google_compute_instance.instances : instance.id => instance.labels
+  }
+}

testing/testdata/host/providers.tf

@@ -12,4 +12,5 @@ terraform {
 
 provider "google" {
   region = var.region
+  project = var.project_id
 }

testing/testdata/host/variables.tf

@@ -0,0 +1,50 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+data "google_client_openid_userinfo" "current" {}
+
+variable "labels" {
+  type = map(string)
+  default = {
+    repository = "boundary-plugin-gcp"
+    purpose    = "testing"
+  }
+}
+
+variable "project_id" {
+  type = string
+}
+
+variable "region" {
+  type    = string
+}
+
+variable "zone" {
+  type    = string
+}
+
+variable "num_instances" {
+  type    = number
+  default = 3
+}
+
+variable "service_account_email" {
+  type    = string
+  default = null
+}
+
+variable "rolesList" {
+  type = list(string)
+  description = "List of roles to assign to the service account"
+  default = ["roles/compute.viewer","roles/iam.serviceAccountKeyAdmin"]
+}
+
+variable "user_prefix" {
+  type = string
+  default = "boundary-gcp-plugin"
+}
+
+variable "num_service_account_keys" {
+  type    = number
+  default = 6
+}

testing/testing.go

@@ -0,0 +1,513 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package testing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os/exec"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary-plugin-gcp/internal/credential"
+	"github.com/hashicorp/boundary-plugin-gcp/internal/values"
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+const (
+	// expectedServiceAccountKeys is the number of service account keys
+	// expected to be created in the Terraform project.
+	expectedServiceAccountKeys = 6
+)
+
+// TestTerraformer is an object representing the lifecycle of a
+// particular Terraform project.
+//
+// Note that Terraform must be installed in the system PATH (not to
+// be confused with the working directory below). This object does
+// not manage anything with regards to installation or version
+// management.
+type TestTerraformer struct {
+	// The full path to the Terraform binary, set during initialization
+	// to the binary found in the system path.
+	tfPath string
+
+	// The working directory for Terraform commands.
+	workDir string
+
+	// The Terraform state. Use the Output method to look up particular
+	// outputs from the Terraform state.
+	state *tfjson.State
+
+	// Keeps track of whether or not we have run the init command
+	// during this run of Terraform. This only needs to be done once.
+	// Do not use initErr directly; it stores the init error if one was
+	// encountered on init.
+	initOnce sync.Once
+	initErr  error
+}
+
+// NewTestTerraformer initializes a Terraform directory. It just
+// returns an initialized project, it does not perform any actions on
+// it.
+func NewTestTerraformer(workDir string) (*TestTerraformer, error) {
+	tfPath, err := exec.LookPath("terraform")
+	if err != nil {
+		return nil, fmt.Errorf("LookPath error: %w", err)
+	}
+
+	if workDir == "" {
+		return nil, errors.New("workDir is empty")
+	}
+
+	return &TestTerraformer{
+		tfPath:  tfPath,
+		workDir: workDir,
+	}, nil
+}
+
+// Deploy runs the following commands, in order: init, apply, plan,
+// json. The output of the JSON command is then imported to the
+// internal state.
+func (t *TestTerraformer) Deploy() error {
+	if err := t.init(); err != nil {
+		return fmt.Errorf("error running Terraform init: %w", err)
+	}
+
+	if err := t.apply(); err != nil {
+		return fmt.Errorf("error running Terraform apply: %w", err)
+	}
+
+	if err := t.plan(); err != nil {
+		return fmt.Errorf("error running Terraform plan: %w", err)
+	}
+
+	output, err := t.showJson()
+	if err != nil {
+		return fmt.Errorf("error getting Terraform plan JSON data: %w", err)
+	}
+
+	newState := new(tfjson.Plan)
+	if err := json.Unmarshal(output, newState); err != nil {
+		return fmt.Errorf("error reading new Terraform plan JSON data: %w", err)
+	}
+
+	t.state = newState.PriorState
+	return nil
+}
+
+// Destroy runs the following commands, in order: init, destroy. The
+// state is also cleared.
+func (t *TestTerraformer) Destroy() error {
+	if err := t.init(); err != nil {
+		return fmt.Errorf("error running Terraform init: %w", err)
+	}
+
+	if err := t.destroy(); err != nil {
+		return fmt.Errorf("error running Terraform destroy: %w", err)
+	}
+
+	t.state = nil
+	return nil
+}
+
+// GetOutput returns the value of the output located at key.
+//
+// Outputs for the root module are supported only. It's an error to
+// run this when no state exists.
+func (t *TestTerraformer) GetOutput(key string) (any, error) {
+	if key == "" {
+		return nil, errors.New("key must not be empty")
+	}
+
+	if t.state == nil {
+		return nil, errors.New("state is nil")
+	}
+
+	if t.state.Values == nil {
+		return nil, errors.New("state integrity error: no state values")
+	}
+
+	if t.state.Values.Outputs == nil {
+		return nil, errors.New("state integrity error: outputs is nil")
+	}
+
+	output, ok := t.state.Values.Outputs[key]
+	if !ok {
+		return nil, fmt.Errorf("output with key %q not found", key)
+	}
+
+	if output == nil {
+		return nil, fmt.Errorf("state integrity error: output at key %q is nil", key)
+	}
+
+	return output.Value, nil
+}
+
+// GetOutputString wraps GetOutput and returns the value as a string. It's an
+// error if the return value is actually not a string.
+func (t *TestTerraformer) GetOutputString(key string) (string, error) {
+	value, err := t.GetOutput(key)
+	if err != nil {
+		return "", err
+	}
+
+	str, ok := value.(string)
+	if !ok {
+		return "", fmt.Errorf("expected output value at key %q to be string, got %T", key, value)
+	}
+
+	return str, nil
+}
+
+// GetOutputMap wraps GetOutput and returns the value as a map. It's an
+// error if the return value is actually not a map.
+func (t *TestTerraformer) GetOutputMap(key string) (map[string]any, error) {
+	value, err := t.GetOutput(key)
+	if err != nil {
+		return nil, err
+	}
+
+	m, ok := value.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("expected output value at key %q to be map, got %T", key, value)
+	}
+
+	return m, nil
+}
+
+// GetOutputSlice wraps GetOutput and returns the value as a slice. It's an
+// error if the return value is actually not a slice.
+func (t *TestTerraformer) GetOutputSlice(key string) ([]any, error) {
+	value, err := t.GetOutput(key)
+	if err != nil {
+		return nil, err
+	}
+
+	m, ok := value.([]any)
+	if !ok {
+		return nil, fmt.Errorf("expected output value at key %q to be slice, got %T", key, value)
+	}
+
+	return m, nil
+}
+
+// init runs "terraform init" in the project's working directory.
+//
+// This is only ever done once.
+func (t *TestTerraformer) init() error {
+	t.initOnce.Do(func() {
+		cmd := exec.Command(t.tfPath, "init")
+		cmd.Dir = t.workDir
+		stdoutStderr, err := cmd.CombinedOutput()
+		if err != nil {
+			t.initErr = fmt.Errorf("%s\n%s", stdoutStderr, err)
+		}
+	})
+
+	return t.initErr
+}
+
+// apply runs "terraform apply" in the project's working directory.
+func (t *TestTerraformer) apply() error {
+	cmd := exec.Command(t.tfPath, "apply", "-input=false", "-auto-approve")
+	cmd.Dir = t.workDir
+	stdoutStderr, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s\n%s", stdoutStderr, err)
+	}
+
+	return nil
+}
+
+// destroy runs "terraform destroy" in the project's working directory.
+func (t *TestTerraformer) destroy() error {
+	cmd := exec.Command(t.tfPath, "destroy", "-input=false", "-auto-approve")
+	cmd.Dir = t.workDir
+	stdoutStderr, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s\n%s", stdoutStderr, err)
+	}
+
+	return nil
+}
+
+// plan runs "terraform plan" in the project's working directory.
+//
+// Note that plan is not run as part of the apply workflow; rather,
+// it's run to get the data for for importing the state via
+// terraform-json. The apply method ignores the plan generated here.
+func (t *TestTerraformer) plan() error {
+	cmd := exec.Command(t.tfPath, "plan", "-input=false", "-out=plan.tfplan")
+	cmd.Dir = t.workDir
+	stdoutStderr, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%s\n%s", stdoutStderr, err)
+	}
+
+	return nil
+}
+
+// showJson runs "terraform show -json". The output is returned along
+// with any error.
+//
+// Output is only the results of stdout and should be suitable for
+// parsing.
+//
+// This method expects the plan.tfplan file to be generated in the
+// working directory already. This can be done with the plan method.
+func (t *TestTerraformer) showJson() ([]byte, error) {
+	cmd := exec.Command(t.tfPath, "show", "-json", "plan.tfplan")
+	cmd.Dir = t.workDir
+	output, err := cmd.Output()
+	if err != nil {
+		exitErr, ok := err.(*exec.ExitError)
+		if !ok {
+			return nil, err
+		}
+
+		return nil, fmt.Errorf("%s\n%s", exitErr.Stderr, exitErr)
+	}
+
+	return output, nil
+}
+
+func requireCredentialsInvalid(
+	t *testing.T,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string) {
+	t.Helper()
+	require := require.New(t)
+
+	c, err := credential.NewConfig(
+		credential.WithProjectId(projectId),
+		credential.WithClientEmail(clientEmail),
+		credential.WithPrivateKeyId(privateKeyId),
+		credential.WithPrivateKey(privateKey))
+	require.NoError(err)
+
+	creds, err := c.GenerateCredentials(context.Background())
+	require.NoError(err)
+
+	// We need to wait for invalidation as while gcp waits for
+	// credential creation, deletion of the old credentials returns
+	// immediately.
+	timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+	defer cancel()
+waitErr:
+	for {
+		_, err = c.ValidateIamPermissions(timeoutCtx, []string{credential.ComputeInstancesListPermission}, option.WithTokenSource(creds.TokenSource))
+		if err != nil {
+			break
+		}
+
+		select {
+		case <-time.After(time.Second):
+			// pass
+
+		case <-timeoutCtx.Done():
+			break waitErr
+		}
+	}
+
+	require.Error(err)
+	require.Contains(err.Error(), "failed to validate IAM permissions")
+}
+
+func requireCredentialsValid(
+	t *testing.T,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string) {
+	t.Helper()
+	require := require.New(t)
+
+	c, err := credential.NewConfig(
+		credential.WithProjectId(projectId),
+		credential.WithClientEmail(clientEmail),
+		credential.WithPrivateKeyId(privateKeyId),
+		credential.WithPrivateKey(privateKey))
+	require.NoError(err)
+
+	creds, err := c.GenerateCredentials(context.Background())
+	require.NoError(err)
+
+	_, err = c.ValidateIamPermissions(context.Background(), []string{credential.ComputeInstancesListPermission}, option.WithTokenSource(creds.TokenSource))
+	require.NoError(err)
+}
+
+func validatePersistedSecrets(
+	t *testing.T,
+	persistedSecrets *structpb.Struct,
+	projectId string,
+	clientEmail string,
+	privateKeyId string,
+	privateKey string,
+	rotate bool) (string, string) {
+	t.Helper()
+	require := require.New(t)
+
+	require.NotNil(persistedSecrets)
+	persistedPrivateKeyId, err := values.GetStringValue(persistedSecrets, credential.ConstPrivateKeyId, true)
+	require.NoError(err)
+	require.NotZero(persistedPrivateKeyId)
+	if rotate {
+		require.NotEqual(privateKeyId, persistedPrivateKeyId)
+	} else {
+		require.Equal(privateKeyId, persistedPrivateKeyId)
+	}
+
+	persistedPrivateKey, err := values.GetStringValue(persistedSecrets, credential.ConstPrivateKey, true)
+	require.NoError(err)
+	require.NotZero(persistedPrivateKey)
+	if rotate {
+		require.NotEqual(privateKey, persistedPrivateKey)
+	} else {
+		require.Equal(privateKey, persistedPrivateKey)
+	}
+
+	persistedCredsLastRotatedTime, err := values.GetTimeValue(persistedSecrets, credential.ConstCredsLastRotatedTime)
+	require.NoError(err)
+	if rotate {
+		require.NotZero(persistedCredsLastRotatedTime)
+		requireCredentialsInvalid(t, projectId, clientEmail, privateKeyId, privateKey)
+	} else {
+		require.Zero(persistedCredsLastRotatedTime)
+	}
+
+	return persistedPrivateKeyId, persistedPrivateKey
+}
+
+func validateUpdateSecrets(
+	t *testing.T,
+	projectId string,
+	clientEmail string,
+	persistedSecrets *structpb.Struct,
+	currentCredsLastRotatedTime time.Time,
+	currentPrivateKeyId string,
+	currentPrivateKey string,
+	newPrivateKeyId string,
+	newPrivateKey string,
+	rotated bool,
+	rotate bool) (string, string) {
+	t.Helper()
+	require := require.New(t)
+
+	// Complex checks based on the scenarios.
+	persistedPrivateKeyId, err := values.GetStringValue(persistedSecrets, credential.ConstPrivateKeyId, true)
+	require.NoError(err)
+	require.NotZero(persistedPrivateKeyId)
+	persistedPrivateKey, err := values.GetStringValue(persistedSecrets, credential.ConstPrivateKey, true)
+	require.NoError(err)
+	require.NotZero(persistedPrivateKey)
+	persistedCredsLastRotatedTime, err := values.GetTimeValue(persistedSecrets, credential.ConstCredsLastRotatedTime)
+	require.NoError(err)
+
+	// Our test scenarios are complex due the multi-dimensional nature
+	// of criteria, so we lay them out in a switch below.
+	switch {
+	case newPrivateKeyId != "" && rotated && rotate:
+		// The new private key ID was provided, we had previously rotated
+		// the credentials before, and the new credential set is to be
+		// rotated as well. In this case, the old credentials should have
+		// been deleted, and the new credentials should have been rotated,
+		// hence, should not match the new credentials initially
+		// provided. Rotation time should be non-zero and updated.
+		requireCredentialsInvalid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.NotEqual(persistedPrivateKeyId, newPrivateKeyId)
+		require.NotEqual(persistedPrivateKey, newPrivateKey)
+		require.NotZero(persistedCredsLastRotatedTime)
+		require.True(persistedCredsLastRotatedTime.After(currentCredsLastRotatedTime))
+
+	case newPrivateKeyId != "" && rotated && !rotate:
+		// The new private key ID was provided, we had previously rotated
+		// the credentials before, and the new credential is *not*
+		// rotated. In this case, the old credentials should have
+		// been deleted, But the new credentials should have not been
+		// rotated, and hence should be the same. Rotation time should be
+		// zero.
+		requireCredentialsInvalid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.Equal(persistedPrivateKeyId, newPrivateKeyId)
+		require.Equal(persistedPrivateKey, newPrivateKey)
+		require.Zero(persistedCredsLastRotatedTime)
+
+	case newPrivateKeyId != "" && !rotated && rotate:
+		// The new private key was provided, we *have not* previously
+		// rotated the credentials, and the new credential set is to be
+		// rotated. In this case, the old credentials should have been
+		// left alone, and the new credentials should have been rotated,
+		// hence, should not match the new credentials initially
+		// provided. Rotation time should be non-zero, but updated.
+		requireCredentialsValid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.NotEqual(persistedPrivateKeyId, newPrivateKeyId)
+		require.NotEqual(persistedPrivateKey, newPrivateKey)
+		require.NotZero(persistedCredsLastRotatedTime)
+		require.True(persistedCredsLastRotatedTime.After(currentCredsLastRotatedTime))
+
+	case newPrivateKeyId != "" && !rotated && !rotate:
+		// The new private key was provided, but we have not rotated
+		// the credentials previously and we still don't plan on rotating
+		// them. In this case, the old credentials should still be valid,
+		// and the persisted ones should match the new ones provided.
+		// Rotation time should be zero.
+		requireCredentialsValid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.Equal(persistedPrivateKeyId, newPrivateKeyId)
+		require.Equal(persistedPrivateKey, newPrivateKey)
+		require.Zero(persistedCredsLastRotatedTime)
+
+	case newPrivateKeyId == "" && rotated && rotate:
+		// No new credentials have been provided, but we have previously
+		// rotated and are still rotating credentials. This is a no-op.
+		// Existing credentials should still be valid and match the ones
+		// persisted to state. Rotation time should be identical since
+		// no new rotation occurred.
+		requireCredentialsValid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.Equal(persistedPrivateKeyId, currentPrivateKeyId)
+		require.Equal(persistedPrivateKey, currentPrivateKey)
+		require.NotZero(persistedCredsLastRotatedTime)
+		require.True(currentCredsLastRotatedTime.Equal(persistedCredsLastRotatedTime))
+
+	case newPrivateKeyId == "" && rotated && !rotate:
+		// No new credentials have been provided, and we have previously
+		// rotated the credentials. This is actually an error, but we
+		// don't test it here; it's covered in unit testing (see
+		// plugin_test.go).
+		require.FailNow("testing rotated-to-not-rotated scenario not implemented by this helper")
+
+	case newPrivateKeyId == "" && !rotated && rotate:
+		// No new credentials have been provided, and while we did not
+		// rotate before, we want to switch to rotation. In this case,
+		// the existing persisted credentials should have been rotated,
+		// with a new non-zero timestamp.
+		requireCredentialsInvalid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.NotEqual(persistedPrivateKeyId, currentPrivateKeyId)
+		require.NotEqual(persistedPrivateKey, currentPrivateKey)
+		require.NotZero(persistedCredsLastRotatedTime)
+
+	case newPrivateKeyId == "" && !rotated && !rotate:
+		// No new credentials have been provided and we have not, nor do
+		// not, plan on rotating the credentials. This is a no-op.
+		// Existing credentials should still be valid and match the ones
+		// persisted to state. Rotation time should remain at zero.
+		requireCredentialsValid(t, projectId, clientEmail, currentPrivateKeyId, currentPrivateKey)
+		require.Equal(persistedPrivateKeyId, currentPrivateKeyId)
+		require.Equal(persistedPrivateKey, currentPrivateKey)
+		require.Zero(persistedCredsLastRotatedTime)
+
+	default:
+		// Scenario was reached that was not covered by this function.
+		require.FailNow("unknown test scenario")
+	}
+
+	return persistedPrivateKeyId, persistedPrivateKey
+}