chore: bump deps (#56)

This reverts commit 899c62ce06.

.gitignore

@@ -4,3 +4,9 @@ plan.tfplan
 *.log
 *.out
 *.html
+
+# IntelliJ IDEA project files
+.idea
+*.iprg
+
+.DS_Store

CODEOWNERS

@@ -1,5 +1,4 @@
 # These owners will be the default owners for everything in
 # the repo, unless a later match takes precedence.
 
-@hashicorp/boundary
-
+* @hashicorp/boundary

go.mod

@@ -1,6 +1,6 @@
 module github.com/hashicorp/boundary-plugin-aws
 
-go 1.21
+go 1.23.3
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.20.1
@@ -11,7 +11,7 @@ require (
 	github.com/aws/smithy-go v1.14.1
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.4.0
-	github.com/hashicorp/boundary/sdk v0.0.43-0.20240717182311-a20aae98794a
+	github.com/hashicorp/boundary/sdk v0.0.49
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.0.0
 	github.com/hashicorp/terraform-json v0.14.0

go.sum

@@ -78,8 +78,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/boundary/sdk v0.0.43-0.20240717182311-a20aae98794a h1:SwVze6sYE5o+J9qdcgj29auY1k6O8oSo9DC2+2Gb9rw=
-github.com/hashicorp/boundary/sdk v0.0.43-0.20240717182311-a20aae98794a/go.mod h1:9iOT7kDM6mYcSkKxNuZlv8rP7U5BG1kXoevjLLL8lNQ=
+github.com/hashicorp/boundary/sdk v0.0.49 h1:XOb6mSKyrU/wI20+5xTYBHQUP7eIeKcLxKSCpCs4yzM=
+github.com/hashicorp/boundary/sdk v0.0.49/go.mod h1:IHP79to8aIi22FiY58pgBqJL96/U9D8ZAUhS2DdC+Us=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

internal/credential/attributes.go

@@ -37,7 +37,7 @@ type CredentialAttributes struct {
 // CredentialsConfig used for configuring an AWS session. An status error is returned
 // with an InvalidArgument code if any unrecognized fields are found in the protobuf
 // struct input.
-func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes, required bool) (*awsutil.CredentialsConfig, error) {
+func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes) (*awsutil.CredentialsConfig, error) {
 	// initialize secrets if it is nil
 	// secrets can be nil because static credentials are optional
 	if secrets == nil {
@@ -49,13 +49,13 @@ func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes,
 	unknownFields := values.StructFields(secrets)
 	badFields := make(map[string]string)
 
-	accessKey, err := values.GetStringValue(secrets, ConstAccessKeyId, required)
+	accessKey, err := values.GetStringValue(secrets, ConstAccessKeyId, false)
 	if err != nil {
 		badFields[fmt.Sprintf("secrets.%s", ConstAccessKeyId)] = err.Error()
 	}
 	delete(unknownFields, ConstAccessKeyId)
 
-	secretKey, err := values.GetStringValue(secrets, ConstSecretAccessKey, required)
+	secretKey, err := values.GetStringValue(secrets, ConstSecretAccessKey, false)
 	if err != nil {
 		badFields[fmt.Sprintf("secrets.%s", ConstSecretAccessKey)] = err.Error()
 	}

internal/credential/attributes_test.go

@@ -105,7 +105,6 @@ func TestGetCredentialsConfig(t *testing.T) {
 	cases := []struct {
 		name                string
 		secrets             *structpb.Struct
-		required            bool
 		attrs               *CredentialAttributes
 		expected            *awsutil.CredentialsConfig
 		expectedErrContains string
@@ -136,32 +135,6 @@ func TestGetCredentialsConfig(t *testing.T) {
 				Region:    "us-west-2",
 			},
 		},
-		{
-			name: "missing access key",
-			secrets: &structpb.Struct{
-				Fields: map[string]*structpb.Value{
-					ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
-				},
-			},
-			attrs: &CredentialAttributes{
-				Region: "us-west-2",
-			},
-			required:            true,
-			expectedErrContains: "secrets.access_key_id: missing required value",
-		},
-		{
-			name: "missing secret key",
-			secrets: &structpb.Struct{
-				Fields: map[string]*structpb.Value{
-					ConstAccessKeyId: structpb.NewStringValue("AKIAfoobar"),
-				},
-			},
-			required: true,
-			attrs: &CredentialAttributes{
-				Region: "us-west-2",
-			},
-			expectedErrContains: "secrets.secret_access_key: missing required value",
-		},
 		{
 			name: "unknown fields",
 			secrets: &structpb.Struct{
@@ -260,7 +233,7 @@ func TestGetCredentialsConfig(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			require := require.New(t)
 
-			actual, err := GetCredentialsConfig(tc.secrets, tc.attrs, tc.required)
+			actual, err := GetCredentialsConfig(tc.secrets, tc.attrs)
 			if tc.expectedErrContains != "" {
 				require.Error(err)
 				require.Contains(err.Error(), tc.expectedErrContains)

internal/credential/state.go

@@ -114,7 +114,7 @@ func (s *AwsCredentialPersistedState) ValidateCreds(ctx context.Context) error {
 		return errors.BadRequestStatus("missing credentials config")
 	}
 	if _, err := s.CredentialsConfig.GetCallerIdentity(ctx, s.testOpts...); err != nil {
-		st, _ := errors.ParseAWSError(err, "validating credentials")
+		st, _ := errors.ParseAWSError("validating credentials", err)
 		return st.Err()
 	}
 	return nil
@@ -135,7 +135,7 @@ func (s *AwsCredentialPersistedState) RotateCreds(ctx context.Context) error {
 	if err := s.CredentialsConfig.RotateKeys(ctx, append([]awsutil.Option{
 		awsutil.WithValidityCheckTimeout(rotationWaitTimeout),
 	}, s.testOpts...)...); err != nil {
-		st, _ := errors.ParseAWSError(err, "error rotating credentials")
+		st, _ := errors.ParseAWSError("rotating credentials", err)
 		return st.Err()
 	}
 	s.CredsLastRotatedTime = time.Now()
@@ -192,7 +192,7 @@ func (s *AwsCredentialPersistedState) DeleteCreds(ctx context.Context) error {
 		}
 
 		// Otherwise treat it as an actual error.
-		st, _ := errors.ParseAWSError(err, "failed to delete access key")
+		st, _ := errors.ParseAWSError("deleting credentials", err)
 		return st.Err()
 	}
 
@@ -252,17 +252,17 @@ func AwsCredentialPersistedStateFromProto(secrets *structpb.Struct, attrs *Crede
 
 	accessKeyId, err := values.GetStringValue(secrets, ConstAccessKeyId, false)
 	if err != nil {
-		return nil, fmt.Errorf("persisted state integrity error: %w", err)
+		return nil, fmt.Errorf("persisted state integrity: %w", err)
 	}
 
 	secretAccessKey, err := values.GetStringValue(secrets, ConstSecretAccessKey, false)
 	if err != nil {
-		return nil, fmt.Errorf("persisted state integrity error: %w", err)
+		return nil, fmt.Errorf("persisted state integrity: %w", err)
 	}
 
 	credsLastRotatedTime, err := values.GetTimeValue(secrets, ConstCredsLastRotatedTime)
 	if err != nil {
-		return nil, fmt.Errorf("persisted state integrity error: %w", err)
+		return nil, fmt.Errorf("persisted state integrity: %w", err)
 	}
 
 	s, err := NewAwsCredentialPersistedState(opts...)

internal/credential/state_test.go

@@ -186,7 +186,7 @@ func TestAwsCredentialPersistedStateFromProto(t *testing.T) {
 			attrs: &CredentialAttributes{
 				Region: "us-west-2",
 			},
-			expectedErr: "persisted state integrity error: could not parse time in value \"creds_last_rotated_time\": parsing time \"notatime\" as \"2006-01-02T15:04:05.999999999Z07:00\": cannot parse \"notatime\" as \"2006\"",
+			expectedErr: "persisted state integrity: could not parse time in value \"creds_last_rotated_time\": parsing time \"notatime\" as \"2006-01-02T15:04:05.999999999Z07:00\": cannot parse \"notatime\" as \"2006\"",
 		},
 		{
 			name: "option error",
@@ -458,7 +458,7 @@ func TestAwsCatalogPersistedState_RotateCreds(t *testing.T) {
 					),
 				},
 			},
-			expectedErr:        "aws service unknown: unknown error: error rotating credentials",
+			expectedErr:        "aws service unknown: unknown error: rotating credentials",
 			expectedStatusCode: codes.Unknown,
 		},
 		{
@@ -568,7 +568,7 @@ func TestAwsCatalogPersistedState_ReplaceCreds(t *testing.T) {
 				},
 			},
 			credentialConfig:   &awsutil.CredentialsConfig{},
-			expectedErr:        "aws service unknown: unknown error: failed to delete access key",
+			expectedErr:        "aws service unknown: unknown error: deleting credentials",
 			expectedStatusCode: codes.Unknown,
 		},
 		{
@@ -693,7 +693,7 @@ func TestAwsCatalogPersistedState_DeleteCreds(t *testing.T) {
 					),
 				},
 			},
-			expectedErr:        "aws service unknown: unknown error: failed to delete access key",
+			expectedErr:        "aws service unknown: unknown error: deleting credentials",
 			expectedStatusCode: codes.Unknown,
 		},
 		{

internal/errors/error.go

@@ -81,11 +81,12 @@ func InvalidArgumentError(msg string, f map[string]string) error {
 // and credentials. This method will fallback to parsing the http status code
 // when it cannot match an aws error code. This method does not handle specific
 // service type errors such as S3 or EC2.
-func ParseAWSError(err error, msg string) (st *status.Status, permission *pb.Permission) {
+func ParseAWSError(op string, err error) (st *status.Status, permission *pb.Permission) {
 	if err == nil {
 		return nil, nil
 	}
 
+	msg := fmt.Sprintf("%s: %v", op, err)
 	// find the service name of the aws api
 	serviceName := "unknown"
 	var oe *smithy.OperationError
@@ -113,7 +114,7 @@ func ParseAWSError(err error, msg string) (st *status.Status, permission *pb.Per
 		Codes: retry.DefaultThrottleErrorCodes,
 	}
 	if throttleErr.IsErrorThrottle(err).Bool() {
-		statusMsg := fmt.Sprintf("aws service %s: throttling error: %s", serviceName, msg)
+		statusMsg := fmt.Sprintf("aws service %s: throttling: %s", serviceName, msg)
 		return status.New(codes.Unavailable, statusMsg), nil
 	}
 
@@ -126,31 +127,31 @@ func ParseAWSError(err error, msg string) (st *status.Status, permission *pb.Per
 		case awsErrorInvalidAccessKeyId:
 			fallthrough
 		case awsErrorExpiredToken:
-			statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: invalid credentials: %s", serviceName, msg)
 			return status.New(codes.PermissionDenied, statusMsg), &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: apiErr.ErrorMessage(),
 				CheckedAt:    timestamppb.Now(),
 			}
 		case awsErrorNoSuchBucket:
-			statusMsg := fmt.Sprintf("aws %s error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: %s", serviceName, msg)
 			return status.New(codes.NotFound, statusMsg), &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: apiErr.ErrorMessage(),
 				CheckedAt:    timestamppb.Now(),
 			}
 		case awsErrorBadDigest:
-			statusMsg := fmt.Sprintf("aws %s error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: %s", serviceName, msg)
 			return status.New(codes.Aborted, statusMsg), nil
 		case awsErrorNoSuchKey:
 			fallthrough
 		case awsErrorInvalidObjectState:
-			statusMsg := fmt.Sprintf("aws %s error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: %s", serviceName, msg)
 			return status.New(codes.NotFound, statusMsg), nil
 		case awsErrorRequestTimeout:
 			fallthrough
 		case awsErrorRequestTimeoutException:
-			statusMsg := fmt.Sprintf("aws %s error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: %s", serviceName, msg)
 			return status.New(codes.DeadlineExceeded, statusMsg), nil
 		}
 	}
@@ -165,39 +166,39 @@ func ParseAWSError(err error, msg string) (st *status.Status, permission *pb.Per
 		defer httpErr.Response.Body.Close()
 		switch httpErr.HTTPStatusCode() {
 		case http.StatusBadRequest:
-			statusMsg := fmt.Sprintf("aws service %s: bad request error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: bad request: %s", serviceName, msg)
 			return status.New(codes.InvalidArgument, statusMsg), nil
 		case http.StatusUnauthorized:
-			statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: invalid credentials: %s", serviceName, msg)
 			return status.New(codes.PermissionDenied, statusMsg), &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: buf.String(),
 				CheckedAt:    timestamppb.Now(),
 			}
 		case http.StatusForbidden:
-			statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: invalid credentials: %s", serviceName, msg)
 			return status.New(codes.PermissionDenied, statusMsg), &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: buf.String(),
 				CheckedAt:    timestamppb.Now(),
 			}
 		case http.StatusNotFound:
-			statusMsg := fmt.Sprintf("aws service %s: resource not found error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: resource not found: %s", serviceName, msg)
 			return status.New(codes.NotFound, statusMsg), nil
 		case http.StatusTooManyRequests:
-			statusMsg := fmt.Sprintf("aws service %s: throttling error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: throttling: %s", serviceName, msg)
 			return status.New(codes.Unavailable, statusMsg), nil
 		case http.StatusInternalServerError:
 			fallthrough
 		case http.StatusBadGateway:
 			fallthrough
 		case http.StatusServiceUnavailable:
-			statusMsg := fmt.Sprintf("aws service %s: connectivity error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: connectivity: %s", serviceName, msg)
 			return status.New(codes.Unavailable, statusMsg), nil
 		case http.StatusRequestTimeout:
 			fallthrough
 		case http.StatusGatewayTimeout:
-			statusMsg := fmt.Sprintf("aws service %s: timeout error: %s", serviceName, msg)
+			statusMsg := fmt.Sprintf("aws service %s: timeout: %s", serviceName, msg)
 			return status.New(codes.DeadlineExceeded, statusMsg), nil
 		}
 	}

internal/errors/error_test.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package errors
 
 import (
@@ -41,7 +44,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-bad-request",
 			err:                TestAwsHttpResponseError(http.StatusBadRequest, "bad request"),
 			expectedStatusCode: codes.InvalidArgument,
-			expectedStatusMsg:  "aws service unknown: bad request error: test",
+			expectedStatusMsg:  "aws service unknown: bad request: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -50,7 +53,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-unauthorized",
 			err:                TestAwsHttpResponseError(http.StatusUnauthorized, "unauthorized request"),
 			expectedStatusCode: codes.PermissionDenied,
-			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedStatusMsg:  "aws service unknown: invalid credentials: test",
 			expectedPermission: &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: "unauthorized request",
@@ -61,7 +64,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-forbidden",
 			err:                TestAwsHttpResponseError(http.StatusForbidden, "forbidden request"),
 			expectedStatusCode: codes.PermissionDenied,
-			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedStatusMsg:  "aws service unknown: invalid credentials: test",
 			expectedPermission: &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: "forbidden request",
@@ -72,7 +75,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-not-found",
 			err:                TestAwsHttpResponseError(http.StatusNotFound, "not found request"),
 			expectedStatusCode: codes.NotFound,
-			expectedStatusMsg:  "aws service unknown: resource not found error: test",
+			expectedStatusMsg:  "aws service unknown: resource not found: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -81,7 +84,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-too-many-requests",
 			err:                TestAwsHttpResponseError(http.StatusTooManyRequests, "too many requests"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -90,7 +93,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-request-timeout",
 			err:                TestAwsHttpResponseError(http.StatusRequestTimeout, "request timeout"),
 			expectedStatusCode: codes.DeadlineExceeded,
-			expectedStatusMsg:  "aws service unknown: timeout error: test",
+			expectedStatusMsg:  "aws service unknown: timeout: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -99,7 +102,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-internal-server-error",
 			err:                TestAwsHttpResponseError(http.StatusInternalServerError, "internal server error"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedStatusMsg:  "aws service unknown: connectivity: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -108,7 +111,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-bad-gateway",
 			err:                TestAwsHttpResponseError(http.StatusBadGateway, "bad gateway"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedStatusMsg:  "aws service unknown: connectivity: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -117,7 +120,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-service-unavailable",
 			err:                TestAwsHttpResponseError(http.StatusServiceUnavailable, "service unavailable"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedStatusMsg:  "aws service unknown: connectivity: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -126,7 +129,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "http-status-code-gateway-timeout",
 			err:                TestAwsHttpResponseError(http.StatusGatewayTimeout, "gateway timeout"),
 			expectedStatusCode: codes.DeadlineExceeded,
-			expectedStatusMsg:  "aws service unknown: timeout error: test",
+			expectedStatusMsg:  "aws service unknown: timeout: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -135,7 +138,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-throttling",
 			err:                TestAwsError("Throttling", "throttling error"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -144,7 +147,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-throttling-exception",
 			err:                TestAwsError("ThrottlingException", "throttling exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -153,7 +156,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-throttled-exception",
 			err:                TestAwsError("ThrottledException", "throttled exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -162,7 +165,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-throttled-exception",
 			err:                TestAwsError("RequestThrottledException", "request throttled exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -171,7 +174,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-too-many-requests-exception",
 			err:                TestAwsError("TooManyRequestsException", "too many requests exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -180,7 +183,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-provisioned-throughput-exceeded-exception",
 			err:                TestAwsError("ProvisionedThroughputExceededException", "provisioned throughput exceeded exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -189,7 +192,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-transaction-in-progress-exception",
 			err:                TestAwsError("TransactionInProgressException", "transaction in progress exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -198,7 +201,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-limit-exceeded",
 			err:                TestAwsError("RequestLimitExceeded", "request limit exceeded"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -207,7 +210,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-bandwidth-limit-exceeded",
 			err:                TestAwsError("BandwidthLimitExceeded", "bandwidth limit exceeded"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -216,7 +219,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-limit-exceeded-exception",
 			err:                TestAwsError("LimitExceededException", "limit exceeded exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -225,7 +228,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-throttled",
 			err:                TestAwsError("RequestThrottled", "request throttled"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -234,7 +237,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-slow-down",
 			err:                TestAwsError("SlowDown", "slow down"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -243,7 +246,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-prior-request-not-complete",
 			err:                TestAwsError("PriorRequestNotComplete", "prior request not complete"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -252,7 +255,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-ec2-throttled-exception",
 			err:                TestAwsError("EC2ThrottledException", "ec2 throttled exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedStatusMsg:  "aws service unknown: throttling: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -261,7 +264,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-timeout",
 			err:                TestAwsError("RequestTimeout", "request timeout"),
 			expectedStatusCode: codes.DeadlineExceeded,
-			expectedStatusMsg:  "aws unknown error: test",
+			expectedStatusMsg:  "aws service unknown: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -270,7 +273,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-timeout-exception",
 			err:                TestAwsError("RequestTimeoutException", "request timeout exception"),
 			expectedStatusCode: codes.DeadlineExceeded,
-			expectedStatusMsg:  "aws unknown error: test",
+			expectedStatusMsg:  "aws service unknown: test",
 			expectedPermission: &pb.Permission{
 				State: pb.StateType_STATE_TYPE_UNKNOWN,
 			},
@@ -279,7 +282,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-access-denied",
 			err:                TestAwsError(awsErrorAccessDenied, "not authorized to perform action"),
 			expectedStatusCode: codes.PermissionDenied,
-			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedStatusMsg:  "aws service unknown: invalid credentials: test",
 			expectedPermission: &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: "not authorized to perform action",
@@ -290,7 +293,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-invalid-access-key-id",
 			err:                TestAwsError(awsErrorInvalidAccessKeyId, "The AWS Access Key Id you provided does not exist in our records."),
 			expectedStatusCode: codes.PermissionDenied,
-			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedStatusMsg:  "aws service unknown: invalid credentials: test",
 			expectedPermission: &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: "The AWS Access Key Id you provided does not exist in our records.",
@@ -301,7 +304,7 @@ func Test_ParseAWSError(t *testing.T) {
 			name:               "aws-request-expired-token",
 			err:                TestAwsError(awsErrorExpiredToken, "The security token included in the request is expired."),
 			expectedStatusCode: codes.PermissionDenied,
-			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedStatusMsg:  "aws service unknown: invalid credentials: test",
 			expectedPermission: &pb.Permission{
 				State:        pb.StateType_STATE_TYPE_ERROR,
 				ErrorDetails: "The security token included in the request is expired.",
@@ -313,14 +316,14 @@ func Test_ParseAWSError(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			require, assert := require.New(t), assert.New(t)
 
-			actualStatus, permission := ParseAWSError(tt.err, "test")
+			actualStatus, permission := ParseAWSError("test", tt.err)
 			if tt.expectedStatusCode == codes.OK {
 				require.Nil(actualStatus)
 				return
 			}
 
 			require.Equal(tt.expectedStatusCode, actualStatus.Code())
-			require.Equal(tt.expectedStatusMsg, actualStatus.Message())
+			require.Contains(actualStatus.Message(), tt.expectedStatusMsg)
 			require.Len(actualStatus.Details(), 0)
 			CheckSimilarPermission(assert, tt.expectedPermission, permission, true)
 		})

internal/errors/testing.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package errors
 
 import (

plugin/plugin.go

@@ -32,6 +32,6 @@ type AwsPlugin struct {
 func NewAwsPlugin() *AwsPlugin {
 	return &AwsPlugin{
 		HostPlugin:    &host.HostPlugin{},
-		StoragePlugin: &storage.StoragePlugin{},
+		StoragePlugin: storage.New(),
 	}
 }

plugin/service/host/README.md

@@ -108,6 +108,14 @@ The following `attributes` are valid on an AWS host catalog resource:
   be performed. See the [Credential Rotation](../../../README.md#credential-rotation) readme for more information.
 - `region` (string): The region to configure the host catalog for. All host sets
   in this catalog will be configured for this region.
+- `role_arn` (string): The AWS Role ARN used for `AssumeRole` authentication. If
+  provided, `disable_credential_rotation` must be also set to `true.
+- `role_external_id` (string):  The external id configured for the `AssumeRole`
+  provider.
+- `role_session_name` (string): The session name configured for the `AssumeRole`
+  provider.
+- `role_tags` (object): The key-value pair tags configured for the `AssumeRole`
+  provider.
 
 The following `secrets` are required on an AWS host catalog resource:
 

plugin/service/host/plugin.go

@@ -5,22 +5,19 @@ package host
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
-	"github.com/aws/smithy-go"
+	"github.com/hashicorp/boundary-plugin-aws/internal/credential"
+	"github.com/hashicorp/boundary-plugin-aws/internal/errors"
 	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/structpb"
-
-	"github.com/hashicorp/boundary-plugin-aws/internal/credential"
-	cred "github.com/hashicorp/boundary-plugin-aws/internal/credential"
 )
 
 // HostPlugin implements the HostPluginServiceServer interface for the
@@ -42,24 +39,19 @@ var _ pb.HostPluginServiceServer = (*HostPlugin)(nil)
 func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalogRequest) (*pb.OnCreateCatalogResponse, error) {
 	catalog := req.GetCatalog()
 	if catalog == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog is nil")
-	}
-
-	secrets := catalog.GetSecrets()
-	if secrets == nil {
-		return nil, status.Error(codes.InvalidArgument, "secrets are required")
+		return nil, errors.BadRequestStatus("catalog is required")
 	}
 
 	attrs := catalog.GetAttributes()
 	if attrs == nil {
-		return nil, status.Error(codes.InvalidArgument, "attributes are required")
+		return nil, errors.BadRequestStatus("attributes are required")
 	}
 
 	catalogAttributes, err := getCatalogAttributes(attrs)
 	if err != nil {
 		return nil, err
 	}
-	credConfig, err := cred.GetCredentialsConfig(secrets, catalogAttributes.CredentialAttributes, true)
+	credConfig, err := credential.GetCredentialsConfig(catalog.GetSecrets(), catalogAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -71,7 +63,14 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 		}, p.testCredStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
+	}
+
+	// Try to rotate AWS static credentials
+	if credential.GetCredentialType(credState.CredentialsConfig) == credential.StaticAWS && !catalogAttributes.DisableCredentialRotation {
+		if err := credState.RotateCreds(ctx); err != nil {
+			return nil, err
+		}
 	}
 
 	catalogState, err := newAwsCatalogPersistedState(
@@ -80,25 +79,17 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 		}, p.testCatalogStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
 	}
 
-	if cred.GetCredentialType(credState.CredentialsConfig) == cred.StaticAWS {
-		if !catalogAttributes.DisableCredentialRotation {
-			if err := credState.RotateCreds(ctx); err != nil {
-				return nil, status.Errorf(codes.InvalidArgument, "error during credential rotation: %s", err)
-			}
-		} else {
-			// Simply validate if we aren't rotating.
-			if err := credState.ValidateCreds(ctx); err != nil {
-				return nil, status.Errorf(codes.InvalidArgument, "error during credential validation: %s", err)
-			}
-		}
+	// perform dry run to ensure we can interact with AWS as expected.
+	if st := dryRunValidation(ctx, catalogState); st != nil {
+		return nil, st.Err()
 	}
 
 	persistedProto, err := catalogState.toProto()
 	if err != nil {
-		return nil, status.Error(codes.InvalidArgument, err.Error())
+		return nil, errors.BadRequestStatus(err.Error())
 	}
 
 	return &pb.OnCreateCatalogResponse{
@@ -111,43 +102,100 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 	// For host catalogs we only need to really validate what is
 	// currently in the new copy of the host catalog data, so skip
 	// fetching the new copy to keep things a little less messy.
-	catalog := req.GetNewCatalog()
-	if catalog == nil {
-		return nil, status.Error(codes.InvalidArgument, "new catalog is nil")
+	currentCatalog := req.GetCurrentCatalog()
+	if currentCatalog == nil {
+		return nil, errors.BadRequestStatus("current catalog is required")
+	}
+	newCatalog := req.GetNewCatalog()
+	if newCatalog == nil {
+		return nil, errors.BadRequestStatus("new catalog is required")
 	}
 
-	var updateSecrets bool
-	secrets := catalog.GetSecrets()
-	if secrets != nil {
-		// We will be updating secrets this run, but what exactly that
-		// means will be determined later.
-		updateSecrets = true
+	oldAttrs := currentCatalog.GetAttributes()
+	if oldAttrs == nil {
+		return nil, errors.BadRequestStatus("current catalog attributes are required")
 	}
-
-	attrs := catalog.GetAttributes()
-	if attrs == nil {
-		return nil, status.Error(codes.InvalidArgument, "new catalog missing attributes")
+	oldCatalogAttributes, err := getCatalogAttributes(oldAttrs)
+	if err != nil {
+		return nil, err
 	}
-
-	catalogAttributes, err := getCatalogAttributes(attrs)
+	newAttrs := newCatalog.GetAttributes()
+	if newAttrs == nil {
+		return nil, errors.BadRequestStatus("new catalog attributes are required")
+	}
+	newCatalogAttributes, err := getCatalogAttributes(newAttrs)
 	if err != nil {
 		return nil, err
 	}
 
-	// Get the persisted data.
-	// NOTE: We might need to change this at a later time. I'm not too
-	// sure *exactly* what scenarios we might encounter that would
-	// ultimately mean that we would have to handle an empty or missing
-	// state in update, but what we are ultimately assuming here
-	// (implicitly through awsCatalogPersistedStateFromProto) is that
-	// the state will exist and be populated. Personally I think this
-	// is fine and important, but this may change in the future.
 	credState, err := credential.AwsCredentialPersistedStateFromProto(
 		req.GetPersisted().GetSecrets(),
-		catalogAttributes.CredentialAttributes,
+		oldCatalogAttributes.CredentialAttributes,
 		p.testCredStateOpts...)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
+	}
+
+	// Verify the incoming credentials are valid and return any errors to the
+	// user if they're not. Note this doesn't validate the credentials against
+	// AWS - it only does logical validation on the fields.
+	updatedCredentials, err := credential.GetCredentialsConfig(newCatalog.GetSecrets(), newCatalogAttributes.CredentialAttributes)
+	if err != nil {
+		return nil, err
+	}
+
+	// We will be updating credentials when one of two changes occur:
+	// 1. the new catalog has secrets for static credentials, in which case we need to delete the old static credentials
+	// 2. the new catalog has a different roleARN value for dynamic credentials
+	if newCatalog.GetSecrets() != nil || newCatalogAttributes.RoleArn != oldCatalogAttributes.RoleArn {
+		// Ensure the incoming credentials are valid for interaction with the S3
+		// Bucket before replacing them.
+		newCredState, err := credential.NewAwsCredentialPersistedState(
+			append([]credential.AwsCredentialPersistedStateOption{
+				credential.WithCredentialsConfig(updatedCredentials),
+			}, p.testCredStateOpts...)...,
+		)
+		if err != nil {
+			return nil, errors.BadRequestStatus("error setting up new credential persisted state: %s", err)
+		}
+		newCatalogState, err := newAwsCatalogPersistedState(
+			append([]awsCatalogPersistedStateOption{
+				withCredentials(newCredState),
+			}, p.testCatalogStateOpts...)...,
+		)
+		if err != nil {
+			return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
+		}
+		if st := dryRunValidation(ctx, newCatalogState); st != nil {
+			return nil, st.Err()
+		}
+
+		// Replace the existing credential state.
+		// This checks the timestamp on the last rotation time as well
+		// and deletes the credentials if we are managing them
+		// (ie: if we've rotated them before).
+		if err := credState.ReplaceCreds(ctx, updatedCredentials); err != nil {
+			return nil, err
+		}
+	}
+
+	if credential.GetCredentialType(credState.CredentialsConfig) == credential.StaticAWS {
+		// This is a validate check to make sure that we aren't disabling
+		// rotation for credentials currently being managed by rotation.
+		// This is not allowed.
+		if newCatalogAttributes.DisableCredentialRotation && newCatalog.GetSecrets() == nil {
+			if !credState.CredsLastRotatedTime.IsZero() {
+				return nil, errors.BadRequestStatus("cannot disable rotation for already-rotated credentials")
+			}
+		}
+
+		// If we're enabling rotation now but didn't before, or have
+		// freshly replaced credentials, we can rotate here.
+		if !newCatalogAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
+			if err := credState.RotateCreds(ctx); err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	catalogState, err := newAwsCatalogPersistedState(
@@ -156,45 +204,17 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		}, p.testCatalogStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
-	if cred.GetCredentialType(credState.CredentialsConfig) == cred.StaticAWS {
-		if catalogAttributes.DisableCredentialRotation && !updateSecrets {
-			// This is a validate check to make sure that we aren't disabling
-			// rotation for credentials currently being managed by rotation.
-			// This is not allowed.
-			if !credState.CredsLastRotatedTime.IsZero() {
-				return nil, status.Error(codes.FailedPrecondition, "cannot disable rotation for already-rotated credentials")
-			}
-		}
-
-		if updateSecrets {
-			catalogSecrets, err := credential.GetCredentialsConfig(secrets, catalogAttributes.CredentialAttributes, true)
-			if err != nil {
-				return nil, err
-			}
-
-			// Replace the credentials. This checks the timestamp on the last
-			// rotation time as well and deletes the credentials if we are
-			// managing them (ie: if we've rotated them before).
-			if err := credState.ReplaceCreds(ctx, catalogSecrets); err != nil {
-				return nil, status.Errorf(codes.InvalidArgument, "error attempting to replace credentials: %s", err)
-			}
-		}
-
-		if !catalogAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
-			// If we're enabling rotation now but didn't before, or have
-			// freshly replaced credentials, we can rotate here.
-			if err := credState.RotateCreds(ctx); err != nil {
-				return nil, status.Errorf(codes.InvalidArgument, "error during credential rotation: %s", err)
-			}
-		}
+	// perform dry run to ensure we can interact with AWS as expected.
+	if st := dryRunValidation(ctx, catalogState); st != nil {
+		return nil, st.Err()
 	}
 
 	persistedProto, err := catalogState.toProto()
 	if err != nil {
-		return nil, status.Error(codes.InvalidArgument, err.Error())
+		return nil, errors.BadRequestStatus(err.Error())
 	}
 
 	return &pb.OnUpdateCatalogResponse{
@@ -206,12 +226,12 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 func (p *HostPlugin) OnDeleteCatalog(ctx context.Context, req *pb.OnDeleteCatalogRequest) (*pb.OnDeleteCatalogResponse, error) {
 	catalog := req.GetCatalog()
 	if catalog == nil {
-		return nil, status.Error(codes.InvalidArgument, "new catalog is nil")
+		return nil, errors.BadRequestStatus("catalog is required")
 	}
 
 	attrs := catalog.GetAttributes()
 	if attrs == nil {
-		return nil, status.Error(codes.InvalidArgument, "new catalog missing attributes")
+		return nil, errors.BadRequestStatus("catalog attributes are required")
 	}
 
 	catalogAttributes, err := getCatalogAttributes(attrs)
@@ -230,7 +250,7 @@ func (p *HostPlugin) OnDeleteCatalog(ctx context.Context, req *pb.OnDeleteCatalo
 		catalogAttributes.CredentialAttributes,
 		p.testCredStateOpts...)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	_, err = newAwsCatalogPersistedState(
@@ -239,17 +259,17 @@ func (p *HostPlugin) OnDeleteCatalog(ctx context.Context, req *pb.OnDeleteCatalo
 		}, p.testCatalogStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	// try to delete static credentials
-	if cred.GetCredentialType(credState.CredentialsConfig) == cred.StaticAWS {
+	if credential.GetCredentialType(credState.CredentialsConfig) == credential.StaticAWS {
 		if !credState.CredsLastRotatedTime.IsZero() {
 			// Delete old/existing credentials. This is done with the same
 			// credentials to ensure that it has the proper permissions to do
 			// it.
 			if err := credState.DeleteCreds(ctx); err != nil {
-				return nil, status.Errorf(codes.Aborted, "error removing rotated credentials during catalog deletion: %s", err)
+				return nil, err
 			}
 		}
 	}
@@ -288,12 +308,12 @@ func (p *HostPlugin) NormalizeSetData(ctx context.Context, req *pb.NormalizeSetD
 func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest) (*pb.OnCreateSetResponse, error) {
 	catalog := req.GetCatalog()
 	if catalog == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog is nil")
+		return nil, errors.BadRequestStatus("catalog is required")
 	}
 
 	catalogAttrsRaw := catalog.GetAttributes()
 	if catalogAttrsRaw == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog missing attributes")
+		return nil, errors.BadRequestStatus("catalog attributes are required")
 	}
 
 	catalogAttributes, err := getCatalogAttributes(catalogAttrsRaw)
@@ -306,7 +326,7 @@ func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest
 		catalogAttributes.CredentialAttributes,
 		p.testCredStateOpts...)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	catalogState, err := newAwsCatalogPersistedState(
@@ -315,56 +335,43 @@ func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest
 		}, p.testCatalogStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	set := req.GetSet()
 	if set == nil {
-		return nil, status.Error(codes.InvalidArgument, "set is nil")
+		return nil, errors.BadRequestStatus("set is required")
 	}
 
 	if set.GetAttributes() == nil {
-		return nil, status.Error(codes.InvalidArgument, "set missing attributes")
+		return nil, errors.BadRequestStatus("set attributes are required")
 	}
 	setAttrs, err := getSetAttributes(set.GetAttributes())
 	if err != nil {
 		return nil, err
 	}
 
-	ec2Client, err := catalogState.EC2Client(ctx)
+	describeInstanceFilters, err := buildFilters(setAttrs)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error getting EC2 client: %s", err)
+		return nil, errors.BadRequestStatus("error building set filters: %s", err)
 	}
 
-	input, err := buildDescribeInstancesInput(setAttrs, true)
-	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error building DescribeInstances parameters: %s", err)
+	if st := dryRunValidation(ctx, catalogState, describeInstanceFilters...); st != nil {
+		return nil, st.Err()
 	}
-
-	_, err = ec2Client.DescribeInstances(ctx, input)
-	if err == nil {
-		return nil, status.Error(codes.FailedPrecondition, "query error: DescribeInstances DryRun should have returned error, but none was found")
-	}
-
-	var apiErr smithy.APIError
-	if errors.As(err, &apiErr) && apiErr.ErrorCode() == "DryRunOperation" {
-		// Success
-		return &pb.OnCreateSetResponse{}, nil
-	}
-
-	return nil, status.Errorf(codes.InvalidArgument, "error performing dry run of DescribeInstances: %s", err)
+	return &pb.OnCreateSetResponse{}, nil
 }
 
 // OnUpdateSet is called when a dynamic host set is updated.
 func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest) (*pb.OnUpdateSetResponse, error) {
 	catalog := req.GetCatalog()
 	if catalog == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog is nil")
+		return nil, errors.BadRequestStatus("catalog is required")
 	}
 
 	catalogAttrsRaw := catalog.GetAttributes()
 	if catalogAttrsRaw == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog missing attributes")
+		return nil, errors.BadRequestStatus("catalog attributes are required")
 	}
 
 	catalogAttributes, err := getCatalogAttributes(catalogAttrsRaw)
@@ -377,7 +384,7 @@ func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest
 		catalogAttributes.CredentialAttributes,
 		p.testCredStateOpts...)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	catalogState, err := newAwsCatalogPersistedState(
@@ -386,46 +393,34 @@ func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest
 		}, p.testCatalogStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	// As with catalog, we don't need to really look at the old host
 	// set here, just need to work off of/validate the new set config
 	set := req.GetNewSet()
 	if set == nil {
-		return nil, status.Error(codes.InvalidArgument, "new set is nil")
+		return nil, errors.BadRequestStatus("new set is required")
 	}
 
 	if set.GetAttributes() == nil {
-		return nil, status.Error(codes.InvalidArgument, "new set missing attributes")
+		return nil, errors.BadRequestStatus("new set attributes are required")
 	}
 	setAttrs, err := getSetAttributes(set.GetAttributes())
 	if err != nil {
 		return nil, err
 	}
 
-	ec2Client, err := catalogState.EC2Client(ctx)
+	describeInstanceFilters, err := buildFilters(setAttrs)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error getting EC2 client: %s", err)
+		return nil, errors.BadRequestStatus("error building set filters: %s", err)
 	}
 
-	input, err := buildDescribeInstancesInput(setAttrs, true)
-	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error building DescribeInstances parameters: %s", err)
+	if st := dryRunValidation(ctx, catalogState, describeInstanceFilters...); st != nil {
+		return nil, st.Err()
 	}
 
-	_, err = ec2Client.DescribeInstances(ctx, input)
-	if err == nil {
-		return nil, status.Error(codes.FailedPrecondition, "query error: DescribeInstances DryRun should have returned error, but none was found")
-	}
-
-	var apiErr smithy.APIError
-	if errors.As(err, &apiErr) && apiErr.ErrorCode() == "DryRunOperation" {
-		// Success
-		return &pb.OnUpdateSetResponse{}, nil
-	}
-
-	return nil, status.Errorf(codes.InvalidArgument, "error performing dry run of DescribeInstances: %s", err)
+	return &pb.OnUpdateSetResponse{}, nil
 }
 
 // OnDeleteSet is called when a dynamic host set is deleted.
@@ -439,12 +434,12 @@ func (p *HostPlugin) OnDeleteSet(ctx context.Context, req *pb.OnDeleteSetRequest
 func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*pb.ListHostsResponse, error) {
 	catalog := req.GetCatalog()
 	if catalog == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog is nil")
+		return nil, errors.BadRequestStatus("catalog is required")
 	}
 
 	catalogAttrsRaw := catalog.GetAttributes()
 	if catalogAttrsRaw == nil {
-		return nil, status.Error(codes.InvalidArgument, "catalog missing attributes")
+		return nil, errors.BadRequestStatus("catalog attributes are required")
 	}
 
 	catalogAttributes, err := getCatalogAttributes(catalogAttrsRaw)
@@ -457,7 +452,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 		catalogAttributes.CredentialAttributes,
 		p.testCredStateOpts...)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	catalogState, err := newAwsCatalogPersistedState(
@@ -466,12 +461,12 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 		}, p.testCatalogStateOpts...)...,
 	)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
+		return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
 	}
 
 	sets := req.GetSets()
 	if sets == nil {
-		return nil, status.Error(codes.InvalidArgument, "sets is nil")
+		return nil, errors.BadRequestStatus("sets are required")
 	}
 
 	// Build all the queries in advance.
@@ -486,11 +481,11 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 	for i, set := range sets {
 		// Validate Id since we use it in output
 		if set.GetId() == "" {
-			return nil, status.Error(codes.InvalidArgument, "set missing id")
+			return nil, errors.BadRequestStatus("set id is required")
 		}
 
 		if set.GetAttributes() == nil {
-			return nil, status.Error(codes.InvalidArgument, "set missing attributes")
+			return nil, errors.BadRequestStatus("set %s attributes are required", set.GetId())
 		}
 		setAttrs, err := getSetAttributes(set.GetAttributes())
 		if err != nil {
@@ -499,7 +494,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 
 		input, err := buildDescribeInstancesInput(setAttrs, false)
 		if err != nil {
-			return nil, status.Errorf(codes.InvalidArgument, "error building DescribeInstances parameters for host set id %q: %s", set.GetId(), err)
+			return nil, errors.BadRequestStatus("error building DescribeInstances parameters for host set id %q: %s", set.GetId(), err)
 		}
 		queries[i] = hostSetQuery{
 			Id:    set.GetId(),
@@ -509,7 +504,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 
 	ec2Client, err := catalogState.EC2Client(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "error getting EC2 client: %s", err)
+		return nil, errors.BadRequestStatus("error getting EC2 client: %s", err)
 	}
 
 	// Run all queries now and assemble output.
@@ -517,7 +512,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 	for i, query := range queries {
 		output, err := ec2Client.DescribeInstances(ctx, query.Input)
 		if err != nil {
-			return nil, status.Errorf(codes.InvalidArgument, "error running DescribeInstances for host set id %q: %s", query.Id, err)
+			return nil, errors.BadRequestStatus("error running DescribeInstances for host set id %q: %s", query.Id, err)
 		}
 
 		queries[i].Output = output
@@ -528,7 +523,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
 			for _, instance := range reservation.Instances {
 				host, err := awsInstanceToHost(instance)
 				if err != nil {
-					return nil, status.Errorf(codes.InvalidArgument, "error processing host results for host set id %q: %s", query.Id, err)
+					return nil, errors.BadRequestStatus("error processing host results for host set id %q: %s", query.Id, err)
 				}
 
 				queries[i].OutputHosts = append(queries[i].OutputHosts, host)
@@ -626,7 +621,7 @@ func buildDescribeInstancesInput(attrs *SetAttributes, dryRun bool) (*ec2.Descri
 // populated.
 func awsInstanceToHost(instance types.Instance) (*pb.ListHostsResponseHost, error) {
 	if aws.ToString(instance.InstanceId) == "" {
-		return nil, errors.New("response integrity error: missing instance id")
+		return nil, fmt.Errorf("response integrity error: missing instance id")
 	}
 
 	result := new(pb.ListHostsResponseHost)
@@ -674,6 +669,28 @@ func awsInstanceToHost(instance types.Instance) (*pb.ListHostsResponseHost, erro
 	return result, nil
 }
 
+// dryRunValidation performs an AWS DescribeInstances call to verify the state's
+// credentials, the host listing functionality as well as the filters, if any
+// are passed in. This function can therefore be used for both host catalog and
+// host set validation.
+func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, filters ...types.Filter) *status.Status {
+	if state == nil {
+		return status.New(codes.InvalidArgument, "persisted state is required")
+	}
+
+	ec2Client, err := state.EC2Client(ctx)
+	if err != nil {
+		return status.New(codes.InvalidArgument, fmt.Sprintf("error getting EC2 client: %s", err))
+	}
+
+	_, err = ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{Filters: filters})
+	if err != nil {
+		return status.New(codes.FailedPrecondition, fmt.Sprintf("aws describe instances failed: %s", err))
+	}
+
+	return nil
+}
+
 // appendDistinct will append the elements to the slice
 // if an element is not nil, empty, and does not exist in slice.
 func appendDistinct(slice []string, elems ...*string) []string {

plugin/service/storage/error.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package storage
 
 import (
@@ -47,12 +50,12 @@ func setStatePermission(req any, permission *pb.Permission) *pb.StorageBucketCre
 // credentials, and connectivity error types.
 //
 // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-func parseS3Error(err error, msg string, req any) *status.Status {
+func parseS3Error(op string, err error, req any) *status.Status {
 	if err == nil {
 		return nil
 	}
 
-	st, p := errors.ParseAWSError(err, msg)
+	st, p := errors.ParseAWSError(op, err)
 	if p != nil {
 		state := setStatePermission(req, p)
 		if st, err = st.WithDetails(state); err != nil {

plugin/service/storage/error_test.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package storage
 
 import (
@@ -31,7 +34,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.GetObjectRequest{},
 			err:                TestAwsS3Error("NoSuchBucket", "GetObject", "no such bucket"),
 			expectedStatusCode: codes.NotFound,
-			expectedStatusMsg:  "aws s3 error: test",
+			expectedStatusMsg:  "aws service s3: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -47,7 +50,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.GetObjectRequest{},
 			err:                TestAwsS3Error("NoSuchKey", "GetObject", "no such key"),
 			expectedStatusCode: codes.NotFound,
-			expectedStatusMsg:  "aws s3 error: test",
+			expectedStatusMsg:  "aws service s3: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -63,7 +66,7 @@ func Test_ParseS3Error(t *testing.T) {
 			err:                errors.TestAwsHttpResponseError(404, "not found"),
 			expectedStatusCode: codes.NotFound,
 			// cannot embed services within mock http errors, therefore unknown service
-			expectedStatusMsg: "aws service unknown: resource not found error: test",
+			expectedStatusMsg: "aws service unknown: resource not found: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -78,7 +81,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.GetObjectRequest{},
 			err:                TestAwsS3Error("InvalidObjectState", "GetObject", "invalid object state"),
 			expectedStatusCode: codes.NotFound,
-			expectedStatusMsg:  "aws s3 error: test",
+			expectedStatusMsg:  "aws service s3: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -93,7 +96,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.PutObjectRequest{},
 			err:                TestAwsS3Error(s3ErrorBadDigest, "PutObject", "checksum mismatch"),
 			expectedStatusCode: codes.Aborted,
-			expectedStatusMsg:  "aws s3 error: test",
+			expectedStatusMsg:  "aws service s3: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Write: &pb.Permission{
@@ -108,7 +111,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.GetObjectRequest{},
 			err:                TestAwsS3Error("Throttling", "action", "throttling exception"),
 			expectedStatusCode: codes.Unavailable,
-			expectedStatusMsg:  "aws service s3: throttling error: test",
+			expectedStatusMsg:  "aws service s3: throttling: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -123,7 +126,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.GetObjectRequest{},
 			err:                TestAwsS3Error("RequestTimeout", "action", "request timeout"),
 			expectedStatusCode: codes.DeadlineExceeded,
-			expectedStatusMsg:  "aws s3 error: test",
+			expectedStatusMsg:  "aws service s3: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -138,7 +141,7 @@ func Test_ParseS3Error(t *testing.T) {
 			req:                &pb.GetObjectRequest{},
 			err:                TestAwsS3Error("AccessDenied", "action", "access denied"),
 			expectedStatusCode: codes.PermissionDenied,
-			expectedStatusMsg:  "aws service s3: invalid credentials error: test",
+			expectedStatusMsg:  "aws service s3: invalid credentials: test:",
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
 					Read: &pb.Permission{
@@ -154,14 +157,14 @@ func Test_ParseS3Error(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			require, assert := require.New(t), assert.New(t)
 
-			actualStatus := parseS3Error(tt.err, "test", tt.req)
+			actualStatus := parseS3Error("test", tt.err, tt.req)
 			if tt.expectedStatusCode == codes.OK {
 				require.Nil(actualStatus)
 				return
 			}
 
 			require.Equal(tt.expectedStatusCode, actualStatus.Code())
-			require.Equal(tt.expectedStatusMsg, actualStatus.Message())
+			require.Contains(actualStatus.Message(), tt.expectedStatusMsg)
 			require.Len(actualStatus.Details(), 1)
 
 			foundDetail := false

plugin/service/storage/plugin.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"sync"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -32,6 +33,14 @@ import (
 // Ensure that we are implementing StoragePluginServiceServer
 var _ pb.StoragePluginServiceServer = (*StoragePlugin)(nil)
 
+// clientCache caches previously initialized S3API clients created with a RoleARN attribute.
+type clientCache struct {
+	// cache is a map of cached s3 clients. The key of the map
+	// is the public id of the storage bucket
+	cache map[string]S3API
+	sync.RWMutex
+}
+
 // StoragePlugin implements the StoragePluginServiceServer interface for the
 // AWS storage service plugin.
 type StoragePlugin struct {
@@ -42,6 +51,112 @@ type StoragePlugin struct {
 
 	// testStorageStateOpts are passed in to the stored state to control test behavior
 	testStorageStateOpts []awsStoragePersistedStateOption
+
+	clients *clientCache
+}
+
+// New creates a new StoragePlugin
+func New() *StoragePlugin {
+	c := &clientCache{
+		cache: make(map[string]S3API),
+	}
+	return &StoragePlugin{
+		clients: c,
+	}
+}
+
+// getClient returns an S3API client for the given storage bucket id.
+func (p *StoragePlugin) getClient(ctx context.Context,
+	storageBucketId string,
+	storageState *awsStoragePersistedState,
+	opt ...s3Option) (S3API, error) {
+	if storageBucketId == "" {
+		// No storage bucket ID to key cache on, create new client and return
+		client, err := storageState.s3Client(ctx, opt...)
+		if err != nil {
+			return nil, errors.BadRequestStatus("error creating S3 client: %s", err)
+		}
+		return client, nil
+	}
+
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing options in cache: %w", err)
+	}
+
+	if opts.withCacheRefresh {
+		// Forced cache refresh
+		p.clients.Lock()
+		defer p.clients.Unlock()
+		client, err := storageState.s3Client(ctx, opt...)
+		if err != nil {
+			return nil, errors.BadRequestStatus("error creating S3 client: %s", err)
+		}
+		p.clients.cache[storageBucketId] = client
+		return client, nil
+	}
+
+	p.clients.RLock()
+	client, ok := p.clients.cache[storageBucketId]
+	p.clients.RUnlock()
+
+	if !ok || client.Credentials().Expired() {
+		// We got a cache miss or the credentials has expired, time to refresh
+		p.clients.Lock()
+		defer p.clients.Unlock()
+
+		// Check cache again in case another caller updated it since we got the lock
+		client, ok = p.clients.cache[storageBucketId]
+		if ok && !client.Credentials().Expired() {
+			// Got a cache hit with valid credentials, return cached client
+			return client, nil
+		}
+
+		// Create new client and cache it
+		var err error
+		client, err = storageState.s3Client(ctx, opt...)
+		if err != nil {
+			return nil, errors.BadRequestStatus("error creating S3 client: %s", err)
+		}
+		p.clients.cache[storageBucketId] = client
+	}
+	return client, nil
+}
+
+type s3Caller func(client S3API) (any, error)
+
+// call gets the s3client and passes it to the s3caller function, if a permission error is returned
+// it forces a cache refresh and tries again.
+func (p *StoragePlugin) call(
+	ctx context.Context,
+	fn s3Caller,
+	storageBucketId string,
+	storageState *awsStoragePersistedState,
+	opts ...s3Option) (any, error) {
+
+	s3Client, err := p.getClient(ctx, storageBucketId, storageState, opts...)
+	if err != nil {
+		return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
+	}
+	firstAttempt := true
+	for {
+		resp, err := fn(s3Client)
+		if err != nil {
+			st, _ := errors.ParseAWSError("", err)
+			if st.Code() == codes.PermissionDenied && firstAttempt {
+				// We got a permission error, this is the first time so refresh cache and try again
+				firstAttempt = false
+				opts = append(opts, WithCacheRefresh(true))
+				s3Client, err = p.getClient(ctx, storageBucketId, storageState, opts...)
+				if err != nil {
+					return nil, errors.BadRequestStatus("error refreshing S3 client: %s", err)
+				}
+				continue
+			}
+			return nil, err
+		}
+		return resp, nil
+	}
 }
 
 // OnCreateStorageBucket is called when a storage bucket is created.
@@ -64,7 +179,7 @@ func (p *StoragePlugin) OnCreateStorageBucket(ctx context.Context, req *pb.OnCre
 		return nil, err
 	}
 
-	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
+	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -155,7 +270,7 @@ func (p *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUpd
 	// Verify the incoming credentials are valid and return any errors to the
 	// user if they're not. Note this doesn't validate the credentials against
 	// AWS - it only does logical validation on the fields.
-	updatedCredentials, err := cred.GetCredentialsConfig(newBucket.GetSecrets(), newStorageAttributes.CredentialAttributes, false)
+	updatedCredentials, err := cred.GetCredentialsConfig(newBucket.GetSecrets(), newStorageAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -318,7 +433,7 @@ func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectReques
 		return nil, err
 	}
 
-	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
+	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -346,19 +461,20 @@ func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectReques
 	if storageAttributes.EndpointUrl != "" {
 		opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
 	}
-	s3Client, err := storageState.S3Client(ctx, opts...)
-	if err != nil {
-		return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
-	}
 
 	objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
-	resp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
-		Bucket: aws.String(bucket.GetBucketName()),
-		Key:    aws.String(objectKey),
-	})
-	if err != nil {
-		return nil, parseS3Error(err, "failed to head object", req).Err()
+	headCall := func(s3Client S3API) (any, error) {
+		return s3Client.HeadObject(ctx, &s3.HeadObjectInput{
+			Bucket: aws.String(bucket.GetBucketName()),
+			Key:    aws.String(objectKey),
+		})
 	}
+	headResp, err := p.call(ctx, headCall, bucket.GetId(), storageState, opts...)
+	if err != nil {
+		return nil, parseS3Error("head object", err, req).Err()
+	}
+	resp := headResp.(*s3.HeadObjectOutput)
+
 	return &pb.HeadObjectResponse{
 		ContentLength: resp.ContentLength,
 		LastModified:  timestamppb.New(*resp.LastModified),
@@ -386,7 +502,7 @@ func (p *StoragePlugin) ValidatePermissions(ctx context.Context, req *pb.Validat
 		return nil, err
 	}
 
-	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
+	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -444,7 +560,7 @@ func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePlu
 		return err
 	}
 
-	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
+	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes)
 	if err != nil {
 		return err
 	}
@@ -472,19 +588,19 @@ func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePlu
 	if storageAttributes.EndpointUrl != "" {
 		opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
 	}
-	s3Client, err := storageState.S3Client(stream.Context(), opts...)
-	if err != nil {
-		return errors.BadRequestStatus("error getting S3 client: %s", err)
-	}
 
 	objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
-	resp, err := s3Client.GetObject(stream.Context(), &s3.GetObjectInput{
-		Bucket: aws.String(bucket.GetBucketName()),
-		Key:    aws.String(objectKey),
-	})
-	if err != nil {
-		return parseS3Error(err, "failed to get object", req).Err()
+	getCall := func(s3Client S3API) (any, error) {
+		return s3Client.GetObject(stream.Context(), &s3.GetObjectInput{
+			Bucket: aws.String(bucket.GetBucketName()),
+			Key:    aws.String(objectKey),
+		})
 	}
+	getResp, err := p.call(stream.Context(), getCall, bucket.GetId(), storageState, opts...)
+	if err != nil {
+		return parseS3Error("get object", err, req).Err()
+	}
+	resp := getResp.(*s3.GetObjectOutput)
 
 	defer resp.Body.Close()
 	reader := bufio.NewReader(resp.Body)
@@ -555,7 +671,7 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
 		return nil, err
 	}
 
-	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
+	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -583,10 +699,6 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
 	if storageAttributes.EndpointUrl != "" {
 		opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
 	}
-	s3Client, err := storageState.S3Client(ctx, opts...)
-	if err != nil {
-		return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
-	}
 
 	hash := sha256.New()
 	if _, err := io.Copy(hash, file); err != nil {
@@ -598,18 +710,23 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
 	if err != nil {
 		return nil, errors.UnknownStatus("failed to rewind file pointer")
 	}
-
 	objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
-	resp, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
-		Bucket:            aws.String(bucket.GetBucketName()),
-		Key:               aws.String(objectKey),
-		Body:              file,
-		ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
-		ChecksumSHA256:    aws.String(checksum),
-	})
-	if err != nil {
-		return nil, parseS3Error(err, "failed to put object", req).Err()
+
+	putCall := func(s3Client S3API) (any, error) {
+		return s3Client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket:            aws.String(bucket.GetBucketName()),
+			Key:               aws.String(objectKey),
+			Body:              file,
+			ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
+			ChecksumSHA256:    aws.String(checksum),
+		})
 	}
+	putResp, err := p.call(ctx, putCall, bucket.GetId(), storageState, opts...)
+	if err != nil {
+		return nil, parseS3Error("put object", err, req).Err()
+	}
+	resp := putResp.(*s3.PutObjectOutput)
+
 	if resp.ChecksumSHA256 == nil {
 		return nil, errors.UnknownStatus("missing checksum response from aws")
 	}
@@ -650,7 +767,7 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects
 		return nil, err
 	}
 
-	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
+	credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -678,10 +795,6 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects
 	if storageAttributes.EndpointUrl != "" {
 		opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
 	}
-	client, err := storageState.S3Client(ctx, opts...)
-	if err != nil {
-		return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
-	}
 
 	prefix := path.Join(bucket.GetBucketPrefix(), req.GetKeyPrefix())
 	if strings.HasSuffix(req.GetKeyPrefix(), "/") {
@@ -691,12 +804,19 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects
 	}
 
 	if !req.Recursive {
-		_, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
-			Bucket: aws.String(bucket.GetBucketName()),
-			Key:    aws.String(prefix),
-		})
+		deleteCall := func(s3Client S3API) (any, error) {
+			return s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(bucket.GetBucketName()),
+				Key:    aws.String(prefix),
+			})
+		}
+		_, err := p.call(ctx, deleteCall, bucket.GetId(), storageState, opts...)
 		if err != nil {
-			return nil, parseS3Error(err, "failed to delete object", req).Err()
+			return nil, parseS3Error("delete object", err, req).Err()
+		}
+
+		if err != nil {
+			return nil, parseS3Error("delete object", err, req).Err()
 		}
 		return &pb.DeleteObjectsResponse{
 			ObjectsDeleted: uint32(1),
@@ -704,10 +824,13 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects
 	}
 
 	const maxkeys = 1000
-
 	objects := []types.ObjectIdentifier{}
+	client, err := p.getClient(ctx, bucket.GetId(), storageState, opts...)
+	if err != nil {
+		return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
+	}
 	var conToken *string
-	truncated := true
+	truncated, firstAttempt := true, true
 	for truncated {
 		res, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
 			Bucket:            aws.String(bucket.GetBucketName()),
@@ -716,7 +839,19 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects
 			ContinuationToken: conToken,
 		})
 		if err != nil {
-			return nil, parseS3Error(err, "failed to list objects", req).Err()
+			st, _ := errors.ParseAWSError("", err)
+			if st.Code() == codes.PermissionDenied && firstAttempt {
+				// This is first attempt and we get a permission error, refresh cache and try again
+				firstAttempt = false
+				opts = append(opts, WithCacheRefresh(true))
+				client, err = p.getClient(ctx, bucket.GetId(), storageState, opts...)
+				if err != nil {
+					return nil, errors.BadRequestStatus("error refreshing S3 client: %s", err)
+				}
+				continue
+			}
+
+			return nil, parseS3Error("list objects", err, req).Err()
 		}
 		truncated = res.IsTruncated
 		conToken = res.NextContinuationToken
@@ -738,7 +873,7 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects
 			},
 		})
 		if err != nil {
-			return nil, parseS3Error(err, "failed to delete objects", req).Err()
+			return nil, parseS3Error("delete objects", err, req).Err()
 		}
 		deleted += len(res.Deleted)
 	}
@@ -770,7 +905,7 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr
 		opts = append(opts, WithEndpoint(attrs.EndpointUrl))
 	}
 
-	client, err := state.S3Client(ctx, opts...)
+	client, err := state.s3Client(ctx, opts...)
 	if err != nil {
 		return status.New(codes.InvalidArgument, fmt.Sprintf("error getting S3 client: %s", err))
 	}
@@ -786,7 +921,7 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr
 		Body:   bytes.NewReader([]byte("hashicorp boundary aws plugin access test")),
 	}); err != nil {
 		var st *status.Status
-		st, permissions.Write = errors.ParseAWSError(err, "failed to put object")
+		st, permissions.Write = errors.ParseAWSError("put object", err)
 		errs = multierror.Append(errs, st.Err())
 	} else {
 		permissions.Write = &pb.Permission{State: pb.StateType_STATE_TYPE_OK, CheckedAt: timestamppb.Now()}
@@ -797,7 +932,7 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr
 		Key:    aws.String(objectKey),
 	}); err != nil {
 		var st *status.Status
-		st, permissions.Read = errors.ParseAWSError(err, "failed to get object")
+		st, permissions.Read = errors.ParseAWSError("get object", err)
 		errs = multierror.Append(errs, st.Err())
 	} else {
 		permissions.Read = &pb.Permission{State: pb.StateType_STATE_TYPE_OK, CheckedAt: timestamppb.Now()}
@@ -808,7 +943,7 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr
 		Key:    aws.String(objectKey),
 	}); err != nil {
 		var st *status.Status
-		st, permissions.Read = errors.ParseAWSError(err, "failed to head object")
+		st, permissions.Read = errors.ParseAWSError("head object", err)
 		errs = multierror.Append(errs, st.Err())
 	}
 
@@ -817,10 +952,11 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr
 		Prefix: aws.String(objectKey),
 	}); err != nil {
 		var st *status.Status
-		st, permissions.Read = errors.ParseAWSError(err, "failed to list object")
+		st, permissions.Read = errors.ParseAWSError("list object", err)
 		errs = multierror.Append(errs, st.Err())
 	} else if res == nil || len(res.Contents) != 1 || *res.Contents[0].Key != objectKey {
-		return status.New(codes.Aborted, fmt.Sprintf("list response did not contain the expected key: %+v", res))
+		permissions.Read = &pb.Permission{State: pb.StateType_STATE_TYPE_UNKNOWN, CheckedAt: timestamppb.Now()}
+		errs = multierror.Append(errs, status.New(codes.Aborted, fmt.Sprintf("list response did not contain the expected key: %+v", res)).Err())
 	}
 
 	if _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
@@ -828,7 +964,7 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr
 		Key:    aws.String(objectKey),
 	}); err != nil {
 		var st *status.Status
-		st, permissions.Delete = errors.ParseAWSError(err, "failed to delete object")
+		st, permissions.Delete = errors.ParseAWSError("delete object", err)
 		errs = multierror.Append(errs, st.Err())
 	} else {
 		permissions.Delete = &pb.Permission{State: pb.StateType_STATE_TYPE_OK, CheckedAt: timestamppb.Now()}

plugin/service/storage/plugin_test.go

@@ -153,7 +153,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					),
 				}),
 			},
-			expectedErrContains: "aws service unknown: unknown error: error rotating credentials",
+			expectedErrContains: "aws service unknown: unknown error: rotating credentials:",
 			expectedErrCode:     codes.Unknown,
 			expectedDetails:     nil,
 		},
@@ -178,7 +178,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					}),
 				)),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to put object",
+			expectedErrContains: "aws service s3: invalid credentials: put object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -211,7 +211,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					}),
 				)),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to put object",
+			expectedErrContains: "aws service s3: throttling: put object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -245,7 +245,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to get object",
+			expectedErrContains: "aws service s3: invalid credentials: get object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -280,7 +280,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to get object",
+			expectedErrContains: "aws service s3: throttling: get object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -314,7 +314,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to head object",
+			expectedErrContains: "aws service s3: invalid credentials: head object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -349,7 +349,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to head object",
+			expectedErrContains: "aws service s3: throttling: head object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -384,7 +384,7 @@ func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to delete object",
+			expectedErrContains: "aws service s3: invalid credentials: delete object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -857,7 +857,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				}),
 			},
-			expectedErrContains: "aws service unknown: unknown error: failed to delete access key",
+			expectedErrContains: "aws service unknown: unknown error: deleting credentials:",
 			expectedErrCode:     codes.Unknown,
 		},
 		{
@@ -896,7 +896,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				}),
 			},
-			expectedErrContains: "aws service unknown: unknown error: error rotating credentials",
+			expectedErrContains: "aws service unknown: unknown error: rotating credentials",
 			expectedErrCode:     codes.Unknown,
 		},
 		{
@@ -920,7 +920,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					testMockS3WithPutObjectError(TestAwsS3Error("AccessDenied", "PutObject", "not authorized")),
 				)),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to put object",
+			expectedErrContains: "aws service s3: invalid credentials: put object:",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -953,7 +953,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					testMockS3WithPutObjectError(TestAwsS3Error("ThrottlingException", "PutObject", "throttling exception")),
 				)),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to put object",
+			expectedErrContains: "aws service s3: throttling: put object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -987,7 +987,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to get object",
+			expectedErrContains: "aws service s3: invalid credentials: get object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1022,7 +1022,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to get object",
+			expectedErrContains: "aws service s3: throttling: get object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1056,7 +1056,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to head object",
+			expectedErrContains: "aws service s3: invalid credentials: head object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1091,7 +1091,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to head object",
+			expectedErrContains: "aws service s3: throttling: head object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1255,7 +1255,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to head object",
+			expectedErrContains: "aws service s3: invalid credentials: head object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1363,7 +1363,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to put object",
+			expectedErrContains: "aws service s3: invalid credentials: put object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1486,7 +1486,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				}),
 			},
-			expectedErrContains: "aws service unknown: unknown error: failed to delete access key",
+			expectedErrContains: "aws service unknown: unknown error: deleting credentials",
 			expectedErrCode:     codes.Unknown,
 		},
 		{
@@ -1599,7 +1599,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to put object",
+			expectedErrContains: "aws service s3: invalid credentials: put object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -1738,7 +1738,7 @@ func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) {
 					),
 				}),
 			},
-			expectedErrContains: "aws service unknown: unknown error: error rotating credentials",
+			expectedErrContains: "aws service unknown: unknown error: rotating credentials",
 			expectedErrCode:     codes.Unknown,
 		},
 		{
@@ -1973,7 +1973,7 @@ func TestStoragePlugin_OnDeleteStorageBucket(t *testing.T) {
 					),
 				}),
 			},
-			expectedErrContains: "aws service unknown: unknown error: failed to delete access key",
+			expectedErrContains: "aws service unknown: unknown error: deleting credentials",
 			expectedErrCode:     codes.Unknown,
 		},
 		{
@@ -2172,7 +2172,7 @@ func TestStoragePlugin_HeadObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to head object",
+			expectedErrContains: "aws service s3: invalid credentials: head object",
 			expectedErrCode:     codes.PermissionDenied,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2196,7 +2196,7 @@ func TestStoragePlugin_HeadObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to head object",
+			expectedErrContains: "aws service s3: throttling: head object",
 			expectedErrCode:     codes.Unavailable,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2219,7 +2219,7 @@ func TestStoragePlugin_HeadObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws s3 error: failed to head object",
+			expectedErrContains: "aws service s3: head object",
 			expectedErrCode:     codes.NotFound,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2425,7 +2425,7 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 					}),
 				)),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to put object",
+			expectedErrContains: "aws service s3: invalid credentials: put object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2458,7 +2458,7 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 					}),
 				)),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to put object",
+			expectedErrContains: "aws service s3: throttling: put object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2492,7 +2492,7 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to get object",
+			expectedErrContains: "aws service s3: invalid credentials: get object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2527,7 +2527,7 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to get object",
+			expectedErrContains: "aws service s3: throttling: get object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2561,7 +2561,7 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to head object",
+			expectedErrContains: "aws service s3: invalid credentials: head object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2596,7 +2596,7 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to head object",
+			expectedErrContains: "aws service s3: throttling: head object",
 			expectedErrCode:     codes.FailedPrecondition,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2666,6 +2666,43 @@ func TestStoragePlugin_ValidatePermissions(t *testing.T) {
 				),
 			},
 		},
+		{
+			name:     "dryRunValidation failed putObject with missing list expected key",
+			req:      validRequest(),
+			credOpts: validSTSMock(),
+			storageOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(
+					nil,
+					testMockS3WithPutObjectError(TestAwsS3Error("AccessDenied", "PutObject", "not authorized")),
+					testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}),
+					testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{}),
+					testMockS3WithListObjectsV2OutputFunc(func(i *s3.ListObjectsV2Input) *s3.ListObjectsV2Output {
+						return &s3.ListObjectsV2Output{
+							Contents: []s3types.Object{},
+						}
+					}),
+				)),
+			},
+			expectedErrContains: "aws service s3: invalid credentials: put object",
+			expectedErrCode:     codes.FailedPrecondition,
+			expectedDetails: &pb.StorageBucketCredentialState{
+				State: &pb.Permissions{
+					Read: &pb.Permission{
+						State:     pb.StateType_STATE_TYPE_UNKNOWN,
+						CheckedAt: timestamppb.Now(),
+					},
+					Write: &pb.Permission{
+						State:        pb.StateType_STATE_TYPE_ERROR,
+						ErrorDetails: "not authorized",
+						CheckedAt:    timestamppb.Now(),
+					},
+					Delete: &pb.Permission{
+						State:     pb.StateType_STATE_TYPE_OK,
+						CheckedAt: timestamppb.Now(),
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range cases {
@@ -2814,7 +2851,7 @@ func TestStoragePlugin_GetObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to get object",
+			expectedErrContains: "aws service s3: invalid credentials: get object",
 			expectedErrCode:     codes.PermissionDenied,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2838,7 +2875,7 @@ func TestStoragePlugin_GetObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws s3 error: failed to get object",
+			expectedErrContains: "aws service s3: get object",
 			expectedErrCode:     codes.NotFound,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2861,7 +2898,7 @@ func TestStoragePlugin_GetObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws s3 error: failed to get object",
+			expectedErrContains: "aws service s3: get object",
 			expectedErrCode:     codes.NotFound,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2884,7 +2921,7 @@ func TestStoragePlugin_GetObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws s3 error: failed to get object",
+			expectedErrContains: "aws service s3: get object",
 			expectedErrCode:     codes.NotFound,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -2908,7 +2945,7 @@ func TestStoragePlugin_GetObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to get object",
+			expectedErrContains: "aws service s3: throttling: get object",
 			expectedErrCode:     codes.Unavailable,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3263,7 +3300,7 @@ func TestStoragePlugin_PutObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to put object",
+			expectedErrContains: "aws service s3: invalid credentials: put object",
 			expectedErrCode:     codes.PermissionDenied,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3287,7 +3324,7 @@ func TestStoragePlugin_PutObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws s3 error: failed to put object",
+			expectedErrContains: "aws service s3: put object",
 			expectedErrCode:     codes.NotFound,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3311,7 +3348,7 @@ func TestStoragePlugin_PutObject(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to put object",
+			expectedErrContains: "aws service s3: throttling: put object",
 			expectedErrCode:     codes.Unavailable,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3603,7 +3640,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to delete object",
+			expectedErrContains: "aws service s3: invalid credentials: delete object",
 			expectedErrCode:     codes.PermissionDenied,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3627,7 +3664,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to list objects",
+			expectedErrContains: "aws service s3: invalid credentials: list objects",
 			expectedErrCode:     codes.PermissionDenied,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3659,7 +3696,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: invalid credentials error: failed to delete objects",
+			expectedErrContains: "aws service s3: invalid credentials: delete objects",
 			expectedErrCode:     codes.PermissionDenied,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3683,7 +3720,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to delete object",
+			expectedErrContains: "aws service s3: throttling: delete object",
 			expectedErrCode:     codes.Unavailable,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3706,7 +3743,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to list objects",
+			expectedErrContains: "aws service s3: throttling: list objects",
 			expectedErrCode:     codes.Unavailable,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3737,7 +3774,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 					),
 				),
 			},
-			expectedErrContains: "aws service s3: throttling error: failed to delete objects",
+			expectedErrContains: "aws service s3: throttling: delete objects",
 			expectedErrCode:     codes.Unavailable,
 			expectedDetails: &pb.StorageBucketCredentialState{
 				State: &pb.Permissions{
@@ -3880,3 +3917,89 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) {
 		})
 	}
 }
+
+func TestS3ClientCache(t *testing.T) {
+	ctx := context.Background()
+	assert, require := assert.New(t), require.New(t)
+
+	cred1, err := awsutil.NewCredentialsConfig(
+		awsutil.WithRegion("us-east-1"),
+		awsutil.WithAccessKey("cred1"),
+		awsutil.WithSecretKey("secret1"),
+	)
+	require.NoError(err)
+	cred2, err := awsutil.NewCredentialsConfig(
+		awsutil.WithRegion("us-east-1"),
+		awsutil.WithAccessKey("cred2"),
+		awsutil.WithSecretKey("secret2"),
+	)
+	require.NoError(err)
+	cred3, err := awsutil.NewCredentialsConfig(
+		awsutil.WithRegion("us-east-1"),
+		awsutil.WithAccessKey("cred3"),
+		awsutil.WithSecretKey("secret3"),
+	)
+	require.NoError(err)
+
+	p := New()
+	cred, err := credential.NewAwsCredentialPersistedState(
+		credential.WithCredentialsConfig(cred1),
+	)
+	require.NoError(err)
+
+	state, err := newAwsStoragePersistedState(withCredentials(cred))
+	require.NoError(err)
+
+	// Cache should be empty before first call
+	assert.Empty(p.clients.cache)
+
+	client, err := p.getClient(ctx, "id", state)
+	require.NoError(err)
+	require.NotNil(client)
+	assert.Equal(cred1.AccessKey, client.Credentials().AccessKeyID)
+	assert.Equal(cred1.SecretKey, client.Credentials().SecretAccessKey)
+
+	// Update state with new credentials
+	cred, err = credential.NewAwsCredentialPersistedState(
+		credential.WithCredentialsConfig(cred2),
+	)
+	require.NoError(err)
+	state, err = newAwsStoragePersistedState(withCredentials(cred))
+	require.NoError(err)
+
+	// Client is cached and creds valid so it should still grab initial client
+	client, err = p.getClient(ctx, "id", state)
+	require.NoError(err)
+	require.NotNil(client)
+	assert.Equal(cred1.AccessKey, client.Credentials().AccessKeyID)
+	assert.Equal(cred1.SecretKey, client.Credentials().SecretAccessKey)
+
+	// Force credentials to expire
+	c := client.(*s3Client)
+	c.creds.CanExpire = true
+	p.clients.Lock()
+	p.clients.cache["id"] = c
+	p.clients.Unlock()
+
+	// Getting client should now create new client on cred2 since the credentials have expired
+	client, err = p.getClient(ctx, "id", state)
+	require.NoError(err)
+	require.NotNil(client)
+	assert.Equal(cred2.AccessKey, client.Credentials().AccessKeyID)
+	assert.Equal(cred2.SecretKey, client.Credentials().SecretAccessKey)
+
+	// Update state with new credentials again
+	cred, err = credential.NewAwsCredentialPersistedState(
+		credential.WithCredentialsConfig(cred3),
+	)
+	require.NoError(err)
+	state, err = newAwsStoragePersistedState(withCredentials(cred))
+	require.NoError(err)
+
+	// Getting client and force cache refresh should return cred3 based client
+	client, err = p.getClient(ctx, "id", state, WithCacheRefresh(true))
+	require.NoError(err)
+	require.NotNil(client)
+	assert.Equal(cred3.AccessKey, client.Credentials().AccessKeyID)
+	assert.Equal(cred3.SecretKey, client.Credentials().SecretAccessKey)
+}

plugin/service/storage/state.go

@@ -27,6 +27,16 @@ type S3API interface {
 	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
 	DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error)
 	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
+	Credentials() aws.Credentials
+}
+
+type s3Client struct {
+	*s3.Client
+	creds aws.Credentials
+}
+
+func (c *s3Client) Credentials() aws.Credentials {
+	return c.creds
 }
 
 var customClient = &http.Client{
@@ -91,9 +101,9 @@ func (s *awsStoragePersistedState) toProto() (*storagebuckets.StorageBucketPersi
 	return &storagebuckets.StorageBucketPersisted{Data: data}, nil
 }
 
-// S3Client returns a configured S3 client based on the session
+// s3Client returns a configured S3 client based on the session
 // information stored in the state.
-func (s *awsStoragePersistedState) S3Client(ctx context.Context, opt ...s3Option) (S3API, error) {
+func (s *awsStoragePersistedState) s3Client(ctx context.Context, opt ...s3Option) (S3API, error) {
 	opts, err := getOpts(opt...)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing options when fetching S3 client: %w", err)
@@ -114,7 +124,16 @@ func (s *awsStoragePersistedState) S3Client(ctx context.Context, opt ...s3Option
 	if s.testS3APIFunc != nil {
 		return s.testS3APIFunc(*awsCfg)
 	}
-	return s3.NewFromConfig(*awsCfg, s3Opts...), nil
+
+	// Retrieve the credentials provider from the client
+	credsProvider := awsCfg.Credentials
+	creds, err := credsProvider.Retrieve(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve credentials: %v\n", err)
+	}
+
+	c := s3.NewFromConfig(*awsCfg, s3Opts...)
+	return &s3Client{Client: c, creds: creds}, nil
 }
 
 // getOpts iterates the inbound s3Options and returns a struct
@@ -136,13 +155,22 @@ type s3Option func(*s3Options) error
 
 // options = how options are represented
 type s3Options struct {
-	withEndpoint string
+	withEndpoint     string
+	withCacheRefresh bool
 }
 
 func getDefaultS3Options() s3Options {
 	return s3Options{}
 }
 
+// WithCacheRefresh controls if the cache should be forced to refresh
+func WithCacheRefresh(refresh bool) s3Option {
+	return func(o *s3Options) error {
+		o.withCacheRefresh = refresh
+		return nil
+	}
+}
+
 // WithEndpoint contains the endpoint to use
 func WithEndpoint(with string) s3Option {
 	return func(o *s3Options) error {

plugin/service/storage/state_test.go

@@ -64,7 +64,7 @@ func TestAwsStoragePersistedStateS3Client(t *testing.T) {
 			)
 			require.NoError(err)
 
-			clientRaw, err := state.S3Client(context.Background(), tc.s3Opts...)
+			clientRaw, err := state.s3Client(context.Background(), tc.s3Opts...)
 			require.NoError(err)
 			require.NotNil(clientRaw)
 			client, ok := clientRaw.(*testMockS3)

testing/e2e_storage_test.go

@@ -17,12 +17,16 @@ import (
 	"github.com/hashicorp/boundary-plugin-aws/plugin/service/storage"
 	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets"
 	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/structpb"
 )
 
+// NOTE: this test does not vaildate against dynamic credentials because
+// this credential type can only be tested on an EC2 instance.
 func TestStoragePlugin(t *testing.T) {
 	region := os.Getenv("AWS_REGION")
 	if region == "" {
@@ -75,6 +79,12 @@ func TestStoragePlugin(t *testing.T) {
 	iamSecretAccessKeyMissingPutObject, err := tf.GetOutputString("iam_secret_access_key_missing_put_obj")
 	require.NoError(err)
 
+	iamAccessKeyMissingDeleteObject, err := tf.GetOutputString("iam_access_key_missing_delete_obj")
+	require.NoError(err)
+
+	iamSecretAccessKeyMissingDeleteObject, err := tf.GetOutputString("iam_secret_access_key_missing_delete_obj")
+	require.NoError(err)
+
 	// Start the workflow now. Set up the storage bucket. Note that this
 	// will cause the state to go out of drift above in the sense that
 	// the access key ID/secret access key will no longer be valid. We
@@ -121,7 +131,7 @@ func TestStoragePlugin(t *testing.T) {
 	testPluginOnDeleteStorageBucket(ctx, t, p, bucketName, region, keyid, secret, true)
 
 	// ********************
-	// * Object Methods: PutObject, GetObject, HeadObject
+	// * Object Methods: PutObject, GetObject, HeadObject, DeleteObjects
 	// ********************
 	//
 	// Reassign the keyid and secret first.
@@ -132,11 +142,59 @@ func TestStoragePlugin(t *testing.T) {
 	// * Validate Permissions
 	// ********************
 	//
+	var expectedSBCState *pb.StorageBucketCredentialState
+
 	// Validate error is returned for missing get object permission
-	testPluginValidatePermissions(ctx, t, p, bucketName, region, iamAccessKeyMissingGetObject, iamSecretAccessKeyMissingGetObject)
+	expectedSBCState = &pb.StorageBucketCredentialState{
+		State: &pb.Permissions{
+			Write: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_OK,
+			},
+			Read: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_ERROR,
+			},
+			Delete: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_OK,
+			},
+		},
+	}
+	testPluginValidatePermissions(ctx, t, p, bucketName, region, iamAccessKeyMissingGetObject, iamSecretAccessKeyMissingGetObject, expectedSBCState)
 
 	// Validate error is returned for missing put object permission
-	testPluginValidatePermissions(ctx, t, p, bucketName, region, iamAccessKeyMissingPutObject, iamSecretAccessKeyMissingPutObject)
+	expectedSBCState = &pb.StorageBucketCredentialState{
+		State: &pb.Permissions{
+			Write: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_ERROR,
+			},
+			Read: &pb.Permission{
+				// Since we cannot write an object to S3,
+				// we cannot test against the ability to read from S3.
+				State: pb.StateType_STATE_TYPE_UNKNOWN,
+			},
+			Delete: &pb.Permission{
+				// We can still try deleting an object that does
+				// not exist in the S3 bucket to validate access.
+				State: pb.StateType_STATE_TYPE_OK,
+			},
+		},
+	}
+	testPluginValidatePermissions(ctx, t, p, bucketName, region, iamAccessKeyMissingPutObject, iamSecretAccessKeyMissingPutObject, expectedSBCState)
+
+	// Validate error is returned for missing delete object permission
+	expectedSBCState = &pb.StorageBucketCredentialState{
+		State: &pb.Permissions{
+			Write: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_OK,
+			},
+			Read: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_OK,
+			},
+			Delete: &pb.Permission{
+				State: pb.StateType_STATE_TYPE_ERROR,
+			},
+		},
+	}
+	testPluginValidatePermissions(ctx, t, p, bucketName, region, iamAccessKeyMissingDeleteObject, iamSecretAccessKeyMissingDeleteObject, expectedSBCState)
 }
 
 func testPluginOnCreateStorageBucket(ctx context.Context, t *testing.T, p *storage.StoragePlugin, bucketName, region, accessKeyId, secretAccessKey string, rotate bool) (string, string) {
@@ -289,10 +347,12 @@ func testPluginOnDeleteStorageBucket(ctx context.Context, t *testing.T, p *stora
 	}
 }
 
-func testPluginValidatePermissions(ctx context.Context, t *testing.T, p *storage.StoragePlugin, bucketName, region, accessKeyId, secretAccessKey string) {
+func testPluginValidatePermissions(ctx context.Context, t *testing.T, p *storage.StoragePlugin, bucketName, region, accessKeyId, secretAccessKey string, expectedState *pb.StorageBucketCredentialState) {
 	t.Helper()
 	t.Logf("testing ValidatePermissions")
-	require := require.New(t)
+	require, assert := require.New(t), assert.New(t)
+
+	require.NotNil(expectedState)
 
 	reqAttrs, err := structpb.NewStruct(map[string]any{
 		credential.ConstRegion:                    region,
@@ -315,6 +375,51 @@ func testPluginValidatePermissions(ctx context.Context, t *testing.T, p *storage
 
 	_, err = p.ValidatePermissions(ctx, req)
 	require.Error(err)
+
+	var actualSBCState *pb.StorageBucketCredentialState
+	if st, ok := status.FromError(err); ok {
+		for _, detail := range st.Details() {
+			if statusDetail, ok := detail.(*pb.StorageBucketCredentialState); ok {
+				actualSBCState = statusDetail
+				break
+			}
+		}
+	}
+	require.NotNil(actualSBCState)
+	require.NotNil(actualSBCState.GetState())
+
+	expectedReadState := expectedState.GetState().GetRead()
+	if expectedReadState != nil {
+		actualReadState := actualSBCState.GetState().GetRead()
+		require.NotNil(actualReadState)
+		assert.Equal(expectedReadState.GetState(), actualReadState.GetState())
+		assert.NotEmpty(actualReadState.GetCheckedAt())
+		if expectedReadState.GetState() == pb.StateType_STATE_TYPE_ERROR {
+			assert.NotEmpty(actualReadState.GetErrorDetails())
+		}
+	}
+
+	expectedWriteState := expectedState.GetState().GetWrite()
+	if expectedWriteState != nil {
+		actualWriteState := actualSBCState.GetState().GetWrite()
+		require.NotNil(actualWriteState)
+		assert.Equal(expectedWriteState.GetState(), actualWriteState.GetState())
+		assert.NotEmpty(actualWriteState.GetCheckedAt())
+		if expectedWriteState.GetState() == pb.StateType_STATE_TYPE_ERROR {
+			assert.NotEmpty(actualWriteState.GetErrorDetails())
+		}
+	}
+
+	expectedDeleteState := expectedState.GetState().GetDelete()
+	if expectedDeleteState != nil {
+		actualDeleteState := actualSBCState.GetState().GetDelete()
+		require.NotNil(actualDeleteState)
+		assert.Equal(expectedDeleteState.GetState(), actualDeleteState.GetState())
+		assert.NotEmpty(actualDeleteState.GetCheckedAt())
+		if expectedDeleteState.GetState() == pb.StateType_STATE_TYPE_ERROR {
+			assert.NotEmpty(actualDeleteState.GetErrorDetails())
+		}
+	}
 }
 
 func testPluginObjectMethods(ctx context.Context, t *testing.T, p *storage.StoragePlugin, bucketName, region, accessKeyId, secretAccessKey string) {
@@ -369,7 +474,7 @@ func testPluginObjectMethods(ctx context.Context, t *testing.T, p *storage.Stora
 
 	// Need to create a GRPC server to test GetObject
 	t.Logf("testing GetObject")
-	lis, err := net.Listen("tcp", "localhost:2030")
+	lis, err := net.Listen("tcp", "[::1]:2030")
 	require.NoError(err)
 	grpcServer := grpc.NewServer()
 	pb.RegisterStoragePluginServiceServer(grpcServer, &storage.StoragePlugin{})
@@ -377,7 +482,7 @@ func testPluginObjectMethods(ctx context.Context, t *testing.T, p *storage.Stora
 		grpcServer.Serve(lis)
 	}()
 
-	conn, err := grpc.Dial("localhost:2030", grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.Dial("[::1]:2030", grpc.WithTransportCredentials(insecure.NewCredentials()))
 	require.NoError(err)
 	defer conn.Close()
 	client := pb.NewStoragePluginServiceClient(conn)

testing/testdata/storage/iam.tf

@@ -32,7 +32,9 @@ resource "aws_iam_policy" "valid" {
       "Action": [
         "s3:PutObject",
         "s3:GetObject",
-        "s3:GetObjectAttributes"
+        "s3:DeleteObject",
+        "s3:GetObjectAttributes",
+        "s3:ListBucket"
       ],
       "Effect": "Allow",
       "Resource": "${aws_s3_bucket.test.arn}*"
@@ -79,7 +81,9 @@ resource "aws_iam_policy" "missing_put_obj" {
     {
       "Action": [
         "s3:GetObject",
-        "s3:GetObjectAttributes"
+        "s3:GetObjectAttributes",
+        "s3:ListBucket",
+        "s3:DeleteObject"
       ],
       "Effect": "Allow",
       "Resource": "${aws_s3_bucket.test.arn}*"
@@ -125,7 +129,7 @@ resource "aws_iam_policy" "missing_get_obj" {
     {
       "Action": [
         "s3:PutObject",
-        "s3:GetObjectAttributes"
+        "s3:DeleteObject"
       ],
       "Effect": "Allow",
       "Resource": "${aws_s3_bucket.test.arn}*"
@@ -141,18 +145,69 @@ resource "aws_iam_role_policy_attachment" "missing_get_obj" {
   policy_arn = aws_iam_policy.missing_get_obj.arn
 }
 
+resource "aws_iam_role" "missing_delete_obj" {
+  name = "${random_id.prefix.dec}-missing-delete-obj"
+  tags = local.tags
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          AWS: data.aws_caller_identity.current.user_id
+        }
+      },
+    ]
+  })
+}
+
+resource "aws_iam_policy" "missing_delete_obj" {
+  name = "${random_id.prefix.dec}-missing-delete-obj"
+  tags = local.tags
+
+  policy = <<EOT
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:GetObjectAttributes",
+        "s3:ListBucket"
+      ],
+      "Effect": "Allow",
+      "Resource": "${aws_s3_bucket.test.arn}*"
+    }
+  ]
+
+}
+EOT
+}
+
+resource "aws_iam_role_policy_attachment" "missing_delete_obj" {
+  role       = aws_iam_role.missing_delete_obj.name
+  policy_arn = aws_iam_policy.missing_delete_obj.arn
+}
+
 # Static Credential Testing
 resource "random_id" "user_name" {
   count       = var.iam_user_count
-  prefix      = random_id.prefix.dec
+  prefix      = "demo-${local.hashicorp_email}-boundary-iam-user"
   byte_length = 4
 }
 
 resource "aws_iam_user" "test" {
-  count         = var.iam_user_count
-  name          = random_id.user_name[count.index].dec
-  force_destroy = true
-  tags          = local.tags
+  count                = var.iam_user_count
+  name                 = random_id.user_name[count.index].dec
+  force_destroy        = true
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
+  tags = {
+    "boundary-demo" = local.hashicorp_email
+  }
 }
 
 resource "aws_iam_access_key" "test" {
@@ -195,17 +250,44 @@ resource "aws_iam_user_policy_attachment" "test_credentials" {
   policy_arn = aws_iam_policy.credentials[count.index].arn
 }
 
+# Edge case for missing DeleteObject permission
+resource "random_id" "missing_delete_obj" {
+  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
+  byte_length = 4
+}
+
+resource "aws_iam_user" "missing_delete_obj" {
+  name          = random_id.missing_delete_obj.dec
+  force_destroy = true
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
+  tags = {
+    "boundary-demo" = local.hashicorp_email
+  }
+}
+
+resource "aws_iam_access_key" "missing_delete_obj" {
+  user  = aws_iam_user.missing_delete_obj.name
+}
+
+resource "aws_iam_user_policy_attachment" "missing_delete_obj" {
+  user       = aws_iam_user.missing_delete_obj.name
+  policy_arn = aws_iam_policy.missing_delete_obj.arn
+}
+
 # Edge case for missing GetObject permission
 
 resource "random_id" "missing_get_obj" {
-  prefix      = random_id.prefix.dec
+  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
   byte_length = 4
 }
 
 resource "aws_iam_user" "missing_get_obj" {
   name          = random_id.missing_get_obj.dec
   force_destroy = true
-  tags          = local.tags
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
+  tags = {
+    "boundary-demo" = local.hashicorp_email
+  }
 }
 
 resource "aws_iam_access_key" "missing_get_obj" {
@@ -220,14 +302,17 @@ resource "aws_iam_user_policy_attachment" "missing_get_obj" {
 # Edge case for missing PutObject permission
 
 resource "random_id" "missing_put_obj" {
-  prefix      = random_id.prefix.dec
+  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
   byte_length = 4
 }
 
 resource "aws_iam_user" "missing_put_obj" {
   name          = random_id.missing_put_obj.dec
   force_destroy = true
-  tags          = local.tags
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
+  tags = {
+    "boundary-demo" = local.hashicorp_email
+  }
 }
 
 resource "aws_iam_access_key" "missing_put_obj" {

testing/testdata/storage/output.tf

@@ -13,10 +13,14 @@ output "missing_put_obj_policy_arn" {
     value = aws_iam_role.missing_put_obj.arn
 }
 
-output "missing_put_get_policy_arn" {
+output "missing_get_obj_policy_arn" {
     value = aws_iam_role.missing_get_obj.arn
 }
 
+output "missing_delete_obj_policy_arn" {
+    value = aws_iam_role.missing_delete_obj.arn
+}
+
 output "iam_user_names" {
   value = aws_iam_user.test.*.name
 }
@@ -53,4 +57,14 @@ output "iam_access_key_missing_get_obj" {
 output "iam_secret_access_key_missing_get_obj" {
   value     = aws_iam_access_key.missing_get_obj.secret
   sensitive = true
+}
+
+output "iam_access_key_missing_delete_obj" {
+  value     = aws_iam_access_key.missing_delete_obj.id
+  sensitive = true
+}
+
+output "iam_secret_access_key_missing_delete_obj" {
+  value     = aws_iam_access_key.missing_delete_obj.secret
+  sensitive = true
 }

testing/testdata/storage/s3.tf

@@ -4,6 +4,14 @@
 resource "aws_s3_bucket" "test" {
   bucket        = random_id.prefix.dec
   force_destroy = true
-  acl           = "private"
   tags          = local.tags
+}
+
+resource "aws_s3_bucket_public_access_block" "secure_access" {
+  bucket = aws_s3_bucket.test.id
+  
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
 }

testing/testdata/storage/variable.tf

@@ -4,7 +4,7 @@
 data "aws_caller_identity" "current" {}
 
 resource "random_id" "prefix" {
-  prefix      = "storage-e2e-test"
+  prefix      = "aws-plugin-e2e-test"
   byte_length = 4
 }
 
@@ -13,8 +13,10 @@ variable "iam_user_count" {
 }
 
 locals {
+  hashicorp_email = split(":", data.aws_caller_identity.current.user_id)[1]
   tags = {
-    user_id = split(":", data.aws_caller_identity.current.user_id)[1]
-    repo    = "boundary-plugin-aws" 
+    User        = split(":", data.aws_caller_identity.current.user_id)[1]
+    Repo        = "boundary-plugin-aws" 
+    Environment = "Demo"
   }
-}
+}