bug(go) AuthFailure when performing static credential rotation

**Problem**
* While adding e2e tests for AssumeRole functionality, it was noticed that some of the static credential e2e tests were flaky.
* The tests would sometimes fail with an AuthFailure.

**Solution**
* It was discovered that this error happened sometimes after a static credential rotation and that AuthFailure occurs if EC2 is invoked with the wrong credentials.
* Due to IAM's eventual consistency model, it can take some time (up to several seconds) for changes, including rotated credentials, to get propagated to all AWS services, including EC2.
* The tests were intermittently failing because sometimes the credentials would be propagated in time before invoking EC2 and other times they were not propagated in time.
* The fix is to add a retry mechanism when calling DescribeInstances. A 30 second window is defined in which the retries can take place.
* The retries happen only for AuthFailures. Any other failure will not trigger a retry.

**Testing**
* After making this change, the intermittent failures were no longer seen for the e2e tests
* The e2e tests cover updating/creating catalogs and sets as well us updating the host list. All pass.
* New unit tests were added to test the retry mechanism. Support had to be added to mock a series of DescribeInstances calls vs a single call
* New and existing unit tests passed

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/boundary/sdk v0.0.59
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.1.3
+	github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.1.2
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0
 	github.com/hashicorp/terraform-json v0.23.0
 	github.com/mitchellh/mapstructure v1.5.0

go.sum

@@ -8,8 +8,12 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go-v2 v1.39.1 h1:fWZhGAwVRK/fAN2tmt7ilH4PPAE11rDj7HytrmbZ2FE=
+github.com/aws/aws-sdk-go-v2 v1.39.1/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY=
 github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
 github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
 github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0=
@@ -18,26 +22,42 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgb
 github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.8 h1:6bgAZgRyT4RoFWhxS+aoGMFyE0cD1bSzFnEEi4bFPGI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.8/go.mod h1:KcGkXFVU8U28qS4KvLEcPxytPZPBcRawaH2Pf/0jptE=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.8 h1:HhJYoES3zOz34yWEpGENqJvRVPqpmJyR3+AFg9ybhdY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.8/go.mod h1:JnA+hPWeYAVbDssp83tv+ysAG8lTfLVXvSsyKg/7xNA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.8 h1:1/bT9kDdLQzfZ1e6J6hpW+SfNDd6xrV8F3M2CuGyUz8=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.8/go.mod h1:RbdwTONAIi59ej/+1H+QzZORt5bcyAtbrS7FQb2pvz0=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.254.0 h1:fTLR6dLDTGChAjecRPlVrKeznT0rVdzR4yn9Z68MTGk=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.254.0/go.mod h1:V0jbRy1/IPapnkqgXSwVOFB+u5pnCwd9S+R3pKWULC4=
 github.com/aws/aws-sdk-go-v2/service/iam v1.47.6 h1:EWehQXACWr+6hzfZPwZChlfoVhiUCfLHE0Xh3kAfzWQ=
 github.com/aws/aws-sdk-go-v2/service/iam v1.47.6/go.mod h1:qRXgEBWPIltrWHQwU+HkyBvwh1QgeigFcaCGCIVrWk0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.8 h1:tIN8MFT1z5STK5kTdOT1TCfMN/bn5fSEnlKsTL8qBOU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.8/go.mod h1:VKS56txtNWjKI8FqD/hliL0BcshyF4ZaLBa1rm2Y+5s=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.8 h1:M6JI2aGFEzYxsF6CXIuRBnkge9Wf9a2xU39rNeXgu10=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.8/go.mod h1:Fw+MyTwlwjFsSTE31mH211Np+CUslml8mzc0AFEG09s=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.8 h1:AgYCo1Rb8XChJXA871BXHDNxNWOTAr6V5YdsRIBbgv0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.8/go.mod h1:Au9dvIGm1Hbqnt29d3VakOCQuN9l0WrkDDTRq8biWS4=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.88.2 h1:T7b3qniouutV5Wwa9B1q7gW+Y8s1B3g9RE9qa7zLBIM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.88.2/go.mod h1:tW9TsLb6t1eaTdBE6LITyJW1m/+DjQPU78Q/jT2FJu8=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
 github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM=
@@ -46,6 +66,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc=
 github.com/aws/aws-sdk-go-v2/service/sts v1.38.5 h1:+LVB0xBqEgjQoqr9bGZbRzvg212B0f17JdflleJRNR4=
 github.com/aws/aws-sdk-go-v2/service/sts v1.38.5/go.mod h1:xoaxeqnnUaZjPjaICgIy5B+MHCSb/ZSOn4MvkFNOUA0=
+github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE=
+github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
 github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
@@ -94,8 +116,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA=
 github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8=
-github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.1.3 h1:mLe7IMysLSedhuu00W7AXsbfb90AA7UKfZmw3OkJ8n4=
-github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.1.3/go.mod h1:6+rVulOPNCQbL3Xv2iLCqM0JmU2WO2wRzP1C6hBKeB8=
+github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.1.2 h1:KlDrZWx/wYpnUcFPB0s9uhPjyNZoeIfhrV2MN/Ly76s=
+github.com/hashicorp/go-secure-stdlib/awsutil/v2 v2.1.2/go.mod h1:6+rVulOPNCQbL3Xv2iLCqM0JmU2WO2wRzP1C6hBKeB8=
 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 h1:ET4pqyjiGmY09R5y+rSd70J2w45CtbWDNvGqWp/R3Ng=
 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2/go.mod h1:EdWO6czbmthiwZ3/PUsDV+UD1D5IRU4ActiaWGwt0Yw=
 github.com/hashicorp/go-secure-stdlib/configutil/v2 v2.0.13 h1:TayxZ5drfMP0G6T++WvnLESGLOWeHtdWDWWTAi2e3Qk=

internal/credential/state.go

@@ -112,35 +112,10 @@ func NewAwsCredentialPersistedState(opts ...AwsCredentialPersistedStateOption) (
 // and runs sts.GetCallerIdentity for the current credentials, which is done
 // to check that the credentials are valid. This method returns a status error
 // with PluginError details.
-//
-// When RoleARN is set and no static key is present (ambient role path),
-// credentials are eagerly retrieved before calling GetCallerIdentity. This
-// forces sts:AssumeRole to be called immediately so that any role errors
-// (e.g. no such role, access denied) surface here rather than being deferred
-// to the first real AWS API call.
 func (s *AwsCredentialPersistedState) ValidateCreds(ctx context.Context) error {
 	if s.CredentialsConfig == nil {
 		return errors.BadRequestStatus("missing credentials config")
 	}
-
-	// For the ambient role path (RoleARN set, no static key), eagerly resolve
-	// the credential chain so that sts:AssumeRole is called now. Without this,
-	// the AssumeRoleProvider inside the chain is lazy and GetCallerIdentity
-	// would succeed even with an invalid role ARN (it uses the mock/ambient
-	// identity directly rather than the assumed one).
-	cc := s.CredentialsConfig
-	if cc.RoleARN != "" && cc.AccessKey == "" {
-		cfg, err := s.GenerateCredentialChain(ctx)
-		if err != nil {
-			st, _ := errors.ParseAWSError("validating credentials", err)
-			return st.Err()
-		}
-		if _, err := cfg.Credentials.Retrieve(ctx); err != nil {
-			st, _ := errors.ParseAWSError("validating credentials", err)
-			return st.Err()
-		}
-	}
-
 	if _, err := s.CredentialsConfig.GetCallerIdentity(ctx, s.testOpts...); err != nil {
 		st, _ := errors.ParseAWSError("validating credentials", err)
 		return st.Err()

internal/errors/error.go

@@ -23,6 +23,11 @@ import (
 )
 
 const (
+	// AwsErrorAuthFailure is returned by EC2 when credentials exist in IAM
+	// but have not yet propagated to EC2. This is a transient eventual
+	// consistency issue that can resolve itself by retrying the request.
+	AwsErrorAuthFailure = "AuthFailure"
+
 	// awsErrorAccessDenied is returned when the credentials do not have
 	// the required permissions to do a specific action on a resource.
 	// This error will persist until the IAM Role Policy is updated by

plugin/service/host/plugin.go

@@ -8,6 +8,7 @@ import (
 	goerrors "errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
@@ -46,8 +47,6 @@ type hostSetQuery struct {
 
 const defaultSessionName = "boundary-default"
 const dryRunOperationErrorCode = "DryRunOperation"
-const errBuildDescribeInstancesInput = "error building DescribeInstances input: %s"
-const errDryRunFailed = "error performing DescribeInstances dry run: %s"
 
 // Ensure that we are implementing HostPluginServiceServer
 var _ pb.HostPluginServiceServer = (*HostPlugin)(nil)
@@ -90,15 +89,6 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 		}
 	}
 
-	// Validate assumed-role credentials at catalog-creation time so that an
-	// invalid role_arn is rejected immediately rather than silently falling
-	// back to ambient credentials.
-	if credential.GetCredentialType(credState.CredentialsConfig) == credential.DynamicAWS {
-		if err := credState.ValidateCreds(ctx); err != nil {
-			return nil, err
-		}
-	}
-
 	catalogState, err := newAwsCatalogPersistedState(
 		append([]awsCatalogPersistedStateOption{
 			withCredentials(credState),
@@ -108,18 +98,13 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
 		return nil, errors.BadRequestStatusf("error setting up persisted state: %s", err)
 	}
 
+	// perform dry run to ensure we can interact with AWS as expected.
 	opts := []ec2Option{}
 	if catalogAttributes.DualStack {
 		opts = append(opts, WithDualStack(catalogAttributes.DualStack))
 	}
-	input, err := buildDescribeInstancesInput(&SetAttributes{}, true)
-	if err != nil {
-		return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
-	}
-	// Run a DescribeInstances request with DryRun set to true to ensure
-	// we can interact with AWS resources as expected.
-	if _, err := checkHosts(ctx, catalogState, input, opts, catalogAttributes.TargetAccount); err != nil {
-		return nil, fmt.Errorf(errDryRunFailed, err)
+	if st := dryRunValidation(ctx, catalogState, opts); st != nil {
+		return nil, st.Err()
 	}
 
 	persistedProto, err := catalogState.toProto()
@@ -194,15 +179,6 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		if err != nil {
 			return nil, errors.BadRequestStatusf("error setting up new credential persisted state: %s", err)
 		}
-
-		// Validate assumed-role credentials when the role_arn changes so that
-		// an invalid ARN is caught before replacing the existing credential state.
-		if credential.GetCredentialType(newCredState.CredentialsConfig) == credential.DynamicAWS {
-			if err := newCredState.ValidateCreds(ctx); err != nil {
-				return nil, err
-			}
-		}
-
 		newCatalogState, err := newAwsCatalogPersistedState(
 			append([]awsCatalogPersistedStateOption{
 				withCredentials(newCredState),
@@ -211,19 +187,12 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		if err != nil {
 			return nil, errors.BadRequestStatusf("error loading persisted state: %s", err)
 		}
-
 		opts := []ec2Option{}
 		if newCatalogAttributes.DualStack {
 			opts = append(opts, WithDualStack(newCatalogAttributes.DualStack))
 		}
-		input, err := buildDescribeInstancesInput(&SetAttributes{}, true)
-		if err != nil {
-			return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
-		}
-		// Run a DescribeInstances request with DryRun set to true to ensure
-		// we can interact with AWS resources as expected.
-		if _, err := checkHosts(ctx, newCatalogState, input, opts, newCatalogAttributes.TargetAccount); err != nil {
-			return nil, fmt.Errorf(errDryRunFailed, err)
+		if st := dryRunValidation(ctx, newCatalogState, opts); st != nil {
+			return nil, st.Err()
 		}
 
 		// Replace the existing credential state.
@@ -263,18 +232,13 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
 		return nil, errors.BadRequestStatusf("error loading persisted state: %s", err)
 	}
 
+	// perform dry run to ensure we can interact with AWS as expected.
 	opts := []ec2Option{}
 	if newCatalogAttributes.DualStack {
 		opts = append(opts, WithDualStack(newCatalogAttributes.DualStack))
 	}
-	input, err := buildDescribeInstancesInput(&SetAttributes{}, true)
-	if err != nil {
-		return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
-	}
-	// Run a DescribeInstances request with DryRun set to true to ensure
-	// we can interact with AWS resources as expected.
-	if _, err := checkHosts(ctx, catalogState, input, opts, newCatalogAttributes.TargetAccount); err != nil {
-		return nil, fmt.Errorf(errDryRunFailed, err)
+	if st := dryRunValidation(ctx, catalogState, opts); st != nil {
+		return nil, st.Err()
 	}
 
 	persistedProto, err := catalogState.toProto()
@@ -418,20 +382,18 @@ func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest
 		return nil, err
 	}
 
+	describeInstanceFilters, err := buildFilters(setAttrs)
+	if err != nil {
+		return nil, errors.BadRequestStatusf("error building set filters: %s", err)
+	}
+
 	opts := []ec2Option{}
 	if catalogAttributes.DualStack {
 		opts = append(opts, WithDualStack(catalogAttributes.DualStack))
 	}
-	input, err := buildDescribeInstancesInput(setAttrs, true)
-	if err != nil {
-		return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
+	if st := dryRunValidation(ctx, catalogState, opts, describeInstanceFilters...); st != nil {
+		return nil, st.Err()
 	}
-	// Run a DescribeInstances request with DryRun set to true to ensure
-	// we can interact with AWS resources as expected.
-	if _, err := checkHosts(ctx, catalogState, input, opts, catalogAttributes.TargetAccount); err != nil {
-		return nil, fmt.Errorf(errDryRunFailed, err)
-	}
-
 	return &pb.OnCreateSetResponse{}, nil
 }
 
@@ -485,18 +447,17 @@ func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest
 		return nil, err
 	}
 
+	describeInstanceFilters, err := buildFilters(setAttrs)
+	if err != nil {
+		return nil, errors.BadRequestStatusf("error building set filters: %s", err)
+	}
+
 	opts := []ec2Option{}
 	if catalogAttributes.DualStack {
 		opts = append(opts, WithDualStack(catalogAttributes.DualStack))
 	}
-	input, err := buildDescribeInstancesInput(setAttrs, true)
-	if err != nil {
-		return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
-	}
-	// Run a DescribeInstances request with DryRun set to true to ensure
-	// we can interact with AWS resources as expected.
-	if _, err := checkHosts(ctx, catalogState, input, opts, catalogAttributes.TargetAccount); err != nil {
-		return nil, fmt.Errorf(errDryRunFailed, err)
+	if st := dryRunValidation(ctx, catalogState, opts, describeInstanceFilters...); st != nil {
+		return nil, st.Err()
 	}
 
 	return &pb.OnUpdateSetResponse{}, nil
@@ -651,11 +612,11 @@ func buildFilters(attrs *SetAttributes) ([]types.Filter, error) {
 		splitFilter := strings.Split(filterAttr, "=")
 		switch {
 		case len(splitFilter) != 2:
-			return nil, errors.BadRequestStatusf("expected filter %q to contain a single equal sign", filterAttr)
+			return nil, fmt.Errorf("expected filter %q to contain a single equal sign", filterAttr)
 		case len(splitFilter[0]) == 0:
-			return nil, errors.BadRequestStatusf("filter %q contains an empty filter key", filterAttr)
+			return nil, fmt.Errorf("filter %q contains an empty filter key", filterAttr)
 		case len(splitFilter[1]) == 0:
-			return nil, errors.BadRequestStatusf("filter %q contains an empty value", filterAttr)
+			return nil, fmt.Errorf("filter %q contains an empty value", filterAttr)
 		}
 
 		filterKey, filterValue := splitFilter[0], splitFilter[1]
@@ -684,13 +645,9 @@ func buildFilters(attrs *SetAttributes) ([]types.Filter, error) {
 }
 
 func buildDescribeInstancesInput(attrs *SetAttributes, dryRun bool) (*ec2.DescribeInstancesInput, error) {
-	if attrs == nil {
-		attrs = &SetAttributes{}
-	}
-
 	filters, err := buildFilters(attrs)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error building filters: %w", err)
 	}
 
 	return &ec2.DescribeInstancesInput{
@@ -872,6 +829,58 @@ func ec2ClientForTarget(
 	return ec2.NewFromConfig(targetCfg, ec2Opts...), nil
 }
 
+// dryRunValidationTimeout is the maximum time to wait for credentials to
+// propagate across AWS services after rotation before giving up.
+const dryRunValidationTimeout = 30 * time.Second
+
+// dryRunValidation performs an AWS DescribeInstances call to verify the state's
+// credentials, the host listing functionality as well as the filters, if any
+// are passed in. This function can therefore be used for both host catalog and
+// host set validation.
+//
+// After credential rotation, IAM keys may be valid for STS (GetCallerIdentity)
+// but not yet propagated to EC2. This function retries on AuthFailure to
+// tolerate that eventual consistency window.
+func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, ec2Opts []ec2Option, filters ...types.Filter) *status.Status {
+	if state == nil {
+		return status.New(codes.InvalidArgument, "persisted state is required")
+	}
+
+	ec2Client, err := state.EC2Client(ctx, ec2Opts...)
+	if err != nil {
+		return status.New(codes.InvalidArgument, fmt.Sprintf("error getting EC2 client: %s", err))
+	}
+
+	deadline := time.Now().Add(dryRunValidationTimeout)
+	delay := time.Second
+	for {
+		_, err = ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{Filters: filters})
+		if err == nil {
+			return nil
+		}
+
+		// AuthFailure indicates the credentials exist but haven't propagated
+		// to EC2 yet — a transient eventual consistency issue after rotation.
+		// Retry until the deadline.
+		if strings.Contains(err.Error(), errors.AwsErrorAuthFailure) && time.Now().Before(deadline) {
+			select {
+			case <-time.After(delay):
+				// Wait before retrying to give credentials time to propagate.
+			case <-ctx.Done():
+				// The caller cancelled — break out of the select.
+				break
+			}
+			if ctx.Err() != nil {
+				break
+			}
+			continue
+		}
+
+		break
+	}
+	return status.New(codes.FailedPrecondition, fmt.Sprintf("aws describe instances failed: %s", err))
+}
+
 // appendDistinct will append the elements to the slice
 // if an element is not nil, empty, and does not exist in slice.
 func appendDistinct(slice []string, elems ...*string) []string {

plugin/service/host/plugin_test.go

@@ -16,10 +16,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/aws/aws-sdk-go-v2/service/iam"
 	iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
-	"github.com/aws/aws-sdk-go-v2/service/sts"
-	stsTypes "github.com/aws/aws-sdk-go-v2/service/sts/types"
 	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/boundary-plugin-aws/internal/credential"
+	pluginErrors "github.com/hashicorp/boundary-plugin-aws/internal/errors"
 	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
 	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
 	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
@@ -30,38 +29,6 @@ import (
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/structpb"
 )
-// testRoleARN is a syntactically valid ARN used for role_arn test cases.
-// All STS calls against it are handled by injected mocks — no real AWS
-// account is contacted.
-const testRoleARN = "arn:aws:iam::123456789012:role/BoundaryTestRole"
-
-// validAssumeRoleOutput returns a mock AssumeRoleOutput with synthetic
-// temporary credentials for use with WithSTSAPIFunc.
-func validAssumeRoleOutput() *sts.AssumeRoleOutput {
-	return &sts.AssumeRoleOutput{
-		Credentials: &stsTypes.Credentials{
-			AccessKeyId:     aws.String("ASIAmockassumedkey"),
-			SecretAccessKey: aws.String("mockassumedsecret"),
-			SessionToken:    aws.String("mockassumedsessiontoken"),
-			Expiration:      aws.Time(time.Now().Add(time.Hour)),
-		},
-	}
-}
-
-// catalogAttrsStruct builds a catalog attributes structpb for the given
-// region. roleArn is written only when non-empty. disableRotation must be
-// set explicitly to avoid implicit rotation behaviour in tests.
-func catalogAttrsStruct(region, roleArn string, disableRotation bool) *structpb.Struct {
-	fields := map[string]*structpb.Value{
-		credential.ConstRegion:                    structpb.NewStringValue(region),
-		credential.ConstDisableCredentialRotation: structpb.NewBoolValue(disableRotation),
-	}
-	if roleArn != "" {
-		fields[credential.ConstRoleArn] = structpb.NewStringValue(roleArn)
-	}
-	return &structpb.Struct{Fields: fields}
-}
-
 
 func TestPluginOnCreateCatalogSuccess(t *testing.T) {
 	tests := []struct {
@@ -172,13 +139,17 @@ func TestPluginOnCreateCatalogSuccess(t *testing.T) {
 			},
 		},
 		{
-			// role_arn set, no static keys. AssumeRole mock succeeds.
-			// Persisted secrets are empty — dynamic credentials are not stored.
 			name: "usingAssumeRole",
 			req: &pb.OnCreateCatalogRequest{
 				Catalog: &hostcatalogs.HostCatalog{
 					Attrs: &hostcatalogs.HostCatalog_Attributes{
-						Attributes: catalogAttrsStruct("us-east-1", testRoleARN, true),
+						Attributes: &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								credential.ConstRegion:                    structpb.NewStringValue("us-east-1"),
+								credential.ConstRoleArn:                   structpb.NewStringValue("arn:0123:test:rolearn"),
+								credential.ConstDisableCredentialRotation: structpb.NewBoolValue(true),
+							},
+						},
 					},
 				},
 			},
@@ -188,13 +159,6 @@ func TestPluginOnCreateCatalogSuccess(t *testing.T) {
 					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
 				)),
 			},
-			credOpts: []credential.AwsCredentialPersistedStateOption{
-				credential.WithStateTestOpts([]awsutil.Option{
-					awsutil.WithSTSAPIFunc(awsutil.NewMockSTS(
-						awsutil.WithAssumeRoleOutput(validAssumeRoleOutput()),
-					)),
-				}),
-			},
 			expRsp: &pb.OnCreateCatalogResponse{Persisted: &pb.HostCatalogPersisted{Secrets: &structpb.Struct{}}},
 		},
 	}
@@ -352,20 +316,6 @@ func TestPluginOnUpdateCatalogSuccess(t *testing.T) {
 					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
 				)),
 			},
-			credOpts: []credential.AwsCredentialPersistedStateOption{
-				credential.WithStateTestOpts([]awsutil.Option{
-					awsutil.WithSTSAPIFunc(awsutil.NewMockSTS(
-						awsutil.WithAssumeRoleOutput(&sts.AssumeRoleOutput{
-							Credentials: &stsTypes.Credentials{
-								AccessKeyId:     aws.String("ASIAfoobar"),
-								SecretAccessKey: aws.String("secretkeyfoobar"),
-								SessionToken:    aws.String("sessiontokenfoobar"),
-								Expiration:      aws.Time(time.Now().Add(time.Hour)),
-							},
-						}),
-					)),
-				}),
-			},
 			expRsp: &pb.OnUpdateCatalogResponse{
 				Persisted: &pb.HostCatalogPersisted{Secrets: &structpb.Struct{}},
 			},
@@ -472,16 +422,6 @@ func TestPluginOnUpdateCatalogSuccess(t *testing.T) {
 			credOpts: []credential.AwsCredentialPersistedStateOption{
 				credential.WithStateTestOpts([]awsutil.Option{
 					awsutil.WithIAMAPIFunc(awsutil.NewMockIAM()),
-					awsutil.WithSTSAPIFunc(awsutil.NewMockSTS(
-						awsutil.WithAssumeRoleOutput(&sts.AssumeRoleOutput{
-							Credentials: &stsTypes.Credentials{
-								AccessKeyId:     aws.String("ASIAfoobar"),
-								SecretAccessKey: aws.String("secretkeyfoobar"),
-								SessionToken:    aws.String("sessiontokenfoobar"),
-								Expiration:      aws.Time(time.Now().Add(time.Hour)),
-							},
-						}),
-					)),
 				}),
 			},
 			expRsp: &pb.OnUpdateCatalogResponse{
@@ -803,40 +743,8 @@ func TestPluginOnCreateCatalogErr(t *testing.T) {
 					testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
 				)),
 			},
-			// checkHosts returns the DescribeInstances error as-is rather than
-			// wrapping it in a gRPC FailedPrecondition status like the old
-			// dryRunValidation did. A plain Go error maps to codes.Unknown.
-			expectedErrContains: testDescribeInstancesError,
-			expectedErrCode:     codes.Unknown,
-		},
-		{
-			// BUG FIX: role_arn set, no static keys. AssumeRole mock returns
-			// an error. ValidateCreds now eagerly calls Retrieve() which forces
-			// sts:AssumeRole, so the error surfaces here at creation time.
-			// Previously this succeeded silently via the ambient credential chain.
-			name: "role_arn AssumeRole error — correctly rejected",
-			req: &pb.OnCreateCatalogRequest{
-				Catalog: &hostcatalogs.HostCatalog{
-					Attrs: &hostcatalogs.HostCatalog_Attributes{
-						Attributes: catalogAttrsStruct("us-east-1", testRoleARN, true),
-					},
-				},
-			},
-			credOpts: []credential.AwsCredentialPersistedStateOption{
-				credential.WithStateTestOpts([]awsutil.Option{
-					awsutil.WithSTSAPIFunc(awsutil.NewMockSTS(
-						awsutil.WithAssumeRoleError(errors.New("simulated AssumeRole failure")),
-					)),
-				}),
-			},
-			catalogOpts: []awsCatalogPersistedStateOption{
-				withTestEC2APIFunc(newTestMockEC2(
-					nil,
-					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
-				)),
-			},
-			expectedErrContains: "simulated AssumeRole failure",
-			expectedErrCode:     codes.Unknown,
+			expectedErrContains: "aws describe instances failed: DescribeInstances error",
+			expectedErrCode:     codes.FailedPrecondition,
 		},
 	}
 
@@ -1087,11 +995,8 @@ func TestPluginOnUpdateCatalogErr(t *testing.T) {
 					testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
 				)),
 			},
-			//checkHosts returns the DesribeInstances errror as-is rather than wrapping it
-			// in a gRpc FailedPrecodndition status like the old dryRunValidiation did.
-			// A plain Go error maps to codeds.Unknown.
-			expectedErrContains: testDescribeInstancesError,
-			expectedErrCode:     codes.Unknown,
+			expectedErrContains: "aws describe instances failed: DescribeInstances error",
+			expectedErrCode:     codes.FailedPrecondition,
 		},
 		{
 			name: "new incoming dynamic credential update dry run error",
@@ -1126,27 +1031,14 @@ func TestPluginOnUpdateCatalogErr(t *testing.T) {
 					},
 				},
 			},
-			// ValidateCreds is called for DynamicAWS credentials before the dry-run
-			// DescribeInstances. Provide a mock STS that returns a successful AssumeRole
-			// so the validate step passes and execution reaches the EC2 mock.
-			credOpts: []credential.AwsCredentialPersistedStateOption{
-				credential.WithStateTestOpts([]awsutil.Option{
-					awsutil.WithSTSAPIFunc(awsutil.NewMockSTS(
-						awsutil.WithAssumeRoleOutput(validAssumeRoleOutput()),
-					)),
-				}),
-			},
 			catalogOpts: []awsCatalogPersistedStateOption{
 				withTestEC2APIFunc(newTestMockEC2(
 					nil,
 					testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
 				)),
 			},
-			// checkHosts returns the DescribeInstances error as-is rather than wrapping it
-			// in a gRPC FailedPrecondition status like the old dryRunValidiation did.
-			// A plain Go error maps to codes.Unknown.
-			expectedErrContains: testDescribeInstancesError,
-			expectedErrCode:     codes.Unknown,
+			expectedErrContains: "aws describe instances failed: DescribeInstances error",
+			expectedErrCode:     codes.FailedPrecondition,
 		},
 		{
 			name: "replace creds error",
@@ -1320,11 +1212,8 @@ func TestPluginOnUpdateCatalogErr(t *testing.T) {
 					testMockEC2WithDescribeInstancesError(fmt.Errorf("oops there was an error")),
 				)),
 			},
-			// checkHosts returns the DescribeInstances error as-is rather than wrapping it
-			// in a gRPC FailedPrecondition status like the old dryRunValidiation did.
-			// A plain Go error maps to codes.Unknown.
-			expectedErrContains: "oops there was an error",
-			expectedErrCode:     codes.Unknown,
+			expectedErrContains: "aws describe instances failed: oops there was an error",
+			expectedErrCode:     codes.FailedPrecondition,
 		},
 	}
 
@@ -1677,8 +1566,8 @@ func TestPluginOnCreateSetErr(t *testing.T) {
 					testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
 				)),
 			},
-			expectedErrContains: testDescribeInstancesError,
-			expectedErrCode:     codes.Unknown,
+			expectedErrContains: fmt.Sprintf("aws describe instances failed: %s", testDescribeInstancesError),
+			expectedErrCode:     codes.FailedPrecondition,
 		},
 	}
 
@@ -1940,8 +1829,8 @@ func TestPluginOnUpdateSetErr(t *testing.T) {
 					testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
 				)),
 			},
-			expectedErrContains: testDescribeInstancesError,
-			expectedErrCode:     codes.Unknown,
+			expectedErrContains: fmt.Sprintf("aws describe instances failed: %s", testDescribeInstancesError),
+			expectedErrCode:     codes.FailedPrecondition,
 		},
 	}
 
@@ -2745,21 +2634,6 @@ func TestBuildDescribeInstancesInput(t *testing.T) {
 			require.Equal(tc.expected, actual)
 		})
 	}
-
-	t.Run("nil attrs, dry run", func(t *testing.T) {
-		require := require.New(t)
-		actual, err := buildDescribeInstancesInput(nil, true)
-		require.NoError(err)
-		require.Equal(&ec2.DescribeInstancesInput{
-			DryRun: aws.Bool(true),
-			Filters: []types.Filter{
-				{
-					Name:   aws.String("instance-state-name"),
-					Values: []string{string(types.InstanceStateNameRunning)},
-				},
-			},
-		}, actual)
-	})
 }
 
 func TestAwsInstanceToHost(t *testing.T) {
@@ -3198,3 +3072,116 @@ func TestAppendDistinct(t *testing.T) {
 		})
 	}
 }
+
+func TestDryRunValidation(t *testing.T) {
+	t.Run("nil credential state", func(t *testing.T) {
+		st := dryRunValidation(context.Background(), nil, nil)
+		require.NotNil(t, st)
+		require.Equal(t, codes.InvalidArgument.String(), st.Code().String())
+		require.Equal(t, "persisted state is required", st.Message())
+	})
+
+	t.Run("ec2ClientErr", func(t *testing.T) {
+		st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
+			AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
+				CredentialsConfig: &awsutil.CredentialsConfig{},
+			},
+			testEC2APIFunc: func(...aws.Config) (EC2API, error) {
+				return nil, fmt.Errorf("oops ec2 client err")
+			},
+		}, []ec2Option{})
+		require.NotNil(t, st)
+		require.Equal(t, codes.InvalidArgument.String(), st.Code().String())
+		require.Equal(t, "error getting EC2 client: oops ec2 client err", st.Message())
+	})
+
+	t.Run("describeInstancesErr", func(t *testing.T) {
+		st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
+			AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
+				CredentialsConfig: &awsutil.CredentialsConfig{
+					AccessKey: "AKIAfoo",
+					SecretKey: "baz",
+				},
+			},
+			testEC2APIFunc: newTestMockEC2(nil, testMockEC2WithDescribeInstancesError(fmt.Errorf("oops describe instances error"))),
+		}, []ec2Option{})
+		require.NotNil(t, st)
+		require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
+		require.Equal(t, "aws describe instances failed: oops describe instances error", st.Message())
+	})
+
+	t.Run("success", func(t *testing.T) {
+		st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
+			AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
+				CredentialsConfig: &awsutil.CredentialsConfig{
+					AccessKey: "AKIAfoo",
+					SecretKey: "baz",
+				},
+			},
+			testEC2APIFunc: newTestMockEC2(nil, testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{})),
+		}, []ec2Option{})
+		require.Nil(t, st)
+	})
+
+	t.Run("authFailureThenSuccess", func(t *testing.T) {
+		// Simulates the IAM eventual consistency window: EC2 returns AuthFailure
+		// twice before the credentials propagate and the call succeeds.
+		state := &testMockEC2State{}
+		st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
+			AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
+				CredentialsConfig: &awsutil.CredentialsConfig{
+					AccessKey: "AKIAfoo",
+					SecretKey: "baz",
+				},
+			},
+			testEC2APIFunc: newTestMockEC2(state,
+				testMockEC2WithDescribeInstancesErrorSequence(
+					fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure),
+					fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure),
+				),
+				testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
+			),
+		}, []ec2Option{})
+		require.Nil(t, st)
+		require.Equal(t, 3, state.DescribeInstancesCallCount)
+	})
+
+	t.Run("authFailureContextCancelled", func(t *testing.T) {
+		// If the context is cancelled while waiting to retry, the function should
+		// return a FailedPrecondition error immediately rather than hanging.
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel() // cancel immediately so the first retry select hits ctx.Done()
+		st := dryRunValidation(ctx, &awsCatalogPersistedState{
+			AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
+				CredentialsConfig: &awsutil.CredentialsConfig{
+					AccessKey: "AKIAfoo",
+					SecretKey: "baz",
+				},
+			},
+			testEC2APIFunc: newTestMockEC2(nil,
+				testMockEC2WithDescribeInstancesError(fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure)),
+			),
+		}, []ec2Option{})
+		require.NotNil(t, st)
+		require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
+	})
+
+	t.Run("nonAuthFailureNoRetry", func(t *testing.T) {
+		// A non-AuthFailure error should fail immediately without retrying.
+		state := &testMockEC2State{}
+		st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
+			AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
+				CredentialsConfig: &awsutil.CredentialsConfig{
+					AccessKey: "AKIAfoo",
+					SecretKey: "baz",
+				},
+			},
+			testEC2APIFunc: newTestMockEC2(state,
+				testMockEC2WithDescribeInstancesError(fmt.Errorf("UnauthorizedOperation: access denied")),
+			),
+		}, []ec2Option{})
+		require.NotNil(t, st)
+		require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
+		require.Equal(t, 1, state.DescribeInstancesCallCount)
+	})
+}

plugin/service/host/testing.go

@@ -43,6 +43,10 @@ type testMockEC2 struct {
 
 	DescribeInstancesOutput *ec2.DescribeInstancesOutput
 	DescribeInstancesError  error
+
+	// DescribeInstancesErrorSequence, if set, returns each error in order on
+	// successive calls. Once exhausted, DescribeInstancesOutput is returned.
+	DescribeInstancesErrorSequence []error
 }
 
 type testMockEC2Option func(m *testMockEC2) error
@@ -61,6 +65,16 @@ func testMockEC2WithDescribeInstancesError(e error) testMockEC2Option {
 	}
 }
 
+// testMockEC2WithDescribeInstancesErrorSequence sets a sequence of errors to
+// return on successive DescribeInstances calls. Once the sequence is exhausted,
+// the mock returns DescribeInstancesOutput (nil error).
+func testMockEC2WithDescribeInstancesErrorSequence(errs ...error) testMockEC2Option {
+	return func(m *testMockEC2) error {
+		m.DescribeInstancesErrorSequence = errs
+		return nil
+	}
+}
+
 func newTestMockEC2(state *testMockEC2State, opts ...testMockEC2Option) ec2APIFunc {
 	return func(cfgs ...aws.Config) (EC2API, error) {
 		m := &testMockEC2{
@@ -96,6 +110,12 @@ func (m *testMockEC2) DescribeInstances(ctx context.Context, input *ec2.Describe
 		m.State.DescribeInstancesInputs = append(m.State.DescribeInstancesInputs, input)
 	}
 
+	if len(m.DescribeInstancesErrorSequence) > 0 {
+		err := m.DescribeInstancesErrorSequence[0]
+		m.DescribeInstancesErrorSequence = m.DescribeInstancesErrorSequence[1:]
+		return nil, err
+	}
+
 	if m.DescribeInstancesError != nil {
 		return nil, m.DescribeInstancesError
 	}