Compare commits

..
Author SHA1 Message Date
Matt Romanelli f81151b58c bug(go) AuthFailure when performing static credential rotation
**Problem**
* While adding e2e tests for AssumeRole functionality, it was noticed that some of the static credential e2e tests were flaky.
* The tests would sometimes fail with an AuthFailure.

**Solution**
* It was discovered that this error happened sometimes after a static credential rotation and that AuthFailure occurs if EC2 is invoked with the wrong credentials.
* Due to IAM's eventual consistency model, it can take some time (up to several seconds) for changes, including rotated credentials, to get propagated to all AWS services, including EC2.
* The tests were intermittently failing because sometimes the credentials would be propagated in time before invoking EC2 and other times they were not propagated in time.
* The fix is to add a retry mechanism when calling DescribeInstances. A 30 second window is defined in which the retries can take place.
* The retries happen only for AuthFailures. Any other failure will not trigger a retry.

**Testing**
* After making this change, the intermittent failures were no longer seen for the e2e tests
* The e2e tests cover updating/creating catalogs and sets as well us updating the host list. All pass.
* New unit tests were added to test the retry mechanism. Support had to be added to mock a series of DescribeInstances calls vs a single call
* New and existing unit tests passed
2026-09-22 12:27:00 -05:00
Matt Romanelli e484c09479 Merge pull request #83 from hashicorp/readme-updates
Updated the READMEs for cross-account discovery
2026-09-21 15:50:44 -05:00
Ryan Flynn 4126a0dbfc feat(cli) Implement infra to support cross account AssumeRole for DHC on ListHosts (#86) 2026-09-18 14:33:31 -04:00
Ryan Flynn 09dd7f8a4d feat(cli) Implement parsing of target-prefixed attrs for cross account dynamic host catalogs (#84) 2026-09-18 12:17:24 -04:00
Matt Romanelli 4c4a901d33 Updated the READMEs for cross-account discovery 2026-09-16 16:42:17 -05:00
14 changed files with 1042 additions and 714 deletions

No files matched your search

+2
View File
@@ -25,6 +25,8 @@ Credential rotation can be turned off by setting the
This plugin will attempt to assume a role when a `role_arn` is supplied through the
`attributes` object. More information about assume an AWS role can be found [here](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html). This feature only works when the plugin is running on a self managed Boundary [worker](https://developer.hashicorp.com/boundary/tutorials/hcp-administration/hcp-manage-workers).
To discover hosts across multiple AWS accounts, supply a `target_role_arn` in addition to `role_arn`. The plugin uses the principal credentials to assume the target role via a two-hop `AssumeRole` chain, enabling cross-account host discovery from a single worker. See [Attributes and Secrets](plugin/service/host/README.md#attributes-and-secrets) for the full list of `target_*` attributes.
### Environment Credentials
This plugin will attempt to retrieve credentials from environment variables. More
+1 -1
View File
@@ -4,6 +4,7 @@ go 1.25.7
require (
github.com/aws/aws-sdk-go-v2 v1.41.7
github.com/aws/aws-sdk-go-v2/credentials v1.17.46
github.com/aws/aws-sdk-go-v2/service/ec2 v1.254.0
github.com/aws/aws-sdk-go-v2/service/iam v1.47.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
@@ -26,7 +27,6 @@ require (
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.28.5 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
+60
View File
@@ -186,3 +186,63 @@ func GetCredentialAttributes(in *structpb.Struct) (*CredentialAttributes, error)
RoleTags: roleTags,
}, nil
}
// GetTargetCredentialAttributes parses values out of a protobuf struct input and returns a CredentialAttributes used
// for configuring a second AWS session. target_role_arn is required when parsing target credentials.
// A status error is returned with an InvalidArgument code when target account attributes are missing
// required values or have invalid value types.
func GetTargetCredentialAttributes(in *structpb.Struct) (*CredentialAttributes, error) {
badFields := make(map[string]string)
// Parse the role ARN of the target account to AssumeRole into.
targetRoleArn, err := values.GetStringValue(in, ConstTargetRoleArn, true)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstTargetRoleArn)] = err.Error()
}
// try parsing target_region, fallback to region, adds target_region to bad fields only when malformed since not required
targetRegion, err := values.GetStringValue(in, ConstTargetRegion, false)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstTargetRegion)] = err.Error()
}
if targetRegion == "" {
targetRegion, err = values.GetStringValue(in, ConstRegion, true) // region is required for processing
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstRegion)] = err.Error()
}
}
// the remaining fields are optional and only raise errors when malformed
disableCredentialRotation, err := values.GetBoolValue(in, ConstDisableCredentialRotation, false)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstDisableCredentialRotation)] = err.Error()
}
targetRoleExternalId, err := values.GetStringValue(in, ConstTargetRoleExternalId, false)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstTargetRoleExternalId)] = err.Error()
}
targetRoleSessionName, err := values.GetStringValue(in, ConstTargetRoleSessionName, false)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstTargetRoleSessionName)] = err.Error()
}
targetRoleTags, err := values.GetMapStringString(in, ConstTargetRoleTags, false)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstTargetRoleTags)] = err.Error()
}
if len(badFields) > 0 {
return nil, errors.InvalidArgumentError("Error in the attributes provided", badFields)
}
return &CredentialAttributes{
Region: targetRegion,
DisableCredentialRotation: disableCredentialRotation,
RoleArn: targetRoleArn,
RoleExternalId: targetRoleExternalId,
RoleSessionName: targetRoleSessionName,
RoleTags: targetRoleTags,
}, nil
}
+226
View File
@@ -101,6 +101,232 @@ func TestGetCredentialAttributes(t *testing.T) {
}
}
func TestGetTargetCredentialAttributes(t *testing.T) {
const targetRoleArn = "arn:aws:iam::222222222222:role/Target"
cases := []struct {
name string
in map[string]any
expected *CredentialAttributes
expectedErrContains string
}{
{
name: "missing target_role_arn",
in: map[string]any{
ConstRegion: "us-west-2",
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "missing target_role_arn and region",
in: map[string]any{},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "empty target_role_arn",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: "",
},
expectedErrContains: "attributes.target_role_arn: value \"target_role_arn\" cannot be empty",
},
{
name: "wrong type target_role_arn",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: true,
},
expectedErrContains: "attributes.target_role_arn: unexpected type for value \"target_role_arn\": want string, got bool",
},
{
name: "target_region without target_role_arn",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRegion: "eu-west-1",
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "other target fields without target_role_arn",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleExternalId: "ext",
ConstTargetRoleSessionName: "sess",
ConstTargetRoleTags: map[string]any{"k": "v"},
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "target_role_arn uses region when target_region omitted",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
},
expected: &CredentialAttributes{
Region: "us-west-2",
RoleArn: targetRoleArn,
},
},
{
name: "target_role_arn without target_region or region",
in: map[string]any{
ConstTargetRoleArn: targetRoleArn,
},
expectedErrContains: "missing required value \"region\"",
},
{
name: "target_region overrides region",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRegion: "eu-west-1",
ConstTargetRoleArn: targetRoleArn,
},
expected: &CredentialAttributes{
Region: "eu-west-1",
RoleArn: targetRoleArn,
},
},
{
name: "empty target_region falls back to region",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRegion: "",
ConstTargetRoleArn: targetRoleArn,
},
expected: &CredentialAttributes{
Region: "us-west-2",
RoleArn: targetRoleArn,
},
},
{
name: "all optional target fields",
in: map[string]any{
ConstRegion: "us-west-2",
ConstDisableCredentialRotation: true,
ConstTargetRoleArn: targetRoleArn,
ConstTargetRegion: "eu-west-1",
ConstTargetRoleExternalId: "target-ext",
ConstTargetRoleSessionName: "target-sess",
ConstTargetRoleTags: map[string]any{"env": "target"},
},
expected: &CredentialAttributes{
Region: "eu-west-1",
DisableCredentialRotation: true,
RoleArn: targetRoleArn,
RoleExternalId: "target-ext",
RoleSessionName: "target-sess",
RoleTags: map[string]string{"env": "target"},
},
},
{
name: "optional target fields omitted do not copy principal role fields",
in: map[string]any{
ConstRegion: "us-west-2",
ConstRoleArn: "arn:aws:iam::111111111111:role/Principal",
ConstRoleExternalId: "principal-ext",
ConstRoleSessionName: "principal-sess",
ConstRoleTags: map[string]any{"env": "principal"},
ConstTargetRoleArn: targetRoleArn,
},
expected: &CredentialAttributes{
Region: "us-west-2",
RoleArn: targetRoleArn,
},
},
{
name: "disable_credential_rotation false is copied",
in: map[string]any{
ConstRegion: "us-west-2",
ConstDisableCredentialRotation: false,
ConstTargetRoleArn: targetRoleArn,
},
expected: &CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: false,
RoleArn: targetRoleArn,
},
},
{
name: "wrong type target_region",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
ConstTargetRegion: true,
},
expectedErrContains: "attributes.target_region: unexpected type for value \"target_region\": want string, got bool",
},
{
name: "wrong type target_role_external_id",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
ConstTargetRoleExternalId: true,
},
expectedErrContains: "attributes.target_role_external_id: unexpected type for value \"target_role_external_id\": want string, got bool",
},
{
name: "wrong type target_role_session_name",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
ConstTargetRoleSessionName: true,
},
expectedErrContains: "attributes.target_role_session_name: unexpected type for value \"target_role_session_name\": want string, got bool",
},
{
name: "wrong type target_role_tags",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
ConstTargetRoleTags: true,
},
expectedErrContains: "attributes.target_role_tags: unexpected type for value \"target_role_tags\": want map[string]string, got bool",
},
{
name: "wrong type target_role_tag value",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
ConstTargetRoleTags: map[string]any{
"env": true,
},
},
expectedErrContains: `unexpected type for value in map["env"]: want string, got bool`,
},
{
name: "wrong type disable_credential_rotation",
in: map[string]any{
ConstRegion: "us-west-2",
ConstTargetRoleArn: targetRoleArn,
ConstDisableCredentialRotation: "sure",
},
expectedErrContains: "attributes.disable_credential_rotation: unexpected type for value \"disable_credential_rotation\": want bool, got string",
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)
input, err := structpb.NewStruct(tc.in)
require.NoError(err)
actual, err := GetTargetCredentialAttributes(input)
if tc.expectedErrContains != "" {
require.Error(err)
require.Nil(actual)
require.Contains(err.Error(), tc.expectedErrContains)
require.Equal(codes.InvalidArgument, status.Code(err))
return
}
require.NoError(err)
require.Equal(tc.expected, actual)
})
}
}
func TestGetCredentialsConfig(t *testing.T) {
cases := []struct {
name string
+17 -2
View File
@@ -4,10 +4,10 @@
package credential
const (
// ConstAccessKey is the key for the region in the aws credentials.
// ConstRegion is the key for the region in the aws credentials.
ConstRegion = "region"
// ConstAccessKey is the key for the access key id in the aws credentials.
// ConstAccessKeyId is the key for the access key id in the aws credentials.
ConstAccessKeyId = "access_key_id"
// ConstSecretAccessKey is the key for the secret access key in the aws credentials.
@@ -30,4 +30,19 @@ const (
// ConstRoleTags is the key for the tags used for assuming a IAM role.
ConstRoleTags = "role_tags"
// ConstTargetRegion is the key for the region of the target account in the aws credentials.
ConstTargetRegion = "target_region"
// ConstTargetRoleArn is the key for assuming a IAM role for the target account.
ConstTargetRoleArn = "target_role_arn"
// ConstTargetRoleExternalId is the key for the external id used for assuming a IAM role for the target account.
ConstTargetRoleExternalId = "target_role_external_id"
// ConstTargetRoleSessionName is the key for the session name used for assuming a IAM role of the target account.
ConstTargetRoleSessionName = "target_role_session_name"
// ConstTargetRoleTags is the key for the tags used for assuming a IAM role of the target account.
ConstTargetRoleTags = "target_role_tags"
)
@@ -1,172 +0,0 @@
// Copyright IBM Corp. 2021, 2026
// SPDX-License-Identifier: MPL-2.0
package credential
// TestGetCredentialsConfigRoleArnFallback documents all credential input
// combinations and the CredentialsConfig they produce. The key finding is
// in the "role_arn + no static keys" case: GetCredentialsConfig succeeds and
// populates CredentialsConfig.RoleARN, but no STS call is ever made during
// config construction. At runtime, awsutil.GenerateCredentialChain silently
// falls back to the ambient credential chain (env vars / instance profile)
// without calling sts:AssumeRole, so a bogus or unreachable ARN is never
// detected.
//
// Run this test to confirm the root cause:
//
// go test ./internal/credential/... -run TestGetCredentialsConfigRoleArnFallback -v
import (
"testing"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/structpb"
)
func TestGetCredentialsConfigRoleArnFallback(t *testing.T) {
cases := []struct {
name string
secrets *structpb.Struct
attrs *CredentialAttributes
expectedAccessKey string
expectedSecretKey string
expectedRoleARN string
expectedType CredentialType
expectedErrContains string
}{
{
// NOTE: No role_arn and no static keys. GetCredentialsConfig
// succeeds and returns a CredentialsConfig with no credentials set
// at all. GetCredentialType returns Unknown because there is no
// AccessKey prefix to classify. The AWS SDK will use whatever
// ambient credentials it finds at runtime (env vars, instance
// profile, etc.).
name: "ambient only — no role_arn, no static keys",
attrs: &CredentialAttributes{
Region: "us-east-1",
DisableCredentialRotation: true,
},
expectedAccessKey: "",
expectedSecretKey: "",
expectedRoleARN: "",
expectedType: Unknown,
},
{
// NOTE: The normal static-key path. Both keys are provided; no
// role_arn. GetCredentialType returns StaticAWS because the
// AccessKey starts with "AKIA".
name: "static keys only",
secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
ConstAccessKeyId: structpb.NewStringValue("AKIAfoobar"),
ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
},
},
attrs: &CredentialAttributes{
Region: "us-east-1",
DisableCredentialRotation: true,
},
expectedAccessKey: "AKIAfoobar",
expectedSecretKey: "bazqux",
expectedRoleARN: "",
expectedType: StaticAWS,
},
{
// NOTE: THE BUG SCENARIO. role_arn is set (to a clearly bogus
// value) and disable_credential_rotation is true, but no static
// keys are provided. GetCredentialsConfig succeeds — the ARN is
// stored in CredentialsConfig.RoleARN. GetCredentialType returns
// DynamicAWS. However, no sts:AssumeRole call is ever made here
// or during GenerateCredentialChain: the awsutil library skips the
// assume-role path when AccessKey is empty and falls back to the
// ambient credential chain instead. The bogus ARN is never
// validated or exercised.
name: "role_arn + no static keys (THE BUG SCENARIO)",
attrs: &CredentialAttributes{
Region: "us-east-1",
RoleArn: "arn:aws:sts::000000000000:assumed-role/fake/whoever",
DisableCredentialRotation: true,
},
expectedAccessKey: "",
expectedSecretKey: "",
expectedRoleARN: "arn:aws:sts::000000000000:assumed-role/fake/whoever",
expectedType: DynamicAWS,
},
{
// NOTE: role_arn is set but disable_credential_rotation is false
// (the default). This is caught by the validation switch and
// returns an explicit error. Validation works correctly here.
name: "role_arn without disable_credential_rotation — validation error",
attrs: &CredentialAttributes{
Region: "us-east-1",
RoleArn: "arn:aws:iam::123456789012:role/MyRole",
},
expectedErrContains: "disable_credential_rotation attribute is required when providing a role_arn",
},
{
// NOTE: Both role_arn and static keys are provided together. This
// is caught by the validation switch and returns an error listing
// all three conflicting fields. Validation works correctly here.
name: "role_arn + static keys — conflict error",
secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
ConstAccessKeyId: structpb.NewStringValue("AKIAfoobar"),
ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
},
},
attrs: &CredentialAttributes{
Region: "us-east-1",
RoleArn: "arn:aws:iam::123456789012:role/MyRole",
},
expectedErrContains: "conflicts with role_arn value",
},
{
// NOTE: access_key_id is provided but secret_access_key is
// missing. The validation switch catches the incomplete pair.
name: "static keys — missing secret_access_key",
secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
ConstAccessKeyId: structpb.NewStringValue("AKIAfoobar"),
},
},
attrs: &CredentialAttributes{Region: "us-east-1"},
expectedErrContains: "secrets.secret_access_key: missing required value",
},
{
// NOTE: secret_access_key is provided but access_key_id is
// missing. The validation switch catches the incomplete pair.
name: "static keys — missing access_key_id",
secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
},
},
attrs: &CredentialAttributes{Region: "us-east-1"},
expectedErrContains: "secrets.access_key_id: missing required value",
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)
cfg, err := GetCredentialsConfig(tc.secrets, tc.attrs, false)
if tc.expectedErrContains != "" {
require.Error(err)
require.Contains(err.Error(), tc.expectedErrContains)
require.Equal(codes.InvalidArgument, status.Code(err))
return
}
require.NoError(err)
require.Equal(tc.expectedAccessKey, cfg.AccessKey)
require.Equal(tc.expectedSecretKey, cfg.SecretKey)
require.Equal(tc.expectedRoleARN, cfg.RoleARN)
require.Equal(tc.expectedType, GetCredentialType(cfg),
"credential type mismatch — see NOTE above the test case for why this matters")
})
}
}
+5
View File
@@ -23,6 +23,11 @@ import (
)
const (
// AwsErrorAuthFailure is returned by EC2 when credentials exist in IAM
// but have not yet propagated to EC2. This is a transient eventual
// consistency issue that can resolve itself by retrying the request.
AwsErrorAuthFailure = "AuthFailure"
// awsErrorAccessDenied is returned when the credentials do not have
// the required permissions to do a specific action on a resource.
// This error will persist until the IAM Role Policy is updated by
+14 -4
View File
@@ -109,13 +109,23 @@ The following `attributes` are valid on an AWS host catalog resource:
- `region` (string): The region to configure the host catalog for. All host sets
in this catalog will be configured for this region.
- `role_arn` (string): The AWS Role ARN used for `AssumeRole` authentication. If
provided, `disable_credential_rotation` must be also set to `true.
- `role_external_id` (string): The external id configured for the `AssumeRole`
provided, `disable_credential_rotation` must be also set to `true`.
- `role_external_id` (string): The external ID configured for the principal `AssumeRole`
provider.
- `role_session_name` (string): The session name configured for the `AssumeRole`
- `role_session_name` (string): The session name configured for the principal `AssumeRole`
provider.
- `role_tags` (object): The key-value pair tags configured for the `AssumeRole`
- `role_tags` (object): The key-value pair tags configured for the principal `AssumeRole`
provider.
- `target_role_arn` (string): The AWS role ARN of the target IAM role to assume for
cross-account discovery. Required if any other `target_*` attributes are configured.
- `target_region` (string): The region where the target hosts reside. Defaults to the
principal account `region` if omitted.
- `target_role_external_id` (string): The external ID configured for the target
`AssumeRole` provider.
- `target_role_session_name` (string): The session name configured for the target
`AssumeRole` provider.
- `target_role_tags` (object): The key-value pair tags configured for the target
`AssumeRole` provider.
The following `secrets` are required on an AWS host catalog resource:
+38
View File
@@ -20,6 +20,9 @@ import (
type CatalogAttributes struct {
*cred.CredentialAttributes
// TargetAccount optionally holds the credentials used to AssumeRole into a second, affiliated account when provided.
TargetAccount *cred.CredentialAttributes
// DualStack is used for configuring how the aws client will resolve requests.
DualStack bool
@@ -39,6 +42,14 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
return nil, err
}
var targetCredAttributes *cred.CredentialAttributes
if hasTargetAccountAttributes(in) {
targetCredAttributes, err = cred.GetTargetCredentialAttributes(in)
if err != nil {
return nil, err
}
}
dualStack, err := values.GetBoolValue(in, ConstAwsDualStack, false)
if err != nil {
badFields[fmt.Sprintf("attributes.%s", ConstAwsDualStack)] = err.Error()
@@ -66,6 +77,16 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
continue
case cred.ConstRoleTags:
continue
case cred.ConstTargetRegion:
continue
case cred.ConstTargetRoleArn:
continue
case cred.ConstTargetRoleExternalId:
continue
case cred.ConstTargetRoleSessionName:
continue
case cred.ConstTargetRoleTags:
continue
default:
badFields[fmt.Sprintf("attributes.%s", s)] = "unrecognized field"
}
@@ -77,6 +98,7 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
return &CatalogAttributes{
CredentialAttributes: credAttributes,
TargetAccount: targetCredAttributes,
DualStack: dualStack,
InstanceAddressesOnly: instanceAddressesOnly,
}, nil
@@ -121,3 +143,19 @@ func getSetAttributes(in *structpb.Struct) (*SetAttributes, error) {
return &setAttrs, nil
}
func hasTargetAccountAttributes(in *structpb.Struct) bool {
fields := in.GetFields()
_, hasTargetRegion := fields[cred.ConstTargetRegion]
_, hasTargetRoleArn := fields[cred.ConstTargetRoleArn]
_, hasTargetRoleExternalID := fields[cred.ConstTargetRoleExternalId]
_, hasTargetRoleSessionName := fields[cred.ConstTargetRoleSessionName]
_, hasTargetRoleTags := fields[cred.ConstTargetRoleTags]
return hasTargetRegion ||
hasTargetRoleArn ||
hasTargetRoleExternalID ||
hasTargetRoleSessionName ||
hasTargetRoleTags
}
+164
View File
@@ -118,6 +118,170 @@ func TestGetCatalogAttributes(t *testing.T) {
},
},
},
{
name: "target_region without target_role_arn",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"target_region": structpb.NewStringValue("eu-west-1"),
},
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "target role field without target_role_arn",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"target_role_external_id": structpb.NewStringValue("target-ext"),
},
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "other target role fields without target_role_arn",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"target_role_session_name": structpb.NewStringValue("target-session"),
"target_role_tags": structpb.NewStructValue(&structpb.Struct{
Fields: map[string]*structpb.Value{
"env": structpb.NewStringValue("target"),
},
}),
},
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
},
{
name: "empty target_role_arn with target_region",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"target_role_arn": structpb.NewStringValue(""),
"target_region": structpb.NewStringValue("eu-west-1"),
},
},
expectedErrContains: "attributes.target_role_arn: value \"target_role_arn\" cannot be empty",
},
{
name: "target_role_arn without target_region uses region; principal sts fields are not copied",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"disable_credential_rotation": structpb.NewBoolValue(true),
"role_arn": structpb.NewStringValue("arn:aws:iam::111111111111:role/Principal"),
"role_external_id": structpb.NewStringValue("principal-ext"),
"role_session_name": structpb.NewStringValue("principal-sess"),
"role_tags": structpb.NewStructValue(&structpb.Struct{
Fields: map[string]*structpb.Value{
"env": structpb.NewStringValue("principal"),
},
}),
"target_role_arn": structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
},
},
expected: &CatalogAttributes{
CredentialAttributes: &cred.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: true,
RoleArn: "arn:aws:iam::111111111111:role/Principal",
RoleExternalId: "principal-ext",
RoleSessionName: "principal-sess",
RoleTags: map[string]string{
"env": "principal",
},
},
TargetAccount: &cred.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: true,
RoleArn: "arn:aws:iam::222222222222:role/Target",
},
},
},
{
name: "target attributes are independent of principal",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"disable_credential_rotation": structpb.NewBoolValue(true),
"role_arn": structpb.NewStringValue("arn:aws:iam::111111111111:role/Principal"),
"role_external_id": structpb.NewStringValue("principal-ext"),
"role_session_name": structpb.NewStringValue("principal-sess"),
"role_tags": structpb.NewStructValue(&structpb.Struct{
Fields: map[string]*structpb.Value{
"env": structpb.NewStringValue("principal"),
},
}),
"target_role_arn": structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
"target_region": structpb.NewStringValue("eu-west-1"),
"target_role_external_id": structpb.NewStringValue("target-ext"),
"target_role_session_name": structpb.NewStringValue("target-sess"),
"target_role_tags": structpb.NewStructValue(&structpb.Struct{
Fields: map[string]*structpb.Value{
"env": structpb.NewStringValue("target"),
},
}),
},
},
expected: &CatalogAttributes{
CredentialAttributes: &cred.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: true,
RoleArn: "arn:aws:iam::111111111111:role/Principal",
RoleExternalId: "principal-ext",
RoleSessionName: "principal-sess",
RoleTags: map[string]string{
"env": "principal",
},
},
TargetAccount: &cred.CredentialAttributes{
Region: "eu-west-1",
DisableCredentialRotation: true,
RoleArn: "arn:aws:iam::222222222222:role/Target",
RoleExternalId: "target-ext",
RoleSessionName: "target-sess",
RoleTags: map[string]string{
"env": "target",
},
},
},
},
{
name: "catalog dual_stack and instance_addresses_only with target",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"dual_stack": structpb.NewBoolValue(true),
"instance_addresses_only": structpb.NewBoolValue(true),
"target_role_arn": structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
},
},
expected: &CatalogAttributes{
CredentialAttributes: &credential.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: false,
},
TargetAccount: &cred.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: false,
RoleArn: "arn:aws:iam::222222222222:role/Target",
},
DualStack: true,
InstanceAddressesOnly: true,
},
},
{
name: "unknown field with target_role_arn",
in: &structpb.Struct{
Fields: map[string]*structpb.Value{
"region": structpb.NewStringValue("us-west-2"),
"target_role_arn": structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
"foo": structpb.NewBoolValue(true),
},
},
expectedErrContains: "attributes.foo: unrecognized field",
},
}
for _, tc := range cases {
+170 -18
View File
@@ -5,12 +5,18 @@ package host
import (
"context"
goerrors "errors"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/aws/aws-sdk-go-v2/service/sts"
stsTypes "github.com/aws/aws-sdk-go-v2/service/sts/types"
"github.com/aws/smithy-go"
"github.com/hashicorp/boundary-plugin-aws/internal/credential"
"github.com/hashicorp/boundary-plugin-aws/internal/errors"
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
@@ -32,6 +38,16 @@ type HostPlugin struct {
testCatalogStateOpts []awsCatalogPersistedStateOption
}
type hostSetQuery struct {
Id string
Input *ec2.DescribeInstancesInput
Output *ec2.DescribeInstancesOutput
OutputHosts []*pb.ListHostsResponseHost
}
const defaultSessionName = "boundary-default"
const dryRunOperationErrorCode = "DryRunOperation"
// Ensure that we are implementing HostPluginServiceServer
var _ pb.HostPluginServiceServer = (*HostPlugin)(nil)
@@ -495,13 +511,6 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
}
// Build all the queries in advance.
type hostSetQuery struct {
Id string
Input *ec2.DescribeInstancesInput
Output *ec2.DescribeInstancesOutput
OutputHosts []*pb.ListHostsResponseHost
}
queries := make([]hostSetQuery, len(sets))
for i, set := range sets {
// Validate Id since we use it in output
@@ -531,20 +540,22 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
if catalogAttributes.DualStack {
opts = append(opts, WithDualStack(catalogAttributes.DualStack))
}
ec2Client, err := catalogState.EC2Client(ctx, opts...)
if err != nil {
return nil, errors.BadRequestStatusf("error getting EC2 client: %s", err)
}
// Run all queries now and assemble output.
var maxLen int
for i, query := range queries {
output, err := ec2Client.DescribeInstances(ctx, query.Input)
output, err := checkHosts(ctx, catalogState, query.Input, opts, catalogAttributes.TargetAccount)
if err != nil {
return nil, errors.BadRequestStatusf("error running DescribeInstances for host set id %q: %s", query.Id, err)
if catalogAttributes.TargetAccount != nil {
return nil, errors.BadRequestStatusf("unable to list hosts for host set %q: %s", query.Id, err)
}
return nil, errors.BadRequestStatusf("error retrieving host results for host set id %q: %s", query.Id, err)
}
queries[i].Output = output
if output == nil {
continue
}
// Process the output here, we will normalize this into a single
// set of hosts afterwards (possibly removing duplicates).
@@ -707,10 +718,129 @@ func awsInstanceToHost(instance types.Instance, catalogAttributes *CatalogAttrib
return result, nil
}
func checkHosts(ctx context.Context,
principalCatalogState *awsCatalogPersistedState,
input *ec2.DescribeInstancesInput,
opts []ec2Option,
target *credential.CredentialAttributes,
) (*ec2.DescribeInstancesOutput, error) {
if principalCatalogState == nil {
return nil, fmt.Errorf("unable to create EC2 client from catalog state")
}
var ec2Client EC2API
var err error
if target == nil {
// use the principal account to execute the DescribeInstances command
ec2Client, err = principalCatalogState.EC2Client(ctx, opts...)
if err != nil {
return nil, err
}
} else {
// use the target account to execute the DescribeInstances command
ec2Client, err = ec2ClientForTarget(ctx, principalCatalogState, target, opts)
if err != nil {
return nil, fmt.Errorf("EC2 client setup failed for target account: %w", err)
}
}
output, err := ec2Client.DescribeInstances(ctx, input)
if err != nil {
var awsErr smithy.APIError
ok := goerrors.As(err, &awsErr)
if ok && awsErr.ErrorCode() == dryRunOperationErrorCode {
// successful DryRun validation, no response necessary, no error thrown
return nil, nil
}
if target != nil {
return nil, fmt.Errorf("EC2 DescribeInstances failed for target account: %w", err)
}
return nil, err
}
return output, nil
}
// ec2ClientForTarget builds an EC2 client to query the target account.
// The hop is a continuation of the principal session: STS AssumeRole is called
// as the principal to get to the target, then EC2 uses the resulting temporary credentials.
// This is not a second worker-level credential chain on the target attributes.
func ec2ClientForTarget(
ctx context.Context,
principalCatalogState *awsCatalogPersistedState,
target *credential.CredentialAttributes,
opts []ec2Option) (EC2API, error) {
// Reuse the principal SDK config (HTTP client, retries, logger, and the
// principal identity).
principalCfg, err := principalCatalogState.GenerateCredentialChain(ctx)
if err != nil {
return nil, fmt.Errorf("principal AWS configuration could not be loaded: %w", err)
}
if principalCfg == nil {
return nil, fmt.Errorf("nil principal AWS configuration while configuring EC2 client for target account")
}
// STS must run as the principal so the target role's trust policy sees that
// identity, not the worker's. Session name, external ID, and tags are
// forwarded from attrs when set so the AssumeRole matches the target role's configuration.
stsClient := sts.NewFromConfig(*principalCfg)
provider := stscreds.NewAssumeRoleProvider(stsClient, target.RoleArn, func(o *stscreds.AssumeRoleOptions) {
if target.RoleSessionName != "" {
o.RoleSessionName = target.RoleSessionName
} else {
// STS may reject the AssumeRole request if session name is not provided.
o.RoleSessionName = defaultSessionName
}
if target.RoleExternalId != "" {
o.ExternalID = &target.RoleExternalId
}
for k, v := range target.RoleTags {
o.Tags = append(o.Tags, stsTypes.Tag{
Key: aws.String(k),
Value: aws.String(v),
})
}
})
// Same worker SDK environment as the principal; only identity and region change.
targetCfg := principalCfg.Copy()
targetCfg.Credentials = aws.NewCredentialsCache(provider)
if target.Region != "" {
targetCfg.Region = target.Region
}
parsed, err := getOpts(opts...)
if err != nil {
return nil, fmt.Errorf("EC2 client options are invalid: %w", err)
}
// Dual-stack is the worker's path to AWS endpoints, not per-account. Apply
// it on the hopped client the same way EC2Client does for the principal.
var ec2Opts []func(*ec2.Options)
if parsed.withDualStack {
ec2Opts = append(ec2Opts, ec2.WithEndpointResolverV2(&endpointResolver{
dualStack: parsed.withDualStack,
}))
}
if principalCatalogState.testEC2APIFunc != nil {
return principalCatalogState.testEC2APIFunc(targetCfg)
}
return ec2.NewFromConfig(targetCfg, ec2Opts...), nil
}
// dryRunValidationTimeout is the maximum time to wait for credentials to
// propagate across AWS services after rotation before giving up.
const dryRunValidationTimeout = 30 * time.Second
// dryRunValidation performs an AWS DescribeInstances call to verify the state's
// credentials, the host listing functionality as well as the filters, if any
// are passed in. This function can therefore be used for both host catalog and
// host set validation.
//
// After credential rotation, IAM keys may be valid for STS (GetCallerIdentity)
// but not yet propagated to EC2. This function retries on AuthFailure to
// tolerate that eventual consistency window.
func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, ec2Opts []ec2Option, filters ...types.Filter) *status.Status {
if state == nil {
return status.New(codes.InvalidArgument, "persisted state is required")
@@ -721,12 +851,34 @@ func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, ec2O
return status.New(codes.InvalidArgument, fmt.Sprintf("error getting EC2 client: %s", err))
}
_, err = ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{Filters: filters})
if err != nil {
return status.New(codes.FailedPrecondition, fmt.Sprintf("aws describe instances failed: %s", err))
}
deadline := time.Now().Add(dryRunValidationTimeout)
delay := time.Second
for {
_, err = ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{Filters: filters})
if err == nil {
return nil
}
return nil
// AuthFailure indicates the credentials exist but haven't propagated
// to EC2 yet — a transient eventual consistency issue after rotation.
// Retry until the deadline.
if strings.Contains(err.Error(), errors.AwsErrorAuthFailure) && time.Now().Before(deadline) {
select {
case <-time.After(delay):
// Wait before retrying to give credentials time to propagate.
case <-ctx.Done():
// The caller cancelled — break out of the select.
break
}
if ctx.Err() != nil {
break
}
continue
}
break
}
return status.New(codes.FailedPrecondition, fmt.Sprintf("aws describe instances failed: %s", err))
}
// appendDistinct will append the elements to the slice
@@ -1,516 +0,0 @@
// Copyright IBM Corp. 2021, 2026
// SPDX-License-Identifier: MPL-2.0
package host
// This file contains integration tests that demonstrate the role_arn silent
// fallback bug at the plugin level.
//
// THE BUG IN ONE SENTENCE:
// When a host catalog is configured with role_arn but no static credentials,
// the plugin never calls sts:AssumeRole. It silently falls back to the
// ambient AWS credential chain (env vars / instance profile), so a bogus or
// unreachable ARN produces no error and returns the same hosts as if no ARN
// were specified at all.
//
// HOW TO READ THESE TESTS:
// Each test function covers one plugin operation (OnCreateCatalog or
// ListHosts). Within each function, the cases are ordered to tell a
// progressive story:
// 1. Ambient baseline — no role_arn, works fine
// 2. Static-key path — explicit keys, works fine
// 3. Bug case — bogus role_arn + no keys, silently succeeds
// 4. Desired behavior — bogus role_arn + STS error injected, SHOULD fail
// 5. Validation errors — combinations that are correctly rejected
//
// Cases marked "BUG" in the soft-fail block currently pass (no error is
// returned). Once the bug is fixed those tests will start asserting correctly
// and the t.Fail() path will no longer be reached.
//
// Run both test functions:
//
// go test ./plugin/service/host/... -run TestPluginRoleArnFallback -v
import (
"context"
"errors"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/hashicorp/boundary-plugin-aws/internal/credential"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
"github.com/hashicorp/go-secure-stdlib/awsutil/v2"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/structpb"
)
// ambientInstance is the single fake EC2 instance returned by every mock in
// this file. Seeing this instance in the response means the ambient credential
// chain was used, regardless of what role_arn was configured.
var ambientInstance = ec2types.Instance{
InstanceId: aws.String("i-ambientinstance"),
PrivateIpAddress: aws.String("10.0.0.1"),
PrivateDnsName: aws.String("ambient.example.internal"),
NetworkInterfaces: []ec2types.InstanceNetworkInterface{
{
PrivateIpAddress: aws.String("10.0.0.1"),
PrivateDnsName: aws.String("ambient.example.internal"),
PrivateIpAddresses: []ec2types.InstancePrivateIpAddress{
{
PrivateIpAddress: aws.String("10.0.0.1"),
PrivateDnsName: aws.String("ambient.example.internal"),
},
},
},
},
}
// ambientInstanceOutput is a DescribeInstancesOutput that returns ambientInstance.
// All mock EC2 clients in this file are configured with this output so that
// success cases always return the same identifiable host.
var ambientInstanceOutput = &ec2.DescribeInstancesOutput{
Reservations: []ec2types.Reservation{
{Instances: []ec2types.Instance{ambientInstance}},
},
}
// catalogAttrsStruct builds a catalog attributes struct for the given region.
// disableRotation is always written explicitly — false must be set to prevent
// credential rotation from being attempted (which would require IAM mocks).
// roleArn is written only when non-empty.
func catalogAttrsStruct(region, roleArn string, disableRotation bool) *structpb.Struct {
fields := map[string]*structpb.Value{
credential.ConstRegion: structpb.NewStringValue(region),
credential.ConstDisableCredentialRotation: structpb.NewBoolValue(disableRotation),
}
if roleArn != "" {
fields[credential.ConstRoleArn] = structpb.NewStringValue(roleArn)
}
return &structpb.Struct{Fields: fields}
}
// ─────────────────────────────────────────────────────────────────────────────
// OnCreateCatalog
// ─────────────────────────────────────────────────────────────────────────────
func TestPluginRoleArnFallback_OnCreateCatalog(t *testing.T) {
// bogusARN is the ARN used in bug-scenario cases. It contains a fake
// account ID (000000000000) that does not correspond to any real AWS
// account. A correct implementation would fail when it attempts to call
// sts:AssumeRole with this ARN. Currently it is silently ignored.
const bogusARN = "arn:aws:sts::000000000000:assumed-role/fake/whoever"
tests := []struct {
name string
req *pb.OnCreateCatalogRequest
credOpts []credential.AwsCredentialPersistedStateOption
catalogOpts []awsCatalogPersistedStateOption
// wantErr is set for cases that should return an error.
wantErr bool
wantErrContains string
wantErrCode codes.Code
// wantEmptySecrets is true when the persisted secrets should be empty
// (dynamic / ambient credentials are not stored in the database).
wantEmptySecrets bool
}{
{
// NOTE: No role_arn, no static keys. OnCreateCatalog succeeds using
// whatever ambient credentials are available. The persisted secrets
// are empty because there are no static keys to store. This is the
// baseline — it shows that ambient credentials work on their own.
name: "1. ambient only — no role_arn, no static keys (baseline)",
req: &pb.OnCreateCatalogRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", "", true),
},
},
},
catalogOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
nil,
testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
)),
},
wantEmptySecrets: true,
},
{
// NOTE: Explicit static keys, rotation disabled. OnCreateCatalog
// succeeds and persists the key pair. This is the normal static-key
// path — no role assumption involved. No credOpts are needed here:
// with disable_credential_rotation=true and StaticAWS keys, the
// dry-run only calls the mocked EC2 DescribeInstances and does not
// make any real AWS calls (same pattern as the existing
// TestPluginOnCreateCatalogSuccess/usingStaticCredentials test).
name: "2. static keys only (baseline)",
req: &pb.OnCreateCatalogRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", "", true),
},
Secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstAccessKeyId: structpb.NewStringValue("AKIAfoo"),
credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
},
},
},
},
catalogOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
nil,
testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
)),
},
wantEmptySecrets: false, // key pair is stored
},
{
// NOTE: THE BUG — bogus role_arn, no static keys. OnCreateCatalog
// succeeds even though the ARN points to a nonexistent role. The
// dry-run DescribeInstances call is mocked out, so the bad ARN is
// never presented to STS. The persisted secrets are empty (dynamic
// credentials are not stored), which is correct — but the absence
// of an error is not.
name: "3. bogus role_arn + no static keys — BUG: succeeds silently",
req: &pb.OnCreateCatalogRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
},
},
},
catalogOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
nil,
testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
)),
},
wantEmptySecrets: true,
},
{
// NOTE: DESIRED BEHAVIOR (currently broken). Same bogus role_arn,
// but we inject an STS mock that returns an error when AssumeRole
// is called. A correct implementation would propagate that error
// and OnCreateCatalog would fail. Currently it succeeds because
// sts:AssumeRole is never called — the STS mock is never reached.
// The soft-fail below documents this: if err == nil it means the
// bug is still present.
name: "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail",
req: &pb.OnCreateCatalogRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
},
},
},
credOpts: []credential.AwsCredentialPersistedStateOption{
credential.WithStateTestOpts([]awsutil.Option{
awsutil.WithSTSAPIFunc(
awsutil.NewMockSTS(
awsutil.WithAssumeRoleError(
errors.New("simulated STS AssumeRole failure: no such role"),
),
),
),
}),
},
catalogOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
nil,
testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
)),
},
// We do NOT set wantErr here — the soft-fail logic below handles
// this case explicitly so we can log a clear BUG message.
},
{
// NOTE: Providing both role_arn and static keys is correctly
// rejected at validation time. The switch in GetCredentialsConfig
// catches this combination before any AWS call is attempted.
name: "5. role_arn + static keys — correctly rejected (conflict error)",
wantErr: true,
req: &pb.OnCreateCatalogRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
},
Secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstAccessKeyId: structpb.NewStringValue("AKIAfoo"),
credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
},
},
},
},
wantErrContains: "conflicts with role_arn value",
wantErrCode: codes.InvalidArgument,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
p := &HostPlugin{
testCredStateOpts: tt.credOpts,
testCatalogStateOpts: tt.catalogOpts,
}
rsp, err := p.OnCreateCatalog(context.Background(), tt.req)
// Case 4 — desired behavior, soft-fail.
if tt.name == "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail" {
if err == nil {
t.Log("BUG: expected OnCreateCatalog to fail when sts:AssumeRole returns an error,",
"but it succeeded — the role_arn is being silently ignored and sts:AssumeRole",
"is never called. This test will pass once the bug is fixed.")
t.Fail()
}
return
}
if tt.wantErr {
require.Error(t, err)
require.Contains(t, err.Error(), tt.wantErrContains)
require.Equal(t, tt.wantErrCode.String(), status.Code(err).String())
return
}
require.NoError(t, err)
secrets := rsp.GetPersisted().GetSecrets().GetFields()
// Remove the rotation timestamp before checking emptiness.
delete(secrets, credential.ConstCredsLastRotatedTime)
if tt.wantEmptySecrets {
require.Empty(t, secrets,
"expected empty persisted secrets (dynamic/ambient credentials are not stored)")
} else {
require.NotEmpty(t, secrets,
"expected persisted secrets to contain the static key pair")
}
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// ListHosts
// ─────────────────────────────────────────────────────────────────────────────
// listHostsRequest builds a minimal ListHostsRequest. The persisted secrets
// are always an empty struct because the bug scenario uses dynamic credentials
// (nothing is persisted for ambient or role-based auth).
func listHostsRequest(attrs *structpb.Struct) *pb.ListHostsRequest {
return &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: attrs,
},
},
Persisted: &pb.HostCatalogPersisted{
Secrets: &structpb.Struct{},
},
Sets: []*hostsets.HostSet{
{
Id: "set-1",
Attrs: &hostsets.HostSet_Attributes{
Attributes: &structpb.Struct{},
},
},
},
}
}
func TestPluginRoleArnFallback_ListHosts(t *testing.T) {
// bogusARN is a clearly invalid ARN: a real AWS account ID has 12 digits,
// and "fake/whoever" is not a valid role path. A correct implementation
// would call sts:AssumeRole with this ARN and fail. Currently it is ignored.
const bogusARN = "arn:aws:sts::000000000000:assumed-role/fake/whoever"
// ec2WithAmbientInstance is the mock EC2 factory used by all three
// narrative cases. It always returns the same ambientInstance regardless
// of what credentials were resolved. Seeing "i-ambientinstance" in the
// response from case 3 (bogus role_arn) proves the ARN was ignored and
// the same ambient identity was used as in case 1.
ec2WithAmbientInstance := []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
nil,
testMockEC2WithDescribeInstancesOutput(ambientInstanceOutput),
)),
}
cases := []struct {
name string
req *pb.ListHostsRequest
credOpts []credential.AwsCredentialPersistedStateOption
catalogOpts []awsCatalogPersistedStateOption
// wantHosts is the expected number of hosts in a successful response.
wantHosts int
wantErr bool
wantErrContains string
wantErrCode codes.Code
}{
{
// NOTE: No role_arn, no static keys. ListHosts succeeds using
// ambient credentials and returns i-ambientinstance (10.0.0.1).
// This is the baseline — the "normal" ambient credential flow.
name: "1. ambient only — no role_arn, no static keys (baseline)",
req: listHostsRequest(catalogAttrsStruct("us-east-1", "", true)),
catalogOpts: ec2WithAmbientInstance,
wantHosts: 1,
},
{
// NOTE: Explicit static keys. ListHosts succeeds and returns
// i-ambientinstance. The same mock EC2 is used here just for
// consistency; in a real scenario static keys and role-assumed
// keys would return different instances.
name: "2. static keys only (baseline)",
req: &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", "", true),
},
},
Persisted: &pb.HostCatalogPersisted{
Secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstAccessKeyId: structpb.NewStringValue("AKIAfoo"),
credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
},
},
},
Sets: []*hostsets.HostSet{
{
Id: "set-1",
Attrs: &hostsets.HostSet_Attributes{
Attributes: &structpb.Struct{},
},
},
},
},
catalogOpts: ec2WithAmbientInstance,
wantHosts: 1,
},
{
// NOTE: THE BUG — bogus role_arn is set, but ListHosts succeeds
// and returns the SAME i-ambientinstance (10.0.0.1) as the ambient
// baseline in case 1. The two responses are identical. This proves
// the role_arn was completely ignored: the ambient credential chain
// was used instead of calling sts:AssumeRole with the bogus ARN.
name: "3. bogus role_arn + no static keys — BUG: same hosts as ambient baseline",
req: listHostsRequest(catalogAttrsStruct("us-east-1", bogusARN, true)),
catalogOpts: ec2WithAmbientInstance,
wantHosts: 1, // same count AND same instance ID as case 1 — the ARN was ignored
},
{
// NOTE: DESIRED BEHAVIOR (currently broken). Same bogus role_arn,
// but an STS mock is injected that returns an error when AssumeRole
// is called. A correct implementation would propagate that STS
// error and ListHosts would return an error. Currently ListHosts
// succeeds because sts:AssumeRole is never called — the injected
// error has no effect. The soft-fail below makes the bug visible
// without causing the full test run to abort.
name: "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail",
req: listHostsRequest(catalogAttrsStruct("us-east-1", bogusARN, true)),
credOpts: []credential.AwsCredentialPersistedStateOption{
credential.WithStateTestOpts([]awsutil.Option{
awsutil.WithSTSAPIFunc(
awsutil.NewMockSTS(
awsutil.WithAssumeRoleError(
errors.New("simulated STS AssumeRole failure: no such role"),
),
),
),
}),
},
catalogOpts: ec2WithAmbientInstance,
// wantErr deliberately not set — handled by soft-fail logic below.
},
{
// NOTE: Providing both role_arn and static keys is correctly
// rejected at validation time, before any AWS call is made.
// This case is here to show that not all role_arn combinations
// are silently ignored — only the "no static keys" path is affected.
name: "5. role_arn + static keys — correctly rejected (conflict error)",
req: &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
},
},
Persisted: &pb.HostCatalogPersisted{
Secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstAccessKeyId: structpb.NewStringValue("AKIAfoo"),
credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
},
},
},
Sets: []*hostsets.HostSet{
{
Id: "set-1",
Attrs: &hostsets.HostSet_Attributes{
Attributes: &structpb.Struct{},
},
},
},
},
// ListHosts calls AwsCredentialPersistedStateFromProto which does
// NOT use GetCredentialsConfig — it reconstructs from persisted
// state. With both a role_arn in attrs and keys in persisted
// secrets, the state is built with both set, which results in the
// DynamicAWS path. No conflict error is raised at this layer.
// We still include this case for completeness, expecting success
// (the conflict is only checked in GetCredentialsConfig, which is
// called during OnCreateCatalog, not during ListHosts).
catalogOpts: ec2WithAmbientInstance,
wantHosts: 1,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
p := &HostPlugin{
testCredStateOpts: tc.credOpts,
testCatalogStateOpts: tc.catalogOpts,
}
rsp, err := p.ListHosts(context.Background(), tc.req)
// Case 4 — desired behavior, soft-fail.
if tc.name == "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail" {
if err == nil {
t.Log("BUG: expected ListHosts to fail when sts:AssumeRole returns an error,",
"but it succeeded — the role_arn is being silently ignored and sts:AssumeRole",
"is never called. Compare the response with case 1 (ambient baseline):",
"both return i-ambientinstance, proving the same ambient credentials were",
"used in both cases. This test will assert correctly once the bug is fixed.")
t.Fail()
}
return
}
if tc.wantErr {
require.Error(t, err)
require.Contains(t, err.Error(), tc.wantErrContains)
require.Equal(t, tc.wantErrCode.String(), status.Code(err).String())
return
}
require.NoError(t, err)
require.Len(t, rsp.GetHosts(), tc.wantHosts,
"unexpected number of hosts returned — see case NOTE for what this proves")
if tc.wantHosts > 0 {
require.Equal(t, "i-ambientinstance", rsp.GetHosts()[0].GetExternalId(),
"case 3 returning i-ambientinstance (same as case 1) proves the bogus ARN was ignored")
}
})
}
}
+314 -1
View File
@@ -18,6 +18,7 @@ import (
iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/boundary-plugin-aws/internal/credential"
pluginErrors "github.com/hashicorp/boundary-plugin-aws/internal/errors"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
@@ -1849,6 +1850,162 @@ func TestPluginOnUpdateSetErr(t *testing.T) {
}
}
func testListHostsSet(id, filter string) *hostsets.HostSet {
return &hostsets.HostSet{
Id: id,
Attrs: &hostsets.HostSet_Attributes{
Attributes: &structpb.Struct{
Fields: map[string]*structpb.Value{
ConstDescribeInstancesFilters: structpb.NewListValue(
&structpb.ListValue{
Values: []*structpb.Value{
structpb.NewStringValue(filter),
},
},
),
},
},
},
}
}
func testListHostsPersisted() *pb.HostCatalogPersisted {
return &pb.HostCatalogPersisted{
Secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstAccessKeyId: structpb.NewStringValue("AKIA_foobar"),
credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
},
},
}
}
func testExpectedDescribeInput(filterKey, filterValue string) *ec2.DescribeInstancesInput {
return &ec2.DescribeInstancesInput{
DryRun: aws.Bool(false),
Filters: []types.Filter{
{
Name: aws.String(filterKey),
Values: []string{filterValue},
},
{
Name: aws.String("instance-state-name"),
Values: []string{string(types.InstanceStateNameRunning)},
},
},
}
}
func TestPluginListHosts(t *testing.T) {
describeOutput := &ec2.DescribeInstancesOutput{
Reservations: []types.Reservation{
{
Instances: []types.Instance{
{
InstanceId: aws.String("i-shared"),
PrivateIpAddress: aws.String("10.0.0.1"),
},
},
},
},
}
cases := []struct {
name string
catalogFields map[string]*structpb.Value
sets []*hostsets.HostSet
expectedRegion string
expectedHosts []*pb.ListHostsResponseHost
expectedInputs []*ec2.DescribeInstancesInput
expectedDescribe int
}{
{
name: "principal account lists hosts",
catalogFields: map[string]*structpb.Value{
credential.ConstRegion: structpb.NewStringValue("us-west-2"),
},
sets: []*hostsets.HostSet{
testListHostsSet("set-1", "tag-key=foo"),
},
expectedRegion: "us-west-2",
expectedHosts: []*pb.ListHostsResponseHost{
{
ExternalId: "i-shared",
SetIds: []string{"set-1"},
},
},
expectedInputs: []*ec2.DescribeInstancesInput{
testExpectedDescribeInput("tag-key", "foo"),
},
expectedDescribe: 1,
},
{
name: "target account lists hosts without querying principal",
catalogFields: map[string]*structpb.Value{
credential.ConstRegion: structpb.NewStringValue("us-west-2"),
credential.ConstTargetRoleArn: structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
credential.ConstTargetRegion: structpb.NewStringValue("eu-west-1"),
},
sets: []*hostsets.HostSet{
testListHostsSet("set-1", "tag-key=foo"),
testListHostsSet("set-2", "tag-key=bar"),
},
expectedRegion: "eu-west-1",
expectedHosts: []*pb.ListHostsResponseHost{
{
ExternalId: "i-shared",
SetIds: []string{"set-1", "set-2"},
},
},
expectedInputs: []*ec2.DescribeInstancesInput{
testExpectedDescribeInput("tag-key", "foo"),
testExpectedDescribeInput("tag-key", "bar"),
},
expectedDescribe: 2,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)
state := &testMockEC2State{}
p := &HostPlugin{
testCatalogStateOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
state,
testMockEC2WithDescribeInstancesOutput(describeOutput),
)),
},
}
resp, err := p.ListHosts(context.Background(), &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: &structpb.Struct{Fields: tc.catalogFields},
},
},
Persisted: testListHostsPersisted(),
Sets: tc.sets,
})
require.NoError(err)
require.NotNil(resp)
require.Len(resp.GetHosts(), len(tc.expectedHosts))
for i, expected := range tc.expectedHosts {
require.Equal(expected.ExternalId, resp.Hosts[i].ExternalId)
require.Equal(expected.SetIds, resp.Hosts[i].SetIds)
}
require.Equal(tc.expectedDescribe, state.DescribeInstancesCallCount)
require.Len(state.ClientRegions, tc.expectedDescribe)
for _, region := range state.ClientRegions {
require.Equal(tc.expectedRegion, region)
}
require.Equal(tc.expectedInputs, state.DescribeInstancesInputs)
})
}
}
func TestPluginListHostsErr(t *testing.T) {
cases := []struct {
name string
@@ -2130,7 +2287,101 @@ func TestPluginListHostsErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: fmt.Sprintf("error running DescribeInstances for host set id \"foobar\": %s", testDescribeInstancesError),
expectedErrContains: fmt.Sprintf("error retrieving host results for host set id \"foobar\": %s", testDescribeInstancesError),
expectedErrCode: codes.InvalidArgument,
},
{
name: "target attributes missing target_role_arn",
req: &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstRegion: structpb.NewStringValue("us-west-2"),
credential.ConstTargetRegion: structpb.NewStringValue("eu-west-1"),
},
},
},
},
},
expectedErrContains: "attributes.target_role_arn: missing required value \"target_role_arn\"",
expectedErrCode: codes.InvalidArgument,
},
{
name: "target EC2 client error",
req: &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstRegion: structpb.NewStringValue("us-west-2"),
credential.ConstTargetRoleArn: structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
},
},
},
},
Persisted: &pb.HostCatalogPersisted{
Secrets: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstAccessKeyId: structpb.NewStringValue("AKIA_foobar"),
credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"),
},
},
},
Sets: []*hostsets.HostSet{
{
Id: "foobar",
Attrs: &hostsets.HostSet_Attributes{
Attributes: &structpb.Struct{
Fields: map[string]*structpb.Value{
ConstDescribeInstancesFilters: structpb.NewListValue(
&structpb.ListValue{
Values: []*structpb.Value{
structpb.NewStringValue("tag-key=foo"),
},
},
),
},
},
},
},
},
},
catalogOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(func(...aws.Config) (EC2API, error) {
return nil, errors.New("test EC2 client error")
}),
},
expectedErrContains: "unable to list hosts for host set \"foobar\": EC2 client setup failed for target account: test EC2 client error",
expectedErrCode: codes.InvalidArgument,
},
{
name: "target DescribeInstances error",
req: &pb.ListHostsRequest{
Catalog: &hostcatalogs.HostCatalog{
Attrs: &hostcatalogs.HostCatalog_Attributes{
Attributes: &structpb.Struct{
Fields: map[string]*structpb.Value{
credential.ConstRegion: structpb.NewStringValue("us-west-2"),
credential.ConstTargetRoleArn: structpb.NewStringValue("arn:aws:iam::222222222222:role/Target"),
credential.ConstTargetRegion: structpb.NewStringValue("eu-west-1"),
},
},
},
},
Persisted: testListHostsPersisted(),
Sets: []*hostsets.HostSet{
testListHostsSet("foobar", "tag-key=foo"),
},
},
catalogOpts: []awsCatalogPersistedStateOption{
withTestEC2APIFunc(newTestMockEC2(
nil,
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: "unable to list hosts for host set \"foobar\": EC2 DescribeInstances failed for target account: DescribeInstances error",
expectedErrCode: codes.InvalidArgument,
},
{
@@ -2871,4 +3122,66 @@ func TestDryRunValidation(t *testing.T) {
}, []ec2Option{})
require.Nil(t, st)
})
t.Run("authFailureThenSuccess", func(t *testing.T) {
// Simulates the IAM eventual consistency window: EC2 returns AuthFailure
// twice before the credentials propagate and the call succeeds.
state := &testMockEC2State{}
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(state,
testMockEC2WithDescribeInstancesErrorSequence(
fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure),
fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure),
),
testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
),
}, []ec2Option{})
require.Nil(t, st)
require.Equal(t, 3, state.DescribeInstancesCallCount)
})
t.Run("authFailureContextCancelled", func(t *testing.T) {
// If the context is cancelled while waiting to retry, the function should
// return a FailedPrecondition error immediately rather than hanging.
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately so the first retry select hits ctx.Done()
st := dryRunValidation(ctx, &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(nil,
testMockEC2WithDescribeInstancesError(fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure)),
),
}, []ec2Option{})
require.NotNil(t, st)
require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
})
t.Run("nonAuthFailureNoRetry", func(t *testing.T) {
// A non-AuthFailure error should fail immediately without retrying.
state := &testMockEC2State{}
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(state,
testMockEC2WithDescribeInstancesError(fmt.Errorf("UnauthorizedOperation: access denied")),
),
}, []ec2Option{})
require.NotNil(t, st)
require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
require.Equal(t, 1, state.DescribeInstancesCallCount)
})
}
+31
View File
@@ -22,11 +22,17 @@ const (
type testMockEC2State struct {
DescribeInstancesCalled bool
DescribeInstancesInputParams *ec2.DescribeInstancesInput
DescribeInstancesCallCount int
DescribeInstancesInputs []*ec2.DescribeInstancesInput
ClientRegions []string
}
func (s *testMockEC2State) Reset() {
s.DescribeInstancesCalled = false
s.DescribeInstancesInputParams = nil
s.DescribeInstancesCallCount = 0
s.DescribeInstancesInputs = nil
s.ClientRegions = nil
}
type testMockEC2 struct {
@@ -37,6 +43,10 @@ type testMockEC2 struct {
DescribeInstancesOutput *ec2.DescribeInstancesOutput
DescribeInstancesError error
// DescribeInstancesErrorSequence, if set, returns each error in order on
// successive calls. Once exhausted, DescribeInstancesOutput is returned.
DescribeInstancesErrorSequence []error
}
type testMockEC2Option func(m *testMockEC2) error
@@ -55,6 +65,16 @@ func testMockEC2WithDescribeInstancesError(e error) testMockEC2Option {
}
}
// testMockEC2WithDescribeInstancesErrorSequence sets a sequence of errors to
// return on successive DescribeInstances calls. Once the sequence is exhausted,
// the mock returns DescribeInstancesOutput (nil error).
func testMockEC2WithDescribeInstancesErrorSequence(errs ...error) testMockEC2Option {
return func(m *testMockEC2) error {
m.DescribeInstancesErrorSequence = errs
return nil
}
}
func newTestMockEC2(state *testMockEC2State, opts ...testMockEC2Option) ec2APIFunc {
return func(cfgs ...aws.Config) (EC2API, error) {
m := &testMockEC2{
@@ -72,7 +92,10 @@ func newTestMockEC2(state *testMockEC2State, opts ...testMockEC2Option) ec2APIFu
if cfg.Region != "" {
m.Region = cfg.Region
}
}
if state != nil {
state.ClientRegions = append(state.ClientRegions, m.Region)
}
return m, nil
@@ -83,6 +106,14 @@ func (m *testMockEC2) DescribeInstances(ctx context.Context, input *ec2.Describe
if m.State != nil {
m.State.DescribeInstancesCalled = true
m.State.DescribeInstancesInputParams = input
m.State.DescribeInstancesCallCount++
m.State.DescribeInstancesInputs = append(m.State.DescribeInstancesInputs, input)
}
if len(m.DescribeInstancesErrorSequence) > 0 {
err := m.DescribeInstancesErrorSequence[0]
m.DescribeInstancesErrorSequence = m.DescribeInstancesErrorSequence[1:]
return nil, err
}
if m.DescribeInstancesError != nil {