Compare commits

..
6 changed files with 76 additions and 246 deletions

No files matched your search

-2
View File
@@ -25,8 +25,6 @@ Credential rotation can be turned off by setting the
This plugin will attempt to assume a role when a `role_arn` is supplied through the
`attributes` object. More information about assume an AWS role can be found [here](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html). This feature only works when the plugin is running on a self managed Boundary [worker](https://developer.hashicorp.com/boundary/tutorials/hcp-administration/hcp-manage-workers).
To discover hosts across multiple AWS accounts, supply a `target_role_arn` in addition to `role_arn`. The plugin uses the principal credentials to assume the target role via a two-hop `AssumeRole` chain, enabling cross-account host discovery from a single worker. See [Attributes and Secrets](plugin/service/host/README.md#attributes-and-secrets) for the full list of `target_*` attributes.
### Environment Credentials
This plugin will attempt to retrieve credentials from environment variables. More
-5
View File
@@ -23,11 +23,6 @@ import (
)
const (
// AwsErrorAuthFailure is returned by EC2 when credentials exist in IAM
// but have not yet propagated to EC2. This is a transient eventual
// consistency issue that can resolve itself by retrying the request.
AwsErrorAuthFailure = "AuthFailure"
// awsErrorAccessDenied is returned when the credentials do not have
// the required permissions to do a specific action on a resource.
// This error will persist until the IAM Role Policy is updated by
+4 -14
View File
@@ -109,23 +109,13 @@ The following `attributes` are valid on an AWS host catalog resource:
- `region` (string): The region to configure the host catalog for. All host sets
in this catalog will be configured for this region.
- `role_arn` (string): The AWS Role ARN used for `AssumeRole` authentication. If
provided, `disable_credential_rotation` must be also set to `true`.
- `role_external_id` (string): The external ID configured for the principal `AssumeRole`
provided, `disable_credential_rotation` must be also set to `true.
- `role_external_id` (string): The external id configured for the `AssumeRole`
provider.
- `role_session_name` (string): The session name configured for the principal `AssumeRole`
- `role_session_name` (string): The session name configured for the `AssumeRole`
provider.
- `role_tags` (object): The key-value pair tags configured for the principal `AssumeRole`
- `role_tags` (object): The key-value pair tags configured for the `AssumeRole`
provider.
- `target_role_arn` (string): The AWS role ARN of the target IAM role to assume for
cross-account discovery. Required if any other `target_*` attributes are configured.
- `target_region` (string): The region where the target hosts reside. Defaults to the
principal account `region` if omitted.
- `target_role_external_id` (string): The external ID configured for the target
`AssumeRole` provider.
- `target_role_session_name` (string): The session name configured for the target
`AssumeRole` provider.
- `target_role_tags` (object): The key-value pair tags configured for the target
`AssumeRole` provider.
The following `secrets` are required on an AWS host catalog resource:
+48 -79
View File
@@ -8,7 +8,6 @@ import (
goerrors "errors"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
@@ -47,6 +46,8 @@ type hostSetQuery struct {
const defaultSessionName = "boundary-default"
const dryRunOperationErrorCode = "DryRunOperation"
const errBuildDescribeInstancesInput = "error building DescribeInstances input: %s"
const errDryRunFailed = "error performing DescribeInstances dry run: %s"
// Ensure that we are implementing HostPluginServiceServer
var _ pb.HostPluginServiceServer = (*HostPlugin)(nil)
@@ -98,13 +99,18 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo
return nil, errors.BadRequestStatusf("error setting up persisted state: %s", err)
}
// perform dry run to ensure we can interact with AWS as expected.
opts := []ec2Option{}
if catalogAttributes.DualStack {
opts = append(opts, WithDualStack(catalogAttributes.DualStack))
}
if st := dryRunValidation(ctx, catalogState, opts); st != nil {
return nil, st.Err()
input, err := buildDescribeInstancesInput(&SetAttributes{}, true)
if err != nil {
return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
}
// Run a DescribeInstances request with DryRun set to true to ensure
// we can interact with AWS resources as expected.
if _, err := checkHosts(ctx, catalogState, input, opts, catalogAttributes.TargetAccount); err != nil {
return nil, fmt.Errorf(errDryRunFailed, err)
}
persistedProto, err := catalogState.toProto()
@@ -187,12 +193,19 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
if err != nil {
return nil, errors.BadRequestStatusf("error loading persisted state: %s", err)
}
opts := []ec2Option{}
if newCatalogAttributes.DualStack {
opts = append(opts, WithDualStack(newCatalogAttributes.DualStack))
}
if st := dryRunValidation(ctx, newCatalogState, opts); st != nil {
return nil, st.Err()
input, err := buildDescribeInstancesInput(&SetAttributes{}, true)
if err != nil {
return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
}
// Run a DescribeInstances request with DryRun set to true to ensure
// we can interact with AWS resources as expected.
if _, err := checkHosts(ctx, newCatalogState, input, opts, newCatalogAttributes.TargetAccount); err != nil {
return nil, fmt.Errorf(errDryRunFailed, err)
}
// Replace the existing credential state.
@@ -232,13 +245,18 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo
return nil, errors.BadRequestStatusf("error loading persisted state: %s", err)
}
// perform dry run to ensure we can interact with AWS as expected.
opts := []ec2Option{}
if newCatalogAttributes.DualStack {
opts = append(opts, WithDualStack(newCatalogAttributes.DualStack))
}
if st := dryRunValidation(ctx, catalogState, opts); st != nil {
return nil, st.Err()
input, err := buildDescribeInstancesInput(&SetAttributes{}, true)
if err != nil {
return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
}
// Run a DescribeInstances request with DryRun set to true to ensure
// we can interact with AWS resources as expected.
if _, err := checkHosts(ctx, catalogState, input, opts, newCatalogAttributes.TargetAccount); err != nil {
return nil, fmt.Errorf(errDryRunFailed, err)
}
persistedProto, err := catalogState.toProto()
@@ -382,18 +400,20 @@ func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest
return nil, err
}
describeInstanceFilters, err := buildFilters(setAttrs)
if err != nil {
return nil, errors.BadRequestStatusf("error building set filters: %s", err)
}
opts := []ec2Option{}
if catalogAttributes.DualStack {
opts = append(opts, WithDualStack(catalogAttributes.DualStack))
}
if st := dryRunValidation(ctx, catalogState, opts, describeInstanceFilters...); st != nil {
return nil, st.Err()
input, err := buildDescribeInstancesInput(setAttrs, true)
if err != nil {
return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
}
// Run a DescribeInstances request with DryRun set to true to ensure
// we can interact with AWS resources as expected.
if _, err := checkHosts(ctx, catalogState, input, opts, catalogAttributes.TargetAccount); err != nil {
return nil, fmt.Errorf(errDryRunFailed, err)
}
return &pb.OnCreateSetResponse{}, nil
}
@@ -447,17 +467,18 @@ func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest
return nil, err
}
describeInstanceFilters, err := buildFilters(setAttrs)
if err != nil {
return nil, errors.BadRequestStatusf("error building set filters: %s", err)
}
opts := []ec2Option{}
if catalogAttributes.DualStack {
opts = append(opts, WithDualStack(catalogAttributes.DualStack))
}
if st := dryRunValidation(ctx, catalogState, opts, describeInstanceFilters...); st != nil {
return nil, st.Err()
input, err := buildDescribeInstancesInput(setAttrs, true)
if err != nil {
return nil, status.Errorf(codes.Unknown, errBuildDescribeInstancesInput, err)
}
// Run a DescribeInstances request with DryRun set to true to ensure
// we can interact with AWS resources as expected.
if _, err := checkHosts(ctx, catalogState, input, opts, catalogAttributes.TargetAccount); err != nil {
return nil, fmt.Errorf(errDryRunFailed, err)
}
return &pb.OnUpdateSetResponse{}, nil
@@ -612,11 +633,11 @@ func buildFilters(attrs *SetAttributes) ([]types.Filter, error) {
splitFilter := strings.Split(filterAttr, "=")
switch {
case len(splitFilter) != 2:
return nil, fmt.Errorf("expected filter %q to contain a single equal sign", filterAttr)
return nil, errors.BadRequestStatusf("expected filter %q to contain a single equal sign", filterAttr)
case len(splitFilter[0]) == 0:
return nil, fmt.Errorf("filter %q contains an empty filter key", filterAttr)
return nil, errors.BadRequestStatusf("filter %q contains an empty filter key", filterAttr)
case len(splitFilter[1]) == 0:
return nil, fmt.Errorf("filter %q contains an empty value", filterAttr)
return nil, errors.BadRequestStatusf("filter %q contains an empty value", filterAttr)
}
filterKey, filterValue := splitFilter[0], splitFilter[1]
@@ -647,7 +668,7 @@ func buildFilters(attrs *SetAttributes) ([]types.Filter, error) {
func buildDescribeInstancesInput(attrs *SetAttributes, dryRun bool) (*ec2.DescribeInstancesInput, error) {
filters, err := buildFilters(attrs)
if err != nil {
return nil, fmt.Errorf("error building filters: %w", err)
return nil, err
}
return &ec2.DescribeInstancesInput{
@@ -829,58 +850,6 @@ func ec2ClientForTarget(
return ec2.NewFromConfig(targetCfg, ec2Opts...), nil
}
// dryRunValidationTimeout is the maximum time to wait for credentials to
// propagate across AWS services after rotation before giving up.
const dryRunValidationTimeout = 30 * time.Second
// dryRunValidation performs an AWS DescribeInstances call to verify the state's
// credentials, the host listing functionality as well as the filters, if any
// are passed in. This function can therefore be used for both host catalog and
// host set validation.
//
// After credential rotation, IAM keys may be valid for STS (GetCallerIdentity)
// but not yet propagated to EC2. This function retries on AuthFailure to
// tolerate that eventual consistency window.
func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, ec2Opts []ec2Option, filters ...types.Filter) *status.Status {
if state == nil {
return status.New(codes.InvalidArgument, "persisted state is required")
}
ec2Client, err := state.EC2Client(ctx, ec2Opts...)
if err != nil {
return status.New(codes.InvalidArgument, fmt.Sprintf("error getting EC2 client: %s", err))
}
deadline := time.Now().Add(dryRunValidationTimeout)
delay := time.Second
for {
_, err = ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{Filters: filters})
if err == nil {
return nil
}
// AuthFailure indicates the credentials exist but haven't propagated
// to EC2 yet — a transient eventual consistency issue after rotation.
// Retry until the deadline.
if strings.Contains(err.Error(), errors.AwsErrorAuthFailure) && time.Now().Before(deadline) {
select {
case <-time.After(delay):
// Wait before retrying to give credentials time to propagate.
case <-ctx.Done():
// The caller cancelled — break out of the select.
break
}
if ctx.Err() != nil {
break
}
continue
}
break
}
return status.New(codes.FailedPrecondition, fmt.Sprintf("aws describe instances failed: %s", err))
}
// appendDistinct will append the elements to the slice
// if an element is not nil, empty, and does not exist in slice.
func appendDistinct(slice []string, elems ...*string) []string {
+24 -126
View File
@@ -18,7 +18,6 @@ import (
iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/boundary-plugin-aws/internal/credential"
pluginErrors "github.com/hashicorp/boundary-plugin-aws/internal/errors"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
@@ -743,8 +742,11 @@ func TestPluginOnCreateCatalogErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: "aws describe instances failed: DescribeInstances error",
expectedErrCode: codes.FailedPrecondition,
// checkHosts returns the DescribeInstances error as-is rather than
// wrapping it in a gRPC FailedPrecondition status like the old
// dryRunValidation did. A plain Go error maps to codes.Unknown.
expectedErrContains: testDescribeInstancesError,
expectedErrCode: codes.Unknown,
},
}
@@ -995,8 +997,11 @@ func TestPluginOnUpdateCatalogErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: "aws describe instances failed: DescribeInstances error",
expectedErrCode: codes.FailedPrecondition,
//checkHosts returns the DesribeInstances errror as-is rather than wrapping it
// in a gRpc FailedPrecodndition status like the old dryRunValidiation did.
// A plain Go error maps to codeds.Unknown.
expectedErrContains: testDescribeInstancesError,
expectedErrCode: codes.Unknown,
},
{
name: "new incoming dynamic credential update dry run error",
@@ -1037,8 +1042,11 @@ func TestPluginOnUpdateCatalogErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: "aws describe instances failed: DescribeInstances error",
expectedErrCode: codes.FailedPrecondition,
// checkHosts returns the DescribeInstances error as-is rather than wrapping it
// in a gRPC FailedPrecondition status like the old dryRunValidiation did.
// A plain Go error maps to codes.Unknown.
expectedErrContains: testDescribeInstancesError,
expectedErrCode: codes.Unknown,
},
{
name: "replace creds error",
@@ -1212,8 +1220,11 @@ func TestPluginOnUpdateCatalogErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(fmt.Errorf("oops there was an error")),
)),
},
expectedErrContains: "aws describe instances failed: oops there was an error",
expectedErrCode: codes.FailedPrecondition,
// checkHosts returns the DescribeInstances error as-is rather than wrapping it
// in a gRPC FailedPrecondition status like the old dryRunValidiation did.
// A plain Go error maps to codes.Unknown.
expectedErrContains: "oops there was an error",
expectedErrCode: codes.Unknown,
},
}
@@ -1566,8 +1577,8 @@ func TestPluginOnCreateSetErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: fmt.Sprintf("aws describe instances failed: %s", testDescribeInstancesError),
expectedErrCode: codes.FailedPrecondition,
expectedErrContains: testDescribeInstancesError,
expectedErrCode: codes.Unknown,
},
}
@@ -1829,8 +1840,8 @@ func TestPluginOnUpdateSetErr(t *testing.T) {
testMockEC2WithDescribeInstancesError(errors.New(testDescribeInstancesError)),
)),
},
expectedErrContains: fmt.Sprintf("aws describe instances failed: %s", testDescribeInstancesError),
expectedErrCode: codes.FailedPrecondition,
expectedErrContains: testDescribeInstancesError,
expectedErrCode: codes.Unknown,
},
}
@@ -3072,116 +3083,3 @@ func TestAppendDistinct(t *testing.T) {
})
}
}
func TestDryRunValidation(t *testing.T) {
t.Run("nil credential state", func(t *testing.T) {
st := dryRunValidation(context.Background(), nil, nil)
require.NotNil(t, st)
require.Equal(t, codes.InvalidArgument.String(), st.Code().String())
require.Equal(t, "persisted state is required", st.Message())
})
t.Run("ec2ClientErr", func(t *testing.T) {
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{},
},
testEC2APIFunc: func(...aws.Config) (EC2API, error) {
return nil, fmt.Errorf("oops ec2 client err")
},
}, []ec2Option{})
require.NotNil(t, st)
require.Equal(t, codes.InvalidArgument.String(), st.Code().String())
require.Equal(t, "error getting EC2 client: oops ec2 client err", st.Message())
})
t.Run("describeInstancesErr", func(t *testing.T) {
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(nil, testMockEC2WithDescribeInstancesError(fmt.Errorf("oops describe instances error"))),
}, []ec2Option{})
require.NotNil(t, st)
require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
require.Equal(t, "aws describe instances failed: oops describe instances error", st.Message())
})
t.Run("success", func(t *testing.T) {
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(nil, testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{})),
}, []ec2Option{})
require.Nil(t, st)
})
t.Run("authFailureThenSuccess", func(t *testing.T) {
// Simulates the IAM eventual consistency window: EC2 returns AuthFailure
// twice before the credentials propagate and the call succeeds.
state := &testMockEC2State{}
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(state,
testMockEC2WithDescribeInstancesErrorSequence(
fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure),
fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure),
),
testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
),
}, []ec2Option{})
require.Nil(t, st)
require.Equal(t, 3, state.DescribeInstancesCallCount)
})
t.Run("authFailureContextCancelled", func(t *testing.T) {
// If the context is cancelled while waiting to retry, the function should
// return a FailedPrecondition error immediately rather than hanging.
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately so the first retry select hits ctx.Done()
st := dryRunValidation(ctx, &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(nil,
testMockEC2WithDescribeInstancesError(fmt.Errorf("%s: credentials not yet propagated", pluginErrors.AwsErrorAuthFailure)),
),
}, []ec2Option{})
require.NotNil(t, st)
require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
})
t.Run("nonAuthFailureNoRetry", func(t *testing.T) {
// A non-AuthFailure error should fail immediately without retrying.
state := &testMockEC2State{}
st := dryRunValidation(context.Background(), &awsCatalogPersistedState{
AwsCredentialPersistedState: &credential.AwsCredentialPersistedState{
CredentialsConfig: &awsutil.CredentialsConfig{
AccessKey: "AKIAfoo",
SecretKey: "baz",
},
},
testEC2APIFunc: newTestMockEC2(state,
testMockEC2WithDescribeInstancesError(fmt.Errorf("UnauthorizedOperation: access denied")),
),
}, []ec2Option{})
require.NotNil(t, st)
require.Equal(t, codes.FailedPrecondition.String(), st.Code().String())
require.Equal(t, 1, state.DescribeInstancesCallCount)
})
}
-20
View File
@@ -43,10 +43,6 @@ type testMockEC2 struct {
DescribeInstancesOutput *ec2.DescribeInstancesOutput
DescribeInstancesError error
// DescribeInstancesErrorSequence, if set, returns each error in order on
// successive calls. Once exhausted, DescribeInstancesOutput is returned.
DescribeInstancesErrorSequence []error
}
type testMockEC2Option func(m *testMockEC2) error
@@ -65,16 +61,6 @@ func testMockEC2WithDescribeInstancesError(e error) testMockEC2Option {
}
}
// testMockEC2WithDescribeInstancesErrorSequence sets a sequence of errors to
// return on successive DescribeInstances calls. Once the sequence is exhausted,
// the mock returns DescribeInstancesOutput (nil error).
func testMockEC2WithDescribeInstancesErrorSequence(errs ...error) testMockEC2Option {
return func(m *testMockEC2) error {
m.DescribeInstancesErrorSequence = errs
return nil
}
}
func newTestMockEC2(state *testMockEC2State, opts ...testMockEC2Option) ec2APIFunc {
return func(cfgs ...aws.Config) (EC2API, error) {
m := &testMockEC2{
@@ -110,12 +96,6 @@ func (m *testMockEC2) DescribeInstances(ctx context.Context, input *ec2.Describe
m.State.DescribeInstancesInputs = append(m.State.DescribeInstancesInputs, input)
}
if len(m.DescribeInstancesErrorSequence) > 0 {
err := m.DescribeInstancesErrorSequence[0]
m.DescribeInstancesErrorSequence = m.DescribeInstancesErrorSequence[1:]
return nil, err
}
if m.DescribeInstancesError != nil {
return nil, m.DescribeInstancesError
}