refactor: organize plugin to support multiple services (#1)

Co-authored-by: Michael Li <michael.li@hashicorp.com>
This commit is contained in:
Damian DebkowskiandMichael Li authored and Damian Debkowski committed 2023-04-30 23:30:08 -07:00
1 parent c441058b3e
commit 46c53fb94a
31 files changed
+2951 -2446

No files matched your search

+128
View File
@@ -0,0 +1,128 @@
## Getting Started
Refer to [Attributes and Secrets](#attributes-and-secrets) for more detail on
configuration options for host catalogs and sets.
To create a host catalog (using default scope created by `boundary dev`):
```
boundary host-catalogs create plugin \
-scope-id p_1234567890 \
-name "Example Plugin-Based Host Catalog" \
-description "Description for plugin-based host catalog" \
-plugin-name aws \
-attr region=REGION \
-secret access_key_id='KEY' \
-secret secret_access_key='SECRET'
```
To create a host set, filtering the host set based on tag keys `foo` or `bar`
(either tag can be present), ensuring that any targets set to this host set only
connect to external addresses in the `54.0.0.0/8` class A subnet:
```
boundary host-sets create plugin \
-host-catalog-id HOST_CATALOG_ID \
-name "Example Plugin-Based Host Set" \
-description "Description for plugin-based host set" \
-attr filters=tag-key=foo,bar \
-preferred-endpoint "cidr:54.0.0.0/8"
```
As above, but instances must have both tags (both `foo` and `bar` *must* be
present):
```
boundary host-sets create plugin \
-host-catalog-id HOST_CATALOG_ID \
-name "Example Plugin-Based Host Set" \
-description "Description for plugin-based host set" \
-attr filters=tag-key=foo \
-attr filters=tag-key=bar \
-preferred-endpoint "cidr:54.0.0.0/8"
```
As above, but matching on tag key named "development" with value "prod" or "dev"
and launch date:
```
boundary host-sets create plugin \
-host-catalog-id HOST_CATALOG_ID \
-name "Example Plugin-Based Host Set" \
-description "Description for plugin-based host set" \
-attr filters=tag:development=prod,dev \
-attr filters=launch-time=2022-01-04T* \
-preferred-endpoint "cidr:54.0.0.0/8"
```
## Required IAM Privileges
The following IAM privileges, at the very least, are required to be attached to
a configured IAM user for this provider:
* `ec2:DescribeInstances`, configured to `*` (`DescribeInstances` cannot be
scoped to a resource ARN). Example policy:
```
{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"ec2:DescribeInstances"
],
"Effect": "Allow",
"Resource": "*"
}
]
}
```
* `iam:GetUser`, `iam:CreateAccessKey`, and `iam:DeleteAccessKey`, configured to
the IAM user to allow credential rotation. Example policy:
```
{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"iam:DeleteAccessKey",
"iam:GetUser",
"iam:CreateAccessKey"
],
"Effect": "Allow",
"Resource": "arn:aws:iam::123456789012:user/JohnDoe"
}
]
}
```
## Attributes and Secrets
### Host Catalog
The following `attributes` are valid on an AWS host catalog resource:
- `disable_credential_rotation` (bool): If `true`, credential rotation will not
be performed. See the [Credential Rotation](#credential-rotation) section
above.
- `region` (string): The region to configure the host catalog for. All host sets
in this catalog will be configured for this region.
The following `secrets` are valid on an AWS host catalog resource:
- `access_key_id` (string): The access key ID for the IAM user to use with this
host catalog.
- `secret_access_key` (string): The secret access key for the IAM user to use
with this host catalog.
### Host Set
The following attributes are valid on an AWS host Set resource:
- `filters` (array of strings): An array of string filters in the format
`key=val1,val2`. The key corresponds to a filter option, and the value(s) are
a comma-separated list. For a list of filter options, check out
[describe-instances in the AWS CLI
reference](https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html).
+90
View File
@@ -0,0 +1,90 @@
package host
import (
"fmt"
cred "github.com/hashicorp/boundary-plugin-host-aws/internal/credential"
"github.com/hashicorp/boundary-plugin-host-aws/internal/errors"
"github.com/hashicorp/boundary-plugin-host-aws/internal/values"
"github.com/mitchellh/mapstructure"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/structpb"
)
// CatalogAttributes is a Go-native representation of the Attributes
// map.
type CatalogAttributes struct {
*cred.CredentialAttributes
}
func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
unknownFields := values.StructFields(in)
badFields := make(map[string]string)
var err error
credAttributes, err := cred.GetCredentialAttributes(in)
if err != nil {
return nil, err
}
for s := range unknownFields {
switch s {
// Ignore knownFields from CredentialAttributes
case cred.ConstRegion:
continue
case cred.ConstDisableCredentialRotation:
continue
default:
badFields[fmt.Sprintf("attributes.%s", s)] = "unrecognized field"
}
}
if len(badFields) > 0 {
return nil, errors.InvalidArgumentError("Invalid arguments in catalog attributes", badFields)
}
return &CatalogAttributes{
CredentialAttributes: credAttributes,
}, nil
}
// SetAttributes is a Go-native representation of the Attributes map that can be
// used for decoding the incoming map via mapstructure.
type SetAttributes struct {
Filters []string
}
func getSetAttributes(in *structpb.Struct) (*SetAttributes, error) {
var setAttrs SetAttributes
// Quick validation to ensure that there's no non-filter attributes
// here for now. Make this more complex if we add more attributes
// to host sets.
unknownFields := values.StructFields(in)
badFields := make(map[string]string)
delete(unknownFields, ConstDescribeInstancesFilters)
for a := range unknownFields {
badFields[fmt.Sprintf("attributes.%s", a)] = "unrecognized field"
}
if len(badFields) > 0 {
return nil, errors.InvalidArgumentError("Error in the attributes provided", badFields)
}
// Mapstructure complains if it expects a slice as output and sees a scalar
// value. Rather than use WeakDecode and risk unintended consequences, I'm
// manually making this change if necessary.
inMap := in.AsMap()
if filtersRaw, ok := inMap[ConstDescribeInstancesFilters]; ok {
switch filterVal := filtersRaw.(type) {
case string:
inMap[ConstDescribeInstancesFilters] = []string{filterVal}
}
}
if err := mapstructure.Decode(inMap, &setAttrs); err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error decoding set attributes: %s", err)
}
return &setAttrs, nil
}
+188
View File
@@ -0,0 +1,188 @@
package host
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/boundary-plugin-host-aws/internal/credential"
"github.com/hashicorp/boundary/sdk/pbs/plugin"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/structpb"
)
func TestGetCatalogAttributes(t *testing.T) {
cases := []struct {
name string
in map[string]any
expected *CatalogAttributes
expectedErrContains string
}{
{
name: "missing region",
in: map[string]any{},
expectedErrContains: "missing required value \"region\"",
},
{
name: "bad value for disable_credential_rotation",
in: map[string]any{
credential.ConstRegion: "us-west-2",
credential.ConstDisableCredentialRotation: "sure",
},
expectedErrContains: "unexpected type for value \"disable_credential_rotation\": want bool, got string",
},
{
name: "unknown fields",
in: map[string]any{
credential.ConstRegion: "us-west-2",
"foo": true,
"bar": true,
},
expectedErrContains: "attributes.bar: unrecognized field, attributes.foo: unrecognized field",
},
{
name: "default",
in: map[string]any{
credential.ConstRegion: "us-west-2",
},
expected: &CatalogAttributes{
CredentialAttributes: &credential.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: false,
},
},
},
{
name: "with disable_credential_rotation",
in: map[string]any{
credential.ConstRegion: "us-west-2",
credential.ConstDisableCredentialRotation: true,
},
expected: &CatalogAttributes{
CredentialAttributes: &credential.CredentialAttributes{
Region: "us-west-2",
DisableCredentialRotation: true,
},
},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)
input, err := structpb.NewStruct(tc.in)
require.NoError(err)
actual, err := getCatalogAttributes(input)
if tc.expectedErrContains != "" {
require.Error(err)
require.Contains(err.Error(), tc.expectedErrContains)
require.Equal(status.Code(err), codes.InvalidArgument)
return
}
require.NoError(err)
require.Equal(tc.expected, actual)
})
}
}
func TestGetSetAttributes(t *testing.T) {
cases := []struct {
name string
in map[string]any
normalized map[string]any
expected *SetAttributes
expectedErrContains string
}{
{
name: "missing",
in: map[string]any{},
expected: &SetAttributes{},
},
{
name: "non-string-slice value",
in: map[string]any{
ConstDescribeInstancesFilters: "zip=foo,bar",
},
expected: &SetAttributes{
Filters: []string{"zip=foo,bar"},
},
},
{
name: "bad filter element value",
in: map[string]any{
ConstDescribeInstancesFilters: []any{1},
},
expectedErrContains: "expected type 'string', got unconvertible type 'float64'",
},
{
name: "unknown fields",
in: map[string]any{
"foo": true,
"bar": true,
},
expectedErrContains: "attributes.bar: unrecognized field, attributes.foo: unrecognized field",
},
{
name: "good",
in: map[string]any{
ConstDescribeInstancesFilters: []any{
"foo=bar",
"zip=zap",
},
},
expected: &SetAttributes{
Filters: []string{"foo=bar", "zip=zap"},
},
},
{
name: "good with filter transform",
in: map[string]any{
ConstDescribeInstancesFilters: "foo=bar",
},
normalized: map[string]any{
ConstDescribeInstancesFilters: []any{"foo=bar"},
},
expected: &SetAttributes{
Filters: []string{"foo=bar"},
},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)
input, err := structpb.NewStruct(tc.in)
require.NoError(err)
p := new(HostPlugin)
normalizedOut, err := p.NormalizeSetData(context.Background(), &plugin.NormalizeSetDataRequest{Attributes: input})
require.NoError(err)
if tc.normalized != nil {
normalized, err := structpb.NewStruct(tc.normalized)
require.NoError(err)
require.Empty(cmp.Diff(normalized, normalizedOut.Attributes, protocmp.Transform()))
}
input = normalizedOut.Attributes
actual, err := getSetAttributes(input)
if tc.expectedErrContains != "" {
require.Error(err)
require.Contains(err.Error(), tc.expectedErrContains)
require.Equal(status.Code(err), codes.InvalidArgument)
return
}
require.NoError(err)
require.Equal(tc.expected, actual)
})
}
}
+10
View File
@@ -0,0 +1,10 @@
package host
const (
// ConstDescribeInstancesFilters is the key for the filter in the dynamic host set.
ConstDescribeInstancesFilters = "filters"
// ConstInstanceNameTagKey denotes the special AWS tag key that contains an EC2
// instance's name. This field is to be matched literally.
ConstInstanceNameTagKey = "Name"
)
+725
View File
@@ -0,0 +1,725 @@
package host
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
import (
"context"
"errors"
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/ec2"
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/structpb"
"github.com/hashicorp/boundary-plugin-host-aws/internal/credential"
cred "github.com/hashicorp/boundary-plugin-host-aws/internal/credential"
)
// HostPlugin implements the HostPluginServiceServer interface for the
// AWS host service plugin.
type HostPlugin struct {
pb.UnimplementedHostPluginServiceServer
// testCredStateOpts are passed in to the stored state to control test behavior
testCredStateOpts []credential.AwsCredentialPersistedStateOption
// testCatalogStateOpts are passed in to the stored state to control test behavior
testCatalogStateOpts []awsCatalogPersistedStateOption
}
// Ensure that we are implementing HostPluginServiceServer
var _ pb.HostPluginServiceServer = (*HostPlugin)(nil)
// OnCreateCatalog is called when a dynamic host catalog is created.
func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalogRequest) (*pb.OnCreateCatalogResponse, error) {
catalog := req.GetCatalog()
if catalog == nil {
return nil, status.Error(codes.InvalidArgument, "catalog is nil")
}
secrets := catalog.GetSecrets()
if secrets == nil {
return nil, status.Error(codes.InvalidArgument, "secrets are required")
}
attrs := catalog.GetAttributes()
if attrs == nil {
return nil, status.Error(codes.InvalidArgument, "attributes are required")
}
catalogAttributes, err := getCatalogAttributes(attrs)
if err != nil {
return nil, err
}
credConfig, err := cred.GetCredentialsConfig(secrets, catalogAttributes.Region)
if err != nil {
return nil, err
}
// Set up the creds in our persisted state.
credState, err := credential.NewAwsCredentialPersistedState(
append([]credential.AwsCredentialPersistedStateOption{
credential.WithAccessKeyId(credConfig.AccessKey),
credential.WithSecretAccessKey(credConfig.SecretKey),
credential.WithRegion(credConfig.Region),
}, p.testCredStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err)
}
catalogState, err := newAwsCatalogPersistedState(
append([]awsCatalogPersistedStateOption{
withCredentials(credState),
}, p.testCatalogStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err)
}
// Try to rotate the credentials if we're not skipping them.
if !catalogAttributes.DisableCredentialRotation {
if err := credState.RotateCreds(); err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error during credential rotation: %s", err)
}
} else {
// Simply validate if we aren't rotating.
if err := credState.ValidateCreds(); err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error during credential validation: %s", err)
}
}
persistedProto, err := catalogState.toProto()
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
return &pb.OnCreateCatalogResponse{
Persisted: persistedProto,
}, nil
}
// OnUpdateCatalog is called when a dynamic host catalog is updated.
func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalogRequest) (*pb.OnUpdateCatalogResponse, error) {
// For host catalogs we only need to really validate what is
// currently in the new copy of the host catalog data, so skip
// fetching the new copy to keep things a little less messy.
catalog := req.GetNewCatalog()
if catalog == nil {
return nil, status.Error(codes.InvalidArgument, "new catalog is nil")
}
var updateSecrets bool
secrets := catalog.GetSecrets()
if secrets != nil {
// We will be updating secrets this run, but what exactly that
// means will be determined later.
updateSecrets = true
}
attrs := catalog.GetAttributes()
if attrs == nil {
return nil, status.Error(codes.InvalidArgument, "new catalog missing attributes")
}
catalogAttributes, err := getCatalogAttributes(attrs)
if err != nil {
return nil, err
}
// Get the persisted data.
// NOTE: We might need to change this at a later time. I'm not too
// sure *exactly* what scenarios we might encounter that would
// ultimately mean that we would have to handle an empty or missing
// state in update, but what we are ultimately assuming here
// (implicitly through awsCatalogPersistedStateFromProto) is that
// the state will exist and be populated. Personally I think this
// is fine and important, but this may change in the future.
credState, err := credential.AwsCredentialPersistedStateFromProto(
req.GetPersisted().GetSecrets(),
append(p.testCredStateOpts, credential.WithRegion(catalogAttributes.Region))...)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
catalogState, err := newAwsCatalogPersistedState(
append([]awsCatalogPersistedStateOption{
withCredentials(credState),
}, p.testCatalogStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
if catalogAttributes.DisableCredentialRotation && !updateSecrets {
// This is a validate check to make sure that we aren't disabling
// rotation for credentials currently being managed by rotation.
// This is not allowed.
if !credState.CredsLastRotatedTime.IsZero() {
return nil, status.Error(codes.FailedPrecondition, "cannot disable rotation for already-rotated credentials")
}
}
if updateSecrets {
catalogSecrets, err := credential.GetCredentialsConfig(secrets, catalogAttributes.Region)
if err != nil {
return nil, err
}
// Replace the credentials. This checks the timestamp on the last
// rotation time as well and deletes the credentials if we are
// managing them (ie: if we've rotated them before).
if err := credState.ReplaceCreds(catalogSecrets.AccessKey, catalogSecrets.SecretKey); err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error attempting to replace credentials: %s", err)
}
}
if !catalogAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
// If we're enabling rotation now but didn't before, or have
// freshly replaced credentials, we can rotate here.
if err := credState.RotateCreds(); err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error during credential rotation: %s", err)
}
}
// That's it!
persistedProto, err := catalogState.toProto()
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
return &pb.OnUpdateCatalogResponse{
Persisted: persistedProto,
}, nil
}
// OnDeleteCatalog is called when a dynamic host catalog is deleted.
func (p *HostPlugin) OnDeleteCatalog(ctx context.Context, req *pb.OnDeleteCatalogRequest) (*pb.OnDeleteCatalogResponse, error) {
catalog := req.GetCatalog()
if catalog == nil {
return nil, status.Error(codes.InvalidArgument, "new catalog is nil")
}
attrs := catalog.GetAttributes()
if attrs == nil {
return nil, status.Error(codes.InvalidArgument, "new catalog missing attributes")
}
catalogAttributes, err := getCatalogAttributes(attrs)
if err != nil {
return nil, err
}
// Get the persisted data.
// NOTE: We return on error here, blocking the delete. This may or
// may not be an overzealous approach to maintaining database/state
// integrity. May need to be changed at later time if there are
// scenarios where we might be deleting things and any secret state
// may be corrupt/and or legitimately missing.
credState, err := credential.AwsCredentialPersistedStateFromProto(
req.GetPersisted().GetSecrets(),
append(p.testCredStateOpts, credential.WithRegion(catalogAttributes.Region))...)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
_, err = newAwsCatalogPersistedState(
append([]awsCatalogPersistedStateOption{
withCredentials(credState),
}, p.testCatalogStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
if !credState.CredsLastRotatedTime.IsZero() {
// Delete old/existing credentials. This is done with the same
// credentials to ensure that it has the proper permissions to do
// it.
if err := credState.DeleteCreds(); err != nil {
return nil, status.Errorf(codes.Aborted, "error removing rotated credentials during catalog deletion: %s", err)
}
}
return &pb.OnDeleteCatalogResponse{}, nil
}
// NormalizeSetData currently ensures that "filters" is an array value, even
// though it's accepted as a string value for CLI UX reasons
func (p *HostPlugin) NormalizeSetData(ctx context.Context, req *pb.NormalizeSetDataRequest) (*pb.NormalizeSetDataResponse, error) {
if req.Attributes == nil {
return new(pb.NormalizeSetDataResponse), nil
}
val := req.Attributes.Fields["filters"]
if val == nil {
return &pb.NormalizeSetDataResponse{Attributes: req.Attributes}, nil
}
stringVal, ok := val.Kind.(*structpb.Value_StringValue)
if !ok {
return &pb.NormalizeSetDataResponse{Attributes: req.Attributes}, nil
}
retAttrs := proto.Clone(req.Attributes).(*structpb.Struct)
retAttrs.Fields["filters"] = structpb.NewListValue(
&structpb.ListValue{
Values: []*structpb.Value{
structpb.NewStringValue(stringVal.StringValue),
},
})
return &pb.NormalizeSetDataResponse{Attributes: retAttrs}, nil
}
// OnCreateSet is called when a dynamic host set is created.
func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest) (*pb.OnCreateSetResponse, error) {
catalog := req.GetCatalog()
if catalog == nil {
return nil, status.Error(codes.InvalidArgument, "catalog is nil")
}
catalogAttrsRaw := catalog.GetAttributes()
if catalogAttrsRaw == nil {
return nil, status.Error(codes.InvalidArgument, "catalog missing attributes")
}
catalogAttributes, err := getCatalogAttributes(catalogAttrsRaw)
if err != nil {
return nil, err
}
credState, err := credential.AwsCredentialPersistedStateFromProto(
req.GetPersisted().GetSecrets(),
append(p.testCredStateOpts, credential.WithRegion(catalogAttributes.Region))...)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
catalogState, err := newAwsCatalogPersistedState(
append([]awsCatalogPersistedStateOption{
withCredentials(credState),
}, p.testCatalogStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
set := req.GetSet()
if set == nil {
return nil, status.Error(codes.InvalidArgument, "set is nil")
}
if set.GetAttributes() == nil {
return nil, status.Error(codes.InvalidArgument, "set missing attributes")
}
setAttrs, err := getSetAttributes(set.GetAttributes())
if err != nil {
return nil, err
}
ec2Client, err := catalogState.EC2Client(catalogAttributes.Region)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error getting EC2 client: %s", err)
}
input, err := buildDescribeInstancesInput(setAttrs, true)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error building DescribeInstances parameters: %s", err)
}
_, err = ec2Client.DescribeInstances(input)
if err == nil {
return nil, status.Error(codes.FailedPrecondition, "query error: DescribeInstances DryRun should have returned error, but none was found")
}
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "DryRunOperation" {
// Success
return &pb.OnCreateSetResponse{}, nil
}
return nil, status.Errorf(codes.InvalidArgument, "error performing dry run of DescribeInstances: %s", err)
}
// OnUpdateSet is called when a dynamic host set is updated.
func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest) (*pb.OnUpdateSetResponse, error) {
catalog := req.GetCatalog()
if catalog == nil {
return nil, status.Error(codes.InvalidArgument, "catalog is nil")
}
catalogAttrsRaw := catalog.GetAttributes()
if catalogAttrsRaw == nil {
return nil, status.Error(codes.InvalidArgument, "catalog missing attributes")
}
catalogAttributes, err := getCatalogAttributes(catalogAttrsRaw)
if err != nil {
return nil, err
}
credState, err := credential.AwsCredentialPersistedStateFromProto(
req.GetPersisted().GetSecrets(),
append(p.testCredStateOpts, credential.WithRegion(catalogAttributes.Region))...)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
catalogState, err := newAwsCatalogPersistedState(
append([]awsCatalogPersistedStateOption{
withCredentials(credState),
}, p.testCatalogStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
// As with catalog, we don't need to really look at the old host
// set here, just need to work off of/validate the new set config
set := req.GetNewSet()
if set == nil {
return nil, status.Error(codes.InvalidArgument, "new set is nil")
}
if set.GetAttributes() == nil {
return nil, status.Error(codes.InvalidArgument, "new set missing attributes")
}
setAttrs, err := getSetAttributes(set.GetAttributes())
if err != nil {
return nil, err
}
ec2Client, err := catalogState.EC2Client(catalogAttributes.Region)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error getting EC2 client: %s", err)
}
input, err := buildDescribeInstancesInput(setAttrs, true)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error building DescribeInstances parameters: %s", err)
}
_, err = ec2Client.DescribeInstances(input)
if err == nil {
return nil, status.Error(codes.FailedPrecondition, "query error: DescribeInstances DryRun should have returned error, but none was found")
}
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "DryRunOperation" {
// Success
return &pb.OnUpdateSetResponse{}, nil
}
return nil, status.Errorf(codes.InvalidArgument, "error performing dry run of DescribeInstances: %s", err)
}
// OnDeleteSet is called when a dynamic host set is deleted.
func (p *HostPlugin) OnDeleteSet(ctx context.Context, req *pb.OnDeleteSetRequest) (*pb.OnDeleteSetResponse, error) {
// No-op, AWS host set does not maintain anything that requires
// cleanup
return &pb.OnDeleteSetResponse{}, nil
}
// ListHosts returns the list of ec2 hosts and their descriptions.
func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*pb.ListHostsResponse, error) {
catalog := req.GetCatalog()
if catalog == nil {
return nil, status.Error(codes.InvalidArgument, "catalog is nil")
}
catalogAttrsRaw := catalog.GetAttributes()
if catalogAttrsRaw == nil {
return nil, status.Error(codes.InvalidArgument, "catalog missing attributes")
}
catalogAttributes, err := getCatalogAttributes(catalogAttrsRaw)
if err != nil {
return nil, err
}
credState, err := credential.AwsCredentialPersistedStateFromProto(req.GetPersisted().GetSecrets(), p.testCredStateOpts...)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
catalogState, err := newAwsCatalogPersistedState(
append([]awsCatalogPersistedStateOption{
withCredentials(credState),
}, p.testCatalogStateOpts...)...,
)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err)
}
sets := req.GetSets()
if sets == nil {
return nil, status.Error(codes.InvalidArgument, "sets is nil")
}
// Build all the queries in advance.
type hostSetQuery struct {
Id string
Input *ec2.DescribeInstancesInput
Output *ec2.DescribeInstancesOutput
OutputHosts []*pb.ListHostsResponseHost
}
queries := make([]hostSetQuery, len(sets))
for i, set := range sets {
// Validate Id since we use it in output
if set.GetId() == "" {
return nil, status.Error(codes.InvalidArgument, "set missing id")
}
if set.GetAttributes() == nil {
return nil, status.Error(codes.InvalidArgument, "set missing attributes")
}
setAttrs, err := getSetAttributes(set.GetAttributes())
if err != nil {
return nil, err
}
input, err := buildDescribeInstancesInput(setAttrs, false)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error building DescribeInstances parameters for host set id %q: %s", set.GetId(), err)
}
queries[i] = hostSetQuery{
Id: set.GetId(),
Input: input,
}
}
ec2Client, err := catalogState.EC2Client(catalogAttributes.Region)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error getting EC2 client: %s", err)
}
// Run all queries now and assemble output.
var maxLen int
for i, query := range queries {
output, err := ec2Client.DescribeInstances(query.Input)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error running DescribeInstances for host set id %q: %s", query.Id, err)
}
queries[i].Output = output
// Process the output here, we will normalize this into a single
// set of hosts afterwards (possibly removing duplicates).
for _, reservation := range output.Reservations {
for _, instance := range reservation.Instances {
host, err := awsInstanceToHost(instance)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "error processing host results for host set id %q: %s", query.Id, err)
}
queries[i].OutputHosts = append(queries[i].OutputHosts, host)
maxLen++ // Increment maximum counter for allocation later
}
}
}
// Now de-duplicate the hosts for the output. Maintain two sets:
// * A slice of hosts that will be used in the output
// * A map of hosts indexed by their external ID
//
// The map will is used in de-duplication to determine whether or
// not we've seen the host before to simply add the query's set ID
// to the list of set IDs that the host was seen in.
hostResultSlice := make([]*pb.ListHostsResponseHost, 0, maxLen)
hostResultMap := make(map[string]*pb.ListHostsResponseHost)
for _, query := range queries {
for _, host := range query.OutputHosts {
if existingHost, ok := hostResultMap[host.ExternalId]; ok {
// Existing host, just add the set ID to the list of seen IDs
// and continue
existingHost.SetIds = append(existingHost.SetIds, query.Id)
continue
}
// This will be the first seen entry, so append the set ID to
// this host, and add it.
host.SetIds = append(host.SetIds, query.Id)
hostResultSlice = append(hostResultSlice, host)
hostResultMap[host.ExternalId] = host
}
}
// Done!
return &pb.ListHostsResponse{
Hosts: hostResultSlice,
}, nil
}
func buildFilters(attrs *SetAttributes) ([]*ec2.Filter, error) {
var filters []*ec2.Filter
var foundStateFilter bool
for _, filterAttr := range attrs.Filters {
// Each filter arrives in "k=v1,v2" format. First, split on equal sign.
splitFilter := strings.Split(filterAttr, "=")
switch {
case len(splitFilter) != 2:
return nil, fmt.Errorf("expected filter %q to contain a single equal sign", filterAttr)
case len(splitFilter[0]) == 0:
return nil, fmt.Errorf("filter %q contains an empty filter key", filterAttr)
case len(splitFilter[1]) == 0:
return nil, fmt.Errorf("filter %q contains an empty value", filterAttr)
}
filterKey, filterValue := splitFilter[0], splitFilter[1]
if filterKey == "instance-state-code" || filterKey == "instance-state-name" {
foundStateFilter = true
}
var filterValues []*string
for _, val := range strings.Split(filterValue, ",") {
filterValues = append(filterValues, aws.String(val))
}
filters = append(filters, &ec2.Filter{
Name: aws.String(filterKey),
Values: filterValues,
})
}
if !foundStateFilter {
// if there is no explicit instance state filter, add the
// instance-state-name = ["running"] to the filter set. This
// ensures that we filter on running instances only at the API
// side, saving time when processing results.
filters = append(filters, &ec2.Filter{
Name: aws.String("instance-state-name"),
Values: aws.StringSlice([]string{ec2.InstanceStateNameRunning}),
})
}
return filters, nil
}
func buildDescribeInstancesInput(attrs *SetAttributes, dryRun bool) (*ec2.DescribeInstancesInput, error) {
filters, err := buildFilters(attrs)
if err != nil {
return nil, fmt.Errorf("error building filters: %w", err)
}
return &ec2.DescribeInstancesInput{
DryRun: aws.Bool(dryRun),
Filters: filters,
}, nil
}
// awsInstanceToHost processes data from an ec2.Instance and returns
// a ListHostsResponseHost with the ID and network addresses
// populated.
func awsInstanceToHost(instance *ec2.Instance) (*pb.ListHostsResponseHost, error) {
// Integrity check: some fields should always be non-nil. Check
// those.
if instance == nil {
return nil, errors.New("response integrity error: missing instance entry")
}
if aws.StringValue(instance.InstanceId) == "" {
return nil, errors.New("response integrity error: missing instance id")
}
result := new(pb.ListHostsResponseHost)
// External ID is the instance ID.
result.ExternalId = aws.StringValue(instance.InstanceId)
// External Name is the AWS "Name" tag, if it exists.
for _, t := range instance.Tags {
if aws.StringValue(t.Key) == ConstInstanceNameTagKey {
result.ExternalName = aws.StringValue(t.Value)
break
}
}
// First IP address/dns name are always the private fields if they
// are populated
if aws.StringValue(instance.PrivateIpAddress) != "" {
result.IpAddresses = append(result.IpAddresses, aws.StringValue(instance.PrivateIpAddress))
}
if aws.StringValue(instance.PrivateDnsName) != "" {
result.DnsNames = append(result.DnsNames, aws.StringValue(instance.PrivateDnsName))
}
// Public IP address/dns names are next
if aws.StringValue(instance.PublicIpAddress) != "" {
result.IpAddresses = append(result.IpAddresses, aws.StringValue(instance.PublicIpAddress))
}
if aws.StringValue(instance.PublicDnsName) != "" {
result.DnsNames = append(result.DnsNames, aws.StringValue(instance.PublicDnsName))
}
// Now go through all of the interfaces and log the IP address of
// every interface.
for _, iface := range instance.NetworkInterfaces {
if iface == nil {
// Probably will never happen, but just in case
return nil, errors.New("response integrity error: interface entry is nil")
}
// Populate default IP addresses/DNS name similar to how we do
// for the entire instance.
if aws.StringValue(iface.PrivateIpAddress) != "" && !stringInSlice(result.IpAddresses, aws.StringValue(iface.PrivateIpAddress)) {
result.IpAddresses = append(result.IpAddresses, aws.StringValue(iface.PrivateIpAddress))
}
if aws.StringValue(iface.PrivateDnsName) != "" && !stringInSlice(result.DnsNames, aws.StringValue(iface.PrivateDnsName)) {
result.DnsNames = append(result.DnsNames, aws.StringValue(iface.PrivateDnsName))
}
// Iterate through the private IP addresses and log the
// information.
for _, addr := range iface.PrivateIpAddresses {
if addr == nil {
return nil, errors.New("response integrity error: interface address entry is nil")
}
// Add private address/dns name if they have not been added yet
if aws.StringValue(addr.PrivateIpAddress) != "" && !stringInSlice(result.IpAddresses, aws.StringValue(addr.PrivateIpAddress)) {
result.IpAddresses = append(result.IpAddresses, aws.StringValue(addr.PrivateIpAddress))
}
if aws.StringValue(addr.PrivateDnsName) != "" && !stringInSlice(result.DnsNames, aws.StringValue(addr.PrivateDnsName)) {
result.DnsNames = append(result.DnsNames, aws.StringValue(addr.PrivateDnsName))
}
// Add public address/dns name if they have not been added yet
if addr.Association != nil && aws.StringValue(addr.Association.PublicIp) != "" && !stringInSlice(result.IpAddresses, aws.StringValue(addr.Association.PublicIp)) {
result.IpAddresses = append(result.IpAddresses, aws.StringValue(addr.Association.PublicIp))
}
if addr.Association != nil && aws.StringValue(addr.Association.PublicDnsName) != "" && !stringInSlice(result.DnsNames, aws.StringValue(addr.Association.PublicDnsName)) {
result.DnsNames = append(result.DnsNames, aws.StringValue(addr.Association.PublicDnsName))
}
}
// Add the IPv6 addresses.
for _, addr := range iface.Ipv6Addresses {
if addr == nil {
continue
}
if addr.Ipv6Address != nil && aws.StringValue(addr.Ipv6Address) != "" {
result.IpAddresses = append(result.IpAddresses, aws.StringValue(addr.Ipv6Address))
}
}
}
// Done
return result, nil
}
func stringInSlice(s []string, x string) bool {
for _, y := range s {
if x == y {
return true
}
}
return false
}
File diff suppressed because it is too large. Load diff
+101
View File
@@ -0,0 +1,101 @@
package host
import (
"errors"
"fmt"
"net"
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/hashicorp/boundary-plugin-host-aws/internal/credential"
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
"google.golang.org/protobuf/types/known/structpb"
)
var customClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
},
}
type ec2APIFunc func(*session.Session, ...*aws.Config) (ec2iface.EC2API, error)
type awsCatalogPersistedStateOption func(s *awsCatalogPersistedState) error
type awsCatalogPersistedState struct {
*credential.AwsCredentialPersistedState
// testEC2APIFunc provides a way to provide a factory for a mock EC2 client
testEC2APIFunc ec2APIFunc
}
func withTestEC2APIFunc(f ec2APIFunc) awsCatalogPersistedStateOption {
return func(s *awsCatalogPersistedState) error {
if s.testEC2APIFunc != nil {
return errors.New("test API function already set")
}
s.testEC2APIFunc = f
return nil
}
}
func withCredentials(x *credential.AwsCredentialPersistedState) awsCatalogPersistedStateOption {
return func(s *awsCatalogPersistedState) error {
if s.AwsCredentialPersistedState != nil {
return errors.New("aws credentials already set")
}
s.AwsCredentialPersistedState = x
return nil
}
}
func newAwsCatalogPersistedState(opts ...awsCatalogPersistedStateOption) (*awsCatalogPersistedState, error) {
s := new(awsCatalogPersistedState)
for _, opt := range opts {
if err := opt(s); err != nil {
return nil, err
}
}
return s, nil
}
func (s *awsCatalogPersistedState) toProto() (*pb.HostCatalogPersisted, error) {
data, err := structpb.NewStruct(s.ToMap())
if err != nil {
return nil, fmt.Errorf("error converting state to structpb.Struct: %w", err)
}
return &pb.HostCatalogPersisted{Secrets: data}, nil
}
// EC2Client returns a configured EC2 client based on the session
// information stored in the state.
func (s *awsCatalogPersistedState) EC2Client(region string) (ec2iface.EC2API, error) {
sess, err := s.GetSession()
if err != nil {
return nil, fmt.Errorf("error getting AWS session when fetching EC2 client: %w", err)
}
if s.testEC2APIFunc != nil {
return s.testEC2APIFunc(sess, aws.NewConfig().WithRegion(region))
}
cfg := aws.NewConfig()
if region != "" {
cfg = cfg.WithRegion(region)
}
cfg.HTTPClient = customClient
return ec2.New(sess, cfg), nil
}
+32
View File
@@ -0,0 +1,32 @@
package host
import (
"testing"
"github.com/hashicorp/boundary-plugin-host-aws/internal/credential"
"github.com/stretchr/testify/require"
)
func TestAwsCatalogPersistedStateEC2Client(t *testing.T) {
require := require.New(t)
cred, err := credential.NewAwsCredentialPersistedState(
credential.WithRegion("us-west-2"),
credential.WithAccessKeyId("foobar"),
credential.WithSecretAccessKey("barfoo"),
)
require.NoError(err)
state, err := newAwsCatalogPersistedState(
withCredentials(cred),
withTestEC2APIFunc(newTestMockEC2(nil)),
)
require.NoError(err)
clientRaw, err := state.EC2Client("us-west-2")
require.NoError(err)
require.NotNil(clientRaw)
client, ok := clientRaw.(*testMockEC2)
require.True(ok)
require.NotNil(client)
require.Equal("us-west-2", client.Region)
}
+91
View File
@@ -0,0 +1,91 @@
package host
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
)
const (
testOptionErr = "test option error"
testGetUserErr = "test error for GetUser"
testGetCallerIdentityErr = "test error for GetCallerIdentity"
testDeleteAccessKeyErr = "test error for DeleteAccessKey"
testDescribeInstancesError = "DescribeInstances error"
)
type testMockEC2State struct {
DescribeInstancesCalled bool
DescribeInstancesInputParams *ec2.DescribeInstancesInput
}
func (s *testMockEC2State) Reset() {
s.DescribeInstancesCalled = false
s.DescribeInstancesInputParams = nil
}
type testMockEC2 struct {
ec2iface.EC2API
State *testMockEC2State
Region string
DescribeInstancesOutput *ec2.DescribeInstancesOutput
DescribeInstancesError error
}
type testMockEC2Option func(m *testMockEC2) error
func testMockEC2WithDescribeInstancesOutput(o *ec2.DescribeInstancesOutput) testMockEC2Option {
return func(m *testMockEC2) error {
m.DescribeInstancesOutput = o
return nil
}
}
func testMockEC2WithDescribeInstancesError(e error) testMockEC2Option {
return func(m *testMockEC2) error {
m.DescribeInstancesError = e
return nil
}
}
func newTestMockEC2(state *testMockEC2State, opts ...testMockEC2Option) ec2APIFunc {
return func(sess *session.Session, cfgs ...*aws.Config) (ec2iface.EC2API, error) {
m := &testMockEC2{
State: state,
}
for _, opt := range opts {
if err := opt(m); err != nil {
return nil, err
}
}
for _, cfg := range cfgs {
// Last region takes precedence
m.Region = *cfg.Region
}
return m, nil
}
}
func (m *testMockEC2) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
if m.State != nil {
m.State.DescribeInstancesCalled = true
m.State.DescribeInstancesInputParams = input
}
if m.DescribeInstancesError != nil {
return nil, m.DescribeInstancesError
}
return m.DescribeInstancesOutput, nil
}
type ec2FilterSorter []*ec2.Filter
func (s ec2FilterSorter) Len() int { return len(s) }
func (s ec2FilterSorter) Less(i, j int) bool { return *s[i].Name < *s[j].Name }
func (s ec2FilterSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }