Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4eb3293a9a | ||
|
|
3962754106 | ||
|
|
512f18baaa | ||
|
|
f4406802cb | ||
|
|
163ec69771 | ||
|
|
e8a0b0bf10 | ||
|
|
4c8e95fcd8 | ||
|
|
01f8da94e4 |
No files matched your search
@@ -4,6 +4,8 @@ on:
|
||||
- workflow_dispatch
|
||||
- push
|
||||
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -22,6 +22,11 @@ type CatalogAttributes struct {
|
||||
|
||||
// DualStack is used for configuring how the aws client will resolve requests.
|
||||
DualStack bool
|
||||
|
||||
// InstanceAddressesOnly specifies whether only the instance addresses (private IPv4/DNS,
|
||||
// public IPv4/DNS, and IPv6) are synced, or all addresses, including those from
|
||||
// secondary ENIs, are synced.
|
||||
InstanceAddressesOnly bool
|
||||
}
|
||||
|
||||
func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
|
||||
@@ -40,6 +45,12 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
|
||||
}
|
||||
delete(unknownFields, ConstAwsDualStack)
|
||||
|
||||
instanceAddressesOnly, err := values.GetBoolValue(in, ConstInstanceAddressesOnly, false)
|
||||
if err != nil {
|
||||
badFields[fmt.Sprintf("attributes.%s", ConstInstanceAddressesOnly)] = err.Error()
|
||||
}
|
||||
delete(unknownFields, ConstInstanceAddressesOnly)
|
||||
|
||||
for s := range unknownFields {
|
||||
switch s {
|
||||
// Ignore knownFields from CredentialAttributes
|
||||
@@ -65,8 +76,9 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) {
|
||||
}
|
||||
|
||||
return &CatalogAttributes{
|
||||
CredentialAttributes: credAttributes,
|
||||
DualStack: dualStack,
|
||||
CredentialAttributes: credAttributes,
|
||||
DualStack: dualStack,
|
||||
InstanceAddressesOnly: instanceAddressesOnly,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -59,6 +59,22 @@ func TestGetCatalogAttributes(t *testing.T) {
|
||||
DualStack: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "withInstanceAddressesOnly",
|
||||
in: &structpb.Struct{
|
||||
Fields: map[string]*structpb.Value{
|
||||
"region": structpb.NewStringValue("us-west-2"),
|
||||
"instance_addresses_only": structpb.NewBoolValue(true),
|
||||
},
|
||||
},
|
||||
expected: &CatalogAttributes{
|
||||
CredentialAttributes: &credential.CredentialAttributes{
|
||||
Region: "us-west-2",
|
||||
DisableCredentialRotation: false,
|
||||
},
|
||||
InstanceAddressesOnly: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "default",
|
||||
in: &structpb.Struct{
|
||||
|
||||
@@ -13,4 +13,9 @@ const (
|
||||
|
||||
// ConstAwsDualStack is the key for the dualstack flag in the aws s3 client.
|
||||
ConstAwsDualStack = "dual_stack"
|
||||
|
||||
// ConstInstanceAddressesOnly is the key (instance_addresses_only) that specifies whether
|
||||
// only the instance addresses (private IPv4/DNS, public IPv4/DNS, and IPv6) are synced,
|
||||
// or all addresses, including those from secondary ENIs, are synced.
|
||||
ConstInstanceAddressesOnly = "instance_addresses_only"
|
||||
)
|
||||
@@ -32,6 +32,13 @@ type HostPlugin struct {
|
||||
testCatalogStateOpts []awsCatalogPersistedStateOption
|
||||
}
|
||||
|
||||
type hostSetQuery struct {
|
||||
Id string
|
||||
Input *ec2.DescribeInstancesInput
|
||||
Output *ec2.DescribeInstancesOutput
|
||||
OutputHosts []*pb.ListHostsResponseHost
|
||||
}
|
||||
|
||||
// Ensure that we are implementing HostPluginServiceServer
|
||||
var _ pb.HostPluginServiceServer = (*HostPlugin)(nil)
|
||||
|
||||
@@ -495,13 +502,6 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
|
||||
}
|
||||
|
||||
// Build all the queries in advance.
|
||||
type hostSetQuery struct {
|
||||
Id string
|
||||
Input *ec2.DescribeInstancesInput
|
||||
Output *ec2.DescribeInstancesOutput
|
||||
OutputHosts []*pb.ListHostsResponseHost
|
||||
}
|
||||
|
||||
queries := make([]hostSetQuery, len(sets))
|
||||
for i, set := range sets {
|
||||
// Validate Id since we use it in output
|
||||
@@ -550,7 +550,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (*
|
||||
// set of hosts afterwards (possibly removing duplicates).
|
||||
for _, reservation := range output.Reservations {
|
||||
for _, instance := range reservation.Instances {
|
||||
host, err := awsInstanceToHost(instance)
|
||||
host, err := awsInstanceToHost(instance, catalogAttributes)
|
||||
if err != nil {
|
||||
return nil, errors.BadRequestStatusf("error processing host results for host set id %q: %s", query.Id, err)
|
||||
}
|
||||
@@ -648,10 +648,13 @@ func buildDescribeInstancesInput(attrs *SetAttributes, dryRun bool) (*ec2.Descri
|
||||
// awsInstanceToHost processes data from an ec2.Instance and returns
|
||||
// a ListHostsResponseHost with the ID and network addresses
|
||||
// populated.
|
||||
func awsInstanceToHost(instance types.Instance) (*pb.ListHostsResponseHost, error) {
|
||||
func awsInstanceToHost(instance types.Instance, catalogAttributes *CatalogAttributes) (*pb.ListHostsResponseHost, error) {
|
||||
if aws.ToString(instance.InstanceId) == "" {
|
||||
return nil, fmt.Errorf("response integrity error: missing instance id")
|
||||
}
|
||||
if catalogAttributes == nil {
|
||||
return nil, fmt.Errorf("response integrity error: missing catalog attributes")
|
||||
}
|
||||
|
||||
result := new(pb.ListHostsResponseHost)
|
||||
|
||||
@@ -666,10 +669,16 @@ func awsInstanceToHost(instance types.Instance) (*pb.ListHostsResponseHost, erro
|
||||
}
|
||||
}
|
||||
|
||||
result.IpAddresses = appendDistinct(result.IpAddresses, instance.PrivateIpAddress, instance.PublicIpAddress)
|
||||
// First let's get the instance's Private IPv4/DNS, Public IPv4/DNS, and IPv6 addresses
|
||||
result.IpAddresses = appendDistinct(result.IpAddresses, instance.PrivateIpAddress, instance.PublicIpAddress, instance.Ipv6Address)
|
||||
result.DnsNames = appendDistinct(result.DnsNames, instance.PrivateDnsName, instance.PublicDnsName)
|
||||
|
||||
// Now go through all of the interfaces and log the IP address of
|
||||
if catalogAttributes.InstanceAddressesOnly {
|
||||
// return early as we already have the instance addresses
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Now go through all of the interfaces and sync the IP address of
|
||||
// every interface.
|
||||
for _, iface := range instance.NetworkInterfaces {
|
||||
// Populate default IP addresses/DNS name similar to how we do
|
||||
|
||||
@@ -638,7 +638,8 @@ func TestPluginOnCreateCatalogErr(t *testing.T) {
|
||||
Fields: map[string]*structpb.Value{
|
||||
credential.ConstAccessKeyId: structpb.NewStringValue("AKIA_foobar"),
|
||||
credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
Attrs: &hostcatalogs.HostCatalog_Attributes{
|
||||
Attributes: &structpb.Struct{
|
||||
Fields: map[string]*structpb.Value{
|
||||
@@ -664,7 +665,8 @@ func TestPluginOnCreateCatalogErr(t *testing.T) {
|
||||
Fields: map[string]*structpb.Value{
|
||||
credential.ConstAccessKeyId: structpb.NewStringValue("AKIA_foobar"),
|
||||
credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
Attrs: &hostcatalogs.HostCatalog_Attributes{
|
||||
Attributes: &structpb.Struct{
|
||||
Fields: map[string]*structpb.Value{
|
||||
@@ -694,7 +696,8 @@ func TestPluginOnCreateCatalogErr(t *testing.T) {
|
||||
Fields: map[string]*structpb.Value{
|
||||
credential.ConstAccessKeyId: structpb.NewStringValue("AKIA_foobar"),
|
||||
credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
Attrs: &hostcatalogs.HostCatalog_Attributes{
|
||||
Attributes: &structpb.Struct{
|
||||
Fields: map[string]*structpb.Value{
|
||||
@@ -721,7 +724,8 @@ func TestPluginOnCreateCatalogErr(t *testing.T) {
|
||||
Fields: map[string]*structpb.Value{
|
||||
credential.ConstAccessKeyId: structpb.NewStringValue("AKIA_foobar"),
|
||||
credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
|
||||
}},
|
||||
},
|
||||
},
|
||||
Attrs: &hostcatalogs.HostCatalog_Attributes{
|
||||
Attributes: &structpb.Struct{
|
||||
Fields: map[string]*structpb.Value{
|
||||
@@ -2383,15 +2387,25 @@ func TestBuildDescribeInstancesInput(t *testing.T) {
|
||||
|
||||
func TestAwsInstanceToHost(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
instance types.Instance
|
||||
expected *pb.ListHostsResponseHost
|
||||
expectedErr string
|
||||
name string
|
||||
instance types.Instance
|
||||
catalogAttrs *CatalogAttributes
|
||||
expected *pb.ListHostsResponseHost
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "missing instance id",
|
||||
instance: types.Instance{},
|
||||
expectedErr: "response integrity error: missing instance id",
|
||||
name: "missing instance id",
|
||||
instance: types.Instance{},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expectedErr: "response integrity error: missing instance id",
|
||||
},
|
||||
{
|
||||
name: "missing catalog attributes",
|
||||
instance: types.Instance{
|
||||
InstanceId: aws.String("foobar"),
|
||||
},
|
||||
catalogAttrs: nil,
|
||||
expectedErr: "response integrity error: missing catalog attributes",
|
||||
},
|
||||
{
|
||||
name: "good, single IP w/public addr",
|
||||
@@ -2418,6 +2432,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1"},
|
||||
@@ -2443,6 +2458,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1"},
|
||||
@@ -2484,6 +2500,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1", "10.0.0.2"},
|
||||
@@ -2529,6 +2546,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1", "10.0.0.2", "1.1.1.2"},
|
||||
@@ -2564,6 +2582,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1", "10.0.0.2"},
|
||||
@@ -2578,6 +2597,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
PrivateDnsName: aws.String("test.example.internal"),
|
||||
PublicIpAddress: aws.String("1.1.1.1"),
|
||||
PublicDnsName: aws.String("test.example.com"),
|
||||
Ipv6Address: aws.String("instance::ipv6::address"),
|
||||
NetworkInterfaces: []types.InstanceNetworkInterface{
|
||||
{
|
||||
PrivateIpAddress: aws.String("10.0.0.1"),
|
||||
@@ -2596,9 +2616,63 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1", "some::fake::address"},
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1", "instance::ipv6::address", "some::fake::address"},
|
||||
DnsNames: []string{"test.example.internal", "test.example.com"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "good, instance addresses only excludes interface secondary addresses",
|
||||
instance: types.Instance{
|
||||
InstanceId: aws.String("foobar"),
|
||||
PrivateIpAddress: aws.String("10.0.0.1"),
|
||||
PrivateDnsName: aws.String("test.example.internal"),
|
||||
PublicIpAddress: aws.String("1.1.1.1"),
|
||||
PublicDnsName: aws.String("test.example.com"),
|
||||
Ipv6Address: aws.String("instance::ipv6::address"),
|
||||
NetworkInterfaces: []types.InstanceNetworkInterface{
|
||||
{
|
||||
PrivateIpAddress: aws.String("10.0.0.1"),
|
||||
PrivateDnsName: aws.String("test.example.internal"),
|
||||
PrivateIpAddresses: []types.InstancePrivateIpAddress{
|
||||
{
|
||||
Association: &types.InstanceNetworkInterfaceAssociation{
|
||||
PublicIp: aws.String("1.1.1.1"),
|
||||
PublicDnsName: aws.String("test.example.com"),
|
||||
},
|
||||
PrivateIpAddress: aws.String("10.0.0.1"),
|
||||
PrivateDnsName: aws.String("test.example.internal"),
|
||||
},
|
||||
{
|
||||
PrivateIpAddress: aws.String("10.0.0.2"),
|
||||
PrivateDnsName: aws.String("test2.example.internal"),
|
||||
},
|
||||
},
|
||||
Ipv6Addresses: []types.InstanceIpv6Address{{Ipv6Address: aws.String("iface::ipv6::address")}},
|
||||
},
|
||||
{
|
||||
PrivateIpAddress: aws.String("10.0.1.1"),
|
||||
PrivateDnsName: aws.String("secondary.example.internal"),
|
||||
PrivateIpAddresses: []types.InstancePrivateIpAddress{
|
||||
{
|
||||
Association: &types.InstanceNetworkInterfaceAssociation{
|
||||
PublicIp: aws.String("2.2.2.2"),
|
||||
PublicDnsName: aws.String("secondary.example.com"),
|
||||
},
|
||||
PrivateIpAddress: aws.String("10.0.1.1"),
|
||||
PrivateDnsName: aws.String("secondary.example.internal"),
|
||||
},
|
||||
},
|
||||
Ipv6Addresses: []types.InstanceIpv6Address{{Ipv6Address: aws.String("secondary::ipv6::address")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{InstanceAddressesOnly: true},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
IpAddresses: []string{"10.0.0.1", "1.1.1.1", "instance::ipv6::address"},
|
||||
DnsNames: []string{"test.example.internal", "test.example.com"},
|
||||
},
|
||||
},
|
||||
@@ -2632,6 +2706,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
ExternalName: "test-instance-actual-name",
|
||||
@@ -2669,6 +2744,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
catalogAttrs: &CatalogAttributes{},
|
||||
expected: &pb.ListHostsResponseHost{
|
||||
ExternalId: "foobar",
|
||||
ExternalName: "test-instance-actual-name",
|
||||
@@ -2682,7 +2758,7 @@ func TestAwsInstanceToHost(t *testing.T) {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require := require.New(t)
|
||||
actual, err := awsInstanceToHost(tc.instance)
|
||||
actual, err := awsInstanceToHost(tc.instance, tc.catalogAttrs)
|
||||
if tc.expectedErr != "" {
|
||||
require.EqualError(err, tc.expectedErr)
|
||||
return
|
||||
|
||||
@@ -11,5 +11,5 @@ const (
|
||||
ConstAwsDualStack = "dual_stack"
|
||||
|
||||
// defaultStreamChunkSize is the recommened chunk size for sending data through a stream
|
||||
defaultStreamChunkSize = 65536 // 64 KiB
|
||||
defaultStreamChunkSize = 1 << 20 // 1 MiB
|
||||
)
|
||||
@@ -665,6 +665,23 @@ func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePlu
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
// multipartThreshold is the file size, in bytes, above which PutObject
|
||||
// uploads an object using a multipart upload instead of a single PutObject
|
||||
// request. 8 MiB matches the AWS CLI and boto3 defaults.
|
||||
multipartThreshold = 8 * 1024 * 1024 // 8 MiB
|
||||
|
||||
// defaultMultipartPartSize is the default size, in bytes, of each part
|
||||
// uploaded during a multipart upload. It is kept equal to
|
||||
// multipartThreshold so a multipart upload always has at least two parts
|
||||
// (S3 requires every part except the last to be at least 5 MiB).
|
||||
defaultMultipartPartSize = 8 * 1024 * 1024 // 8 MiB
|
||||
|
||||
// maxMultipartParts is the maximum number of parts S3 permits for a single
|
||||
// multipart upload.
|
||||
maxMultipartParts = 10000
|
||||
)
|
||||
|
||||
// PutObject is called when putting objects into an s3 bucket.
|
||||
func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest) (*pb.PutObjectResponse, error) {
|
||||
if req == nil {
|
||||
@@ -751,6 +768,32 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
|
||||
}
|
||||
objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
|
||||
|
||||
// decodedChecksum is the raw SHA-256 digest of the entire object. Boundary
|
||||
// compares this byte-for-byte against the hash it computed locally before
|
||||
// the upload, so it must always be the whole-object checksum regardless of
|
||||
// whether a single PutObject or a multipart upload is used.
|
||||
decodedChecksum, err := base64.StdEncoding.DecodeString(checksum)
|
||||
if err != nil {
|
||||
return nil, errors.UnknownStatus("failed to decode checksum value")
|
||||
}
|
||||
|
||||
// for larger files, use a multipart upload instead of a single PutObject
|
||||
// request.
|
||||
if info.Size() > multipartThreshold {
|
||||
partSize := int64(defaultMultipartPartSize)
|
||||
// S3 allows at most maxMultipartParts parts per upload, so grow the
|
||||
// part size for very large objects to stay within that limit.
|
||||
if sz := info.Size(); sz/partSize >= maxMultipartParts {
|
||||
partSize = (sz + maxMultipartParts - 1) / maxMultipartParts
|
||||
}
|
||||
if err := p.putObjectMultipart(ctx, req, storageState, bucket.GetBucketName(), objectKey, file, partSize, opts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &pb.PutObjectResponse{
|
||||
ChecksumSha_256: decodedChecksum,
|
||||
}, nil
|
||||
}
|
||||
|
||||
putCall := func(s3Client S3API) (any, error) {
|
||||
return s3Client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket.GetBucketName()),
|
||||
@@ -772,15 +815,126 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
|
||||
if checksum != *resp.ChecksumSHA256 {
|
||||
return nil, checksumMistmatchStatus()
|
||||
}
|
||||
decodedChecksum, err := base64.StdEncoding.DecodeString(*resp.ChecksumSHA256)
|
||||
if err != nil {
|
||||
return nil, errors.UnknownStatus("failed to decode checksum value from aws")
|
||||
}
|
||||
return &pb.PutObjectResponse{
|
||||
ChecksumSha_256: decodedChecksum,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// putObjectMultipart uploads file to the given bucket and key using an S3
|
||||
// multipart upload. The file is split into parts of partSize bytes (the final
|
||||
// part may be smaller) and each part is uploaded with its own SHA-256 checksum
|
||||
// so S3 can verify part integrity on the wire.
|
||||
//
|
||||
// If any step fails the upload is aborted on a best-effort basis to avoid
|
||||
// leaving orphaned parts behind, which would otherwise continue to accrue
|
||||
// storage charges.
|
||||
func (p *StoragePlugin) putObjectMultipart(
|
||||
ctx context.Context,
|
||||
req *pb.PutObjectRequest,
|
||||
storageState *awsStoragePersistedState,
|
||||
bucketName, objectKey string,
|
||||
file *os.File,
|
||||
partSize int64,
|
||||
opts ...s3Option,
|
||||
) error {
|
||||
bucketId := req.GetBucket().GetId()
|
||||
|
||||
createCall := func(s3Client S3API) (any, error) {
|
||||
return s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
|
||||
})
|
||||
}
|
||||
createResp, err := p.call(ctx, createCall, bucketId, storageState, opts...)
|
||||
if err != nil {
|
||||
return parseS3Error("create multipart upload", err, req).Err()
|
||||
}
|
||||
uploadId := createResp.(*s3.CreateMultipartUploadOutput).UploadId
|
||||
|
||||
// once the upload has been created, any subsequent failure must abort it so
|
||||
// uploaded parts are not orphaned.
|
||||
abort := func() {
|
||||
abortCall := func(s3Client S3API) (any, error) {
|
||||
return s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadId,
|
||||
})
|
||||
}
|
||||
// best effort: the failure that triggered the abort is more useful to
|
||||
// the caller than an error from the abort itself.
|
||||
_, _ = p.call(ctx, abortCall, bucketId, storageState, opts...)
|
||||
}
|
||||
|
||||
var completedParts []types.CompletedPart
|
||||
buf := make([]byte, partSize)
|
||||
var partNumber int32 = 1
|
||||
for {
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
if readErr == io.EOF {
|
||||
// previous iteration consumed the final bytes of the file
|
||||
break
|
||||
}
|
||||
if readErr != nil && readErr != io.ErrUnexpectedEOF {
|
||||
abort()
|
||||
return errors.UnknownStatusf("failed to read file: %s", readErr)
|
||||
}
|
||||
|
||||
partData := buf[:n]
|
||||
sum := sha256.Sum256(partData)
|
||||
partChecksum := base64.StdEncoding.EncodeToString(sum[:])
|
||||
|
||||
pn := partNumber
|
||||
uploadCall := func(s3Client S3API) (any, error) {
|
||||
return s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadId,
|
||||
PartNumber: aws.Int32(pn),
|
||||
Body: bytes.NewReader(partData),
|
||||
ContentLength: aws.Int64(int64(n)),
|
||||
ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
|
||||
ChecksumSHA256: aws.String(partChecksum),
|
||||
})
|
||||
}
|
||||
uploadResp, err := p.call(ctx, uploadCall, bucketId, storageState, opts...)
|
||||
if err != nil {
|
||||
abort()
|
||||
return parseS3Error("upload part", err, req).Err()
|
||||
}
|
||||
part := uploadResp.(*s3.UploadPartOutput)
|
||||
completedParts = append(completedParts, types.CompletedPart{
|
||||
ETag: part.ETag,
|
||||
ChecksumSHA256: part.ChecksumSHA256,
|
||||
PartNumber: aws.Int32(pn),
|
||||
})
|
||||
|
||||
partNumber++
|
||||
if readErr == io.ErrUnexpectedEOF {
|
||||
// the final, short part
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
completeCall := func(s3Client S3API) (any, error) {
|
||||
return s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: completedParts,
|
||||
},
|
||||
})
|
||||
}
|
||||
if _, err := p.call(ctx, completeCall, bucketId, storageState, opts...); err != nil {
|
||||
abort()
|
||||
return parseS3Error("complete multipart upload", err, req).Err()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteObjects is used to delete one or many objects from an s3 bucket.
|
||||
func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjectsRequest) (*pb.DeleteObjectsResponse, error) {
|
||||
if req.GetKeyPrefix() == "" {
|
||||
|
||||
@@ -3715,6 +3715,195 @@ func TestStoragePlugin_PutObject(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoragePlugin_PutObjectMultipart(t *testing.T) {
|
||||
td := t.TempDir()
|
||||
|
||||
// genFile writes size bytes of deterministic content to a temp file and
|
||||
// returns its path along with the raw SHA-256 digest of the whole file.
|
||||
genFile := func(t *testing.T, name string, size int) (string, []byte) {
|
||||
t.Helper()
|
||||
data := make([]byte, size)
|
||||
for i := range data {
|
||||
data[i] = byte(i % 251)
|
||||
}
|
||||
fp := path.Join(td, name)
|
||||
require.NoError(t, os.WriteFile(fp, data, 0o600))
|
||||
sum := sha256.Sum256(data)
|
||||
return fp, sum[:]
|
||||
}
|
||||
|
||||
validRequest := func(p string) *pb.PutObjectRequest {
|
||||
return &pb.PutObjectRequest{
|
||||
Bucket: &storagebuckets.StorageBucket{
|
||||
BucketName: "external-obj-store",
|
||||
BucketPrefix: "prefix",
|
||||
Secrets: credential.MockStaticCredentialSecrets(),
|
||||
Attributes: &structpb.Struct{
|
||||
Fields: map[string]*structpb.Value{
|
||||
credential.ConstRegion: structpb.NewStringValue("us-west-2"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Key: "mock-object",
|
||||
Path: p,
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("multipart success spanning multiple parts", func(t *testing.T) {
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
// 20 MiB with 8 MiB parts -> 3 parts (8, 8, 4 MiB).
|
||||
fp, wholeChecksum := genFile(t, "large-file", 20*1024*1024)
|
||||
|
||||
state := &testMockS3State{}
|
||||
p := &StoragePlugin{
|
||||
testCredStateOpts: validSTSMock(),
|
||||
testStorageStateOpts: []awsStoragePersistedStateOption{
|
||||
withTestS3APIFunc(newTestMockS3(state)),
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := p.PutObject(context.Background(), validRequest(fp))
|
||||
require.NoError(err)
|
||||
require.NotNil(resp)
|
||||
|
||||
// Boundary must receive the whole-object SHA-256, not the multipart
|
||||
// composite checksum.
|
||||
assert.True(bytes.Equal(wholeChecksum, resp.GetChecksumSha_256()))
|
||||
|
||||
// The single-shot PutObject path must not have been used.
|
||||
assert.False(state.PutObjectCalled)
|
||||
assert.True(state.CreateMultipartUploadCalled)
|
||||
assert.Equal(3, state.UploadPartCallCount)
|
||||
assert.True(state.CompleteMultipartUploadCalled)
|
||||
assert.False(state.AbortMultipartUploadCalled)
|
||||
|
||||
// The reassembled part bodies must equal the original file content.
|
||||
orig, err := os.ReadFile(fp)
|
||||
require.NoError(err)
|
||||
assert.True(bytes.Equal(orig, state.UploadPartBody))
|
||||
|
||||
// The object key must include the configured bucket prefix.
|
||||
require.NotNil(state.CreateMultipartUploadInputParams)
|
||||
assert.Equal("prefix/mock-object", aws.ToString(state.CreateMultipartUploadInputParams.Key))
|
||||
})
|
||||
|
||||
t.Run("file just above threshold uses two parts", func(t *testing.T) {
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
// 8 MiB + 1 byte -> 2 parts (8 MiB, 1 byte).
|
||||
fp, wholeChecksum := genFile(t, "just-over", 8*1024*1024+1)
|
||||
|
||||
state := &testMockS3State{}
|
||||
p := &StoragePlugin{
|
||||
testCredStateOpts: validSTSMock(),
|
||||
testStorageStateOpts: []awsStoragePersistedStateOption{
|
||||
withTestS3APIFunc(newTestMockS3(state)),
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := p.PutObject(context.Background(), validRequest(fp))
|
||||
require.NoError(err)
|
||||
assert.True(bytes.Equal(wholeChecksum, resp.GetChecksumSha_256()))
|
||||
assert.Equal(2, state.UploadPartCallCount)
|
||||
assert.True(state.CompleteMultipartUploadCalled)
|
||||
})
|
||||
|
||||
t.Run("file at threshold uses single PutObject", func(t *testing.T) {
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
// Exactly 8 MiB is not above the threshold, so the single-shot path is used.
|
||||
fp, wholeChecksum := genFile(t, "at-threshold", 8*1024*1024)
|
||||
checksum := base64.StdEncoding.EncodeToString(wholeChecksum)
|
||||
|
||||
state := &testMockS3State{}
|
||||
p := &StoragePlugin{
|
||||
testCredStateOpts: validSTSMock(),
|
||||
testStorageStateOpts: []awsStoragePersistedStateOption{
|
||||
withTestS3APIFunc(newTestMockS3(
|
||||
state,
|
||||
testMockS3WithPutObjectOutput(&s3.PutObjectOutput{
|
||||
ChecksumSHA256: aws.String(checksum),
|
||||
}),
|
||||
)),
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := p.PutObject(context.Background(), validRequest(fp))
|
||||
require.NoError(err)
|
||||
assert.True(bytes.Equal(wholeChecksum, resp.GetChecksumSha_256()))
|
||||
assert.True(state.PutObjectCalled)
|
||||
assert.False(state.CreateMultipartUploadCalled)
|
||||
})
|
||||
|
||||
t.Run("create multipart upload error", func(t *testing.T) {
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
fp, _ := genFile(t, "create-err", 20*1024*1024)
|
||||
|
||||
state := &testMockS3State{}
|
||||
p := &StoragePlugin{
|
||||
testCredStateOpts: validSTSMock(),
|
||||
testStorageStateOpts: []awsStoragePersistedStateOption{
|
||||
withTestS3APIFunc(newTestMockS3(
|
||||
state,
|
||||
testMockS3WithCreateMultipartUploadError(TestAwsS3Error("AccessDenied", "CreateMultipartUpload", "not authorized")),
|
||||
)),
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := p.PutObject(context.Background(), validRequest(fp))
|
||||
require.Error(err)
|
||||
assert.Nil(resp)
|
||||
assert.Equal(codes.PermissionDenied, status.Code(err))
|
||||
// Nothing was created, so nothing should be aborted.
|
||||
assert.False(state.AbortMultipartUploadCalled)
|
||||
})
|
||||
|
||||
t.Run("upload part error aborts upload", func(t *testing.T) {
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
fp, _ := genFile(t, "upload-err", 20*1024*1024)
|
||||
|
||||
state := &testMockS3State{}
|
||||
p := &StoragePlugin{
|
||||
testCredStateOpts: validSTSMock(),
|
||||
testStorageStateOpts: []awsStoragePersistedStateOption{
|
||||
withTestS3APIFunc(newTestMockS3(
|
||||
state,
|
||||
testMockS3WithUploadPartErrorOnPart(2, TestAwsS3Error("AccessDenied", "UploadPart", "not authorized")),
|
||||
)),
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := p.PutObject(context.Background(), validRequest(fp))
|
||||
require.Error(err)
|
||||
assert.Nil(resp)
|
||||
assert.Equal(codes.PermissionDenied, status.Code(err))
|
||||
assert.True(state.CreateMultipartUploadCalled)
|
||||
assert.False(state.CompleteMultipartUploadCalled)
|
||||
// The failed upload must be aborted to avoid orphaned parts.
|
||||
assert.True(state.AbortMultipartUploadCalled)
|
||||
})
|
||||
|
||||
t.Run("complete multipart upload error aborts upload", func(t *testing.T) {
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
fp, _ := genFile(t, "complete-err", 20*1024*1024)
|
||||
|
||||
state := &testMockS3State{}
|
||||
p := &StoragePlugin{
|
||||
testCredStateOpts: validSTSMock(),
|
||||
testStorageStateOpts: []awsStoragePersistedStateOption{
|
||||
withTestS3APIFunc(newTestMockS3(
|
||||
state,
|
||||
testMockS3WithCompleteMultipartUploadError(TestAwsS3Error("InternalError", "CompleteMultipartUpload", "boom")),
|
||||
)),
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := p.PutObject(context.Background(), validRequest(fp))
|
||||
require.Error(err)
|
||||
assert.Nil(resp)
|
||||
assert.Equal(3, state.UploadPartCallCount)
|
||||
assert.True(state.AbortMultipartUploadCalled)
|
||||
})
|
||||
}
|
||||
|
||||
func TestStoragePlugin_DeleteObjects(t *testing.T) {
|
||||
validRequest := func() *pb.DeleteObjectsRequest {
|
||||
return &pb.DeleteObjectsRequest{
|
||||
|
||||
@@ -7,9 +7,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/retry"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
smithyEndpoints "github.com/aws/smithy-go/endpoints"
|
||||
"github.com/hashicorp/boundary-plugin-aws/internal/credential"
|
||||
@@ -22,6 +26,10 @@ type S3API interface {
|
||||
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
|
||||
UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error)
|
||||
CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
|
||||
DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error)
|
||||
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||
Credentials() aws.Credentials
|
||||
@@ -118,7 +126,10 @@ func (s *awsStoragePersistedState) s3Client(ctx context.Context, opt ...s3Option
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to retrieve credentials: %v\n", err)
|
||||
}
|
||||
|
||||
err = newTransport(awsCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to configure new http transport: %v", err)
|
||||
}
|
||||
c := s3.NewFromConfig(*awsCfg, s3Opts...)
|
||||
return &s3Client{Client: c, creds: creds}, nil
|
||||
}
|
||||
@@ -202,3 +213,30 @@ func (e *endpointResolver) ResolveEndpoint(ctx context.Context, params s3.Endpoi
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// newTransport returns an http Transport object. The http transport object utilizes the
|
||||
// default transport created by minio. The following transport fields are updated to provide
|
||||
// more reliable connectivity when handling larger objects:
|
||||
// MaxResponseHeaderBytes is set to 1MiB to prevent header reads from stalling.
|
||||
// WriteBufferSize is set to 1MiB, matching the default buffer size used in Boundary storage.
|
||||
// ReadBufferSize is set to 1MiB, matching the default buffer size used in Boundary storage.
|
||||
// ForceAttemptHTTP2 is set to true, supporting more resilient multiplexing and avoids HOL blocking on retries.
|
||||
// IdleConnTimeout is set to 2 minutes.
|
||||
func newTransport(cfg *aws.Config) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("missing aws configuration")
|
||||
}
|
||||
cfg.HTTPClient = awshttp.NewBuildableClient().WithTransportOptions(func(transport *http.Transport) {
|
||||
transport.MaxResponseHeaderBytes = 1 << 20 // 1 MiB prevents header read from stalling
|
||||
transport.WriteBufferSize = 1 << 20 // 1 MiB write buffer (default is 4 KiB)
|
||||
transport.ReadBufferSize = 1 << 20 // 1 MiB read buffer (default is 4 KiB)
|
||||
transport.ForceAttemptHTTP2 = true // enables HTTP/2 where supported more resilient multiplexing, avoids HOL blocking on retries
|
||||
transport.IdleConnTimeout = 2 * time.Minute // double the idle connection timeout, default is 1 minute
|
||||
})
|
||||
cfg.Retryer = func() aws.Retryer {
|
||||
return retry.NewStandard(func(o *retry.StandardOptions) {
|
||||
o.MaxAttempts = 20
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6,6 +6,8 @@ package storage
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
@@ -39,6 +41,23 @@ type testMockS3State struct {
|
||||
PutObjectInputParams *s3.PutObjectInput
|
||||
PutObjectBody []byte
|
||||
|
||||
CreateMultipartUploadCalled bool
|
||||
CreateMultipartUploadInputParams *s3.CreateMultipartUploadInput
|
||||
|
||||
UploadPartCalled bool
|
||||
UploadPartCallCount int
|
||||
UploadPartInputParams []*s3.UploadPartInput
|
||||
// UploadPartBody is the concatenation, in part-number order, of every
|
||||
// part body received by UploadPart. It can be compared against the full
|
||||
// object that was expected to be uploaded.
|
||||
UploadPartBody []byte
|
||||
|
||||
CompleteMultipartUploadCalled bool
|
||||
CompleteMultipartUploadInputParams *s3.CompleteMultipartUploadInput
|
||||
|
||||
AbortMultipartUploadCalled bool
|
||||
AbortMultipartUploadInputParams *s3.AbortMultipartUploadInput
|
||||
|
||||
HeadObjectCalled bool
|
||||
HeadObjectInputParams *s3.HeadObjectInput
|
||||
|
||||
@@ -57,6 +76,16 @@ func (s *testMockS3State) Reset() {
|
||||
s.PutObjectCalled = false
|
||||
s.PutObjectInputParams = nil
|
||||
s.PutObjectBody = nil
|
||||
s.CreateMultipartUploadCalled = false
|
||||
s.CreateMultipartUploadInputParams = nil
|
||||
s.UploadPartCalled = false
|
||||
s.UploadPartCallCount = 0
|
||||
s.UploadPartInputParams = nil
|
||||
s.UploadPartBody = nil
|
||||
s.CompleteMultipartUploadCalled = false
|
||||
s.CompleteMultipartUploadInputParams = nil
|
||||
s.AbortMultipartUploadCalled = false
|
||||
s.AbortMultipartUploadInputParams = nil
|
||||
s.HeadObjectCalled = false
|
||||
s.HeadObjectInputParams = nil
|
||||
s.ListObjectsV2Called = false
|
||||
@@ -81,6 +110,19 @@ type testMockS3 struct {
|
||||
PutObjectOutput *s3.PutObjectOutput
|
||||
PutObjectErr error
|
||||
|
||||
// mocked responses for the multipart upload calls
|
||||
CreateMultipartUploadOutput *s3.CreateMultipartUploadOutput
|
||||
CreateMultipartUploadErr error
|
||||
UploadPartOutput *s3.UploadPartOutput
|
||||
UploadPartErr error
|
||||
// UploadPartErrOnPart, when greater than zero, causes UploadPart to return
|
||||
// UploadPartErr (or a generic error) only for the matching part number.
|
||||
UploadPartErrOnPart int32
|
||||
CompleteMultipartUploadOutput *s3.CompleteMultipartUploadOutput
|
||||
CompleteMultipartUploadErr error
|
||||
AbortMultipartUploadOutput *s3.AbortMultipartUploadOutput
|
||||
AbortMultipartUploadErr error
|
||||
|
||||
// mocked responses for headObject
|
||||
HeadObjectOutput *s3.HeadObjectOutput
|
||||
HeadObjectErr error
|
||||
@@ -114,6 +156,63 @@ func testMockS3WithPutObjectError(e error) testMockS3Option {
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithCreateMultipartUploadOutput(o *s3.CreateMultipartUploadOutput) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.CreateMultipartUploadOutput = o
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithCreateMultipartUploadError(e error) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.CreateMultipartUploadErr = e
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithUploadPartOutput(o *s3.UploadPartOutput) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.UploadPartOutput = o
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithUploadPartError(e error) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.UploadPartErr = e
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithUploadPartErrorOnPart(partNumber int32, e error) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.UploadPartErr = e
|
||||
m.UploadPartErrOnPart = partNumber
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithCompleteMultipartUploadOutput(o *s3.CompleteMultipartUploadOutput) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.CompleteMultipartUploadOutput = o
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithCompleteMultipartUploadError(e error) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.CompleteMultipartUploadErr = e
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithAbortMultipartUploadError(e error) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.AbortMultipartUploadErr = e
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func testMockS3WithGetObjectOutput(o *s3.GetObjectOutput) testMockS3Option {
|
||||
return func(m *testMockS3) error {
|
||||
m.GetObjectOutput = o
|
||||
@@ -255,6 +354,95 @@ func (m *testMockS3) PutObject(ctx context.Context, params *s3.PutObjectInput, o
|
||||
return m.PutObjectOutput, nil
|
||||
}
|
||||
|
||||
func (m *testMockS3) CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) {
|
||||
if m.State != nil {
|
||||
m.State.CreateMultipartUploadCalled = true
|
||||
m.State.CreateMultipartUploadInputParams = params
|
||||
}
|
||||
|
||||
if m.CreateMultipartUploadErr != nil {
|
||||
return nil, m.CreateMultipartUploadErr
|
||||
}
|
||||
|
||||
if m.CreateMultipartUploadOutput != nil {
|
||||
return m.CreateMultipartUploadOutput, nil
|
||||
}
|
||||
return &s3.CreateMultipartUploadOutput{
|
||||
UploadId: aws.String("test-upload-id"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *testMockS3) UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) {
|
||||
if m.State != nil {
|
||||
m.State.UploadPartCalled = true
|
||||
m.State.UploadPartCallCount++
|
||||
m.State.UploadPartInputParams = append(m.State.UploadPartInputParams, params)
|
||||
}
|
||||
|
||||
// always drain the body so the part data can be asserted on and so the
|
||||
// reader is not left dangling.
|
||||
var data []byte
|
||||
if params.Body != nil {
|
||||
var err error
|
||||
data, err = io.ReadAll(params.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.State != nil {
|
||||
m.State.UploadPartBody = append(m.State.UploadPartBody, data...)
|
||||
}
|
||||
}
|
||||
|
||||
if m.UploadPartErr != nil {
|
||||
if m.UploadPartErrOnPart == 0 || (params.PartNumber != nil && *params.PartNumber == m.UploadPartErrOnPart) {
|
||||
return nil, m.UploadPartErr
|
||||
}
|
||||
}
|
||||
|
||||
if m.UploadPartOutput != nil {
|
||||
return m.UploadPartOutput, nil
|
||||
}
|
||||
// echo the supplied part checksum so CompleteMultipartUpload can be built
|
||||
// the way the real service would allow.
|
||||
sum := sha256.Sum256(data)
|
||||
return &s3.UploadPartOutput{
|
||||
ETag: aws.String(fmt.Sprintf("etag-%d", aws.ToInt32(params.PartNumber))),
|
||||
ChecksumSHA256: aws.String(base64.StdEncoding.EncodeToString(sum[:])),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *testMockS3) CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
if m.State != nil {
|
||||
m.State.CompleteMultipartUploadCalled = true
|
||||
m.State.CompleteMultipartUploadInputParams = params
|
||||
}
|
||||
|
||||
if m.CompleteMultipartUploadErr != nil {
|
||||
return nil, m.CompleteMultipartUploadErr
|
||||
}
|
||||
|
||||
if m.CompleteMultipartUploadOutput != nil {
|
||||
return m.CompleteMultipartUploadOutput, nil
|
||||
}
|
||||
return &s3.CompleteMultipartUploadOutput{}, nil
|
||||
}
|
||||
|
||||
func (m *testMockS3) AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) {
|
||||
if m.State != nil {
|
||||
m.State.AbortMultipartUploadCalled = true
|
||||
m.State.AbortMultipartUploadInputParams = params
|
||||
}
|
||||
|
||||
if m.AbortMultipartUploadErr != nil {
|
||||
return nil, m.AbortMultipartUploadErr
|
||||
}
|
||||
|
||||
if m.AbortMultipartUploadOutput != nil {
|
||||
return m.AbortMultipartUploadOutput, nil
|
||||
}
|
||||
return &s3.AbortMultipartUploadOutput{}, nil
|
||||
}
|
||||
|
||||
func (m *testMockS3) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||
if m.State != nil {
|
||||
m.State.ListObjectsV2Called = true
|
||||
|
||||
@@ -165,6 +165,9 @@ func TestHostPlugin(t *testing.T) {
|
||||
testPluginListHosts(ctx, t, p, region, keyid, secret, tc, expectedTagInstancesMap)
|
||||
// TODO: add OnDeleteSet if it needs to be implemented
|
||||
}
|
||||
|
||||
// finally lets test instance_addresses_only catalog attribute correctly syncs only instance addresses
|
||||
testInstanceAddressesOnlyListHosts(ctx, t, p, tf, region, keyid, secret)
|
||||
}
|
||||
|
||||
func testPluginOnCreateCatalog(ctx context.Context, t *testing.T, p *host.HostPlugin, region, accessKeyId, secretAccessKey string, rotate bool) (string, string) {
|
||||
@@ -450,3 +453,92 @@ func testPluginListHosts(ctx context.Context, t *testing.T, p *host.HostPlugin,
|
||||
// Success
|
||||
t.Logf("testing ListHosts: success (region=%s, tags=%v, expected/actual=(len=%d, ids=%s))", region, tags, len(actualInstances), actualInstances)
|
||||
}
|
||||
|
||||
func testInstanceAddressesOnlyListHosts(ctx context.Context, t *testing.T, p *host.HostPlugin, tf *TestTerraformer, region, accessKeyId, secretAccessKey string) {
|
||||
t.Helper()
|
||||
t.Logf("testing testInstanceAddressesOnlyListHosts")
|
||||
|
||||
// Get the instance details from tf output
|
||||
instanceID, err := tf.GetOutputString("multi_address_instance_id")
|
||||
require.NoError(t, err)
|
||||
primaryAddresses, err := tf.GetOutputMap("multi_address_primary_addresses")
|
||||
require.NoError(t, err)
|
||||
secondaryAddresses, err := tf.GetOutputMap("multi_address_secondary_addresses")
|
||||
require.NoError(t, err)
|
||||
|
||||
var primary, allAddresses []string
|
||||
for _, addr := range primaryAddresses {
|
||||
primary = append(primary, addr.(string))
|
||||
}
|
||||
allAddresses = append(allAddresses, primary...)
|
||||
for _, addr := range secondaryAddresses {
|
||||
allAddresses = append(allAddresses, addr.(string))
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
instanceAddressesOnly bool
|
||||
expectedIPs []string
|
||||
}{
|
||||
{
|
||||
name: "all addresses",
|
||||
expectedIPs: allAddresses,
|
||||
},
|
||||
{
|
||||
name: "instance addresses only",
|
||||
instanceAddressesOnly: true,
|
||||
expectedIPs: primary,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
catalogAttrMap := map[string]any{
|
||||
credential.ConstRegion: region,
|
||||
credential.ConstDisableCredentialRotation: true,
|
||||
}
|
||||
if tc.instanceAddressesOnly {
|
||||
catalogAttrMap[host.ConstInstanceAddressesOnly] = true
|
||||
}
|
||||
|
||||
catalogAttrs, err := structpb.NewStruct(catalogAttrMap)
|
||||
require.NoError(err)
|
||||
setAttrs, err := structpb.NewStruct(map[string]any{
|
||||
host.ConstDescribeInstancesFilters: []any{fmt.Sprintf("instance-id=%s", instanceID)},
|
||||
})
|
||||
require.NoError(err)
|
||||
|
||||
reqPersistedSecrets, err := structpb.NewStruct(map[string]any{
|
||||
credential.ConstAccessKeyId: accessKeyId,
|
||||
credential.ConstSecretAccessKey: secretAccessKey,
|
||||
credential.ConstCredsLastRotatedTime: (time.Time{}).Format(time.RFC3339Nano),
|
||||
})
|
||||
require.NoError(err)
|
||||
request := &pb.ListHostsRequest{
|
||||
Catalog: &hostcatalogs.HostCatalog{
|
||||
Attrs: &hostcatalogs.HostCatalog_Attributes{
|
||||
Attributes: catalogAttrs,
|
||||
},
|
||||
},
|
||||
Sets: []*hostsets.HostSet{{
|
||||
Id: "hostset-0",
|
||||
Attrs: &hostsets.HostSet_Attributes{Attributes: setAttrs},
|
||||
}},
|
||||
Persisted: &pb.HostCatalogPersisted{
|
||||
Secrets: reqPersistedSecrets,
|
||||
},
|
||||
}
|
||||
|
||||
response, err := p.ListHosts(ctx, request)
|
||||
require.NoError(err)
|
||||
require.Len(response.GetHosts(), 1)
|
||||
hostResp := response.GetHosts()[0]
|
||||
|
||||
require.Equal(instanceID, hostResp.ExternalId)
|
||||
require.ElementsMatch(hostResp.IpAddresses, tc.expectedIPs)
|
||||
})
|
||||
}
|
||||
}
|
||||
Vendored
+1
-1
@@ -26,7 +26,7 @@ resource "random_id" "baz_key" {
|
||||
|
||||
locals {
|
||||
hashicorp_email = split(":", data.aws_caller_identity.current.user_id)[1]
|
||||
|
||||
|
||||
instance_tags = [
|
||||
{
|
||||
"${random_id.foo_key.dec}" = "true"
|
||||
|
||||
Vendored
+121
-4
@@ -20,13 +20,41 @@ data "aws_ami" "ubuntu" {
|
||||
}
|
||||
|
||||
resource "aws_vpc" "vpc" {
|
||||
cidr_block = "10.0.0.0/16"
|
||||
cidr_block = "10.0.0.0/16"
|
||||
assign_generated_ipv6_cidr_block = true
|
||||
}
|
||||
|
||||
resource "aws_subnet" "subnet" {
|
||||
vpc_id = aws_vpc.vpc.id
|
||||
cidr_block = aws_vpc.vpc.cidr_block
|
||||
availability_zone = data.aws_availability_zones.azs.names[0]
|
||||
vpc_id = aws_vpc.vpc.id
|
||||
cidr_block = aws_vpc.vpc.cidr_block
|
||||
ipv6_cidr_block = cidrsubnet(aws_vpc.vpc.ipv6_cidr_block, 8, 0)
|
||||
availability_zone = data.aws_availability_zones.azs.names[0]
|
||||
map_public_ip_on_launch = true
|
||||
assign_ipv6_address_on_creation = true
|
||||
}
|
||||
|
||||
# Need to add a gateway for public IP addr assignments
|
||||
resource "aws_internet_gateway" "igw" {
|
||||
vpc_id = aws_vpc.vpc.id
|
||||
}
|
||||
|
||||
resource "aws_route_table" "public" {
|
||||
vpc_id = aws_vpc.vpc.id
|
||||
|
||||
route {
|
||||
cidr_block = "0.0.0.0/0"
|
||||
gateway_id = aws_internet_gateway.igw.id
|
||||
}
|
||||
|
||||
route {
|
||||
ipv6_cidr_block = "::/0"
|
||||
gateway_id = aws_internet_gateway.igw.id
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_route_table_association" "public" {
|
||||
subnet_id = aws_subnet.subnet.id
|
||||
route_table_id = aws_route_table.public.id
|
||||
}
|
||||
|
||||
resource "aws_instance" "instances" {
|
||||
@@ -38,6 +66,73 @@ resource "aws_instance" "instances" {
|
||||
tags = local.instance_tags[count.index]
|
||||
}
|
||||
|
||||
resource "aws_instance" "multi_address_instance" {
|
||||
ami = data.aws_ami.ubuntu.id
|
||||
instance_type = "t3.nano"
|
||||
subnet_id = aws_subnet.subnet.id
|
||||
secondary_private_ips = ["10.0.10.10"]
|
||||
ipv6_address_count = 1
|
||||
associate_public_ip_address = true
|
||||
|
||||
tags = {
|
||||
Name = "boundary-multi-address-instance"
|
||||
"boundary-demo" = local.hashicorp_email
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_network_interface" "multi_address_secondary_eni" {
|
||||
subnet_id = aws_subnet.subnet.id
|
||||
private_ips = ["10.0.20.10"]
|
||||
ipv6_addresses = [cidrhost(aws_subnet.subnet.ipv6_cidr_block, 32)]
|
||||
|
||||
tags = {
|
||||
Name = "boundary-multi-address-secondary-eni"
|
||||
"boundary-demo" = local.hashicorp_email
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_network_interface_attachment" "multi_address_secondary_eni" {
|
||||
instance_id = aws_instance.multi_address_instance.id
|
||||
network_interface_id = aws_network_interface.multi_address_secondary_eni.id
|
||||
device_index = 1
|
||||
}
|
||||
|
||||
resource "aws_eip" "multi_address_primary" {
|
||||
domain = "vpc"
|
||||
network_interface = aws_instance.multi_address_instance.primary_network_interface_id
|
||||
associate_with_private_ip = aws_instance.multi_address_instance.private_ip
|
||||
depends_on = [aws_internet_gateway.igw, aws_route_table_association.public]
|
||||
|
||||
tags = {
|
||||
Name = "boundary-multi-address-primary-eip"
|
||||
"boundary-demo" = local.hashicorp_email
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_eip" "multi_address_primary_secondary_private" {
|
||||
domain = "vpc"
|
||||
network_interface = aws_instance.multi_address_instance.primary_network_interface_id
|
||||
associate_with_private_ip = "10.0.10.10"
|
||||
depends_on = [aws_internet_gateway.igw, aws_route_table_association.public, aws_instance.multi_address_instance]
|
||||
|
||||
tags = {
|
||||
Name = "boundary-multi-address-primary-secondary-eip"
|
||||
"boundary-demo" = local.hashicorp_email
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_eip" "multi_address_secondary_eni" {
|
||||
domain = "vpc"
|
||||
network_interface = aws_network_interface.multi_address_secondary_eni.id
|
||||
associate_with_private_ip = "10.0.20.10"
|
||||
depends_on = [aws_internet_gateway.igw, aws_route_table_association.public, aws_network_interface_attachment.multi_address_secondary_eni]
|
||||
|
||||
tags = {
|
||||
Name = "boundary-multi-address-secondary-eni-eip"
|
||||
"boundary-demo" = local.hashicorp_email
|
||||
}
|
||||
}
|
||||
|
||||
output "instance_ids" {
|
||||
value = aws_instance.instances.*.id
|
||||
}
|
||||
@@ -61,3 +156,25 @@ output "instance_tag_keys" {
|
||||
random_id.baz_key.dec,
|
||||
]
|
||||
}
|
||||
|
||||
output "multi_address_instance_id" {
|
||||
value = aws_instance.multi_address_instance.id
|
||||
}
|
||||
|
||||
output "multi_address_primary_addresses" {
|
||||
value = {
|
||||
private_ip = aws_instance.multi_address_instance.private_ip
|
||||
public_ip = aws_eip.multi_address_primary.public_ip
|
||||
ipv6 = tolist(aws_instance.multi_address_instance.ipv6_addresses)[0]
|
||||
}
|
||||
}
|
||||
|
||||
output "multi_address_secondary_addresses" {
|
||||
value = {
|
||||
primary_eni_private_ip = "10.0.10.10"
|
||||
primary_eni_public_ip = aws_eip.multi_address_primary_secondary_private.public_ip
|
||||
secondary_eni_private_ip = aws_network_interface.multi_address_secondary_eni.private_ip
|
||||
secondary_eni_public_ip = aws_eip.multi_address_secondary_eni.public_ip
|
||||
secondary_eni_ipv6 = tolist(aws_network_interface.multi_address_secondary_eni.ipv6_addresses)[0]
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user