From 899c62ce0694d1d1315dc39969a2fd9926cd89e3 Mon Sep 17 00:00:00 2001 From: Damian Debkowski Date: Mon, 28 Oct 2024 11:50:18 -0700 Subject: [PATCH] feat: support dual stack requests for aws clients (#52) * feat: enable dual stack in aws clients * refactor: optionally set custom endpoint resolver * feat: update sts & iam client to utilize dualstack * update aws dependencies to remove role policy bug --- go.mod | 28 ++++----- go.sum | 55 ++++++++--------- internal/credential/attributes.go | 13 +++- internal/credential/attributes_test.go | 28 ++++++++- internal/credential/state.go | 36 ++++++++++- internal/credential/state_test.go | 30 +++++++++- plugin/service/host/attributes.go | 10 ++++ plugin/service/host/attributes_test.go | 16 +++++ plugin/service/host/const.go | 3 + plugin/service/host/plugin.go | 49 +++++++++++---- plugin/service/host/plugin_test.go | 10 ++-- plugin/service/host/state.go | 73 ++++++++++++++++++----- plugin/service/storage/attributes.go | 10 ++++ plugin/service/storage/attributes_test.go | 15 +++++ plugin/service/storage/const.go | 3 + plugin/service/storage/plugin.go | 34 +++++++---- plugin/service/storage/plugin_test.go | 12 ++-- plugin/service/storage/state.go | 36 +++++------ 18 files changed, 349 insertions(+), 112 deletions(-) diff --git a/go.mod b/go.mod index b2066c4..ba88a76 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/hashicorp/boundary-plugin-aws go 1.21 require ( - github.com/aws/aws-sdk-go-v2 v1.20.1 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.99.0 - github.com/aws/aws-sdk-go-v2/service/iam v1.22.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 - github.com/aws/smithy-go v1.14.1 + github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0 + github.com/aws/aws-sdk-go-v2/service/iam v1.37.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 + github.com/aws/smithy-go v1.22.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.4.0 github.com/hashicorp/boundary/sdk v0.0.43-0.20240717182311-a20aae98794a @@ -22,18 +22,18 @@ require ( ) require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.33 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.32 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index ad16605..122a06e 100644 --- a/go.sum +++ b/go.sum @@ -8,51 +8,52 @@ github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/ github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg= github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 h1:lN6L3LrYHeZ6xCxaIYtoWCx4GMLk4nRknsh29OMSqHY= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12/go.mod h1:TDCkEAkMTXxTs0oLBGBKpBZbk3NLh8EvAfF0Q3x8/0c= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= github.com/aws/aws-sdk-go-v2/credentials v1.13.32 h1:lIH1eKPcCY1ylR4B6PkBGRWMHO3aVenOKJHWiS4/G2w= github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 h1:DK/9C+UN/X+1+Wm8pqaDksQr2tSLzq+8X1/rI/ZxKEQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 h1:vUh7dBFNS3oFCtVv6CiYKh5hP9ls8+kIpKLeFruIBLk= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1/go.mod h1:sFMeinkhj/SZKQM8BxtvNtSPjJEo0Xrz+w3g2e4FSKI= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.99.0 h1:NXi4pNJWjAaiI56P1Rl8DC9A4jMNRE00WNBsDua5WRg= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.99.0/go.mod h1:L3ZT0N/vBsw77mOAawXmRnREpEjcHd2v5Hzf7AkIH8M= -github.com/aws/aws-sdk-go-v2/service/iam v1.22.2 h1:DPFxx/6Zwes/MiadlDteVqDKov7yQ5v9vuwfhZuJm1s= -github.com/aws/aws-sdk-go-v2/service/iam v1.22.2/go.mod h1:cQTMNdo/Z5t1DDRsUnx0a2j6cPnytMBidUYZw2zks28= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 h1:iV/W5OMBys+66OeXJi/7xIRrKZNsu0ylsLGu+6nbmQE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13/go.mod h1:ReJb6xYmtGyu9KoFtRreWegbN9dZqvZIIv4vWnhcsyI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 h1:QviNkc+vGSuEHx8P+pVNKOdWLXBPIwMFv7p0fphgE4U= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33/go.mod h1:fABTUmOrAgAalG2i9WJpjBvlnk7UK8YmnYaxN+Q2CwE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 h1:dGAseBFEYxth10V23b5e2mAS+tX7oVbfYHD6dnDdAsg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0 h1:n2l2WeV+lEABrGwG/4MsE0WFEbd3j7yKsmZzbnEm5CY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0/go.mod h1:kYXaB4FzyhEJjvrJ84oPnMElLiEAjGxxUunVW2tBSng= +github.com/aws/aws-sdk-go-v2/service/iam v1.37.2 h1:E7vCDUFeDN8uOk8Nb2d4E1howWS1TR4HrKABXsvttIs= +github.com/aws/aws-sdk-go-v2/service/iam v1.37.2/go.mod h1:QzMecFrIFYJ1cyxjlUoIFRzYSDX19gdqYUd0Tyws2J8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 h1:PT6PBCycRwhpEW5hJnRiceCeoWJ+r3bdgXtV+VKG7Pk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1/go.mod h1:TqoxCLwT2nrxrBGA+z7t6OWM7LBkgRckK3gOjYE+7JA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 h1:v346f1h8sUBKXnEbrv43L37MTBlFHyKXQPIZHNAaghA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2/go.mod h1:cwCATiyNrXK9P2FsWdZ89g9mpsYv2rhk0UA/KByl5fY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1 h1:MkQ4unegQEStiQYmfFj+Aq5uTp265ncSmm0XTQwDwi0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 h1:A2RlEMo4SJSwbNoUUgkxTAEMduAy/8wG3eB2b2lP4gY= github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 h1:OJELEgyaT2kmaBGZ+myyZbTTLobfe3ox3FSh5eYK9Qs= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 h1:ympg1+Lnq33XLhcK/xTG4yZHPs1Oyxu+6DEWbl7qOzA= github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/internal/credential/attributes.go b/internal/credential/attributes.go index eac891a..b94af11 100644 --- a/internal/credential/attributes.go +++ b/internal/credential/attributes.go @@ -37,7 +37,7 @@ type CredentialAttributes struct { // CredentialsConfig used for configuring an AWS session. An status error is returned // with an InvalidArgument code if any unrecognized fields are found in the protobuf // struct input. -func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes) (*awsutil.CredentialsConfig, error) { +func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes, dualStack bool) (*awsutil.CredentialsConfig, error) { // initialize secrets if it is nil // secrets can be nil because static credentials are optional if secrets == nil { @@ -117,6 +117,17 @@ func GetCredentialsConfig(secrets *structpb.Struct, attrs *CredentialAttributes) opts = append(opts, awsutil.WithRoleTags(attrs.RoleTags)) } + if dualStack { + opts = append(opts, []awsutil.Option{ + awsutil.WithIamEndpointResolver(&iamEndpointResolver{ + dualStack: dualStack, + }), + awsutil.WithStsEndpointResolver(&stsEndpointResolver{ + dualStack: dualStack, + }), + }...) + } + cfg, err := awsutil.NewCredentialsConfig(opts...) if err != nil { return nil, errors.BadRequestStatus(err.Error()) diff --git a/internal/credential/attributes_test.go b/internal/credential/attributes_test.go index b21a662..3619f0b 100644 --- a/internal/credential/attributes_test.go +++ b/internal/credential/attributes_test.go @@ -106,6 +106,7 @@ func TestGetCredentialsConfig(t *testing.T) { name string secrets *structpb.Struct attrs *CredentialAttributes + dualStack bool expected *awsutil.CredentialsConfig expectedErrContains string }{ @@ -135,6 +136,24 @@ func TestGetCredentialsConfig(t *testing.T) { Region: "us-west-2", }, }, + { + name: "with dualstack", + secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + ConstAccessKeyId: structpb.NewStringValue("AKIAfoobar"), + ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + dualStack: true, + attrs: &CredentialAttributes{ + Region: "us-west-2", + }, + expected: &awsutil.CredentialsConfig{ + AccessKey: "AKIAfoobar", + SecretKey: "bazqux", + Region: "us-west-2", + }, + }, { name: "unknown fields", secrets: &structpb.Struct{ @@ -233,7 +252,7 @@ func TestGetCredentialsConfig(t *testing.T) { t.Run(tc.name, func(t *testing.T) { require := require.New(t) - actual, err := GetCredentialsConfig(tc.secrets, tc.attrs) + actual, err := GetCredentialsConfig(tc.secrets, tc.attrs, tc.dualStack) if tc.expectedErrContains != "" { require.Error(err) require.Contains(err.Error(), tc.expectedErrContains) @@ -249,6 +268,13 @@ func TestGetCredentialsConfig(t *testing.T) { require.Equal(tc.expected.RoleExternalId, actual.RoleExternalId) require.Equal(tc.expected.RoleSessionName, actual.RoleSessionName) require.Equal(tc.expected.RoleTags, actual.RoleTags) + if tc.dualStack { + require.NotNil(actual.IAMEndpointResolver) + require.NotNil(actual.STSEndpointResolver) + } else { + require.Nil(actual.IAMEndpointResolver) + require.Nil(actual.STSEndpointResolver) + } }) } } diff --git a/internal/credential/state.go b/internal/credential/state.go index 15555f5..72b95f6 100644 --- a/internal/credential/state.go +++ b/internal/credential/state.go @@ -11,7 +11,10 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + smithyEndpoints "github.com/aws/smithy-go/endpoints" "github.com/hashicorp/boundary-plugin-aws/internal/errors" "github.com/hashicorp/boundary-plugin-aws/internal/values" "github.com/hashicorp/go-secure-stdlib/awsutil/v2" @@ -237,7 +240,7 @@ func (s *AwsCredentialPersistedState) ToMap() map[string]any { // AwsCredentialPersistedStateFromProto parses values out of a protobuf struct input // and returns a AwsCredentialPersistedState used for configuring an AWS session. -func AwsCredentialPersistedStateFromProto(secrets *structpb.Struct, attrs *CredentialAttributes, opts ...AwsCredentialPersistedStateOption) (*AwsCredentialPersistedState, error) { +func AwsCredentialPersistedStateFromProto(secrets *structpb.Struct, attrs *CredentialAttributes, dualStack bool, opts ...AwsCredentialPersistedStateOption) (*AwsCredentialPersistedState, error) { // initialize secrets if it is nil // secrets can be nil because static credentials are optional if secrets == nil { @@ -292,6 +295,17 @@ func AwsCredentialPersistedStateFromProto(secrets *structpb.Struct, attrs *Crede if len(attrs.RoleTags) != 0 { awsOpts = append(awsOpts, awsutil.WithRoleTags(attrs.RoleTags)) } + if dualStack { + awsOpts = append(awsOpts, []awsutil.Option{ + awsutil.WithIamEndpointResolver(&iamEndpointResolver{ + dualStack: dualStack, + }), + awsutil.WithStsEndpointResolver(&stsEndpointResolver{ + dualStack: dualStack, + }), + }...) + } + credentialsConfig, err := awsutil.NewCredentialsConfig(awsOpts...) if err != nil { return nil, err @@ -321,3 +335,23 @@ func GetCredentialType(cc *awsutil.CredentialsConfig) CredentialType { return Unknown } + +type iamEndpointResolver struct { + dualStack bool +} + +func (e *iamEndpointResolver) ResolveEndpoint(ctx context.Context, params iam.EndpointParameters) (resolver smithyEndpoints.Endpoint, err error) { + params.UseDualStack = aws.Bool(e.dualStack) + resolver, err = iam.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) + return +} + +type stsEndpointResolver struct { + dualStack bool +} + +func (e *stsEndpointResolver) ResolveEndpoint(ctx context.Context, params sts.EndpointParameters) (resolver smithyEndpoints.Endpoint, err error) { + params.UseDualStack = aws.Bool(e.dualStack) + resolver, err = sts.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) + return +} diff --git a/internal/credential/state_test.go b/internal/credential/state_test.go index 2bc8b7d..1cda1a0 100644 --- a/internal/credential/state_test.go +++ b/internal/credential/state_test.go @@ -112,6 +112,7 @@ func TestAwsCredentialPersistedStateFromProto(t *testing.T) { name string secrets *structpb.Struct attrs *CredentialAttributes + dualStack bool opts []AwsCredentialPersistedStateOption expected *AwsCredentialPersistedState expectedErr string @@ -174,6 +175,26 @@ func TestAwsCredentialPersistedStateFromProto(t *testing.T) { }, }, }, + { + name: "with dualstack static credentials", + secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + ConstAccessKeyId: structpb.NewStringValue("foobar"), + ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + dualStack: true, + attrs: &CredentialAttributes{ + Region: "us-west-2", + }, + expected: &AwsCredentialPersistedState{ + CredentialsConfig: &awsutil.CredentialsConfig{ + Region: "us-west-2", + AccessKey: "foobar", + SecretKey: "bazqux", + }, + }, + }, { name: "bad last rotated time", secrets: &structpb.Struct{ @@ -318,7 +339,7 @@ func TestAwsCredentialPersistedStateFromProto(t *testing.T) { t.Run(tc.name, func(t *testing.T) { require := require.New(t) - actual, err := AwsCredentialPersistedStateFromProto(tc.secrets, tc.attrs, tc.opts...) + actual, err := AwsCredentialPersistedStateFromProto(tc.secrets, tc.attrs, tc.dualStack, tc.opts...) if tc.expectedErr != "" { require.EqualError(err, tc.expectedErr) return @@ -336,6 +357,13 @@ func TestAwsCredentialPersistedStateFromProto(t *testing.T) { require.Equal(tc.expected.CredentialsConfig.RoleExternalId, actual.CredentialsConfig.RoleExternalId) require.Equal(tc.expected.CredentialsConfig.RoleSessionName, actual.CredentialsConfig.RoleSessionName) require.Equal(tc.expected.CredentialsConfig.RoleTags, actual.CredentialsConfig.RoleTags) + if tc.dualStack { + require.NotNil(actual.CredentialsConfig.STSEndpointResolver) + require.NotNil(actual.CredentialsConfig.IAMEndpointResolver) + } else { + require.Nil(actual.CredentialsConfig.STSEndpointResolver) + require.Nil(actual.CredentialsConfig.IAMEndpointResolver) + } } }) } diff --git a/plugin/service/host/attributes.go b/plugin/service/host/attributes.go index 3b16d15..66903e1 100644 --- a/plugin/service/host/attributes.go +++ b/plugin/service/host/attributes.go @@ -19,6 +19,9 @@ import ( // map. type CatalogAttributes struct { *cred.CredentialAttributes + + // DualStack is used for configuring how the aws client will resolve requests. + DualStack bool } func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) { @@ -31,6 +34,12 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) { return nil, err } + dualStack, err := values.GetBoolValue(in, ConstAwsDualStack, false) + if err != nil { + badFields[fmt.Sprintf("attributes.%s", ConstAwsDualStack)] = err.Error() + } + delete(unknownFields, ConstAwsDualStack) + for s := range unknownFields { switch s { // Ignore knownFields from CredentialAttributes @@ -57,6 +66,7 @@ func getCatalogAttributes(in *structpb.Struct) (*CatalogAttributes, error) { return &CatalogAttributes{ CredentialAttributes: credAttributes, + DualStack: dualStack, }, nil } diff --git a/plugin/service/host/attributes_test.go b/plugin/service/host/attributes_test.go index e84c042..95780a0 100644 --- a/plugin/service/host/attributes_test.go +++ b/plugin/service/host/attributes_test.go @@ -43,6 +43,22 @@ func TestGetCatalogAttributes(t *testing.T) { }, expectedErrContains: "attributes.bar: unrecognized field, attributes.foo: unrecognized field", }, + { + name: "with dual stack", + in: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "region": structpb.NewStringValue("us-west-2"), + "dual_stack": structpb.NewBoolValue(true), + }, + }, + expected: &CatalogAttributes{ + CredentialAttributes: &credential.CredentialAttributes{ + Region: "us-west-2", + DisableCredentialRotation: false, + }, + DualStack: true, + }, + }, { name: "default", in: &structpb.Struct{ diff --git a/plugin/service/host/const.go b/plugin/service/host/const.go index 660095d..698895b 100644 --- a/plugin/service/host/const.go +++ b/plugin/service/host/const.go @@ -10,4 +10,7 @@ const ( // ConstInstanceNameTagKey denotes the special AWS tag key that contains an EC2 // instance's name. This field is to be matched literally. ConstInstanceNameTagKey = "Name" + + // ConstAwsDualStack is the key for the dualstack flag in the aws s3 client. + ConstAwsDualStack = "dual_stack" ) diff --git a/plugin/service/host/plugin.go b/plugin/service/host/plugin.go index 944523a..f87e6f9 100644 --- a/plugin/service/host/plugin.go +++ b/plugin/service/host/plugin.go @@ -51,7 +51,7 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo if err != nil { return nil, err } - credConfig, err := credential.GetCredentialsConfig(catalog.GetSecrets(), catalogAttributes.CredentialAttributes) + credConfig, err := credential.GetCredentialsConfig(catalog.GetSecrets(), catalogAttributes.CredentialAttributes, catalogAttributes.DualStack) if err != nil { return nil, err } @@ -83,7 +83,11 @@ func (p *HostPlugin) OnCreateCatalog(ctx context.Context, req *pb.OnCreateCatalo } // perform dry run to ensure we can interact with AWS as expected. - if st := dryRunValidation(ctx, catalogState); st != nil { + opts := []ec2Option{} + if catalogAttributes.DualStack { + opts = append(opts, WithDualStack(catalogAttributes.DualStack)) + } + if st := dryRunValidation(ctx, catalogState, opts); st != nil { return nil, st.Err() } @@ -131,6 +135,7 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo credState, err := credential.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetSecrets(), oldCatalogAttributes.CredentialAttributes, + oldCatalogAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -139,7 +144,7 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo // Verify the incoming credentials are valid and return any errors to the // user if they're not. Note this doesn't validate the credentials against // AWS - it only does logical validation on the fields. - updatedCredentials, err := credential.GetCredentialsConfig(newCatalog.GetSecrets(), newCatalogAttributes.CredentialAttributes) + updatedCredentials, err := credential.GetCredentialsConfig(newCatalog.GetSecrets(), newCatalogAttributes.CredentialAttributes, newCatalogAttributes.DualStack) if err != nil { return nil, err } @@ -166,7 +171,11 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) } - if st := dryRunValidation(ctx, newCatalogState); st != nil { + opts := []ec2Option{} + if newCatalogAttributes.DualStack { + opts = append(opts, WithDualStack(newCatalogAttributes.DualStack)) + } + if st := dryRunValidation(ctx, newCatalogState, opts); st != nil { return nil, st.Err() } @@ -208,7 +217,11 @@ func (p *HostPlugin) OnUpdateCatalog(ctx context.Context, req *pb.OnUpdateCatalo } // perform dry run to ensure we can interact with AWS as expected. - if st := dryRunValidation(ctx, catalogState); st != nil { + opts := []ec2Option{} + if newCatalogAttributes.DualStack { + opts = append(opts, WithDualStack(newCatalogAttributes.DualStack)) + } + if st := dryRunValidation(ctx, catalogState, opts); st != nil { return nil, st.Err() } @@ -248,6 +261,7 @@ func (p *HostPlugin) OnDeleteCatalog(ctx context.Context, req *pb.OnDeleteCatalo credState, err := credential.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetSecrets(), catalogAttributes.CredentialAttributes, + catalogAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -324,6 +338,7 @@ func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest credState, err := credential.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetSecrets(), catalogAttributes.CredentialAttributes, + catalogAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -356,7 +371,11 @@ func (p *HostPlugin) OnCreateSet(ctx context.Context, req *pb.OnCreateSetRequest return nil, errors.BadRequestStatus("error building set filters: %s", err) } - if st := dryRunValidation(ctx, catalogState, describeInstanceFilters...); st != nil { + opts := []ec2Option{} + if catalogAttributes.DualStack { + opts = append(opts, WithDualStack(catalogAttributes.DualStack)) + } + if st := dryRunValidation(ctx, catalogState, opts, describeInstanceFilters...); st != nil { return nil, st.Err() } return &pb.OnCreateSetResponse{}, nil @@ -382,6 +401,7 @@ func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest credState, err := credential.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetSecrets(), catalogAttributes.CredentialAttributes, + catalogAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -416,7 +436,11 @@ func (p *HostPlugin) OnUpdateSet(ctx context.Context, req *pb.OnUpdateSetRequest return nil, errors.BadRequestStatus("error building set filters: %s", err) } - if st := dryRunValidation(ctx, catalogState, describeInstanceFilters...); st != nil { + opts := []ec2Option{} + if catalogAttributes.DualStack { + opts = append(opts, WithDualStack(catalogAttributes.DualStack)) + } + if st := dryRunValidation(ctx, catalogState, opts, describeInstanceFilters...); st != nil { return nil, st.Err() } @@ -450,6 +474,7 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (* credState, err := credential.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetSecrets(), catalogAttributes.CredentialAttributes, + catalogAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -502,7 +527,11 @@ func (p *HostPlugin) ListHosts(ctx context.Context, req *pb.ListHostsRequest) (* } } - ec2Client, err := catalogState.EC2Client(ctx) + opts := []ec2Option{} + if catalogAttributes.DualStack { + opts = append(opts, WithDualStack(catalogAttributes.DualStack)) + } + ec2Client, err := catalogState.EC2Client(ctx, opts...) if err != nil { return nil, errors.BadRequestStatus("error getting EC2 client: %s", err) } @@ -673,12 +702,12 @@ func awsInstanceToHost(instance types.Instance) (*pb.ListHostsResponseHost, erro // credentials, the host listing functionality as well as the filters, if any // are passed in. This function can therefore be used for both host catalog and // host set validation. -func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, filters ...types.Filter) *status.Status { +func dryRunValidation(ctx context.Context, state *awsCatalogPersistedState, ec2Opts []ec2Option, filters ...types.Filter) *status.Status { if state == nil { return status.New(codes.InvalidArgument, "persisted state is required") } - ec2Client, err := state.EC2Client(ctx) + ec2Client, err := state.EC2Client(ctx, ec2Opts...) if err != nil { return status.New(codes.InvalidArgument, fmt.Sprintf("error getting EC2 client: %s", err)) } diff --git a/plugin/service/host/plugin_test.go b/plugin/service/host/plugin_test.go index 0c809e2..5ade636 100644 --- a/plugin/service/host/plugin_test.go +++ b/plugin/service/host/plugin_test.go @@ -2747,8 +2747,8 @@ func TestAppendDistinct(t *testing.T) { } func TestDryRunValidation(t *testing.T) { - t.Run("nilState", func(t *testing.T) { - st := dryRunValidation(context.Background(), nil) + t.Run("nil credential state", func(t *testing.T) { + st := dryRunValidation(context.Background(), nil, nil) require.NotNil(t, st) require.Equal(t, codes.InvalidArgument.String(), st.Code().String()) require.Equal(t, "persisted state is required", st.Message()) @@ -2762,7 +2762,7 @@ func TestDryRunValidation(t *testing.T) { testEC2APIFunc: func(...aws.Config) (EC2API, error) { return nil, fmt.Errorf("oops ec2 client err") }, - }) + }, []ec2Option{}) require.NotNil(t, st) require.Equal(t, codes.InvalidArgument.String(), st.Code().String()) require.Equal(t, "error getting EC2 client: oops ec2 client err", st.Message()) @@ -2777,7 +2777,7 @@ func TestDryRunValidation(t *testing.T) { }, }, testEC2APIFunc: newTestMockEC2(nil, testMockEC2WithDescribeInstancesError(fmt.Errorf("oops describe instances error"))), - }) + }, []ec2Option{}) require.NotNil(t, st) require.Equal(t, codes.FailedPrecondition.String(), st.Code().String()) require.Equal(t, "aws describe instances failed: oops describe instances error", st.Message()) @@ -2792,7 +2792,7 @@ func TestDryRunValidation(t *testing.T) { }, }, testEC2APIFunc: newTestMockEC2(nil, testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{})), - }) + }, []ec2Option{}) require.Nil(t, st) }) } diff --git a/plugin/service/host/state.go b/plugin/service/host/state.go index 523e74d..a2e772e 100644 --- a/plugin/service/host/state.go +++ b/plugin/service/host/state.go @@ -7,12 +7,10 @@ import ( "context" "errors" "fmt" - "net" - "net/http" - "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" + smithyEndpoints "github.com/aws/smithy-go/endpoints" "github.com/hashicorp/boundary-plugin-aws/internal/credential" pb "github.com/hashicorp/boundary/sdk/pbs/plugin" "google.golang.org/protobuf/types/known/structpb" @@ -22,17 +20,6 @@ type EC2API interface { DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) } -var customClient = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - }, -} - type awsCatalogPersistedState struct { *credential.AwsCredentialPersistedState @@ -86,7 +73,11 @@ func (s *awsCatalogPersistedState) toProto() (*pb.HostCatalogPersisted, error) { // EC2Client returns a configured EC2 client based on the session // information stored in the state. -func (s *awsCatalogPersistedState) EC2Client(ctx context.Context) (EC2API, error) { +func (s *awsCatalogPersistedState) EC2Client(ctx context.Context, opt ...ec2Option) (EC2API, error) { + opts, err := getOpts(opt...) + if err != nil { + return nil, fmt.Errorf("error parsing options when fetching EC2 client: %w", err) + } awsCfg, err := s.GenerateCredentialChain(ctx) if err != nil { return nil, fmt.Errorf("error getting AWS configuration when fetching EC2 client: %w", err) @@ -94,8 +85,58 @@ func (s *awsCatalogPersistedState) EC2Client(ctx context.Context) (EC2API, error if awsCfg == nil { return nil, fmt.Errorf("nil aws configuration") } + var ec2Opts []func(*ec2.Options) + if opts.withDualStack { + ec2Opts = append(ec2Opts, ec2.WithEndpointResolverV2(&endpointResolver{ + dualStack: opts.withDualStack, + })) + } if s.testEC2APIFunc != nil { return s.testEC2APIFunc(*awsCfg) } - return ec2.NewFromConfig(*awsCfg), nil + return ec2.NewFromConfig(*awsCfg, ec2Opts...), nil +} + +// getOpts iterates the inbound ec2Options and returns a struct +func getOpts(opt ...ec2Option) (ec2Options, error) { + opts := getDefaultEc2Options() + for _, o := range opt { + if o == nil { + continue + } + if err := o(&opts); err != nil { + return ec2Options{}, err + } + } + return opts, nil +} + +// ec2Option - how ec2Options are passed as arguments +type ec2Option func(*ec2Options) error + +// options = how options are represented +type ec2Options struct { + withDualStack bool +} + +func getDefaultEc2Options() ec2Options { + return ec2Options{} +} + +// WithDualStack sets the dual stack resolver +func WithDualStack(with bool) ec2Option { + return func(o *ec2Options) error { + o.withDualStack = with + return nil + } +} + +type endpointResolver struct { + dualStack bool +} + +func (e *endpointResolver) ResolveEndpoint(ctx context.Context, params ec2.EndpointParameters) (resolver smithyEndpoints.Endpoint, err error) { + params.UseDualStack = aws.Bool(e.dualStack) + resolver, err = ec2.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) + return } diff --git a/plugin/service/storage/attributes.go b/plugin/service/storage/attributes.go index 0ce76ac..1ff08f7 100644 --- a/plugin/service/storage/attributes.go +++ b/plugin/service/storage/attributes.go @@ -19,6 +19,9 @@ type StorageAttributes struct { // EndpointUrl is used for configuring how the aws client will resolve requests. EndpointUrl string + + // DualStack is used for configuring how the aws client will resolve requests. + DualStack bool } func getStorageAttributes(in *structpb.Struct) (*StorageAttributes, error) { @@ -37,6 +40,12 @@ func getStorageAttributes(in *structpb.Struct) (*StorageAttributes, error) { } delete(unknownFields, ConstAwsEndpointUrl) + dualStack, err := values.GetBoolValue(in, ConstAwsDualStack, false) + if err != nil { + badFields[fmt.Sprintf("attributes.%s", ConstAwsDualStack)] = err.Error() + } + delete(unknownFields, ConstAwsDualStack) + for s := range unknownFields { switch s { // Ignore knownFields from CredentialAttributes @@ -64,5 +73,6 @@ func getStorageAttributes(in *structpb.Struct) (*StorageAttributes, error) { return &StorageAttributes{ CredentialAttributes: credAttributes, EndpointUrl: endpointUrl, + DualStack: dualStack, }, nil } diff --git a/plugin/service/storage/attributes_test.go b/plugin/service/storage/attributes_test.go index c22e81f..b618c00 100644 --- a/plugin/service/storage/attributes_test.go +++ b/plugin/service/storage/attributes_test.go @@ -41,6 +41,21 @@ func TestGetStorageAttributes(t *testing.T) { }, expectedErrContains: "attributes.bar: unrecognized field, attributes.foo: unrecognized field", }, + { + name: "good w/ dualstack", + in: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "region": structpb.NewStringValue("us-west-2"), + "dual_stack": structpb.NewBoolValue(true), + }, + }, + expected: &StorageAttributes{ + CredentialAttributes: &cred.CredentialAttributes{ + Region: "us-west-2", + }, + DualStack: true, + }, + }, { name: "good w/ endpoint", in: &structpb.Struct{ diff --git a/plugin/service/storage/const.go b/plugin/service/storage/const.go index 3699741..b9d8f7e 100644 --- a/plugin/service/storage/const.go +++ b/plugin/service/storage/const.go @@ -7,6 +7,9 @@ const ( // ConstAwsEndpointUrl is the key for the endpoint url in the aws s3 client. ConstAwsEndpointUrl = "endpoint_url" + // ConstAwsDualStack is the key for the dualstack flag in the aws s3 client. + ConstAwsDualStack = "dual_stack" + // defaultStreamChunkSize is the recommened chunk size for sending data through a stream defaultStreamChunkSize = 65536 // 64 KiB ) diff --git a/plugin/service/storage/plugin.go b/plugin/service/storage/plugin.go index e29e05e..212e9d5 100644 --- a/plugin/service/storage/plugin.go +++ b/plugin/service/storage/plugin.go @@ -64,7 +64,7 @@ func (p *StoragePlugin) OnCreateStorageBucket(ctx context.Context, req *pb.OnCre return nil, err } - credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes) + credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, storageAttributes.DualStack) if err != nil { return nil, err } @@ -147,6 +147,7 @@ func (p *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUpd credState, err := cred.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetData(), oldStorageAttributes.CredentialAttributes, + oldStorageAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -155,7 +156,7 @@ func (p *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUpd // Verify the incoming credentials are valid and return any errors to the // user if they're not. Note this doesn't validate the credentials against // AWS - it only does logical validation on the fields. - updatedCredentials, err := cred.GetCredentialsConfig(newBucket.GetSecrets(), newStorageAttributes.CredentialAttributes) + updatedCredentials, err := cred.GetCredentialsConfig(newBucket.GetSecrets(), newStorageAttributes.CredentialAttributes, newStorageAttributes.DualStack) if err != nil { return nil, err } @@ -264,6 +265,7 @@ func (p *StoragePlugin) OnDeleteStorageBucket(ctx context.Context, req *pb.OnDel credState, err := cred.AwsCredentialPersistedStateFromProto( req.GetPersisted().GetData(), storageAttributes.CredentialAttributes, + storageAttributes.DualStack, p.testCredStateOpts...) if err != nil { return nil, errors.BadRequestStatus("error loading persisted state: %s", err) @@ -318,7 +320,7 @@ func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectReques return nil, err } - credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes) + credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, storageAttributes.DualStack) if err != nil { return nil, err } @@ -346,6 +348,9 @@ func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectReques if storageAttributes.EndpointUrl != "" { opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl)) } + if storageAttributes.DualStack { + opts = append(opts, WithDualStack(storageAttributes.DualStack)) + } s3Client, err := storageState.S3Client(ctx, opts...) if err != nil { return nil, errors.BadRequestStatus("error getting S3 client: %s", err) @@ -360,7 +365,7 @@ func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectReques return nil, parseS3Error("head object", err, req).Err() } return &pb.HeadObjectResponse{ - ContentLength: resp.ContentLength, + ContentLength: aws.ToInt64(resp.ContentLength), LastModified: timestamppb.New(*resp.LastModified), }, nil } @@ -386,7 +391,7 @@ func (p *StoragePlugin) ValidatePermissions(ctx context.Context, req *pb.Validat return nil, err } - credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes) + credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, storageAttributes.DualStack) if err != nil { return nil, err } @@ -444,7 +449,7 @@ func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePlu return err } - credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes) + credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, storageAttributes.DualStack) if err != nil { return err } @@ -472,6 +477,9 @@ func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePlu if storageAttributes.EndpointUrl != "" { opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl)) } + if storageAttributes.DualStack { + opts = append(opts, WithDualStack(storageAttributes.DualStack)) + } s3Client, err := storageState.S3Client(stream.Context(), opts...) if err != nil { return errors.BadRequestStatus("error getting S3 client: %s", err) @@ -555,7 +563,7 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest) return nil, err } - credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes) + credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, storageAttributes.DualStack) if err != nil { return nil, err } @@ -583,6 +591,9 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest) if storageAttributes.EndpointUrl != "" { opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl)) } + if storageAttributes.DualStack { + opts = append(opts, WithDualStack(storageAttributes.DualStack)) + } s3Client, err := storageState.S3Client(ctx, opts...) if err != nil { return nil, errors.BadRequestStatus("error getting S3 client: %s", err) @@ -650,7 +661,7 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects return nil, err } - credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes) + credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, storageAttributes.DualStack) if err != nil { return nil, err } @@ -712,13 +723,13 @@ func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjects res, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(bucket.GetBucketName()), Prefix: aws.String(prefix), - MaxKeys: maxkeys, + MaxKeys: aws.Int32(maxkeys), ContinuationToken: conToken, }) if err != nil { return nil, parseS3Error("list objects", err, req).Err() } - truncated = res.IsTruncated + truncated = aws.ToBool(res.IsTruncated) conToken = res.NextContinuationToken for _, o := range res.Contents { objects = append(objects, types.ObjectIdentifier{ @@ -769,6 +780,9 @@ func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attr if attrs.EndpointUrl != "" { opts = append(opts, WithEndpoint(attrs.EndpointUrl)) } + if attrs.DualStack { + opts = append(opts, WithDualStack(attrs.DualStack)) + } client, err := state.S3Client(ctx, opts...) if err != nil { diff --git a/plugin/service/storage/plugin_test.go b/plugin/service/storage/plugin_test.go index 600afab..510545b 100644 --- a/plugin/service/storage/plugin_test.go +++ b/plugin/service/storage/plugin_test.go @@ -2241,7 +2241,7 @@ func TestStoragePlugin_HeadObject(t *testing.T) { newTestMockS3( nil, testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{ - ContentLength: 1024, + ContentLength: aws.Int64(1024), LastModified: aws.Time(createTime(t, "2006-01-02T15:04:05Z")), }), ), @@ -2272,7 +2272,7 @@ func TestStoragePlugin_HeadObject(t *testing.T) { newTestMockS3( nil, testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{ - ContentLength: 1024, + ContentLength: aws.Int64(1024), LastModified: aws.Time(createTime(t, "2006-01-02T15:04:05Z")), }), ), @@ -3685,7 +3685,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) { newTestMockS3( nil, testMockS3WithListObjectsV2Output(&s3.ListObjectsV2Output{ - IsTruncated: false, + IsTruncated: aws.Bool(false), Contents: []s3types.Object{ { Key: aws.String("abc/abc"), @@ -3763,7 +3763,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) { newTestMockS3( nil, testMockS3WithListObjectsV2Output(&s3.ListObjectsV2Output{ - IsTruncated: false, + IsTruncated: aws.Bool(false), Contents: []s3types.Object{ s3types.Object{ Key: aws.String("abc/abc"), @@ -3808,7 +3808,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) { newTestMockS3( nil, testMockS3WithListObjectsV2Output(&s3.ListObjectsV2Output{ - IsTruncated: false, + IsTruncated: aws.Bool(false), Contents: []s3types.Object{ {Key: aws.String("abc/abc1")}, {Key: aws.String("abc/abc2")}, @@ -3836,7 +3836,7 @@ func TestStoragePlugin_DeleteObjects(t *testing.T) { newTestMockS3( nil, testMockS3WithListObjectsV2Output(&s3.ListObjectsV2Output{ - IsTruncated: false, + IsTruncated: aws.Bool(false), Contents: []s3types.Object{}, }), // no deleted response since it shouldn't be called diff --git a/plugin/service/storage/state.go b/plugin/service/storage/state.go index 1cab6bf..d82d011 100644 --- a/plugin/service/storage/state.go +++ b/plugin/service/storage/state.go @@ -7,10 +7,7 @@ import ( "context" "errors" "fmt" - "net" - "net/http" "net/url" - "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -29,17 +26,6 @@ type S3API interface { ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) } -var customClient = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - }, -} - type awsStoragePersistedState struct { *credential.AwsCredentialPersistedState @@ -106,9 +92,10 @@ func (s *awsStoragePersistedState) S3Client(ctx context.Context, opt ...s3Option return nil, fmt.Errorf("nil aws configuration") } var s3Opts []func(*s3.Options) - if opts.withEndpoint != "" { + if opts.withEndpoint != "" || opts.withDualStack { s3Opts = append(s3Opts, s3.WithEndpointResolverV2(&endpointResolver{ - endpoint: opts.withEndpoint, + endpoint: opts.withEndpoint, + dualStack: opts.withDualStack, })) } if s.testS3APIFunc != nil { @@ -136,7 +123,8 @@ type s3Option func(*s3Options) error // options = how options are represented type s3Options struct { - withEndpoint string + withEndpoint string + withDualStack bool } func getDefaultS3Options() s3Options { @@ -151,18 +139,27 @@ func WithEndpoint(with string) s3Option { } } +// WithDualStack sets the dual stack resolver +func WithDualStack(with bool) s3Option { + return func(o *s3Options) error { + o.withDualStack = with + return nil + } +} + type endpointResolver struct { - endpoint string + endpoint string + dualStack bool } func (e *endpointResolver) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (resolver smithyEndpoints.Endpoint, err error) { var uri *url.URL + params.UseDualStack = aws.Bool(e.dualStack) resolver, err = s3.NewDefaultEndpointResolverV2().ResolveEndpoint(ctx, params) if err != nil { return } if e.endpoint == "" { - // TODO: not sure if we should set an error or ignore the missing endpoint value and continue with the default resolver return } uri, err = url.Parse(e.endpoint) @@ -170,7 +167,6 @@ func (e *endpointResolver) ResolveEndpoint(ctx context.Context, params s3.Endpoi return } if uri == nil { - // TODO: not sure if we should set an error or ignore the missing uri value and continue with the default resolver return } resolver = smithyEndpoints.Endpoint{