From 010b9df595aef7ff6ee6a90d373e4fdb368e4f77 Mon Sep 17 00:00:00 2001 From: Damian Debkowski Date: Thu, 25 May 2023 22:19:43 -0700 Subject: [PATCH] feat(plugin): add support for storage service --- .github/workflows/test.yml | 10 +- README.md | 43 +- go.mod | 23 +- go.sum | 37 + internal/credential/attributes.go | 3 + internal/credential/attributes_test.go | 14 + plugin/plugin.go | 16 +- plugin/service/storage/README.md | 83 + plugin/service/storage/attributes.go | 57 + plugin/service/storage/attributes_test.go | 80 + plugin/service/storage/const.go | 9 + plugin/service/storage/plugin.go | 715 ++++++++ plugin/service/storage/plugin_test.go | 1984 +++++++++++++++++++++ plugin/service/storage/state.go | 189 ++ plugin/service/storage/state_test.go | 40 + plugin/service/storage/testing.go | 300 ++++ 16 files changed, 3589 insertions(+), 14 deletions(-) create mode 100644 plugin/service/storage/README.md create mode 100644 plugin/service/storage/attributes.go create mode 100644 plugin/service/storage/attributes_test.go create mode 100644 plugin/service/storage/const.go create mode 100644 plugin/service/storage/plugin.go create mode 100644 plugin/service/storage/plugin_test.go create mode 100644 plugin/service/storage/state.go create mode 100644 plugin/service/storage/state_test.go create mode 100644 plugin/service/storage/testing.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dbfc449..87ce3ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,13 +6,17 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: [self-hosted, ondemand, linux, small] + env: + GO111MODULE: on + permissions: + id-token: write + contents: read steps: - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: Set up Go uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 with: go-version-file: 'go.mod' cache: true - name: Run Tests - run: go test -v ./plugin/... + run: go test -v ./plugin/... ./internal/... diff --git a/README.md b/README.md index 75eae22..972a679 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,9 @@ Boundary](https://www.boundaryproject.io/). Although credentials are stored encrypted within Boundary, by default this plugin will attempt to rotate credentials when they are supplied through the -`secrets` object on a create or update call to the host catalog resource. The -given credentials will be used to create a new credential, and then the given -credential will be revoked. In this way, after rotation, only Boundary knows the -client secret in use by this plugin. +`secrets` object. The given credentials will be used to create a new credential, +and then the given credential will be revoked. In this way, after rotation, +only Boundary knows the client secret in use by this plugin. Credential rotation can be turned off by setting the `disable_credential_rotation` attribute to `true`. @@ -23,13 +22,41 @@ Host sets created with this plugin define filters which select and group like instances within AWS; these host sets can in turn be added to targets within Boundary as host sources. -At creation or update of a host catalog of this type, configuration of the -plugin is performed via the attribute/secret values passed to the create or -update calls actions. These values are input as JSON objects; the expected types -are indicated below with the valid fields. +At creation, update or deletion of a host catalog of this type, configuration of the +plugin is performed via the attribute/secret values passed to the create, update, or +delete calls actions. The values passed in to the plugin here are the attributes set +on on a host catalog in boundary. The plugin fetches hosts through the [DescribeInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html) call. [Getting Started](https://github.com/hashicorp/boundary-plugin-aws/plugin/service/host/README.md) + +## Storage Bucket + +This plugin supports storing and fetching objects from Amazon S3. + +Files created with this plugin are stored as objects defined by the bucket +name and bucket prefix values configured in the storage bucket resource; +these storage bucket resources can in turn be associated to targets within +Boundary. + +At creation, update or deletion of a storage bucket of this type, configuration of the +plugin is performed via the attribute/secret values passed to the create, update, or +delete calls actions. The values passed in to the plugin here are the attributes set +on on a storage bucket in boundary. + +The plugin fetches files through the +[GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) +call. + +The plugin stores files through the +[PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) +call. + +The plugin fetches metadata about the files through the +[HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html) +call. + +[Getting Started](https://github.com/hashicorp/boundary-plugin-aws/plugin/service/storage/README.md) diff --git a/go.mod b/go.mod index 1e6b5a2..5b0d202 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,16 @@ module github.com/hashicorp/boundary-plugin-host-aws -go 1.18 +go 1.20 require ( github.com/aws/aws-sdk-go v1.44.80 + github.com/aws/aws-sdk-go-v2 v1.17.7 + github.com/aws/aws-sdk-go-v2/config v1.18.19 + github.com/aws/aws-sdk-go-v2/credentials v1.13.18 + github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1 github.com/google/go-cmp v0.5.9 - github.com/hashicorp/boundary/sdk v0.0.31 + github.com/google/uuid v1.3.0 + github.com/hashicorp/boundary/sdk v0.0.30 github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 github.com/hashicorp/terraform-json v0.14.0 github.com/mitchellh/mapstructure v1.5.0 @@ -15,6 +20,20 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.14.1 // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index 8b357c7..7e731da 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,42 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.80 h1:jEXGecSgPdvM5KnyDsSgFhZSm7WwaTp4h544Im4SfhI= github.com/aws/aws-sdk-go v1.44.80/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= +github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= +github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 h1:DWYZIsyqagnWL00f8M/SOr9fN063OEQWn9LLTbdYXsk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23/go.mod h1:uIiFgURZbACBEQJfqTZPb/jxO7R+9LeoHUFudtIdeQI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26 h1:CeuSeq/8FnYpPtnuIeLQEEvDv9zUjneuYi8EghMBdwQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.26/go.mod h1:2UqAAwMUXKeRkAHIlDJqvMVgOWkUi/AUXPk/YIe+Dg4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0 h1:e2ooMhpYGhDnBfSvIyusvAwX7KexuZaHbQY2Dyei7VU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.0/go.mod h1:bh2E0CXKZsQN+faiKVqC40vfNMAWheoULBCnEgO9K+8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1 h1:PJH4I+qYjPXclKRbVCW47iYUvtXEh1u6YmDhn5J8VQE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.31.1/go.mod h1:ncltU6n4Nof5uJttDtcNQ537uNuwYqsZZQcpkd2/GUQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -32,6 +68,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/boundary/sdk v0.0.31 h1:KAPWpic7UFSUc/V4BDOVmXBYtSbSHKwAVrDXA8u/CxE= github.com/hashicorp/boundary/sdk v0.0.31/go.mod h1:6SNSerq1eWX35xsC2NE3VLmFzMjsllCbaVXE7dsueaI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/internal/credential/attributes.go b/internal/credential/attributes.go index 3ec6ee1..77d12df 100644 --- a/internal/credential/attributes.go +++ b/internal/credential/attributes.go @@ -44,6 +44,9 @@ func GetCredentialsConfig(in *structpb.Struct, region string) (*awsutil.Credenti badFields[fmt.Sprintf("attributes.%s", ConstRegion)] = err.Error() } + // the creds_last_rotated_time field will be found in the input struct for + // persisted secrets, this value is not needed for creating the CredentialsConfig + delete(unknownFields, ConstCredsLastRotatedTime) for s := range unknownFields { badFields[fmt.Sprintf("secrets.%s", s)] = "unrecognized field" } diff --git a/internal/credential/attributes_test.go b/internal/credential/attributes_test.go index 27aecfc..14c94d1 100644 --- a/internal/credential/attributes_test.go +++ b/internal/credential/attributes_test.go @@ -119,6 +119,20 @@ func TestGetCredentialsConfig(t *testing.T) { region: "us-west-2", expectedErrContains: "secrets.bar: unrecognized field, secrets.foo: unrecognized field", }, + { + name: "valid ignore creds_last_rotated_time", + in: map[string]any{ + ConstAccessKeyId: "foobar", + ConstSecretAccessKey: "bazqux", + ConstCredsLastRotatedTime: "2006-01-02T15:04:05+07:00", + }, + region: "us-west-2", + expected: &awsutil.CredentialsConfig{ + AccessKey: "foobar", + SecretKey: "bazqux", + Region: "us-west-2", + }, + }, { name: "good", in: map[string]any{ diff --git a/plugin/plugin.go b/plugin/plugin.go index c21e514..9dddc15 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -2,14 +2,17 @@ package plugin import ( "github.com/hashicorp/boundary-plugin-host-aws/plugin/service/host" + "github.com/hashicorp/boundary-plugin-host-aws/plugin/service/storage" pb "github.com/hashicorp/boundary/sdk/pbs/plugin" ) // Ensure that AWSPlugin implementings the following services: // // HostPluginServiceServer +// StoragePluginServiceServer var ( - _ pb.HostPluginServiceServer = (*host.HostPlugin)(nil) + _ pb.HostPluginServiceServer = (*host.HostPlugin)(nil) + _ pb.StoragePluginServiceServer = (*storage.StoragePlugin)(nil) ) // AwsPlugin contains a collection of all aws plugin services. @@ -17,4 +20,15 @@ type AwsPlugin struct { // HostPlugin implements the HostPluginServiceServer interface for // supporting dynamically sourcing hosts from Amazon EC2 *host.HostPlugin + + // StoragePlugin implements the StoragePluginServiceServer interface for + // supporting storing and retrieving BSR files from Amazon S3 + *storage.StoragePlugin +} + +func NewAwsPlugin() *AwsPlugin { + return &AwsPlugin{ + HostPlugin: &host.HostPlugin{}, + StoragePlugin: &storage.StoragePlugin{}, + } } diff --git a/plugin/service/storage/README.md b/plugin/service/storage/README.md new file mode 100644 index 0000000..d9c45f6 --- /dev/null +++ b/plugin/service/storage/README.md @@ -0,0 +1,83 @@ +## Getting Started + +To create a storage bucket (using default scope created by `boundary dev`): + +``` +boundary storage-buckets create \ + -scope-id p_1234567890 \ + -name "Example Plugin-Based Storage Bucket" \ + -description "Description for plugin-based storage bucket" \ + -plugin-name aws \ + -bucket-name="session_recording_storage" \ + -bucket-prefix="foo/bar/zoo" \ + -worker-filter ‘“aws-access” in “/tags/type”’ \ + -attr region=REGION \ + -attr endpoint_url=0.0.0.0 \ + -secret access_key_id='KEY' \ + -secret secret_access_key='SECRET' +``` + +### Required IAM Privileges + +The following IAM privileges, at the very least, are required to be attached to +a configured IAM user for this provider: + +Example policy: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:GetObjectAttributes" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::session_recording_storage/foo/bar/zoo/*" + } + ] +} +``` + +* `iam:GetUser`, `iam:CreateAccessKey`, and `iam:DeleteAccessKey`, configured to + the IAM user to allow credential rotation. Example policy: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "iam:DeleteAccessKey", + "iam:GetUser", + "iam:CreateAccessKey" + ], + "Effect": "Allow", + "Resource": "arn:aws:iam::123456789012:user/JohnDoe" + } + ] +} +``` + +### Attributes + +The following `attributes` are valid on an AWS storage bucket resource: + +- `disable_credential_rotation` (bool): If `true`, credential rotation will not + be performed. See the [Credential Rotation](#credential-rotation) section + above. +- `region` (string): The region to configure the storage bucket for. +- `endpoint-url` (string): The endpoint to configure the storage. + +An example of how to utilize the [endpoint attribute](https://docs.aws.amazon.com/vpc/latest/privatelink/vpc-endpoints-s3.html) + +### Secrets + +The following `secrets` are valid on an AWS storage bucket resource: + +- `access_key_id` (string): The access key ID for the IAM user to use with this + storage bucket. +- `secret_access_key` (string): The secret access key for the IAM user to use + with this storage bucket. \ No newline at end of file diff --git a/plugin/service/storage/attributes.go b/plugin/service/storage/attributes.go new file mode 100644 index 0000000..af5146a --- /dev/null +++ b/plugin/service/storage/attributes.go @@ -0,0 +1,57 @@ +package storage + +import ( + "fmt" + + cred "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/hashicorp/boundary-plugin-host-aws/internal/errors" + "github.com/hashicorp/boundary-plugin-host-aws/internal/values" + "google.golang.org/protobuf/types/known/structpb" +) + +// StorageAttributes is a Go-native representation of the Attributes +// map. +type StorageAttributes struct { + *cred.CredentialAttributes + + // EndpointUrl is used for configuring how the aws client will resolve requests. + EndpointUrl string +} + +func getStorageAttributes(in *structpb.Struct) (*StorageAttributes, error) { + unknownFields := values.StructFields(in) + badFields := make(map[string]string) + + var err error + credAttributes, err := cred.GetCredentialAttributes(in) + if err != nil { + return nil, err + } + + endpointUrl, err := values.GetStringValue(in, ConstAwsEndpointUrl, false) + if err != nil { + badFields[fmt.Sprintf("attributes.%s", ConstAwsEndpointUrl)] = err.Error() + } + delete(unknownFields, ConstAwsEndpointUrl) + + for s := range unknownFields { + switch s { + // Ignore knownFields from CredentialAttributes + case cred.ConstRegion: + continue + case cred.ConstDisableCredentialRotation: + continue + default: + badFields[fmt.Sprintf("attributes.%s", s)] = "unrecognized field" + } + } + + if len(badFields) > 0 { + return nil, errors.InvalidArgumentError("Invalid arguments in storage bucket attributes", badFields) + } + + return &StorageAttributes{ + CredentialAttributes: credAttributes, + EndpointUrl: endpointUrl, + }, nil +} diff --git a/plugin/service/storage/attributes_test.go b/plugin/service/storage/attributes_test.go new file mode 100644 index 0000000..bfae676 --- /dev/null +++ b/plugin/service/storage/attributes_test.go @@ -0,0 +1,80 @@ +package storage + +import ( + "testing" + + cred "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestGetStorageAttributes(t *testing.T) { + cases := []struct { + name string + in map[string]any + expected *StorageAttributes + expectedErrContains string + }{ + { + name: "missing region", + in: map[string]any{}, + expectedErrContains: "missing required value \"region\"", + }, + { + name: "unknown fields", + in: map[string]any{ + "region": "us-west-2", + "foo": true, + "bar": true, + }, + expectedErrContains: "attributes.bar: unrecognized field, attributes.foo: unrecognized field", + }, + { + name: "good w/ endpoint", + in: map[string]any{ + cred.ConstRegion: "us-west-2", + ConstAwsEndpointUrl: "0.0.0.0", + }, + expected: &StorageAttributes{ + CredentialAttributes: &cred.CredentialAttributes{ + Region: "us-west-2", + }, + EndpointUrl: "0.0.0.0", + }, + }, + { + name: "good w/o endpoint", + in: map[string]any{ + cred.ConstRegion: "us-west-2", + }, + expected: &StorageAttributes{ + CredentialAttributes: &cred.CredentialAttributes{ + Region: "us-west-2", + }, + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + + input, err := structpb.NewStruct(tc.in) + require.NoError(err) + + actual, err := getStorageAttributes(input) + if tc.expectedErrContains != "" { + require.Error(err) + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err), codes.InvalidArgument) + return + } + + require.NoError(err) + require.Equal(tc.expected, actual) + }) + } +} diff --git a/plugin/service/storage/const.go b/plugin/service/storage/const.go new file mode 100644 index 0000000..f3a4557 --- /dev/null +++ b/plugin/service/storage/const.go @@ -0,0 +1,9 @@ +package storage + +const ( + // ConstAwsEndpointUrl is the key for the endpoint url in the aws s3 client. + ConstAwsEndpointUrl = "endpoint_url" + + // defaultStreamChunkSize is the recommened chunk size for sending data through a stream + defaultStreamChunkSize = 65536 // 64 KiB +) diff --git a/plugin/service/storage/plugin.go b/plugin/service/storage/plugin.go new file mode 100644 index 0000000..b840f11 --- /dev/null +++ b/plugin/service/storage/plugin.go @@ -0,0 +1,715 @@ +package storage + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "io" + "os" + "path" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/retry" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" + cred "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets" + pb "github.com/hashicorp/boundary/sdk/pbs/plugin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Ensure that we are implementing StoragePluginServiceServer +var _ pb.StoragePluginServiceServer = (*StoragePlugin)(nil) + +// StoragePlugin implements the StoragePluginServiceServer interface for the +// AWS storage service plugin. +type StoragePlugin struct { + pb.UnimplementedStoragePluginServiceServer + + // testCredStateOpts are passed in to the stored state to control test behavior + testCredStateOpts []cred.AwsCredentialPersistedStateOption + + // testStorageStateOpts are passed in to the stored state to control test behavior + testStorageStateOpts []awsStoragePersistedStateOption +} + +// OnCreateStorageBucket is called when a storage bucket is created. +func (p *StoragePlugin) OnCreateStorageBucket(ctx context.Context, req *pb.OnCreateStorageBucketRequest) (*pb.OnCreateStorageBucketResponse, error) { + bucket := req.GetBucket() + if bucket == nil { + return nil, status.Error(codes.InvalidArgument, "bucket is required") + } + if bucket.BucketName == "" { + return nil, status.Error(codes.InvalidArgument, "bucketName is required") + } + + secrets := bucket.GetSecrets() + if secrets == nil { + return nil, status.Error(codes.InvalidArgument, "secrets is required") + } + + attrs := bucket.GetAttributes() + if attrs == nil { + return nil, status.Error(codes.InvalidArgument, "attributes is required") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return nil, err + } + + credConfig, err := cred.GetCredentialsConfig(secrets, storageAttributes.Region) + if err != nil { + return nil, err + } + + // Set up the creds in our persisted state. + credState, err := cred.NewAwsCredentialPersistedState( + append([]cred.AwsCredentialPersistedStateOption{ + cred.WithAccessKeyId(credConfig.AccessKey), + cred.WithSecretAccessKey(credConfig.SecretKey), + cred.WithRegion(credConfig.Region), + }, p.testCredStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + // Try to rotate the credentials if we're not skipping them. + if !storageAttributes.DisableCredentialRotation { + if err := credState.RotateCreds(); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error during credential rotation: %s", err) + } + } else { + // Simply validate if we aren't rotating. + if err := credState.ValidateCreds(); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error during credential validation: %s", err) + } + } + + storageState, err := newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + // perform dry run to ensure we can interact with the bucket as expected. + if err := dryRunValidation(ctx, storageState, storageAttributes, bucket); err != nil { + return nil, err + } + + persistedProto, err := storageState.toProto() + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + return &pb.OnCreateStorageBucketResponse{ + Persisted: persistedProto, + }, nil +} + +// OnUpdateStorageBucket is called when a storage bucket is updated. +func (p *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUpdateStorageBucketRequest) (*pb.OnUpdateStorageBucketResponse, error) { + // For storage buckets we only need to really validate what is + // currently in the new copy of the storage bucket data, so skip + // fetching the new copy to keep things a little less messy. + bucket := req.GetNewBucket() + if bucket == nil { + return nil, status.Error(codes.InvalidArgument, "new bucket is required") + } + if bucket.BucketName == "" { + return nil, status.Error(codes.InvalidArgument, "new bucketName is required") + } + + var updateSecrets bool + secrets := bucket.GetSecrets() + if secrets != nil { + // We will be updating secrets this run, but what exactly that + // means will be determined later. + updateSecrets = true + } + + attrs := bucket.GetAttributes() + if attrs == nil { + return nil, status.Error(codes.InvalidArgument, "new bucket attributes is required") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return nil, err + } + + // Get the persisted data. + // NOTE: We might need to change this at a later time. I'm not too + // sure *exactly* what scenarios we might encounter that would + // ultimately mean that we would have to handle an empty or missing + // state in update, but what we are ultimately assuming here + // (implicitly through awsStoragePersistedStateFromProto) is that + // the state will exist and be populated. Personally I think this + // is fine and important, but this may change in the future. + credState, err := cred.AwsCredentialPersistedStateFromProto( + req.GetPersisted().GetData(), + append(p.testCredStateOpts, cred.WithRegion(storageAttributes.Region))...) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err) + } + + if storageAttributes.DisableCredentialRotation && !updateSecrets { + // This is a validate check to make sure that we aren't disabling + // rotation for credentials currently being managed by rotation. + // This is not allowed. + if !credState.CredsLastRotatedTime.IsZero() { + return nil, status.Error(codes.FailedPrecondition, "cannot disable rotation for already-rotated credentials") + } + } + + if updateSecrets { + storageSecrets, err := cred.GetCredentialsConfig(secrets, storageAttributes.Region) + if err != nil { + return nil, err + } + + // Replace the credentials. This checks the timestamp on the last + // rotation time as well and deletes the credentials if we are + // managing them (ie: if we've rotated them before). + if err := credState.ReplaceCreds(storageSecrets.AccessKey, storageSecrets.SecretKey); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error attempting to replace credentials: %s", err) + } + } + + if !storageAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() { + // If we're enabling rotation now but didn't before, or have + // freshly replaced credentials, we can rotate here. + if err := credState.RotateCreds(); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error during credential rotation: %s", err) + } + } + + storageState, err := newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err) + } + + // perform dry run to ensure we can interact with the bucket as expected. + if err := dryRunValidation(ctx, storageState, storageAttributes, bucket); err != nil { + return nil, err + } + + persistedProto, err := storageState.toProto() + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + return &pb.OnUpdateStorageBucketResponse{ + Persisted: persistedProto, + }, nil +} + +// OnDeleteStorageBucket is called when a storage bucket is deleted. +func (p *StoragePlugin) OnDeleteStorageBucket(ctx context.Context, req *pb.OnDeleteStorageBucketRequest) (*pb.OnDeleteStorageBucketResponse, error) { + bucket := req.GetBucket() + if bucket == nil { + return nil, status.Error(codes.InvalidArgument, "bucket is required") + } + + attrs := bucket.GetAttributes() + if attrs == nil { + return nil, status.Error(codes.InvalidArgument, "attributes is required") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return nil, err + } + + // Get the persisted data. + // NOTE: We return on error here, blocking the delete. This may or + // may not be an overzealous approach to maintaining database/state + // integrity. May need to be changed at later time if there are + // scenarios where we might be deleting things and any secret state + // may be corrupt/and or legitimately missing. + credState, err := cred.AwsCredentialPersistedStateFromProto( + req.GetPersisted().GetData(), + append(p.testCredStateOpts, cred.WithRegion(storageAttributes.Region))...) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err) + } + + _, err = newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error loading persisted state: %s", err) + } + + if !credState.CredsLastRotatedTime.IsZero() { + // Delete old/existing credentials. This is done with the same + // credentials to ensure that it has the proper permissions to do + // it. + if err := credState.DeleteCreds(); err != nil { + return nil, status.Errorf(codes.Aborted, "error removing rotated credentials during storage bucket deletion: %s", err) + } + } + + return &pb.OnDeleteStorageBucketResponse{}, nil +} + +// HeadObject is called to get the metadata of an object. +func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectRequest) (*pb.HeadObjectResponse, error) { + if req.GetKey() == "" { + return nil, status.Error(codes.InvalidArgument, "key is required") + } + + bucket := req.GetBucket() + if bucket == nil { + return nil, status.Error(codes.InvalidArgument, "bucket is required") + } + + if bucket.BucketName == "" { + return nil, status.Error(codes.InvalidArgument, "bucketName is required") + } + + secrets := bucket.GetSecrets() + if secrets == nil { + return nil, status.Error(codes.InvalidArgument, "secrets is required") + } + + attrs := bucket.GetAttributes() + if attrs == nil { + return nil, status.Error(codes.InvalidArgument, "attributes is required") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return nil, err + } + + credConfig, err := cred.GetCredentialsConfig(secrets, storageAttributes.Region) + if err != nil { + return nil, err + } + + // Set up the creds in our persisted state. + credState, err := cred.NewAwsCredentialPersistedState( + append([]cred.AwsCredentialPersistedStateOption{ + cred.WithAccessKeyId(credConfig.AccessKey), + cred.WithSecretAccessKey(credConfig.SecretKey), + cred.WithRegion(credConfig.Region), + }, p.testCredStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + storageState, err := newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + opts := []s3Option{} + if storageAttributes.Region != "" { + opts = append(opts, WithRegion(storageAttributes.Region)) + } + if storageAttributes.EndpointUrl != "" { + opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl)) + } + s3Client, err := storageState.S3Client(ctx, opts...) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error getting S3 client: %s", err) + } + + objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey()) + resp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + }) + if err != nil { + return nil, parseAWSErrCode("error getting head object from s3", err) + } + return &pb.HeadObjectResponse{ + ContentLength: resp.ContentLength, + LastModified: timestamppb.New(*resp.LastModified), + }, nil +} + +// ValidatePermissions is called to validate the secrets associated with the storage bucket. +func (p *StoragePlugin) ValidatePermissions(ctx context.Context, req *pb.ValidatePermissionsRequest) (*pb.ValidatePermissionsResponse, error) { + bucket := req.GetBucket() + if bucket == nil { + return nil, status.Error(codes.InvalidArgument, "bucket is required") + } + + if bucket.BucketName == "" { + return nil, status.Error(codes.InvalidArgument, "bucketName is required") + } + + secrets := bucket.GetSecrets() + if secrets == nil { + return nil, status.Error(codes.InvalidArgument, "secrets is required") + } + + attrs := bucket.GetAttributes() + if attrs == nil { + return nil, status.Error(codes.InvalidArgument, "attributes is required") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return nil, err + } + + credConfig, err := cred.GetCredentialsConfig(secrets, storageAttributes.Region) + if err != nil { + return nil, err + } + + // Set up the creds in our persisted state. + credState, err := cred.NewAwsCredentialPersistedState( + append([]cred.AwsCredentialPersistedStateOption{ + cred.WithAccessKeyId(credConfig.AccessKey), + cred.WithSecretAccessKey(credConfig.SecretKey), + cred.WithRegion(credConfig.Region), + }, p.testCredStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + storageState, err := newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + // perform dry run to ensure we can interact with the bucket as expected. + if err := dryRunValidation(ctx, storageState, storageAttributes, bucket); err != nil { + return nil, err + } + + return &pb.ValidatePermissionsResponse{}, nil +} + +// GetObject is called when retrieving objects from an s3 bucket. +// GetObject is a blocking call until the stream has been recieved in full. +func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePluginService_GetObjectServer) error { + if req.GetKey() == "" { + return status.Error(codes.InvalidArgument, "key is required") + } + + bucket := req.GetBucket() + if bucket == nil { + return status.Error(codes.InvalidArgument, "bucket is required") + } + + if bucket.GetBucketName() == "" { + return status.Error(codes.InvalidArgument, "bucketName is required") + } + + secrets := bucket.GetSecrets() + if secrets == nil { + return status.Error(codes.InvalidArgument, "secrets is required") + } + + attrs := bucket.GetAttributes() + if attrs == nil { + return status.Error(codes.InvalidArgument, "attributes is required") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return err + } + + credConfig, err := cred.GetCredentialsConfig(secrets, storageAttributes.Region) + if err != nil { + return err + } + + // Set up the creds in our persisted state. + credState, err := cred.NewAwsCredentialPersistedState( + append([]cred.AwsCredentialPersistedStateOption{ + cred.WithAccessKeyId(credConfig.AccessKey), + cred.WithSecretAccessKey(credConfig.SecretKey), + cred.WithRegion(credConfig.Region), + }, p.testCredStateOpts...)..., + ) + if err != nil { + return status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + storageState, err := newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + opts := []s3Option{} + if storageAttributes.Region != "" { + opts = append(opts, WithRegion(storageAttributes.Region)) + } + if storageAttributes.EndpointUrl != "" { + opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl)) + } + s3Client, err := storageState.S3Client(stream.Context(), opts...) + if err != nil { + return status.Errorf(codes.InvalidArgument, "error getting S3 client: %s", err) + } + + objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey()) + resp, err := s3Client.GetObject(stream.Context(), &s3.GetObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + }) + if err != nil { + return parseAWSErrCode("error getting object from s3", err) + } + + defer resp.Body.Close() + reader := bufio.NewReader(resp.Body) + chunkSize := req.GetChunkSize() + if chunkSize == 0 { + chunkSize = defaultStreamChunkSize + } + for { + buffer := make([]byte, chunkSize) + n, err := reader.Read(buffer) + if err != nil && err != io.EOF { + return status.Errorf(codes.Internal, "error reading chunk from s3: %s", err) + } + if n > 0 { + if err := stream.Send(&pb.GetObjectResponse{ + FileChunk: buffer[:n], + }); err != nil { + return status.Errorf(codes.Internal, "error sending chunk to client: %s", err) + } + } + if err == io.EOF { + break + } + } + + return nil +} + +// PutObject is called when putting objects into an s3 bucket. +func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest) (*pb.PutObjectResponse, error) { + if req == nil { + return nil, status.Errorf(codes.InvalidArgument, "request is required") + } + if req.GetKey() == "" { + return nil, status.Errorf(codes.InvalidArgument, "key is required") + } + bucket := req.GetBucket() + if bucket == nil { + return nil, status.Error(codes.InvalidArgument, "bucket is required") + } + if bucket.GetBucketName() == "" { + return nil, status.Error(codes.InvalidArgument, "bucketName is required") + } + secrets := bucket.GetSecrets() + if secrets == nil { + return nil, status.Error(codes.InvalidArgument, "secrets is required") + } + attrs := bucket.GetAttributes() + if attrs == nil { + return nil, status.Error(codes.InvalidArgument, "attributes is required") + } + if req.GetPath() == "" { + return nil, status.Errorf(codes.InvalidArgument, "path is required") + } + file, err := os.Open(req.GetPath()) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to open file") + } + info, err := file.Stat() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to read file info") + } + if info.IsDir() { + return nil, status.Errorf(codes.InvalidArgument, "path is not a file") + } + if info.Size() == 0 { + return nil, status.Errorf(codes.InvalidArgument, "file is empty") + } + + storageAttributes, err := getStorageAttributes(attrs) + if err != nil { + return nil, err + } + + credConfig, err := cred.GetCredentialsConfig(secrets, storageAttributes.Region) + if err != nil { + return nil, err + } + + // Set up the creds in our persisted state. + credState, err := cred.NewAwsCredentialPersistedState( + append([]cred.AwsCredentialPersistedStateOption{ + cred.WithAccessKeyId(credConfig.AccessKey), + cred.WithSecretAccessKey(credConfig.SecretKey), + cred.WithRegion(credConfig.Region), + }, p.testCredStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + storageState, err := newAwsStoragePersistedState( + append([]awsStoragePersistedStateOption{ + withCredentials(credState), + }, p.testStorageStateOpts...)..., + ) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error setting up persisted state: %s", err) + } + + opts := []s3Option{} + if storageAttributes.Region != "" { + opts = append(opts, WithRegion(storageAttributes.Region)) + } + if storageAttributes.EndpointUrl != "" { + opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl)) + } + s3Client, err := storageState.S3Client(ctx, opts...) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "error getting S3 client: %s", err) + } + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return nil, status.Errorf(codes.Internal, "failed to calcualte hash") + } + checksum := base64.StdEncoding.EncodeToString(hash.Sum(nil)) + + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to rewind file pointer") + } + + objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey()) + resp, err := s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + Body: file, + ChecksumAlgorithm: types.ChecksumAlgorithmSha256, + ChecksumSHA256: aws.String(checksum), + }) + if err != nil { + return nil, parseAWSErrCode("error putting object into s3", err) + } + if resp.ChecksumSHA256 == nil { + return nil, status.Errorf(codes.Internal, "missing checksum response from aws") + } + if checksum != *resp.ChecksumSHA256 { + return nil, status.Errorf(codes.Internal, "mismatched checksum") + } + decodedChecksum, err := base64.StdEncoding.DecodeString(*resp.ChecksumSHA256) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to decode checksum value from aws") + } + return &pb.PutObjectResponse{ + ChecksumSha_256: decodedChecksum, + }, nil +} + +// dryRunValidation validates that the IAM role policy attached to the given secrets of the +// storage bucket has the minimum required permissions needed for this plugin to function +// as expected. This function will create an object in the s3 bucket to validate it has +// PutObject permissions. This function will read the same file it had uploaded to validate +// it has GetObject permissions. This function will read the metadata of the same file it +// had uploaded to validate it has GetObjectAttributes permissions. +func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attrs *StorageAttributes, bucket *storagebuckets.StorageBucket) error { + if state == nil { + return status.Error(codes.InvalidArgument, "persisted state is required") + } + if attrs == nil { + return status.Error(codes.InvalidArgument, "attributes is required") + } + if bucket == nil { + return status.Error(codes.InvalidArgument, "bucket is required") + } + + opts := []s3Option{} + if attrs.Region != "" { + opts = append(opts, WithRegion(attrs.Region)) + } + if attrs.EndpointUrl != "" { + opts = append(opts, WithEndpoint(attrs.EndpointUrl)) + } + + client, err := state.S3Client(ctx, opts...) + if err != nil { + return status.Errorf(codes.InvalidArgument, "error getting S3 client: %s", err) + } + + objectKey := path.Join(bucket.GetBucketPrefix(), uuid.New().String()) + if _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + Body: bytes.NewReader([]byte("hashicorp boundary aws plugin access test")), + }); err != nil { + return parseAWSErrCode("error failed to put object", err) + } + + if _, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + }); err != nil { + return parseAWSErrCode("error failed to get object", err) + } + + if _, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + }); err != nil { + return parseAWSErrCode("error failed to get head object", err) + } + + // attempt to delete the test object created for the dry run validation, + // this step is allowed to fail because DeleteObject is not a required + // operation for the plugin. + client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket.GetBucketName()), + Key: aws.String(objectKey), + }) + + return nil +} + +// parseAWSErrCode detects if an aws error is a throttle error. +// If the error is a throttle error, then an Unavailable status code +// is returned. Otherwise the default status code is an Internal error. +func parseAWSErrCode(msg string, err error) error { + errCode := codes.Internal + retryErr := retry.ThrottleErrorCode{ + Codes: retry.DefaultThrottleErrorCodes, + } + if retryErr.IsErrorThrottle(err).Bool() { + errCode = codes.Unavailable + } + return status.Errorf(errCode, "%s: %s", msg, err) +} diff --git a/plugin/service/storage/plugin_test.go b/plugin/service/storage/plugin_test.go new file mode 100644 index 0000000..db36775 --- /dev/null +++ b/plugin/service/storage/plugin_test.go @@ -0,0 +1,1984 @@ +package storage + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets" + pb "github.com/hashicorp/boundary/sdk/pbs/plugin" + "github.com/hashicorp/go-secure-stdlib/awsutil" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestStoragePlugin_OnCreateStorageBucket(t *testing.T) { + validRequest := func() *pb.OnCreateStorageBucketRequest { + return &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + credential.ConstDisableCredentialRotation: structpb.NewBoolValue(true), + }, + }, + }, + } + } + cases := []struct { + name string + req *pb.OnCreateStorageBucketRequest + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "nil storage bucket", + req: &pb.OnCreateStorageBucketRequest{}, + expectedErrContains: "bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "empty bucket name", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{}, + }, + expectedErrContains: "bucketName is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil secrets", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + }, + }, + expectedErrContains: "secrets is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil attributes", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + }, + }, + expectedErrContains: "attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: new(structpb.Struct), + }, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading secrets", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + }, + expectedErrContains: "missing required value \"access_key_id\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "persisted state setup error", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "rotation error", + req: &pb.OnCreateStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + }, + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithIAMAPIFunc( + awsutil.NewMockIAM( + awsutil.WithGetUserError(errors.New(testGetUserErr)), + ), + ), + }), + }, + expectedErrContains: fmt.Sprintf("error during credential rotation: error rotating credentials: error calling CreateAccessKey: error calling aws.GetUser: %s", testGetUserErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "validation error", + req: validRequest(), + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithSTSAPIFunc( + awsutil.NewMockSTS( + awsutil.WithGetCallerIdentityError(errors.New(testGetCallerIdentityErr)), + ), + ), + }), + }, + expectedErrContains: fmt.Sprintf("error during credential validation: error validating credentials: %s", testGetCallerIdentityErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "dryRunValidation failed putObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc(newTestMockS3( + nil, + testMockS3WithPutObjectError(errors.New(testPutObjectErr)), + )), + }, + expectedErrContains: "error failed to put object: test error for PutObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation putObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc(newTestMockS3( + nil, + testMockS3WithPutObjectError(new(throttleErr)), + )), + }, + expectedErrContains: "error failed to put object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "dryRunValidation failed getObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectError(errors.New(testGetObjectErr)), + ), + ), + }, + expectedErrContains: "error failed to get object: test error for GetObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed getObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error failed to get object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "dryRunValidation failed headObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectError(errors.New(testHeadObjectErr)), + ), + ), + }, + expectedErrContains: "error failed to get head object: test error for HeadObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed headObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error failed to get head object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "success", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{}), + ), + ), + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + resp, err := p.OnCreateStorageBucket(context.Background(), tc.req) + if tc.expectedErrContains != "" { + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err).String(), tc.expectedErrCode.String()) + return + } + require.NoError(err) + require.NotNil(resp) + require.NotNil(resp.GetPersisted()) + require.NotNil(resp.GetPersisted().GetData()) + require.Equal( + resp.GetPersisted().GetData().AsMap()[credential.ConstAccessKeyId], + tc.req.Bucket.Secrets.AsMap()[credential.ConstAccessKeyId]) + require.Equal( + resp.GetPersisted().GetData().AsMap()[credential.ConstSecretAccessKey], + tc.req.Bucket.Secrets.AsMap()[credential.ConstSecretAccessKey]) + }) + } +} + +func TestStoragePlugin_OnUpdateStorageBucket(t *testing.T) { + validRequest := func() *pb.OnUpdateStorageBucketRequest { + return &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + credential.ConstDisableCredentialRotation: structpb.NewBoolValue(true), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + }, + } + } + cases := []struct { + name string + req *pb.OnUpdateStorageBucketRequest + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "nil new bucket", + req: &pb.OnUpdateStorageBucketRequest{}, + expectedErrContains: "new bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "empty new bucketName", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{}, + }, + expectedErrContains: "new bucketName is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil new bucket attributes", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + }, + }, + expectedErrContains: "new bucket attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: new(structpb.Struct), + }, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "persisted state setup error", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error loading persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "cannot disable rotation for already rotated credentials", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + credential.ConstDisableCredentialRotation: structpb.NewBoolValue(true), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + expectedErrContains: "cannot disable rotation for already-rotated credentials", + expectedErrCode: codes.FailedPrecondition, + }, + { + name: "error reading secrets", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + expectedErrContains: "missing required value \"access_key_id\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "updating secrets, replace error", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("onetwo"), + credential.ConstSecretAccessKey: structpb.NewStringValue("threefour"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + credential.ConstDisableCredentialRotation: structpb.NewBoolValue(true), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithIAMAPIFunc( + awsutil.NewMockIAM( + awsutil.WithDeleteAccessKeyError(errors.New(testDeleteAccessKeyErr)), + ), + ), + }), + }, + expectedErrContains: fmt.Sprintf("error attempting to replace credentials: error deleting old access key: %s", testDeleteAccessKeyErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "rotation error", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + }, + }, + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithIAMAPIFunc( + awsutil.NewMockIAM( + awsutil.WithGetUserError(errors.New(testGetUserErr)), + ), + ), + }), + }, + expectedErrContains: fmt.Sprintf("error during credential rotation: error rotating credentials: error calling CreateAccessKey: error calling aws.GetUser: %s", testGetUserErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "dryRunValidation failed putObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc(newTestMockS3( + nil, + testMockS3WithPutObjectError(errors.New(testPutObjectErr)), + )), + }, + expectedErrContains: "error failed to put object: test error for PutObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed putObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc(newTestMockS3( + nil, + testMockS3WithPutObjectError(new(throttleErr)), + )), + }, + expectedErrContains: "error failed to put object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "dryRunValidation failed getObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectError(errors.New(testGetObjectErr)), + ), + ), + }, + expectedErrContains: "error failed to get object: test error for GetObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed getObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error failed to get object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "dryRunValidation failed headObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectError(errors.New(testHeadObjectErr)), + ), + ), + }, + expectedErrContains: "error failed to get head object: test error for HeadObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed headObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error failed to get head object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "success with credential rotation", + req: &pb.OnUpdateStorageBucketRequest{ + NewBucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + }, + }, + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithSTSAPIFunc( + awsutil.NewMockSTS(), + ), + awsutil.WithIAMAPIFunc( + awsutil.NewMockIAM( + awsutil.WithGetUserOutput( + &iam.GetUserOutput{ + User: &iam.User{ + Arn: aws.String("arn:aws:iam::123456789012:user/JohnDoe"), + UserId: aws.String("AIDAJQABLZS4A3QDU576Q"), + UserName: aws.String("JohnDoe"), + }, + }, + ), + awsutil.WithCreateAccessKeyOutput( + &iam.CreateAccessKeyOutput{ + AccessKey: &iam.AccessKey{ + AccessKeyId: aws.String("one"), + SecretAccessKey: aws.String("two"), + UserName: aws.String("JohnDoe"), + }, + }, + ), + ), + ), + }), + }, + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{}), + ), + ), + }, + }, + { + name: "success", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{}), + ), + ), + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + resp, err := p.OnUpdateStorageBucket(context.Background(), tc.req) + if tc.expectedErrContains != "" { + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err).String(), tc.expectedErrCode.String()) + return + } + require.NoError(err) + require.NotNil(resp) + require.NotNil(resp.GetPersisted()) + require.NotNil(resp.GetPersisted().GetData()) + + var disableCredentialRotation bool + expectedAccessKey := tc.req.Persisted.Data.AsMap()[credential.ConstAccessKeyId] + expectedSecretKey := tc.req.Persisted.Data.AsMap()[credential.ConstSecretAccessKey] + if value, ok := tc.req.GetNewBucket().GetAttributes().AsMap()[credential.ConstDisableCredentialRotation]; ok { + disableCredentialRotation = value.(bool) + } + if !disableCredentialRotation { + expectedAccessKey = "one" + expectedSecretKey = "two" + } + require.Equal(expectedAccessKey, resp.GetPersisted().GetData().AsMap()[credential.ConstAccessKeyId]) + require.Equal(expectedSecretKey, resp.GetPersisted().GetData().AsMap()[credential.ConstSecretAccessKey]) + }) + } +} + +func TestStoragePlugin_OnDeleteStorageBucket(t *testing.T) { + cases := []struct { + name string + req *pb.OnDeleteStorageBucketRequest + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "nil bucket", + req: &pb.OnDeleteStorageBucketRequest{}, + expectedErrContains: "bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil attributes", + req: &pb.OnDeleteStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{}, + }, + expectedErrContains: "attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + req: &pb.OnDeleteStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + Attributes: new(structpb.Struct), + }, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + req: &pb.OnDeleteStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "persisted state setup error", + req: &pb.OnDeleteStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error loading persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "delete error", + req: &pb.OnDeleteStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithIAMAPIFunc( + awsutil.NewMockIAM( + awsutil.WithDeleteAccessKeyError(errors.New(testDeleteAccessKeyErr)), + ), + ), + }), + }, + expectedErrContains: fmt.Sprintf("error removing rotated credentials during storage bucket deletion: error deleting old access key: %s", testDeleteAccessKeyErr), + expectedErrCode: codes.Aborted, + }, + { + name: "success", + req: &pb.OnDeleteStorageBucketRequest{ + Bucket: &storagebuckets.StorageBucket{ + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Persisted: &storagebuckets.StorageBucketPersisted{ + Data: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + credential.ConstCredsLastRotatedTime: structpb.NewStringValue("2006-01-02T15:04:05+07:00"), + }, + }, + }, + }, + credOpts: []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithIAMAPIFunc( + awsutil.NewMockIAM( + awsutil.WithDeleteAccessKeyError(nil), + ), + ), + }), + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + resp, err := p.OnDeleteStorageBucket(context.Background(), tc.req) + if tc.expectedErrContains != "" { + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err).String(), tc.expectedErrCode.String()) + return + } + require.NoError(err) + require.NotNil(resp) + }) + } +} + +func TestStoragePlugin_HeadObject(t *testing.T) { + validRequest := func() *pb.HeadObjectRequest { + return &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + } + } + cases := []struct { + name string + req *pb.HeadObjectRequest + contentLength int64 + lastModified time.Time + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "empty object key", + req: &pb.HeadObjectRequest{}, + expectedErrContains: "key is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil bucket", + req: &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + }, + expectedErrContains: "bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "empty bucket name", + req: &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{}, + }, + expectedErrContains: "bucketName is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil secrets", + req: &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + }, + }, + expectedErrContains: "secrets is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil attributes", + req: &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + }, + }, + expectedErrContains: "attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + req: &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: new(structpb.Struct), + }, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + req: &pb.HeadObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "credential persisted state setup error", + req: validRequest(), + credOpts: []credential.AwsCredentialPersistedStateOption{ + func(s *credential.AwsCredentialPersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "storage persisted state setup error", + req: validRequest(), + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "headObject error", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithHeadObjectError(errors.New(testHeadObjectErr)), + ), + ), + }, + expectedErrContains: "error getting head object from s3: test error for HeadObject", + expectedErrCode: codes.Internal, + }, + { + name: "headObject throttle error", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithHeadObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error getting head object from s3: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "success", + req: validRequest(), + credOpts: validSTSMock(), + contentLength: 1024, + lastModified: createTime(t, "2006-01-02T15:04:05Z"), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{ + ContentLength: 1024, + LastModified: aws.Time(createTime(t, "2006-01-02T15:04:05Z")), + }), + ), + ), + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + resp, err := p.HeadObject(context.Background(), tc.req) + if tc.expectedErrContains != "" { + require.Error(err) + require.Nil(resp) + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err).String(), tc.expectedErrCode.String()) + return + } + require.NoError(err) + require.NotNil(resp) + require.Equal(tc.contentLength, resp.ContentLength) + require.Equal(tc.lastModified.String(), resp.LastModified.AsTime().String()) + }) + } +} + +func TestStoragePlugin_ValidatePermissions(t *testing.T) { + validRequest := func() *pb.ValidatePermissionsRequest { + return &pb.ValidatePermissionsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + } + } + cases := []struct { + name string + req *pb.ValidatePermissionsRequest + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "nil bucket", + req: &pb.ValidatePermissionsRequest{}, + expectedErrContains: "bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "empty bucket name", + req: &pb.ValidatePermissionsRequest{ + Bucket: &storagebuckets.StorageBucket{}, + }, + expectedErrContains: "bucketName is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil secrets", + req: &pb.ValidatePermissionsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + }, + }, + expectedErrContains: "secrets is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil attributes", + req: &pb.ValidatePermissionsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + }, + }, + expectedErrContains: "attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + req: &pb.ValidatePermissionsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: new(structpb.Struct), + }, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + req: &pb.ValidatePermissionsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "credential persisted state setup error", + req: validRequest(), + credOpts: []credential.AwsCredentialPersistedStateOption{ + func(s *credential.AwsCredentialPersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "storage persisted state setup error", + req: validRequest(), + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "dryRunValidation failed putObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc(newTestMockS3( + nil, + testMockS3WithPutObjectError(errors.New(testPutObjectErr)), + )), + }, + expectedErrContains: "error failed to put object: test error for PutObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed putObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc(newTestMockS3( + nil, + testMockS3WithPutObjectError(new(throttleErr)), + )), + }, + expectedErrContains: "error failed to put object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "dryRunValidation failed getObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectError(errors.New(testGetObjectErr)), + ), + ), + }, + expectedErrContains: "error failed to get object: test error for GetObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed getObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error failed to get object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "dryRunValidation failed headObject", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectError(errors.New(testHeadObjectErr)), + ), + ), + }, + expectedErrContains: "error failed to get head object: test error for HeadObject", + expectedErrCode: codes.Internal, + }, + { + name: "dryRunValidation failed headObject throttle", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error failed to get head object: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "success", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{}), + testMockS3WithHeadObjectOutput(&s3.HeadObjectOutput{}), + ), + ), + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + resp, err := p.ValidatePermissions(context.Background(), tc.req) + if tc.expectedErrContains != "" { + require.Error(err) + require.Nil(resp) + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err).String(), tc.expectedErrCode.String()) + return + } + require.NoError(err) + require.NotNil(resp) + }) + } +} + +func TestStoragePlugin_GetObject(t *testing.T) { + validRequest := func() *pb.GetObjectRequest { + return &pb.GetObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }}, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + } + } + cases := []struct { + name string + req *pb.GetObjectRequest + objectData []byte + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "empty object key", + req: &pb.GetObjectRequest{}, + expectedErrContains: "key is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil bucket", + req: &pb.GetObjectRequest{ + Key: "/foo/bar/key", + }, + expectedErrContains: "bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing bucket name", + req: &pb.GetObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{}, + }, + expectedErrContains: "bucketName is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil secrets", + req: &pb.GetObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + }, + }, + expectedErrContains: "secrets is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "nil attributes", + req: &pb.GetObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + }, + }, + expectedErrContains: "attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + req: &pb.GetObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: new(structpb.Struct), + }, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + req: &pb.GetObjectRequest{ + Key: "/foo/bar/key", + Bucket: &storagebuckets.StorageBucket{ + BucketName: "foo", + Secrets: new(structpb.Struct), + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "credential persisted state setup error", + req: validRequest(), + credOpts: []credential.AwsCredentialPersistedStateOption{ + func(s *credential.AwsCredentialPersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "storage persisted state setup error", + req: validRequest(), + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "getObject error", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithGetObjectError(errors.New(testGetObjectErr)), + ), + ), + }, + expectedErrContains: "error getting object from s3: test error for GetObject", + expectedErrCode: codes.Internal, + }, + { + name: "getObject throttle error", + req: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithGetObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error getting object from s3: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "with chunk size", + req: func() *pb.GetObjectRequest { + request := validRequest() + request.ChunkSize = 16384 + return request + }(), + credOpts: validSTSMock(), + objectData: []byte("test"), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{ + Body: ioutil.NopCloser(bytes.NewReader([]byte("test"))), + }), + ), + ), + }, + }, + { + name: "success", + req: validRequest(), + credOpts: validSTSMock(), + objectData: []byte("test"), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithGetObjectOutput(&s3.GetObjectOutput{ + Body: ioutil.NopCloser(bytes.NewReader([]byte("test"))), + }), + ), + ), + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + stream := newGetObjectStreamMock() + err := p.GetObject(tc.req, stream) + if tc.expectedErrContains != "" { + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(status.Code(err).String(), tc.expectedErrCode.String()) + return + } + require.NoError(err) + + resp, err := stream.Recv() + require.NoError(err) + require.NotNil(resp) + require.EqualValues(tc.objectData, resp.FileChunk) + }) + } +} + +func TestStoragePlugin_PutObject(t *testing.T) { + td := t.TempDir() + + emptyFilePath := path.Join(td, "empty-file-test") + emptyFile, err := os.Create(emptyFilePath) + require.NoError(t, err) + require.NoError(t, emptyFile.Close()) + + validFilePath := path.Join(td, "valid-file-test") + validFile, err := os.Create(validFilePath) + require.NoError(t, err) + n, err := validFile.WriteString("CONTENT CHECK") + require.NoError(t, err) + require.Equal(t, len("CONTENT CHECK"), n) + require.NoError(t, validFile.Close()) + + validRequest := func() *pb.PutObjectRequest { + return &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Key: "mock-object", + Path: validFilePath, + } + } + + cases := []struct { + name string + credOpts []credential.AwsCredentialPersistedStateOption + storageOpts []awsStoragePersistedStateOption + request *pb.PutObjectRequest + setup func() + expectedObject []byte + expectedErrContains string + expectedErrCode codes.Code + }{ + { + name: "missing request", + expectedErrContains: "request is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing object key", + request: &pb.PutObjectRequest{}, + expectedErrContains: "key is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing bucket", + request: &pb.PutObjectRequest{ + Key: "mock-object", + }, + expectedErrContains: "bucket is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing bucket name", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{}, + Key: "mock-object", + }, + expectedErrContains: "bucketName is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing bucket secrets", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + }, + Key: "mock-object", + }, + expectedErrContains: "secrets is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing attributes", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + }, + Key: "mock-object", + }, + expectedErrContains: "attributes is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "missing path", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Key: "mock-object", + }, + expectedErrContains: "path is required", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "file does not exist", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Key: "mock-object", + Path: path.Join(td, "file-does-not-exist"), + }, + expectedErrContains: "failed to open file", + expectedErrCode: codes.Internal, + }, + { + name: "path is a directory", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Key: "mock-object", + Path: td, + }, + expectedErrContains: "path is not a file", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "file is empty", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("us-west-2"), + }, + }, + }, + Key: "mock-object", + Path: emptyFilePath, + }, + expectedErrContains: "file is empty", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "error reading attributes", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{}, + }, + }, + Key: "mock-object", + Path: validFilePath, + }, + expectedErrContains: "missing required value \"region\"", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "invalid region", + request: &pb.PutObjectRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "external-obj-store", + Secrets: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstAccessKeyId: structpb.NewStringValue("foobar"), + credential.ConstSecretAccessKey: structpb.NewStringValue("bazqux"), + }, + }, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + credential.ConstRegion: structpb.NewStringValue("foobar"), + }, + }, + }, + Key: "mock-object", + Path: validFilePath, + }, + expectedErrContains: "not a valid region: foobar", + expectedErrCode: codes.InvalidArgument, + }, + { + name: "credential persisted state setup error", + request: validRequest(), + credOpts: []credential.AwsCredentialPersistedStateOption{ + func(s *credential.AwsCredentialPersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "storage persisted state setup error", + request: validRequest(), + storageOpts: []awsStoragePersistedStateOption{ + func(s *awsStoragePersistedState) error { + return errors.New(testOptionErr) + }, + }, + expectedErrContains: fmt.Sprintf("error setting up persisted state: %s", testOptionErr), + expectedErrCode: codes.InvalidArgument, + }, + { + name: "putObject error", + request: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectError(errors.New(testPutObjectErr)), + ), + ), + }, + expectedErrContains: "error putting object into s3: test error for PutObject", + expectedErrCode: codes.Internal, + }, + { + name: "throttle error", + request: validRequest(), + credOpts: validSTSMock(), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + nil, + testMockS3WithPutObjectError(new(throttleErr)), + ), + ), + }, + expectedErrContains: "error putting object into s3: ThrottlingException", + expectedErrCode: codes.Unavailable, + }, + { + name: "missing checksum from aws", + request: validRequest(), + credOpts: validSTSMock(), + expectedObject: []byte("CONTENT CHECK"), + storageOpts: []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + &testMockS3State{ + PutObjectBody: []byte("CONTENT CHECK"), + }, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{}), + ), + ), + }, + expectedErrContains: "missing checksum response from aws", + expectedErrCode: codes.Internal, + }, + { + name: "mismatched checksum", + request: validRequest(), + credOpts: validSTSMock(), + expectedObject: []byte("CONTENT CHECK"), + storageOpts: func() []awsStoragePersistedStateOption { + hash := sha256.New() + _, err := hash.Write([]byte("CONTENT CHECK")) + require.NoError(t, err) + return []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + &testMockS3State{ + PutObjectBody: []byte("CONTENT CHECK"), + }, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{ + ChecksumSHA256: aws.String(string(hash.Sum(nil))), + }), + ), + ), + } + }(), + expectedErrContains: "mismatched checksum", + expectedErrCode: codes.Internal, + }, + { + name: "valid file", + request: validRequest(), + credOpts: validSTSMock(), + expectedObject: []byte("CONTENT CHECK"), + storageOpts: func() []awsStoragePersistedStateOption { + hash := sha256.New() + _, err := hash.Write([]byte("CONTENT CHECK")) + require.NoError(t, err) + checksum := base64.StdEncoding.EncodeToString(hash.Sum(nil)) + return []awsStoragePersistedStateOption{ + withTestS3APIFunc( + newTestMockS3( + &testMockS3State{ + PutObjectBody: []byte("CONTENT CHECK"), + }, + testMockS3WithPutObjectOutput(&s3.PutObjectOutput{ + ChecksumSHA256: aws.String(checksum), + }), + ), + ), + } + }(), + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + p := &StoragePlugin{ + testCredStateOpts: tc.credOpts, + testStorageStateOpts: tc.storageOpts, + } + if tc.request != nil && len(tc.expectedObject) > 0 { + data, err := os.ReadFile(tc.request.Path) + require.NoError(err) + require.ElementsMatch(data, tc.expectedObject) + } + resp, err := p.PutObject(context.Background(), tc.request) + if tc.expectedErrContains != "" { + require.Error(err) + require.Contains(err.Error(), tc.expectedErrContains) + require.Equal(tc.expectedErrCode.String(), status.Code(err).String()) + return + } + require.NoError(err) + require.NotNil(resp) + + hash := sha256.New() + _, err = hash.Write(tc.expectedObject) + require.NoError(err) + require.True(bytes.Equal(hash.Sum(nil), resp.GetChecksumSha_256())) + }) + } +} diff --git a/plugin/service/storage/state.go b/plugin/service/storage/state.go new file mode 100644 index 0000000..fe6b530 --- /dev/null +++ b/plugin/service/storage/state.go @@ -0,0 +1,189 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets" + "google.golang.org/protobuf/types/known/structpb" +) + +type S3API interface { + DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) +} + +var customClient = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + }, +} + +type awsStoragePersistedState struct { + *credential.AwsCredentialPersistedState + + // testS3APIFunc provides a way to provide a factory for a mock EC2 client + testS3APIFunc s3APIFunc +} + +type s3APIFunc func(...aws.Config) (S3API, error) + +type awsStoragePersistedStateOption func(s *awsStoragePersistedState) error + +func withTestS3APIFunc(f s3APIFunc) awsStoragePersistedStateOption { + return func(s *awsStoragePersistedState) error { + if s.testS3APIFunc != nil { + return errors.New("test API function already set") + } + + s.testS3APIFunc = f + return nil + } +} + +func withCredentials(x *credential.AwsCredentialPersistedState) awsStoragePersistedStateOption { + return func(s *awsStoragePersistedState) error { + if s.AwsCredentialPersistedState != nil { + return errors.New("aws credentials already set") + } + + s.AwsCredentialPersistedState = x + return nil + } +} + +func newAwsStoragePersistedState(opts ...awsStoragePersistedStateOption) (*awsStoragePersistedState, error) { + s := new(awsStoragePersistedState) + for _, opt := range opts { + if err := opt(s); err != nil { + return nil, err + } + } + return s, nil +} + +func (s *awsStoragePersistedState) toProto() (*storagebuckets.StorageBucketPersisted, error) { + + data, err := structpb.NewStruct(s.ToMap()) + if err != nil { + return nil, fmt.Errorf("error converting state to structpb.Struct: %w", err) + } + return &storagebuckets.StorageBucketPersisted{Data: data}, nil +} + +// S3Client returns a configured S3 client based on the session +// information stored in the state. +func (s *awsStoragePersistedState) S3Client(ctx context.Context, opt ...s3Option) (S3API, error) { + opts, err := getOpts(opt...) + if err != nil { + return nil, fmt.Errorf("error parsing options when fetching S3 client: %w", err) + } + + awsConfigOpts := []func(*config.LoadOptions) error{} + sess, err := s.GetSession() + if err != nil { + return nil, fmt.Errorf("error getting AWS session when fetching S3 client: %w", err) + } + creds, err := sess.Config.Credentials.Get() + if err != nil { + return nil, fmt.Errorf("error getting AWS session when fetching S3 client: %w", err) + } + customCredentials := config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken), + ) + awsConfigOpts = append(awsConfigOpts, customCredentials) + if sess.Config.Region != nil { + awsConfigOpts = append(awsConfigOpts, config.WithRegion(*sess.Config.Region)) + } + if opts.withRegion != "" { + awsConfigOpts = append(awsConfigOpts, config.WithRegion(opts.withRegion)) + } + if opts.withEndpoint != "" { + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + if service == s3.ServiceID { + if opts.withRegion != "" { + region = opts.withRegion + } + return aws.Endpoint{ + PartitionID: "aws", + URL: opts.withEndpoint, + SigningRegion: region, + }, nil + } + // returning EndpointNotFoundError will allow the service to fallback to it's default resolution + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + }) + awsConfigOpts = append(awsConfigOpts, config.WithEndpointResolverWithOptions(customResolver)) + } + cfg, err := config.LoadDefaultConfig(ctx, awsConfigOpts...) + if err != nil { + return nil, fmt.Errorf("error loading aws configuration when fetching S3 client: %w", err) + } + + if s.testS3APIFunc != nil { + return s.testS3APIFunc(cfg) + } + + cfg.HTTPClient = customClient + + return s3.NewFromConfig(cfg), nil +} + +// getOpts iterates the inbound s3Options and returns a struct +func getOpts(opt ...s3Option) (s3Options, error) { + opts := getDefaultS3Options() + for _, o := range opt { + if o == nil { + continue + } + if err := o(&opts); err != nil { + return s3Options{}, err + } + } + return opts, nil +} + +// s3Option - how s3Options are passed as arguments +type s3Option func(*s3Options) error + +// options = how options are represented +type s3Options struct { + withRegion string + withEndpoint string +} + +func getDefaultS3Options() s3Options { + return s3Options{} +} + +// WithRegion contains the region to use +func WithRegion(with string) s3Option { + return func(o *s3Options) error { + o.withRegion = with + return nil + } +} + +// WithEndpoint contains the endpoint to use +func WithEndpoint(with string) s3Option { + return func(o *s3Options) error { + o.withEndpoint = with + return nil + } +} diff --git a/plugin/service/storage/state_test.go b/plugin/service/storage/state_test.go new file mode 100644 index 0000000..d384dcd --- /dev/null +++ b/plugin/service/storage/state_test.go @@ -0,0 +1,40 @@ +package storage + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/stretchr/testify/require" +) + +func TestAwsStoragePersistedStateS3Client(t *testing.T) { + require := require.New(t) + cred, err := credential.NewAwsCredentialPersistedState( + credential.WithRegion("us-west-2"), + credential.WithAccessKeyId("foobar"), + credential.WithSecretAccessKey("barfoo"), + ) + require.NoError(err) + + state, err := newAwsStoragePersistedState( + withCredentials(cred), + withTestS3APIFunc(newTestMockS3(nil)), + ) + require.NoError(err) + + clientRaw, err := state.S3Client(context.Background(), WithRegion("us-west-2"), WithEndpoint("0.0.0.0")) + require.NoError(err) + require.NotNil(clientRaw) + client, ok := clientRaw.(*testMockS3) + require.True(ok) + require.NotNil(client) + require.Equal("us-west-2", client.Region) + + endpoint, err := client.Endpoint.ResolveEndpoint(s3.ServiceID, client.Region) + require.NoError(err) + require.Equal("0.0.0.0", endpoint.URL) + require.Equal("us-west-2", endpoint.SigningRegion) + require.Equal("aws", endpoint.PartitionID) +} diff --git a/plugin/service/storage/testing.go b/plugin/service/storage/testing.go new file mode 100644 index 0000000..4cb84cc --- /dev/null +++ b/plugin/service/storage/testing.go @@ -0,0 +1,300 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/hashicorp/boundary-plugin-host-aws/internal/credential" + "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets" + pb "github.com/hashicorp/boundary/sdk/pbs/plugin" + "github.com/hashicorp/go-secure-stdlib/awsutil" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/structpb" +) + +const ( + testOptionErr = "test option error" + testGetUserErr = "test error for GetUser" + testGetCallerIdentityErr = "test error for GetCallerIdentity" + testDeleteAccessKeyErr = "test error for DeleteAccessKey" + testGetObjectErr = "test error for GetObject" + testPutObjectErr = "test error for PutObject" + testHeadObjectErr = "test error for HeadObject" +) + +// throttleErr is a mocked error used for testing the aws s3 client +type throttleErr struct{} + +func (e *throttleErr) ErrorCode() string { + return "ThrottlingException" +} + +func (e *throttleErr) Error() string { + return "ThrottlingException" +} + +type testMockS3State struct { + GetObjectCalled bool + GetObjectInputParams *s3.GetObjectInput + + PutObjectCalled bool + PutObjectInputParams *s3.PutObjectInput + PutObjectBody []byte + + HeadObjectCalled bool + HeadObjectInputParams *s3.HeadObjectInput +} + +func (s *testMockS3State) Reset() { + s.GetObjectCalled = false + s.GetObjectInputParams = nil + s.PutObjectCalled = false + s.PutObjectInputParams = nil + s.PutObjectBody = nil + s.HeadObjectCalled = false + s.HeadObjectInputParams = nil +} + +type testMockS3 struct { + S3API + + State *testMockS3State + Region string + Endpoint aws.EndpointResolverWithOptions + + // mocked responses for getObject + GetObjectOutput *s3.GetObjectOutput + GetObjectErr error + + // mocked responses for putObject + PutObjectOutput *s3.PutObjectOutput + PutObjectErr error + + // mocked responses for headObject + HeadObjectOutput *s3.HeadObjectOutput + HeadObjectErr error +} + +type testMockS3Option func(m *testMockS3) error + +func testMockS3WithPutObjectOutput(o *s3.PutObjectOutput) testMockS3Option { + return func(m *testMockS3) error { + m.PutObjectOutput = o + return nil + } +} + +func testMockS3WithPutObjectError(e error) testMockS3Option { + return func(m *testMockS3) error { + m.PutObjectErr = e + return nil + } +} + +func testMockS3WithGetObjectOutput(o *s3.GetObjectOutput) testMockS3Option { + return func(m *testMockS3) error { + m.GetObjectOutput = o + return nil + } +} + +func testMockS3WithGetObjectError(e error) testMockS3Option { + return func(m *testMockS3) error { + m.GetObjectErr = e + return nil + } +} + +func testMockS3WithHeadObjectOutput(o *s3.HeadObjectOutput) testMockS3Option { + return func(m *testMockS3) error { + m.HeadObjectOutput = o + return nil + } +} + +func testMockS3WithHeadObjectError(e error) testMockS3Option { + return func(m *testMockS3) error { + m.HeadObjectErr = e + return nil + } +} + +func newTestMockS3(state *testMockS3State, opts ...testMockS3Option) s3APIFunc { + return func(cfgs ...aws.Config) (S3API, error) { + m := &testMockS3{ + State: state, + } + + for _, opt := range opts { + if err := opt(m); err != nil { + return nil, err + } + } + + for _, cfg := range cfgs { + // Last region takes precedence + if cfg.Region != "" { + m.Region = cfg.Region + } + if cfg.EndpointResolverWithOptions != nil { + m.Endpoint = cfg.EndpointResolverWithOptions + } + } + + return m, nil + } +} + +func (m *testMockS3) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + if m.State != nil { + m.State.GetObjectCalled = true + m.State.GetObjectInputParams = params + } + + if m.GetObjectErr != nil { + return nil, m.GetObjectErr + } + + return m.GetObjectOutput, nil +} + +func (m *testMockS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if m.State != nil { + m.State.PutObjectCalled = true + m.State.PutObjectInputParams = params + } + + if m.PutObjectErr != nil { + return nil, m.PutObjectErr + } + + // Reads and Writes on the pipe are matched one to one except when multiple Reads are needed to consume a single Write. + // Therefore, we need to read the entire body to ensure that the pipe is not blocked. + if params.Body != nil { + data, err := io.ReadAll(params.Body) + if err != nil { + return nil, err + } + if m.State == nil || m.State.PutObjectBody == nil { + return m.PutObjectOutput, nil + } + if !bytes.Equal(data, m.State.PutObjectBody) { + return nil, fmt.Errorf("expected body %q, got %q", m.State.PutObjectBody, data) + } + } + + return m.PutObjectOutput, nil +} + +func (m *testMockS3) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + return &s3.DeleteObjectOutput{}, nil +} + +func (m *testMockS3) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + if m.State != nil { + m.State.HeadObjectCalled = true + m.State.HeadObjectInputParams = params + } + + if m.HeadObjectErr != nil { + return nil, m.HeadObjectErr + } + + return m.HeadObjectOutput, nil +} + +type getObjectStreamMock struct { + grpc.ServerStream + ctx context.Context + err error + data *pb.GetObjectResponse +} + +func (s *getObjectStreamMock) Context() context.Context { + return s.ctx +} + +// Send streams a response message to the client. +// This function should only be called on the server side. +func (g *getObjectStreamMock) Send(req *pb.GetObjectResponse) error { + if req == nil { + return fmt.Errorf(`parameter arg "resp GetObjectResponse" cannot be nil.`) + } + g.data = req + return nil +} + +// Recv streams a response message from the server. +// This function should only be called on the client side. +func (g *getObjectStreamMock) Recv() (*pb.GetObjectResponse, error) { + if g.data == nil { + if g.err != nil { + return nil, g.err + } + return nil, io.EOF + } + resp := &pb.GetObjectResponse{ + FileChunk: append([]byte{}, g.data.FileChunk...), + } + g.data = nil + return resp, nil +} + +func newGetObjectStreamMock() *getObjectStreamMock { + return &getObjectStreamMock{ + ctx: context.Background(), + } +} + +func createTime(t *testing.T, timestamp string) time.Time { + require := require.New(t) + testTime, err := time.Parse(time.RFC3339, timestamp) + require.NoError(err) + return testTime +} + +func deepCopyPutObjectRequest(v *pb.PutObjectRequest) *pb.PutObjectRequest { + deepCopy := &pb.PutObjectRequest{ + Key: v.Key, + Path: v.Path, + } + if v.Bucket != nil { + deepCopy.Bucket = &storagebuckets.StorageBucket{ + BucketName: v.Bucket.BucketName, + BucketPrefix: v.Bucket.BucketPrefix, + } + if v.Bucket.Attributes != nil { + attrs, _ := structpb.NewStruct(v.Bucket.Attributes.AsMap()) + deepCopy.Bucket.Attributes = attrs + } + if v.Bucket.Secrets != nil { + secrets, _ := structpb.NewStruct(v.Bucket.Secrets.AsMap()) + deepCopy.Bucket.Secrets = secrets + } + } + return deepCopy +} + +func validSTSMock() []credential.AwsCredentialPersistedStateOption { + return []credential.AwsCredentialPersistedStateOption{ + credential.WithStateTestOpts([]awsutil.Option{ + awsutil.WithSTSAPIFunc( + awsutil.NewMockSTS( + awsutil.WithGetCallerIdentityOutput(&sts.GetCallerIdentityOutput{ + Account: aws.String("0123456789"), + Arn: aws.String("arn:aws:iam::0123456789:user/test"), + UserId: aws.String("test"), + }), + ), + ), + }), + } +}