Elim Tsiagbey
c2e7c94f9c
* fix: Append list error for permission validation Instead of returning the error right away, append the error when `ListObject` is successful but the key expected if not found. Set the `Read` permission state as `Unknown` if the response from the list calls does not find the expected key. This can be caused if the credentials do not have `s3:PutObject` but have `s3:ListBucket`. Without `s3:PutObject`, the object will not be written to S3 so other calls to find the object will not be successful even if they have valid permissions. This prevents us from returning a `write` error on Boundary because the list call will always fail. Without this, the Permissions are not wrapped and the SBC states are all set to `unknown` even though we know certain permission failed.
849 lines
27 KiB
Go
849 lines
27 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package storage
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/google/uuid"
|
|
cred "github.com/hashicorp/boundary-plugin-aws/internal/credential"
|
|
"github.com/hashicorp/boundary-plugin-aws/internal/errors"
|
|
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets"
|
|
pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
|
|
"github.com/hashicorp/go-multierror"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
)
|
|
|
|
// Ensure that we are implementing StoragePluginServiceServer
|
|
var _ pb.StoragePluginServiceServer = (*StoragePlugin)(nil)
|
|
|
|
// StoragePlugin implements the StoragePluginServiceServer interface for the
|
|
// AWS storage service plugin.
|
|
type StoragePlugin struct {
|
|
pb.UnimplementedStoragePluginServiceServer
|
|
|
|
// testCredStateOpts are passed in to the stored state to control test behavior
|
|
testCredStateOpts []cred.AwsCredentialPersistedStateOption
|
|
|
|
// testStorageStateOpts are passed in to the stored state to control test behavior
|
|
testStorageStateOpts []awsStoragePersistedStateOption
|
|
}
|
|
|
|
// OnCreateStorageBucket is called when a storage bucket is created.
|
|
func (p *StoragePlugin) OnCreateStorageBucket(ctx context.Context, req *pb.OnCreateStorageBucketRequest) (*pb.OnCreateStorageBucketResponse, error) {
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return nil, errors.BadRequestStatus("bucket is required")
|
|
}
|
|
if bucket.BucketName == "" {
|
|
return nil, errors.BadRequestStatus("bucketName is required")
|
|
}
|
|
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return nil, errors.BadRequestStatus("attributes is required")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set up the creds in our persisted state.
|
|
credState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(credConfig),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
// Try to rotate AWS static credentials
|
|
if cred.GetCredentialType(credConfig) == cred.StaticAWS && !storageAttributes.DisableCredentialRotation {
|
|
if err := credState.RotateCreds(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
// perform dry run to ensure we can interact with the bucket as expected.
|
|
if st := dryRunValidation(ctx, storageState, storageAttributes, bucket); st != nil {
|
|
return nil, st.Err()
|
|
}
|
|
|
|
persistedProto, err := storageState.toProto()
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus(err.Error())
|
|
}
|
|
|
|
return &pb.OnCreateStorageBucketResponse{
|
|
Persisted: persistedProto,
|
|
}, nil
|
|
}
|
|
|
|
// OnUpdateStorageBucket is called when a storage bucket is updated.
|
|
func (p *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUpdateStorageBucketRequest) (*pb.OnUpdateStorageBucketResponse, error) {
|
|
// For storage buckets we only need to really validate what is
|
|
// currently in the new copy of the storage bucket data, so skip
|
|
// fetching the new copy to keep things a little less messy.
|
|
newBucket := req.GetNewBucket()
|
|
if newBucket == nil {
|
|
return nil, errors.BadRequestStatus("new bucket is required")
|
|
}
|
|
if newBucket.BucketName == "" {
|
|
return nil, errors.BadRequestStatus("new bucketName is required")
|
|
}
|
|
currentBucket := req.GetCurrentBucket()
|
|
if currentBucket == nil {
|
|
return nil, errors.BadRequestStatus("current bucket is required")
|
|
}
|
|
|
|
oldAttrs := currentBucket.GetAttributes()
|
|
if oldAttrs == nil {
|
|
return nil, errors.BadRequestStatus("current bucket attributes is required")
|
|
}
|
|
oldStorageAttributes, err := getStorageAttributes(oldAttrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
newAttrs := newBucket.GetAttributes()
|
|
if newAttrs == nil {
|
|
return nil, errors.BadRequestStatus("new bucket attributes is required")
|
|
}
|
|
newStorageAttributes, err := getStorageAttributes(newAttrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
credState, err := cred.AwsCredentialPersistedStateFromProto(
|
|
req.GetPersisted().GetData(),
|
|
oldStorageAttributes.CredentialAttributes,
|
|
p.testCredStateOpts...)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
|
|
}
|
|
|
|
// Verify the incoming credentials are valid and return any errors to the
|
|
// user if they're not. Note this doesn't validate the credentials against
|
|
// AWS - it only does logical validation on the fields.
|
|
updatedCredentials, err := cred.GetCredentialsConfig(newBucket.GetSecrets(), newStorageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// We will be updating credentials when one of two changes occur:
|
|
// 1. the new bucket has secrets for static credentials, in which case we need to delete the old static credentials
|
|
// 2. the new bucket has a different roleARN value for dynamic credentials
|
|
if newBucket.GetSecrets() != nil || newStorageAttributes.RoleArn != oldStorageAttributes.RoleArn {
|
|
// Ensure the incoming credentials are valid for interaction with the S3
|
|
// Bucket before replacing them.
|
|
newCredState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(updatedCredentials),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up new credential persisted state: %s", err)
|
|
}
|
|
newStorageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(newCredState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
|
|
}
|
|
if st := dryRunValidation(ctx, newStorageState, newStorageAttributes, newBucket); st != nil {
|
|
return nil, st.Err()
|
|
}
|
|
|
|
// Replace the existing credential state.
|
|
// This checks the timestamp on the last rotation time as well
|
|
// and deletes the credentials if we are managing them
|
|
// (ie: if we've rotated them before).
|
|
if err := credState.ReplaceCreds(ctx, updatedCredentials); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if cred.GetCredentialType(credState.CredentialsConfig) == cred.StaticAWS {
|
|
// This is a validate check to make sure that we aren't disabling
|
|
// rotation for credentials currently being managed by rotation.
|
|
// This is not allowed.
|
|
if newStorageAttributes.DisableCredentialRotation && newBucket.GetSecrets() == nil {
|
|
if !credState.CredsLastRotatedTime.IsZero() {
|
|
return nil, errors.BadRequestStatus("cannot disable rotation for already-rotated credentials")
|
|
}
|
|
}
|
|
|
|
// If we're enabling rotation now but didn't before, or have
|
|
// freshly replaced credentials, we can rotate here.
|
|
if !newStorageAttributes.DisableCredentialRotation && credState.CredsLastRotatedTime.IsZero() {
|
|
if err := credState.RotateCreds(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
|
|
}
|
|
|
|
// perform dry run to ensure we can interact with the bucket as expected.
|
|
if st := dryRunValidation(ctx, storageState, newStorageAttributes, newBucket); st != nil {
|
|
return nil, st.Err()
|
|
}
|
|
|
|
persistedProto, err := storageState.toProto()
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus(err.Error())
|
|
}
|
|
|
|
return &pb.OnUpdateStorageBucketResponse{
|
|
Persisted: persistedProto,
|
|
}, nil
|
|
}
|
|
|
|
// OnDeleteStorageBucket is called when a storage bucket is deleted.
|
|
func (p *StoragePlugin) OnDeleteStorageBucket(ctx context.Context, req *pb.OnDeleteStorageBucketRequest) (*pb.OnDeleteStorageBucketResponse, error) {
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return nil, errors.BadRequestStatus("bucket is required")
|
|
}
|
|
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return nil, errors.BadRequestStatus("attributes is required")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Get the persisted data.
|
|
// NOTE: We return on error here, blocking the delete. This may or
|
|
// may not be an overzealous approach to maintaining database/state
|
|
// integrity. May need to be changed at later time if there are
|
|
// scenarios where we might be deleting things and any secret state
|
|
// may be corrupt/and or legitimately missing.
|
|
credState, err := cred.AwsCredentialPersistedStateFromProto(
|
|
req.GetPersisted().GetData(),
|
|
storageAttributes.CredentialAttributes,
|
|
p.testCredStateOpts...)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
|
|
}
|
|
|
|
_, err = newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error loading persisted state: %s", err)
|
|
}
|
|
|
|
// try to delete static credentials
|
|
if cred.GetCredentialType(credState.CredentialsConfig) == cred.StaticAWS {
|
|
if !credState.CredsLastRotatedTime.IsZero() {
|
|
// Delete old/existing credentials. This is done with the same
|
|
// credentials to ensure that it has the proper permissions to do
|
|
// it.
|
|
if err := credState.DeleteCreds(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
return &pb.OnDeleteStorageBucketResponse{}, nil
|
|
}
|
|
|
|
// HeadObject is called to get the metadata of an object.
|
|
func (p *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectRequest) (*pb.HeadObjectResponse, error) {
|
|
if req.GetKey() == "" {
|
|
return nil, errors.BadRequestStatus("key is required")
|
|
}
|
|
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return nil, errors.BadRequestStatus("bucket is required")
|
|
}
|
|
|
|
if bucket.BucketName == "" {
|
|
return nil, errors.BadRequestStatus("bucketName is required")
|
|
}
|
|
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return nil, errors.BadRequestStatus("attributes is required")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set up the creds in our persisted state.
|
|
credState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(credConfig),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
opts := []s3Option{}
|
|
if storageAttributes.EndpointUrl != "" {
|
|
opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
|
|
}
|
|
s3Client, err := storageState.S3Client(ctx, opts...)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
|
|
}
|
|
|
|
objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
|
|
resp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
if err != nil {
|
|
return nil, parseS3Error(err, "failed to head object", req).Err()
|
|
}
|
|
return &pb.HeadObjectResponse{
|
|
ContentLength: resp.ContentLength,
|
|
LastModified: timestamppb.New(*resp.LastModified),
|
|
}, nil
|
|
}
|
|
|
|
// ValidatePermissions is called to validate the secrets associated with the storage bucket.
|
|
func (p *StoragePlugin) ValidatePermissions(ctx context.Context, req *pb.ValidatePermissionsRequest) (*pb.ValidatePermissionsResponse, error) {
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return nil, errors.BadRequestStatus("bucket is required")
|
|
}
|
|
|
|
if bucket.BucketName == "" {
|
|
return nil, errors.BadRequestStatus("bucketName is required")
|
|
}
|
|
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return nil, errors.BadRequestStatus("attributes is required")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set up the creds in our persisted state.
|
|
credState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(credConfig),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
// perform dry run to ensure we can interact with the bucket as expected.
|
|
if st := dryRunValidation(ctx, storageState, storageAttributes, bucket); st != nil {
|
|
return nil, st.Err()
|
|
}
|
|
|
|
return &pb.ValidatePermissionsResponse{}, nil
|
|
}
|
|
|
|
// GetObject is called when retrieving objects from an s3 bucket.
|
|
// GetObject is a blocking call until the stream has been recieved in full.
|
|
func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePluginService_GetObjectServer) error {
|
|
if req.GetKey() == "" {
|
|
return errors.BadRequestStatus("key is required")
|
|
}
|
|
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return errors.BadRequestStatus("bucket is required")
|
|
}
|
|
|
|
if bucket.GetBucketName() == "" {
|
|
return errors.BadRequestStatus("bucketName is required")
|
|
}
|
|
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return errors.BadRequestStatus("attributes is required")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Set up the creds in our persisted state.
|
|
credState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(credConfig),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
opts := []s3Option{}
|
|
if storageAttributes.EndpointUrl != "" {
|
|
opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
|
|
}
|
|
s3Client, err := storageState.S3Client(stream.Context(), opts...)
|
|
if err != nil {
|
|
return errors.BadRequestStatus("error getting S3 client: %s", err)
|
|
}
|
|
|
|
objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
|
|
resp, err := s3Client.GetObject(stream.Context(), &s3.GetObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
if err != nil {
|
|
return parseS3Error(err, "failed to get object", req).Err()
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
reader := bufio.NewReader(resp.Body)
|
|
chunkSize := req.GetChunkSize()
|
|
if chunkSize == 0 {
|
|
chunkSize = defaultStreamChunkSize
|
|
}
|
|
for {
|
|
buffer := make([]byte, chunkSize)
|
|
n, err := reader.Read(buffer)
|
|
if err != nil && err != io.EOF {
|
|
return errors.UnknownStatus("error reading chunk from s3: %s", err)
|
|
}
|
|
if n > 0 {
|
|
if err := stream.Send(&pb.GetObjectResponse{
|
|
FileChunk: buffer[:n],
|
|
}); err != nil {
|
|
return errors.UnknownStatus("error sending chunk to client: %s", err)
|
|
}
|
|
}
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// PutObject is called when putting objects into an s3 bucket.
|
|
func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest) (*pb.PutObjectResponse, error) {
|
|
if req == nil {
|
|
return nil, errors.BadRequestStatus("request is required")
|
|
}
|
|
if req.GetKey() == "" {
|
|
return nil, errors.BadRequestStatus("key is required")
|
|
}
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return nil, errors.BadRequestStatus("bucket is required")
|
|
}
|
|
if bucket.GetBucketName() == "" {
|
|
return nil, errors.BadRequestStatus("bucketName is required")
|
|
}
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return nil, errors.BadRequestStatus("attributes is required")
|
|
}
|
|
if req.GetPath() == "" {
|
|
return nil, errors.BadRequestStatus("path is required")
|
|
}
|
|
file, err := os.Open(req.GetPath())
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("failed to open file")
|
|
}
|
|
info, err := file.Stat()
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("failed to read file info")
|
|
}
|
|
if info.IsDir() {
|
|
return nil, errors.BadRequestStatus("path is not a file")
|
|
}
|
|
if info.Size() == 0 {
|
|
return nil, errors.BadRequestStatus("file is empty")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set up the creds in our persisted state.
|
|
credState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(credConfig),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
opts := []s3Option{}
|
|
if storageAttributes.EndpointUrl != "" {
|
|
opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
|
|
}
|
|
s3Client, err := storageState.S3Client(ctx, opts...)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
|
|
}
|
|
|
|
hash := sha256.New()
|
|
if _, err := io.Copy(hash, file); err != nil {
|
|
return nil, errors.UnknownStatus("failed to calcualte hash")
|
|
}
|
|
checksum := base64.StdEncoding.EncodeToString(hash.Sum(nil))
|
|
|
|
_, err = file.Seek(0, io.SeekStart)
|
|
if err != nil {
|
|
return nil, errors.UnknownStatus("failed to rewind file pointer")
|
|
}
|
|
|
|
objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
|
|
resp, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
Body: file,
|
|
ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
|
|
ChecksumSHA256: aws.String(checksum),
|
|
})
|
|
if err != nil {
|
|
return nil, parseS3Error(err, "failed to put object", req).Err()
|
|
}
|
|
if resp.ChecksumSHA256 == nil {
|
|
return nil, errors.UnknownStatus("missing checksum response from aws")
|
|
}
|
|
if checksum != *resp.ChecksumSHA256 {
|
|
return nil, checksumMistmatchStatus()
|
|
}
|
|
decodedChecksum, err := base64.StdEncoding.DecodeString(*resp.ChecksumSHA256)
|
|
if err != nil {
|
|
return nil, errors.UnknownStatus("failed to decode checksum value from aws")
|
|
}
|
|
return &pb.PutObjectResponse{
|
|
ChecksumSha_256: decodedChecksum,
|
|
}, nil
|
|
}
|
|
|
|
// DeleteObjects is used to delete one or many objects from an s3 bucket.
|
|
func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjectsRequest) (*pb.DeleteObjectsResponse, error) {
|
|
if req.GetKeyPrefix() == "" {
|
|
return nil, errors.BadRequestStatus("key prefix is required")
|
|
}
|
|
|
|
bucket := req.GetBucket()
|
|
if bucket == nil {
|
|
return nil, errors.BadRequestStatus("bucket is required")
|
|
}
|
|
|
|
if bucket.GetBucketName() == "" {
|
|
return nil, errors.BadRequestStatus("bucketName is required")
|
|
}
|
|
|
|
attrs := bucket.GetAttributes()
|
|
if attrs == nil {
|
|
return nil, errors.BadRequestStatus("attributes is required")
|
|
}
|
|
|
|
storageAttributes, err := getStorageAttributes(attrs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
credConfig, err := cred.GetCredentialsConfig(bucket.GetSecrets(), storageAttributes.CredentialAttributes, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set up the creds in our persisted state.
|
|
credState, err := cred.NewAwsCredentialPersistedState(
|
|
append([]cred.AwsCredentialPersistedStateOption{
|
|
cred.WithCredentialsConfig(credConfig),
|
|
}, p.testCredStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
storageState, err := newAwsStoragePersistedState(
|
|
append([]awsStoragePersistedStateOption{
|
|
withCredentials(credState),
|
|
}, p.testStorageStateOpts...)...,
|
|
)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error setting up persisted state: %s", err)
|
|
}
|
|
|
|
opts := []s3Option{}
|
|
if storageAttributes.EndpointUrl != "" {
|
|
opts = append(opts, WithEndpoint(storageAttributes.EndpointUrl))
|
|
}
|
|
client, err := storageState.S3Client(ctx, opts...)
|
|
if err != nil {
|
|
return nil, errors.BadRequestStatus("error getting S3 client: %s", err)
|
|
}
|
|
|
|
prefix := path.Join(bucket.GetBucketPrefix(), req.GetKeyPrefix())
|
|
if strings.HasSuffix(req.GetKeyPrefix(), "/") {
|
|
// path.Join ends by "cleaning" the path, including removing a trailing slash, if
|
|
// it exists. given that a slash is used to denote a folder, it is required here.
|
|
prefix += "/"
|
|
}
|
|
|
|
if !req.Recursive {
|
|
_, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(prefix),
|
|
})
|
|
if err != nil {
|
|
return nil, parseS3Error(err, "failed to delete object", req).Err()
|
|
}
|
|
return &pb.DeleteObjectsResponse{
|
|
ObjectsDeleted: uint32(1),
|
|
}, nil
|
|
}
|
|
|
|
const maxkeys = 1000
|
|
|
|
objects := []types.ObjectIdentifier{}
|
|
var conToken *string
|
|
truncated := true
|
|
for truncated {
|
|
res, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Prefix: aws.String(prefix),
|
|
MaxKeys: maxkeys,
|
|
ContinuationToken: conToken,
|
|
})
|
|
if err != nil {
|
|
return nil, parseS3Error(err, "failed to list objects", req).Err()
|
|
}
|
|
truncated = res.IsTruncated
|
|
conToken = res.NextContinuationToken
|
|
for _, o := range res.Contents {
|
|
objects = append(objects, types.ObjectIdentifier{
|
|
Key: o.Key,
|
|
})
|
|
}
|
|
}
|
|
|
|
deleted := 0
|
|
|
|
for i := 0; i < len(objects); i += maxkeys {
|
|
toDelete := objects[i:min(len(objects), i+maxkeys)] // min is required to avoid an out of bounds panic
|
|
res, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Delete: &types.Delete{
|
|
Objects: toDelete,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, parseS3Error(err, "failed to delete objects", req).Err()
|
|
}
|
|
deleted += len(res.Deleted)
|
|
}
|
|
|
|
return &pb.DeleteObjectsResponse{
|
|
ObjectsDeleted: uint32(deleted),
|
|
}, nil
|
|
}
|
|
|
|
// dryRunValidation validates that the IAM role policy attached to the given secrets of the
|
|
// storage bucket has the minimum required permissions needed for this plugin to function
|
|
// as expected. This function will create an object in the s3 bucket to validate it has
|
|
// PutObject permissions. This function will read the same file it had uploaded to validate
|
|
// it has GetObject permissions. This function will read the metadata of the same file it
|
|
// had uploaded to validate it has GetObjectAttributes permissions.
|
|
func dryRunValidation(ctx context.Context, state *awsStoragePersistedState, attrs *StorageAttributes, bucket *storagebuckets.StorageBucket) *status.Status {
|
|
if state == nil {
|
|
return status.New(codes.InvalidArgument, "persisted state is required")
|
|
}
|
|
if attrs == nil {
|
|
return status.New(codes.InvalidArgument, "attributes is required")
|
|
}
|
|
if bucket == nil {
|
|
return status.New(codes.InvalidArgument, "bucket is required")
|
|
}
|
|
|
|
opts := []s3Option{}
|
|
if attrs.EndpointUrl != "" {
|
|
opts = append(opts, WithEndpoint(attrs.EndpointUrl))
|
|
}
|
|
|
|
client, err := state.S3Client(ctx, opts...)
|
|
if err != nil {
|
|
return status.New(codes.InvalidArgument, fmt.Sprintf("error getting S3 client: %s", err))
|
|
}
|
|
|
|
var errs *multierror.Error
|
|
permissions := &pb.Permissions{}
|
|
// we track the codes returned by ParseAWSError and pick the most severe one by using the max func
|
|
|
|
objectKey := path.Join(bucket.GetBucketPrefix(), uuid.New().String())
|
|
if _, err := client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
Body: bytes.NewReader([]byte("hashicorp boundary aws plugin access test")),
|
|
}); err != nil {
|
|
var st *status.Status
|
|
st, permissions.Write = errors.ParseAWSError(err, "failed to put object")
|
|
errs = multierror.Append(errs, st.Err())
|
|
} else {
|
|
permissions.Write = &pb.Permission{State: pb.StateType_STATE_TYPE_OK, CheckedAt: timestamppb.Now()}
|
|
}
|
|
|
|
if _, err := client.GetObject(ctx, &s3.GetObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
}); err != nil {
|
|
var st *status.Status
|
|
st, permissions.Read = errors.ParseAWSError(err, "failed to get object")
|
|
errs = multierror.Append(errs, st.Err())
|
|
} else {
|
|
permissions.Read = &pb.Permission{State: pb.StateType_STATE_TYPE_OK, CheckedAt: timestamppb.Now()}
|
|
}
|
|
|
|
if _, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
}); err != nil {
|
|
var st *status.Status
|
|
st, permissions.Read = errors.ParseAWSError(err, "failed to head object")
|
|
errs = multierror.Append(errs, st.Err())
|
|
}
|
|
|
|
if res, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Prefix: aws.String(objectKey),
|
|
}); err != nil {
|
|
var st *status.Status
|
|
st, permissions.Read = errors.ParseAWSError(err, "failed to list object")
|
|
errs = multierror.Append(errs, st.Err())
|
|
} else if res == nil || len(res.Contents) != 1 || *res.Contents[0].Key != objectKey {
|
|
permissions.Read = &pb.Permission{State: pb.StateType_STATE_TYPE_UNKNOWN, CheckedAt: timestamppb.Now()}
|
|
errs = multierror.Append(errs, status.New(codes.Aborted, fmt.Sprintf("list response did not contain the expected key: %+v", res)).Err())
|
|
}
|
|
|
|
if _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucket.GetBucketName()),
|
|
Key: aws.String(objectKey),
|
|
}); err != nil {
|
|
var st *status.Status
|
|
st, permissions.Delete = errors.ParseAWSError(err, "failed to delete object")
|
|
errs = multierror.Append(errs, st.Err())
|
|
} else {
|
|
permissions.Delete = &pb.Permission{State: pb.StateType_STATE_TYPE_OK, CheckedAt: timestamppb.Now()}
|
|
}
|
|
|
|
if errs != nil {
|
|
st := status.New(codes.FailedPrecondition, fmt.Sprintf("failed to validate provided aws credentials: %v", errs.Unwrap()))
|
|
state := &pb.StorageBucketCredentialState{State: permissions}
|
|
if st, err = st.WithDetails(state); err != nil {
|
|
st = status.New(codes.Internal, err.Error())
|
|
}
|
|
return st
|
|
}
|
|
|
|
return nil
|
|
}
|