Merge remote-tracking branch 'origin/hugo-cred-rotation' into test-delete-cred-rotation

internal/values/proto.go

@@ -5,6 +5,7 @@ package values
 
 import (
 	"fmt"
+	"time"
 
 	"google.golang.org/protobuf/types/known/structpb"
 )
@@ -37,6 +38,52 @@ func GetStringValue(in *structpb.Struct, k string, required bool) (string, error
 	return s, nil
 }
 
+// GetBoolValue returns a boolean value and no error if the given key is found
+// in the provided proto struct input. An error is returned if the key is not
+// found and required is true, or the value type is not a boolean.
+func GetBoolValue(in *structpb.Struct, k string, required bool) (bool, error) {
+	mv := in.GetFields()
+	v, ok := mv[k]
+	if !ok {
+		if required {
+			return false, fmt.Errorf("missing required value %q", k)
+		}
+
+		return false, nil
+	}
+
+	b, ok := v.AsInterface().(bool)
+	if !ok {
+		return false, fmt.Errorf("unexpected type for value %q: want bool, got %T", k, v.AsInterface())
+	}
+
+	return b, nil
+}
+
+// GetTimeValue returns a time.Time object and no error if the given key is
+// found in the provided proto struct input. An error is returned if the value
+// in the struct isn't a string or the value is not parsable by time.Parse. The
+// expected time layout is RFC3339Nano.
+func GetTimeValue(in *structpb.Struct, k string) (time.Time, error) {
+	mv := in.GetFields()
+	v, ok := mv[k]
+	if !ok {
+		return time.Time{}, nil
+	}
+
+	tRaw, ok := v.AsInterface().(string)
+	if !ok {
+		return time.Time{}, fmt.Errorf("unexpected type for value %q: want string, got %T", k, v.AsInterface())
+	}
+
+	t, err := time.Parse(time.RFC3339Nano, tRaw)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("could not parse time in value %q: %w", k, err)
+	}
+
+	return t, nil
+}
+
 // StructFields returns a map[string]struct{} of the proto struct input.
 func StructFields(s *structpb.Struct) map[string]struct{} {
 	m := make(map[string]struct{}, len(s.GetFields()))

internal/values/proto_test.go

@@ -5,6 +5,7 @@ package values
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -78,3 +79,132 @@ func TestGetStringValue(t *testing.T) {
 		})
 	}
 }
+
+func TestGetBoolValue(t *testing.T) {
+	cases := []struct {
+		name        string
+		in          map[string]any
+		key         string
+		required    bool
+		expected    bool
+		expectedErr string
+	}{
+		{
+			name:        "required missing",
+			in:          map[string]any{},
+			key:         "foo",
+			required:    true,
+			expectedErr: "missing required value \"foo\"",
+		},
+		{
+			name:     "optional missing",
+			in:       map[string]any{},
+			key:      "foo",
+			expected: false,
+		},
+		{
+			name: "non-bool value",
+			in: map[string]any{
+				"foo": "bar",
+			},
+			key:         "foo",
+			expectedErr: "unexpected type for value \"foo\": want bool, got string",
+		},
+		{
+			name: "good",
+			in: map[string]any{
+				"foo": true,
+			},
+			key:      "foo",
+			expected: true,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			require := require.New(t)
+
+			input, err := structpb.NewStruct(tc.in)
+			require.NoError(err)
+
+			actual, err := GetBoolValue(input, tc.key, tc.required)
+			if tc.expectedErr != "" {
+				require.EqualError(err, tc.expectedErr)
+				return
+			}
+
+			require.NoError(err)
+			require.Equal(tc.expected, actual)
+		})
+	}
+}
+
+func TestGetTimeValue(t *testing.T) {
+	staticTime := time.Now()
+
+	cases := []struct {
+		name        string
+		in          map[string]any
+		key         string
+		expected    time.Time
+		expectedErr string
+	}{
+		{
+			name:     "missing",
+			in:       map[string]any{},
+			key:      "foo",
+			expected: time.Time{},
+		},
+		{
+			name: "non-time value",
+			in: map[string]any{
+				"foo": 1,
+			},
+			key:         "foo",
+			expectedErr: "unexpected type for value \"foo\": want string, got float64",
+		},
+		{
+			name: "bad parse",
+			in: map[string]any{
+				"foo": "bar",
+			},
+			key:         "foo",
+			expectedErr: "could not parse time in value \"foo\": parsing time \"bar\" as \"2006-01-02T15:04:05.999999999Z07:00\": cannot parse \"bar\" as \"2006\"",
+		},
+		{
+			name: "good",
+			in: map[string]any{
+				"foo": staticTime.Format(time.RFC3339Nano),
+			},
+			key: "foo",
+			expected: func() time.Time {
+				u, err := time.Parse(time.RFC3339Nano, staticTime.Format(time.RFC3339Nano))
+				if err != nil {
+					panic(err)
+				}
+
+				return u
+			}(),
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			require := require.New(t)
+
+			input, err := structpb.NewStruct(tc.in)
+			require.NoError(err)
+
+			actual, err := GetTimeValue(input, tc.key)
+			if tc.expectedErr != "" {
+				require.EqualError(err, tc.expectedErr)
+				return
+			}
+
+			require.NoError(err)
+			require.Equal(tc.expected, actual)
+		})
+	}
+}

plugin/service/storage/attributes.go

@@ -6,6 +6,7 @@ package storage
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/boundary-plugin-minio/internal/errors"
 	"github.com/hashicorp/boundary-plugin-minio/internal/values"
@@ -15,10 +16,12 @@ import (
 )
 
 const (
-	ConstEndpointUrl     = "endpoint_url"
+	ConstEndpointUrl               = "endpoint_url"
-	ConstAccessKeyId     = "access_key_id"
+	ConstAccessKeyId               = "access_key_id"
-	ConstSecretAccessKey = "secret_access_key"
+	ConstSecretAccessKey           = "secret_access_key"
-	ConstRegion          = "region"
+	ConstRegion                    = "region"
+	ConstDisableCredentialRotation = "disable_credential_rotation"
+	ConstLastRotatedTime           = "creds_last_rotated_time"
 )
 
 type StorageAttributes struct {
@@ -31,6 +34,11 @@ type StorageAttributes struct {
 	// UseSSL determines if the MinIO SDK will use SSL when communicating with
 	// the server. This field is determined based on the endpoint URL prefix.
 	UseSSL bool
+	// DisableCredentialRotation is used to configure whether this plugin should
+	// manage the MinIO credentials or not. By default, this field is set to
+	// false. See the plugin's OnCreateStorageBucket, OnUpdateStorageBucket and
+	// OnDeleteStorageBucket for how we use this.
+	DisableCredentialRotation bool
 }
 
 type StorageSecrets struct {
@@ -40,6 +48,10 @@ type StorageSecrets struct {
 	// SecretAccessKey is the MinIO Secret Access Key. This field is required
 	// and comes from user input.
 	SecretAccessKey string
+	// LastRotatedTime is the time the secrets contained in this object were
+	// last rotated. Will be set to the zero-value (time.Time{}) if the
+	// credentials haven't been rotated.
+	LastRotatedTime time.Time
 }
 
 // AsMap returns a map StorageAttributes's secret fields as a map.
@@ -47,6 +59,17 @@ func (sa *StorageSecrets) AsMap() map[string]any {
 	return map[string]any{
 		ConstAccessKeyId:     sa.AccessKeyId,
 		ConstSecretAccessKey: sa.SecretAccessKey,
+		ConstLastRotatedTime: sa.LastRotatedTime.Format(time.RFC3339Nano),
+	}
+}
+
+// Clone returns a new StorageSecrets object with the same state as the incoming
+// one.
+func (ss *StorageSecrets) Clone() *StorageSecrets {
+	return &StorageSecrets{
+		AccessKeyId:     ss.AccessKeyId,
+		SecretAccessKey: ss.SecretAccessKey,
+		LastRotatedTime: ss.LastRotatedTime,
 	}
 }
 
@@ -83,6 +106,12 @@ func getStorageAttributes(inAttributes *structpb.Struct) (*StorageAttributes, er
 	}
 	delete(unknownAttrFields, ConstRegion)
 
+	disableCredentialRotation, err := values.GetBoolValue(inAttributes, ConstDisableCredentialRotation, false)
+	if err != nil {
+		badFields[fmt.Sprintf("attributes.%s", ConstDisableCredentialRotation)] = err.Error()
+	}
+	delete(unknownAttrFields, ConstDisableCredentialRotation)
+
 	for s := range unknownAttrFields {
 		badFields[fmt.Sprintf("attributes.%s", s)] = "unrecognized field"
 	}
@@ -91,9 +120,10 @@ func getStorageAttributes(inAttributes *structpb.Struct) (*StorageAttributes, er
 		return nil, errors.InvalidArgumentError("invalid or unrecognized attributes found", badFields)
 	}
 	return &StorageAttributes{
-		EndpointUrl: endpointUrl,
+		EndpointUrl:               endpointUrl,
-		Region:      region,
+		Region:                    region,
-		UseSSL:      useSSL,
+		UseSSL:                    useSSL,
+		DisableCredentialRotation: disableCredentialRotation,
 	}, nil
 }
 
@@ -118,6 +148,12 @@ func getStorageSecrets(inSecrets *structpb.Struct) (*StorageSecrets, error) {
 	}
 	delete(unknownSecretFields, ConstSecretAccessKey)
 
+	lastRotated, err := values.GetTimeValue(inSecrets, ConstLastRotatedTime)
+	if err != nil {
+		badFields[fmt.Sprintf("secrets.%s", ConstLastRotatedTime)] = err.Error()
+	}
+	delete(unknownSecretFields, ConstLastRotatedTime)
+
 	for s := range unknownSecretFields {
 		badFields[fmt.Sprintf("secrets.%s", s)] = "unrecognized field"
 	}
@@ -128,5 +164,6 @@ func getStorageSecrets(inSecrets *structpb.Struct) (*StorageSecrets, error) {
 	return &StorageSecrets{
 		AccessKeyId:     accessKeyId,
 		SecretAccessKey: secretAccessKey,
+		LastRotatedTime: lastRotated,
 	}, nil
 }

plugin/service/storage/attributes_test.go

@@ -5,6 +5,7 @@ package storage
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -89,18 +90,30 @@ func TestGetStorageAttributes(t *testing.T) {
 			},
 			expErrMsg: "attributes.endpoint_url: missing required value \"endpoint_url\"",
 		},
+		{
+			name: "disableCredRotationBadType",
+			inAttributes: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstEndpointUrl:               structpb.NewStringValue("http://bar"),
+					ConstDisableCredentialRotation: structpb.NewStringValue("notbool"),
+				},
+			},
+			expErrMsg: "attributes.disable_credential_rotation: unexpected type for value \"disable_credential_rotation\": want bool, got string",
+		},
 		{
 			name: "successWithNoSSL",
 			inAttributes: &structpb.Struct{
 				Fields: map[string]*structpb.Value{
-					ConstEndpointUrl: structpb.NewStringValue("http://foo.bar"),
+					ConstEndpointUrl:               structpb.NewStringValue("http://foo.bar"),
-					ConstRegion:      structpb.NewStringValue("us-east-1"),
+					ConstRegion:                    structpb.NewStringValue("us-east-1"),
+					ConstDisableCredentialRotation: structpb.NewBoolValue(true),
 				},
 			},
 			expStorageAttributes: &StorageAttributes{
-				EndpointUrl: "foo.bar",
+				EndpointUrl:               "foo.bar",
-				Region:      "us-east-1",
+				Region:                    "us-east-1",
-				UseSSL:      false,
+				UseSSL:                    false,
+				DisableCredentialRotation: true,
 			},
 		},
 		{
@@ -215,6 +228,17 @@ func TestGetStorageSecrets(t *testing.T) {
 			},
 			expErrMsg: "secrets.secret_access_key: unexpected type for value \"secret_access_key\": want string, got bool",
 		},
+		{
+			name: "lastRotatedBadType",
+			inSecrets: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstAccessKeyId:     structpb.NewStringValue("foo"),
+					ConstSecretAccessKey: structpb.NewStringValue("bar"),
+					ConstLastRotatedTime: structpb.NewStringValue("nottime"),
+				},
+			},
+			expErrMsg: "secrets.creds_last_rotated_time: could not parse time in value \"creds_last_rotated_time\"",
+		},
 		{
 			name: "unknownSecret",
 			inSecrets: &structpb.Struct{
@@ -239,6 +263,21 @@ func TestGetStorageSecrets(t *testing.T) {
 				SecretAccessKey: "bar",
 			},
 		},
+		{
+			name: "successWithLastRotatedTime",
+			inSecrets: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstAccessKeyId:     structpb.NewStringValue("foo"),
+					ConstSecretAccessKey: structpb.NewStringValue("bar"),
+					ConstLastRotatedTime: structpb.NewStringValue(time.Date(2024, 01, 02, 03, 04, 05, 06, time.UTC).Format(time.RFC3339Nano)),
+				},
+			},
+			expStorageSecrets: &StorageSecrets{
+				AccessKeyId:     "foo",
+				SecretAccessKey: "bar",
+				LastRotatedTime: time.Date(2024, 01, 02, 03, 04, 05, 06, time.UTC),
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -268,10 +307,12 @@ func TestStorageSecretsAsMap(t *testing.T) {
 			in: &StorageSecrets{
 				AccessKeyId:     "access_key_id_value",
 				SecretAccessKey: "secret_access_key_value",
+				LastRotatedTime: time.Date(2024, 01, 02, 03, 04, 05, 06, time.UTC),
 			},
 			expMap: map[string]any{
-				"access_key_id":     "access_key_id_value",
+				"access_key_id":           "access_key_id_value",
-				"secret_access_key": "secret_access_key_value",
+				"secret_access_key":       "secret_access_key_value",
+				"creds_last_rotated_time": "2024-01-02T03:04:05.000000006Z",
 			},
 		},
 	}
@@ -283,3 +324,18 @@ func TestStorageSecretsAsMap(t *testing.T) {
 		})
 	}
 }
+
+func TestTestStorageSecretsClone(t *testing.T) {
+	now := time.Now()
+	in := &StorageSecrets{
+		AccessKeyId:     "foo",
+		SecretAccessKey: "bar",
+		LastRotatedTime: now,
+	}
+
+	ss := in.Clone()
+	require.Equal(t, in.AccessKeyId, ss.AccessKeyId)
+	require.Equal(t, in.AccessKeyId, ss.AccessKeyId)
+	require.Equal(t, in.LastRotatedTime, ss.LastRotatedTime)
+	require.NotSame(t, in, ss)
+}

plugin/service/storage/plugin.go

@@ -4,6 +4,7 @@
 package storage
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/sha256"
@@ -12,6 +13,8 @@ import (
 	"io"
 	"os"
 	"path"
+	"strings"
+	"sync"
 
 	"github.com/google/uuid"
 	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets"
@@ -27,6 +30,10 @@ import (
 
 var _ pb.StoragePluginServiceServer = (*StoragePlugin)(nil)
 
+const (
+	defaultGetObjectChunkSize = uint32(65536)
+)
+
 // StoragePlugin implements the StoragePluginServiceServer interface for the
 // MinIO storage service plugin.
 type StoragePlugin struct {
@@ -58,6 +65,16 @@ func (sp *StoragePlugin) OnCreateStorageBucket(ctx context.Context, req *pb.OnCr
 		return nil, status.Errorf(codes.InvalidArgument, "failed to ensure service account: %v", err)
 	}
 
+	deleteOldCredsFn := func() error { return nil }
+	if !sa.DisableCredentialRotation {
+		newSec, fn, err := rotateCredentials(ctx, bucket.GetBucketName(), sa, sec)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "failed to rotate minio credentials: %v", err)
+		}
+		sec = newSec
+		deleteOldCredsFn = fn
+	}
+
 	cl, err := minio.New(sa.EndpointUrl, &minio.Options{
 		Creds:  credentials.NewStaticV4(sec.AccessKeyId, sec.SecretAccessKey, ""),
 		Secure: sa.UseSSL,
@@ -76,6 +93,13 @@ func (sp *StoragePlugin) OnCreateStorageBucket(ctx context.Context, req *pb.OnCr
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to convert storage attributes to protobuf struct: %v", err)
 	}
+
+	// Given that at this point everything is OK with the bucket, we shouldn't
+	// let a failure to delete the old credentials fail the entire process. If
+	// we ever establish a way to log non-critical errors in the plugin, log
+	// this.
+	_ = deleteOldCredsFn()
+
 	return &pb.OnCreateStorageBucketResponse{
 		Persisted: &storagebuckets.StorageBucketPersisted{Data: persistedData},
 	}, nil
@@ -90,24 +114,22 @@ func (sp *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUp
 	if newBucket.BucketName == "" {
 		return nil, status.Error(codes.InvalidArgument, "new bucketName is required")
 	}
+
 	oldBucket := req.GetCurrentBucket()
 	if oldBucket == nil {
 		return nil, status.Error(codes.InvalidArgument, "current bucket is required")
 	}
-
-	// current attributes
 	osa, err := getStorageAttributes(oldBucket.GetAttributes())
 	if err != nil {
 		return nil, err
 	}
 
-	// new attributes and secrets
 	nsa, err := getStorageAttributes(newBucket.GetAttributes())
 	if err != nil {
 		return nil, err
 	}
 
-	sec, err := getStorageSecrets(newBucket.GetSecrets())
+	sec, err := getStorageSecrets(req.GetPersisted().GetData())
 	if err != nil {
 		return nil, err
 	}
@@ -123,8 +145,77 @@ func (sp *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUp
 		return nil, status.Errorf(codes.InvalidArgument, "cannot update attribute EndpointUrl")
 	}
 
-	if err = ensureServiceAccount(ctx, nsa, sec); err != nil {
+	if nsa.DisableCredentialRotation && newBucket.GetSecrets() == nil {
-		return nil, status.Errorf(codes.InvalidArgument, "failed to ensure service account: %s", err)
+		if !sec.LastRotatedTime.IsZero() {
+			return nil, status.Error(codes.InvalidArgument, "cannot disable rotation for already-rotated credentials")
+		}
+	}
+
+	deleteRotatedCredsFn := func() error { return nil }
+	deleteOldPersistedCredsFn := func() error { return nil }
+	if newBucket.GetSecrets() != nil {
+		// Preemptively check if we're managing the current persisted
+		// credentials and schedule them for later deletion if we are, since in
+		// both rotation and non-rotation cases below, we'll be replacing these.
+		if !sec.LastRotatedTime.IsZero() {
+			sc := sec.Clone()
+			deleteOldPersistedCredsFn = sync.OnceValue(func() error {
+				cl, err := newMadminClient(nsa, sc)
+				if err != nil {
+					return fmt.Errorf("failed to create new minio admin client: %w", err)
+				}
+				err = cl.DeleteServiceAccount(ctx, sc.AccessKeyId)
+				if err != nil {
+					return fmt.Errorf("failed to delete minio service account: %w", err)
+				}
+
+				return nil
+			})
+		}
+
+		newSec, err := getStorageSecrets(newBucket.GetSecrets())
+		if err != nil {
+			return nil, err
+		}
+
+		err = ensureServiceAccount(ctx, nsa, newSec)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "failed to ensure service account: %s", err)
+		}
+
+		if !nsa.DisableCredentialRotation {
+			// We have new incoming credentials and we're also rotating,
+			// therefore we use incoming credentials to create a new service
+			// account, then delete the incoming credentials. The existing
+			// persisted credentials have already been scheduled for deletion
+			// above (if applicable), so we don't need to do it here.
+			ss, fn, err := rotateCredentials(ctx, newBucket.GetBucketName(), nsa, newSec)
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to rotate minio credentials: %v", err)
+			}
+			sec = ss
+			deleteRotatedCredsFn = fn
+		} else {
+			// We have new incoming credentials but we're not rotating. Set them
+			// directly for validation below. The existing persisted credentials
+			// have already been scheduled for deletion above (if applicable),
+			// so we don't need to do it here.
+			sec = newSec
+		}
+	} else {
+		if osa.DisableCredentialRotation && !nsa.DisableCredentialRotation {
+			// Handle the case where the user enables credential rotation but
+			// provides no new credentials. In this case, we use the existing
+			// persisted credentials to generate a new service account. The
+			// persisted credentials are, by definition, not being managed by us
+			// (credential rotation was disabled beforehand), so don't schedule
+			// them for deletion.
+			ss, _, err := rotateCredentials(ctx, newBucket.GetBucketName(), nsa, sec)
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "failed to rotate minio credentials: %v", err)
+			}
+			sec = ss
+		}
 	}
 
 	cl, err := minio.New(nsa.EndpointUrl, &minio.Options{
@@ -145,14 +236,46 @@ func (sp *StoragePlugin) OnUpdateStorageBucket(ctx context.Context, req *pb.OnUp
 		return nil, status.Errorf(codes.InvalidArgument, "failed to convert storage attributes to protobuf struct: %s", err)
 	}
 
+	// Given that at this point everything is OK with the bucket, we shouldn't
+	// let a failure to delete credentials fail the entire process. If we ever
+	// establish a way to log non-critical errors in the plugin, log this.
+	_ = deleteRotatedCredsFn()
+	_ = deleteOldPersistedCredsFn()
+
 	return &pb.OnUpdateStorageBucketResponse{
 		Persisted: &storagebuckets.StorageBucketPersisted{Data: persisted},
 	}, nil
 }
 
 // OnDeleteStorageBucket is a hook that runs when a storage bucket is deleted.
-// Since this plugin manages no state at the moment, this function is a no-op.
 func (sp *StoragePlugin) OnDeleteStorageBucket(ctx context.Context, req *pb.OnDeleteStorageBucketRequest) (*pb.OnDeleteStorageBucketResponse, error) {
+	bucket := req.GetBucket()
+	if bucket == nil {
+		return nil, status.Error(codes.InvalidArgument, "no storage bucket information found")
+	}
+
+	sec, err := getStorageSecrets(req.GetPersisted().GetData())
+	if err != nil {
+		return nil, err
+	}
+
+	if !sec.LastRotatedTime.IsZero() {
+		sa, err := getStorageAttributes(bucket.GetAttributes())
+		if err != nil {
+			return nil, err
+		}
+
+		ac, err := newMadminClient(sa, sec)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "failed to create minio admin client: %v", err)
+		}
+
+		err = ac.DeleteServiceAccount(ctx, sec.AccessKeyId)
+		if err != nil {
+			return nil, status.Errorf(codes.Unknown, "failed to delete minio service account: %v", err)
+		}
+	}
+
 	return &pb.OnDeleteStorageBucketResponse{}, nil
 }
 
@@ -246,7 +369,71 @@ func (sp *StoragePlugin) HeadObject(ctx context.Context, req *pb.HeadObjectReque
 
 // GetObject is a hook that retrieves objects.
 func (sp *StoragePlugin) GetObject(req *pb.GetObjectRequest, objServer pb.StoragePluginService_GetObjectServer) error {
-	return fmt.Errorf("unimplemented")
+	bucket := req.GetBucket()
+	if bucket == nil {
+		return status.Error(codes.InvalidArgument, "no storage bucket information found")
+	}
+	if bucket.GetBucketName() == "" {
+		return status.Error(codes.InvalidArgument, "storage bucket name is required")
+	}
+
+	if req.GetKey() == "" {
+		return status.Error(codes.InvalidArgument, "object key is required")
+	}
+
+	sa, err := getStorageAttributes(bucket.GetAttributes())
+	if err != nil {
+		return err
+	}
+
+	sec, err := getStorageSecrets(bucket.GetSecrets())
+	if err != nil {
+		return err
+	}
+
+	cl, err := minio.New(sa.EndpointUrl, &minio.Options{
+		Creds:  credentials.NewStaticV4(sec.AccessKeyId, sec.SecretAccessKey, ""),
+		Secure: sa.UseSSL,
+		Region: sa.Region,
+	})
+	if err != nil {
+		return status.Errorf(codes.Unknown, "failed to create minio sdk client: %v", err)
+	}
+
+	objKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
+	obj, err := cl.GetObject(objServer.Context(), bucket.GetBucketName(), objKey, minio.GetObjectOptions{})
+	if err != nil {
+		return status.Errorf(codes.Unknown, "failed to get object: %v", err)
+	}
+	defer obj.Close()
+
+	chunkSize := req.GetChunkSize()
+	if chunkSize == 0 {
+		chunkSize = defaultGetObjectChunkSize
+	}
+
+	rd := bufio.NewReader(obj)
+	for {
+		select {
+		case <-objServer.Context().Done():
+			return status.Errorf(codes.Canceled, "server context done while streaming: %v", objServer.Context().Err())
+		default:
+			buffer := make([]byte, chunkSize)
+			n, err := rd.Read(buffer)
+			if err != nil && err != io.EOF {
+				return status.Errorf(codes.Unknown, "failed to read chunk from minio object: %v", err)
+			}
+			if n > 0 {
+				err := objServer.Send(&pb.GetObjectResponse{FileChunk: buffer[:n]})
+				if err != nil {
+					return status.Errorf(codes.Unknown, "failed to send chunk to client: %v", err)
+				}
+			}
+			if err == io.EOF {
+				return nil
+			}
+		}
+	}
 }
 
 // PutObject is a hook that reads a file stored on local disk and stores it to
@@ -341,7 +528,86 @@ func (sp *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest
 // DeleteObjects deletes one or many files in an external object store via a
 // provided key prefix.
 func (sp *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjectsRequest) (*pb.DeleteObjectsResponse, error) {
-	return nil, fmt.Errorf("unimplemented")
+	bucket := req.GetBucket()
+	if bucket == nil {
+		return nil, status.Error(codes.InvalidArgument, "no storage bucket information found")
+	}
+	if bucket.GetBucketName() == "" {
+		return nil, status.Error(codes.InvalidArgument, "storage bucket name is required")
+	}
+
+	if req.GetKeyPrefix() == "" {
+		return nil, status.Error(codes.InvalidArgument, "key prefix is required")
+	}
+
+	sa, err := getStorageAttributes(bucket.GetAttributes())
+	if err != nil {
+		return nil, err
+	}
+
+	sec, err := getStorageSecrets(bucket.GetSecrets())
+	if err != nil {
+		return nil, err
+	}
+
+	cl, err := minio.New(sa.EndpointUrl, &minio.Options{
+		Creds:  credentials.NewStaticV4(sec.AccessKeyId, sec.SecretAccessKey, ""),
+		Secure: sa.UseSSL,
+		Region: sa.Region,
+	})
+	if err != nil {
+		return nil, status.Errorf(codes.Unknown, "failed to create minio sdk client: %v", err)
+	}
+
+	prefix := path.Join(bucket.GetBucketPrefix(), req.GetKeyPrefix())
+	if strings.HasSuffix(req.GetKeyPrefix(), "/") {
+		// path.Join ends by "cleaning" the path, including removing a trailing slash, if
+		// it exists. given that a slash is used to denote a folder, it is required here.
+		prefix += "/"
+	}
+
+	if !req.Recursive {
+		if err := cl.RemoveObject(ctx, bucket.GetBucketName(), prefix, minio.RemoveObjectOptions{}); err != nil {
+			return nil, status.Errorf(codes.Unknown, "error deleting minio object: %v", err)
+		}
+		return &pb.DeleteObjectsResponse{
+			ObjectsDeleted: uint32(1),
+		}, nil
+	}
+
+	objects := []minio.ObjectInfo{}
+	for obj := range cl.ListObjects(ctx, bucket.GetBucketName(), minio.ListObjectsOptions{
+		Prefix:    prefix,
+		Recursive: true,
+	}) {
+		if obj.Err != nil {
+			// check if context cancelled ?
+			return nil, status.Errorf(codes.Unknown, "error iterating minio bucket contents: %v", err)
+		}
+		objects = append(objects, obj)
+	}
+
+	emitter := make(chan minio.ObjectInfo)
+	go func() {
+		defer close(emitter)
+		// we could loop the listobjects respose here, but we want to check all the errs ahead of time
+		for _, obj := range objects {
+			emitter <- obj
+		}
+	}()
+
+	removed := 0
+	for res := range cl.RemoveObjectsWithResult(ctx, bucket.GetBucketName(), emitter, minio.RemoveObjectsOptions{}) {
+		if res.Err != nil {
+			// check if context cancelled ?
+			return nil, status.Errorf(codes.Unknown, "error deleting minio object(s): %v", err)
+		}
+		removed++
+	}
+
+	return &pb.DeleteObjectsResponse{
+		ObjectsDeleted: uint32(removed),
+	}, nil
 }
 
 func dryRun(ctx context.Context, cl *minio.Client, bucket *storagebuckets.StorageBucket) error {

plugin/service/storage/rotation.go

@@ -0,0 +1,88 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+)
+
+const (
+	defaultPolicyTemplate = `{
+		"Statement": [
+			{
+				"Action": [
+					"s3:GetObject",
+					"s3:GetObjectAttributes",
+					"s3:PutObject",
+					"s3:DeleteObject"
+				],
+				"Effect": "Allow",
+				"Resource": "arn:aws:s3:::%[1]s/*"
+			},
+			{
+				"Action": "s3:ListBucket",
+				"Effect": "Allow",
+				"Resource": "arn:aws:s3:::%[1]s"
+			},
+			{
+				"Action": [
+					"admin:CreateServiceAccount",
+					"admin:RemoveServiceAccount"
+				],
+				"Effect": "Allow"
+			}
+		],
+		"Version": "2012-10-17"
+	}`
+)
+
+type deleteInputCredsFn func() error
+
+// rotateCredentials uses the incoming service account credentials to generate a
+// new service account. It returns a new StorageSecrets object with the new
+// service account credentials and a callback function to delete the service
+// account provided in the input.
+func rotateCredentials(ctx context.Context, bucketName string, sa *StorageAttributes, inSec *StorageSecrets) (*StorageSecrets, deleteInputCredsFn, error) {
+	err := ensureServiceAccount(ctx, sa, inSec)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to ensure minio service account credentials: %w", err)
+	}
+
+	ac, err := newMadminClient(sa, inSec)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create minio admin client: %w", err)
+	}
+
+	policy := json.RawMessage(fmt.Sprintf(defaultPolicyTemplate, bucketName))
+	newCreds, err := ac.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		Name:        "Boundary MinIO Service Account",
+		Description: fmt.Sprintf("Boundary-managed MinIO service account for storage bucket '%s'", bucketName),
+		Policy:      policy,
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create new minio service account: %w", err)
+	}
+
+	newSec := &StorageSecrets{
+		AccessKeyId:     newCreds.AccessKey,
+		SecretAccessKey: newCreds.SecretKey,
+		LastRotatedTime: time.Now(),
+	}
+
+	inCl, newCl := inSec.Clone(), newSec.Clone()
+	return newSec, sync.OnceValue(func() error {
+		ac, err := newMadminClient(sa, newCl)
+		if err != nil {
+			return fmt.Errorf("failed to create minio admin client: %w", err)
+		}
+		err = ac.DeleteServiceAccount(ctx, inCl.AccessKeyId)
+		if err != nil {
+			return fmt.Errorf("failed to delete minio service account: %w", err)
+		}
+		return nil
+	}), nil
+}

plugin/service/storage/rotation_test.go

@@ -0,0 +1,137 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	internaltest "github.com/hashicorp/boundary-plugin-minio/internal/testing"
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRotateCredentials(t *testing.T) {
+	ctx := context.Background()
+	server := internaltest.NewMinioServer(t)
+
+	bucketName := "test-bucket"
+	err := server.Client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
+	require.NoError(t, err)
+
+	tests := []struct {
+		name       string
+		ss         *StorageSecrets
+		expErrMsg  string
+		credDelErr bool
+	}{
+		{
+			name: "notUsingServiceAccount",
+			ss: &StorageSecrets{
+				AccessKeyId:     server.RootUsername,
+				SecretAccessKey: server.RootPassword,
+			},
+			expErrMsg: "failed to ensure minio service account credentials: failed to obtain service account info: The specified service account is not found",
+		},
+		{
+			name: "addServiceAccountFail",
+			ss: func() *StorageSecrets {
+				creds, err := server.AdminClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+					Policy: json.RawMessage(`
+					{
+						"Statement": [
+							{
+								"Action": [
+									"admin:CreateServiceAccount",
+									"admin:RemoveServiceAccount"
+								],
+								"Effect": "Deny"
+							}
+						],
+						"Version": "2012-10-17"
+					}
+					`),
+				})
+				require.NoError(t, err)
+
+				return &StorageSecrets{
+					AccessKeyId:     creds.AccessKey,
+					SecretAccessKey: creds.SecretKey,
+				}
+			}(),
+			expErrMsg: "failed to create new minio service account: Access Denied",
+		},
+		{
+			name: "credentialDeletionFail",
+			ss: func() *StorageSecrets {
+				creds, err := server.AdminClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{})
+				require.NoError(t, err)
+
+				return &StorageSecrets{
+					AccessKeyId:     creds.AccessKey,
+					SecretAccessKey: creds.SecretKey,
+				}
+			}(),
+			credDelErr: true,
+		},
+		{
+			name: "success",
+			ss: func() *StorageSecrets {
+				creds, err := server.AdminClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{})
+				require.NoError(t, err)
+
+				return &StorageSecrets{
+					AccessKeyId:     creds.AccessKey,
+					SecretAccessKey: creds.SecretKey,
+				}
+			}(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sa := &StorageAttributes{
+				EndpointUrl: server.ApiAddr,
+				UseSSL:      false,
+			}
+
+			ss, delFn, err := rotateCredentials(context.Background(), bucketName, sa, tt.ss)
+			if tt.expErrMsg != "" {
+				require.ErrorContains(t, err, tt.expErrMsg)
+				require.Nil(t, ss)
+				require.Nil(t, delFn)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, ss)
+			require.NotNil(t, delFn)
+			require.NotEqual(t, ss.AccessKeyId, tt.ss.AccessKeyId)
+			require.NotEqual(t, ss.SecretAccessKey, tt.ss.SecretAccessKey)
+			require.NotEmpty(t, ss.LastRotatedTime)
+
+			// Ensure old creds still exist since the callback hasn't been
+			// called. We then make the delete call and finally assert that the
+			// credential was removed if there was no error when attempting to
+			// delete.
+			_, err = server.AdminClient.InfoServiceAccount(ctx, tt.ss.AccessKeyId)
+			require.NoError(t, err)
+
+			if tt.credDelErr {
+				sa.EndpointUrl = "localhost:1"
+				require.ErrorContains(t, delFn(), "failed to delete minio service account")
+				require.ErrorContains(t, delFn(), "failed to delete minio service account") // 2nd call should return the same state (due to sync.Once).
+
+				_, err := server.AdminClient.InfoServiceAccount(ctx, tt.ss.AccessKeyId)
+				require.NoError(t, err)
+				return
+			} else {
+				require.NoError(t, delFn())
+				require.NoError(t, delFn()) // 2nd call should return the same state (due to sync.Once).
+			}
+
+			_, err = server.AdminClient.InfoServiceAccount(ctx, tt.ss.AccessKeyId)
+			require.ErrorContains(t, err, "service account is not found")
+		})
+	}
+}