update storage buckets to use a shared persisted secrets type. add options

internal/plugin/loopback/storage.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets"
 	plgpb "github.com/hashicorp/boundary/sdk/pbs/plugin"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -134,7 +135,7 @@ func (l *LoopbackStorage) onCreateStorageBucket(ctx context.Context, req *plgpb.
 		}
 	}
 	return &plgpb.OnCreateStorageBucketResponse{
-		Persisted: &plgpb.StorageBucketPersisted{
+		Persisted: &storagebuckets.StorageBucketPersisted{
 			Data: req.GetBucket().GetSecrets(),
 		},
 	}, nil
@@ -148,7 +149,7 @@ func (l *LoopbackStorage) onUpdateStorageBucket(ctx context.Context, req *plgpb.
 	if req.GetNewBucket() == nil {
 		return nil, status.Errorf(codes.InvalidArgument, "%s: missing storage bucket", op)
 	}
-	if req.GetNewBucket().GetSecrets() == nil {
+	if req.Persisted == nil {
 		return nil, status.Errorf(codes.InvalidArgument, "%s: missing secrets", op)
 	}
 	l.m.Lock()
@@ -161,10 +162,16 @@ func (l *LoopbackStorage) onUpdateStorageBucket(ctx context.Context, req *plgpb.
 			return nil, status.Errorf(err.errCode, "%s: %s", op, err.errMsg)
 		}
 	}
-	return &plgpb.OnUpdateStorageBucketResponse{
-		Persisted: &plgpb.StorageBucketPersisted{
+	var sec *storagebuckets.StorageBucketPersisted
+	if req.NewBucket.Secrets != nil {
+		sec = &storagebuckets.StorageBucketPersisted{
 			Data: req.GetNewBucket().GetSecrets(),
-		},
+		}
+	} else {
+		sec = req.Persisted
+	}
+	return &plgpb.OnUpdateStorageBucketResponse{
+		Persisted: sec,
 	}, nil
 }
 
@@ -176,7 +183,7 @@ func (l *LoopbackStorage) onDeleteStorageBucket(ctx context.Context, req *plgpb.
 	if req.GetBucket() == nil {
 		return nil, status.Errorf(codes.InvalidArgument, "%s: missing storage bucket", op)
 	}
-	if req.GetBucket().GetSecrets() == nil {
+	if req.Persisted == nil {
 		return nil, status.Errorf(codes.InvalidArgument, "%s: missing secrets", op)
 	}
 	return &plgpb.OnDeleteStorageBucketResponse{}, nil

internal/plugin/loopback/storage_test.go

@@ -145,6 +145,9 @@ func TestLoopbackOnUpdateStorageBucket(t *testing.T) {
 			"AWS_SECRET_ACCESS_KEY": structpb.NewStringValue("secret_access_key"),
 		},
 	}
+	persisted := &storagebuckets.StorageBucketPersisted{
+		Data: secrets,
+	}
 
 	tests := []struct {
 		name        string
@@ -165,6 +168,7 @@ func TestLoopbackOnUpdateStorageBucket(t *testing.T) {
 			request: &plgpb.OnUpdateStorageBucketRequest{
 				NewBucket: &storagebuckets.StorageBucket{
 					BucketName: "aws_s3_mock",
+					Secrets:    secrets,
 				},
 			},
 			expectedErr: codes.InvalidArgument,
@@ -176,6 +180,7 @@ func TestLoopbackOnUpdateStorageBucket(t *testing.T) {
 					BucketName: "bucket_dne",
 					Secrets:    secrets,
 				},
+				Persisted: persisted,
 			},
 			expectedErr: codes.NotFound,
 		},
@@ -186,6 +191,7 @@ func TestLoopbackOnUpdateStorageBucket(t *testing.T) {
 					BucketName: "aws_s3_err",
 					Secrets:    secrets,
 				},
+				Persisted: persisted,
 			},
 			expectedErr: codes.PermissionDenied,
 		},
@@ -196,6 +202,7 @@ func TestLoopbackOnUpdateStorageBucket(t *testing.T) {
 					BucketName: "aws_s3_mock",
 					Secrets:    secrets,
 				},
+				Persisted: persisted,
 			},
 		},
 	}
@@ -233,6 +240,9 @@ func TestLoopbackOnDeleteStorageBucket(t *testing.T) {
 			"AWS_SECRET_ACCESS_KEY": structpb.NewStringValue("secret_access_key"),
 		},
 	}
+	persisted := &storagebuckets.StorageBucketPersisted{
+		Data: secrets,
+	}
 
 	tests := []struct {
 		name        string
@@ -253,6 +263,7 @@ func TestLoopbackOnDeleteStorageBucket(t *testing.T) {
 			request: &plgpb.OnDeleteStorageBucketRequest{
 				Bucket: &storagebuckets.StorageBucket{
 					BucketName: "aws_s3_mock",
+					Secrets:    secrets,
 				},
 			},
 			expectedErr: codes.InvalidArgument,
@@ -264,6 +275,7 @@ func TestLoopbackOnDeleteStorageBucket(t *testing.T) {
 					BucketName: "aws_s3_mock",
 					Secrets:    secrets,
 				},
+				Persisted: persisted,
 			},
 		},
 	}

internal/proto/controller/api/resources/storagebuckets/v1/storage_bucket.proto

@@ -104,3 +104,16 @@ message StorageBucket {
   // Output only. The available actions on this resource for this user.
   repeated string authorized_actions = 300 [json_name = "authorized_actions"]; // @gotags: `class:"public"`
 }
+
+// StorageBucketPersisted is data that the plugin can read from and write
+// to that will always be provided by the host.
+message StorageBucketPersisted {
+  // Data has no explicit structure other than valid json.
+  // Data may contain sensitive information, such as
+  // credentials, rotated secrets, or configuration data.
+  // Data can be anything that may need to be provided to
+  // a series of different method calls.
+  // Data is encrypted at-rest by Boundary.
+  // Data is never returned to the end user.
+  google.protobuf.Struct data = 10;
+}

internal/proto/plugin/v1/storage_plugin_service.proto

@@ -6,7 +6,6 @@ syntax = "proto3";
 package plugin.v1;
 
 import "controller/api/resources/storagebuckets/v1/storage_bucket.proto";
-import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 
 option go_package = "github.com/hashicorp/boundary/sdk/pbs/plugin;plugin";
@@ -49,7 +48,7 @@ message OnCreateStorageBucketRequest {
 
 message OnCreateStorageBucketResponse {
   // The persisted data represents state persisted between storage bucket calls.
-  StorageBucketPersisted persisted = 10;
+  controller.api.resources.storagebuckets.v1.StorageBucketPersisted persisted = 10;
 }
 
 message OnUpdateStorageBucketRequest {
@@ -62,12 +61,12 @@ message OnUpdateStorageBucketRequest {
   controller.api.resources.storagebuckets.v1.StorageBucket new_bucket = 20;
 
   // Required. The existing persisted secret data.
-  StorageBucketPersisted persisted = 30;
+  controller.api.resources.storagebuckets.v1.StorageBucketPersisted persisted = 30;
 }
 
 message OnUpdateStorageBucketResponse {
   // The persisted data represents state persisted between storage bucket calls.
-  StorageBucketPersisted persisted = 10;
+  controller.api.resources.storagebuckets.v1.StorageBucketPersisted persisted = 10;
 }
 
 message OnDeleteStorageBucketRequest {
@@ -75,7 +74,7 @@ message OnDeleteStorageBucketRequest {
   controller.api.resources.storagebuckets.v1.StorageBucket bucket = 10;
 
   // Required. The existing persisted secret data.
-  StorageBucketPersisted persisted = 20;
+  controller.api.resources.storagebuckets.v1.StorageBucketPersisted persisted = 20;
 }
 
 message OnDeleteStorageBucketResponse {}
@@ -131,16 +130,3 @@ message PutObjectResponse {
   // 256-bit SHA-256 digest of the object.
   bytes checksum_sha_256 = 10;
 }
-
-// StorageBucketPersisted is data that the plugin can read from and write
-// to that will always be provided by the host.
-message StorageBucketPersisted {
-  // Data has no explicit structure other than valid json.
-  // Data may contain sensitive information, such as
-  // credentials, rotated secrets, or configuration data.
-  // Data can be anything that may need to be provided to
-  // a series of different method calls.
-  // Data is encrypted at-rest by Boundary.
-  // Data is never returned to the end user.
-  google.protobuf.Struct data = 10;
-}

internal/storage/plugin/options.go

@@ -0,0 +1,105 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package plugin
+
+import "google.golang.org/protobuf/types/known/structpb"
+
+const (
+	// DefaultChunkSize is the default chunk size for streaming
+	DefaultChunkSize = 65536 // 64KiB
+)
+
+// getOpts - iterate the inbound Options and return a struct
+func getOpts(opt ...Option) options {
+	opts := getDefaultOptions()
+	for _, o := range opt {
+		o(&opts)
+	}
+	return opts
+}
+
+// Option - how Options are passed as arguments.
+type Option func(*options)
+
+// options = how options are represented
+type options struct {
+	withChunkSize    int
+	withName         string
+	withDescription  string
+	withAttributes   *structpb.Struct
+	withSecrets      *structpb.Struct
+	withWorkerFilter string
+	withBucketPrefix string
+	withLimit        int
+}
+
+func getDefaultOptions() options {
+	return options{
+		withChunkSize:  DefaultChunkSize,
+		withAttributes: &structpb.Struct{},
+	}
+}
+
+// WithChunkSize provides an optional chunkSize to associate
+// with a StorageClient. ChunkSize is the number of bytes to
+// send to the plugin in a single request. If not provided,
+// the default is 64KiB. The recommended chunk size for
+// streamed messages is 16KiB to 64KiB.
+func WithChunkSize(chunkSize int) Option {
+	return func(o *options) {
+		o.withChunkSize = chunkSize
+	}
+}
+
+// WithDescription provides an optional description.
+func WithDescription(desc string) Option {
+	return func(o *options) {
+		o.withDescription = desc
+	}
+}
+
+// WithName provides an optional name.
+func WithName(name string) Option {
+	return func(o *options) {
+		o.withName = name
+	}
+}
+
+// WithAttributes provides an optional attributes field.
+func WithAttributes(attrs *structpb.Struct) Option {
+	return func(o *options) {
+		o.withAttributes = attrs
+	}
+}
+
+// WithSecrets provides an optional secrets field.
+func WithSecrets(secrets *structpb.Struct) Option {
+	return func(o *options) {
+		o.withSecrets = secrets
+	}
+}
+
+// WithBucketPrefix provides an optional bucket prefix.
+func WithBucketPrefix(bp string) Option {
+	return func(o *options) {
+		o.withBucketPrefix = bp
+	}
+}
+
+// WithWorkerFilter provides a worker filter that indicate which workers
+// can support requests for this storage bucket.
+func WithWorkerFilter(wf string) Option {
+	return func(o *options) {
+		o.withWorkerFilter = wf
+	}
+}
+
+// WithLimit provides an option to provide a limit. Intentionally allowing
+// negative integers. If WithLimit < 0, then unlimited results are
+// returned. If WithLimit == 0, then default limits are used for results.
+func WithLimit(l int) Option {
+	return func(o *options) {
+		o.withLimit = l
+	}
+}

internal/storage/plugin/options_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package plugin
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func Test_GetOpts(t *testing.T) {
+	t.Parallel()
+	t.Run("WithChunkSize", func(t *testing.T) {
+		opts := getOpts(WithChunkSize(1024))
+		testOpts := getDefaultOptions()
+		testOpts.withChunkSize = 1024
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithAttributes", func(t *testing.T) {
+		opts := getOpts(WithAttributes(&structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}))
+		testOpts := getDefaultOptions()
+		testOpts.withAttributes = &structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithName", func(t *testing.T) {
+		opts := getOpts(WithName("test"))
+		testOpts := getDefaultOptions()
+		testOpts.withName = "test"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithDescription", func(t *testing.T) {
+		opts := getOpts(WithDescription("test desc"))
+		testOpts := getDefaultOptions()
+		testOpts.withDescription = "test desc"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithSecrets", func(t *testing.T) {
+		opts := getOpts(WithSecrets(&structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}))
+		testOpts := getDefaultOptions()
+		testOpts.withSecrets = &structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithBucketPrefix", func(t *testing.T) {
+		opts := getOpts(WithBucketPrefix("test prefix"))
+		testOpts := getDefaultOptions()
+		testOpts.withBucketPrefix = "test prefix"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithWorkerFilter", func(t *testing.T) {
+		opts := getOpts(WithWorkerFilter("test filter"))
+		testOpts := getDefaultOptions()
+		testOpts.withWorkerFilter = "test filter"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithLimit", func(t *testing.T) {
+		opts := getOpts(WithLimit(12345))
+		testOpts := getDefaultOptions()
+		testOpts.withLimit = 12345
+		assert.Equal(t, opts, testOpts)
+	})
+}

internal/storage/plugin/storage_bucket.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/boundary/internal/oplog"
 	"github.com/hashicorp/boundary/internal/storage/plugin/store"
 	"github.com/hashicorp/boundary/internal/util"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets"
 	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
 	"github.com/hashicorp/go-kms-wrapping/v2/extras/structwrapping"
 	"google.golang.org/protobuf/proto"
@@ -185,6 +186,20 @@ func (sbs *StorageBucketSecret) decrypt(ctx context.Context, cipher wrapping.Wra
 	return nil
 }
 
+func (sbs *StorageBucketSecret) toPersisted(ctx context.Context) (*storagebuckets.StorageBucketPersisted, error) {
+	const op = "plugin.(StorageBucketSecret).toPersisted"
+	if sbs.Secrets == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "secret data not populated")
+	}
+	sec := &storagebuckets.StorageBucketPersisted{
+		Data: &structpb.Struct{},
+	}
+	if err := proto.Unmarshal(sbs.Secrets, sec.Data); err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter))
+	}
+	return sec, nil
+}
+
 type storageBucketAgg struct {
 	PublicId         string `gorm:"primary_key"`
 	ScopeId          string