update storage buckets to use a shared persisted secrets type. add options

internal/storage/plugin/options.go

@@ -0,0 +1,105 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package plugin
+
+import "google.golang.org/protobuf/types/known/structpb"
+
+const (
+	// DefaultChunkSize is the default chunk size for streaming
+	DefaultChunkSize = 65536 // 64KiB
+)
+
+// getOpts - iterate the inbound Options and return a struct
+func getOpts(opt ...Option) options {
+	opts := getDefaultOptions()
+	for _, o := range opt {
+		o(&opts)
+	}
+	return opts
+}
+
+// Option - how Options are passed as arguments.
+type Option func(*options)
+
+// options = how options are represented
+type options struct {
+	withChunkSize    int
+	withName         string
+	withDescription  string
+	withAttributes   *structpb.Struct
+	withSecrets      *structpb.Struct
+	withWorkerFilter string
+	withBucketPrefix string
+	withLimit        int
+}
+
+func getDefaultOptions() options {
+	return options{
+		withChunkSize:  DefaultChunkSize,
+		withAttributes: &structpb.Struct{},
+	}
+}
+
+// WithChunkSize provides an optional chunkSize to associate
+// with a StorageClient. ChunkSize is the number of bytes to
+// send to the plugin in a single request. If not provided,
+// the default is 64KiB. The recommended chunk size for
+// streamed messages is 16KiB to 64KiB.
+func WithChunkSize(chunkSize int) Option {
+	return func(o *options) {
+		o.withChunkSize = chunkSize
+	}
+}
+
+// WithDescription provides an optional description.
+func WithDescription(desc string) Option {
+	return func(o *options) {
+		o.withDescription = desc
+	}
+}
+
+// WithName provides an optional name.
+func WithName(name string) Option {
+	return func(o *options) {
+		o.withName = name
+	}
+}
+
+// WithAttributes provides an optional attributes field.
+func WithAttributes(attrs *structpb.Struct) Option {
+	return func(o *options) {
+		o.withAttributes = attrs
+	}
+}
+
+// WithSecrets provides an optional secrets field.
+func WithSecrets(secrets *structpb.Struct) Option {
+	return func(o *options) {
+		o.withSecrets = secrets
+	}
+}
+
+// WithBucketPrefix provides an optional bucket prefix.
+func WithBucketPrefix(bp string) Option {
+	return func(o *options) {
+		o.withBucketPrefix = bp
+	}
+}
+
+// WithWorkerFilter provides a worker filter that indicate which workers
+// can support requests for this storage bucket.
+func WithWorkerFilter(wf string) Option {
+	return func(o *options) {
+		o.withWorkerFilter = wf
+	}
+}
+
+// WithLimit provides an option to provide a limit. Intentionally allowing
+// negative integers. If WithLimit < 0, then unlimited results are
+// returned. If WithLimit == 0, then default limits are used for results.
+func WithLimit(l int) Option {
+	return func(o *options) {
+		o.withLimit = l
+	}
+}

internal/storage/plugin/options_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package plugin
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func Test_GetOpts(t *testing.T) {
+	t.Parallel()
+	t.Run("WithChunkSize", func(t *testing.T) {
+		opts := getOpts(WithChunkSize(1024))
+		testOpts := getDefaultOptions()
+		testOpts.withChunkSize = 1024
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithAttributes", func(t *testing.T) {
+		opts := getOpts(WithAttributes(&structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}))
+		testOpts := getDefaultOptions()
+		testOpts.withAttributes = &structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithName", func(t *testing.T) {
+		opts := getOpts(WithName("test"))
+		testOpts := getDefaultOptions()
+		testOpts.withName = "test"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithDescription", func(t *testing.T) {
+		opts := getOpts(WithDescription("test desc"))
+		testOpts := getDefaultOptions()
+		testOpts.withDescription = "test desc"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithSecrets", func(t *testing.T) {
+		opts := getOpts(WithSecrets(&structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}))
+		testOpts := getDefaultOptions()
+		testOpts.withSecrets = &structpb.Struct{Fields: map[string]*structpb.Value{"foo": structpb.NewStringValue("bar")}}
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithBucketPrefix", func(t *testing.T) {
+		opts := getOpts(WithBucketPrefix("test prefix"))
+		testOpts := getDefaultOptions()
+		testOpts.withBucketPrefix = "test prefix"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithWorkerFilter", func(t *testing.T) {
+		opts := getOpts(WithWorkerFilter("test filter"))
+		testOpts := getDefaultOptions()
+		testOpts.withWorkerFilter = "test filter"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithLimit", func(t *testing.T) {
+		opts := getOpts(WithLimit(12345))
+		testOpts := getDefaultOptions()
+		testOpts.withLimit = 12345
+		assert.Equal(t, opts, testOpts)
+	})
+}

internal/storage/plugin/storage_bucket.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/boundary/internal/oplog"
 	"github.com/hashicorp/boundary/internal/storage/plugin/store"
 	"github.com/hashicorp/boundary/internal/util"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets"
 	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
 	"github.com/hashicorp/go-kms-wrapping/v2/extras/structwrapping"
 	"google.golang.org/protobuf/proto"
@@ -185,6 +186,20 @@ func (sbs *StorageBucketSecret) decrypt(ctx context.Context, cipher wrapping.Wra
 	return nil
 }
 
+func (sbs *StorageBucketSecret) toPersisted(ctx context.Context) (*storagebuckets.StorageBucketPersisted, error) {
+	const op = "plugin.(StorageBucketSecret).toPersisted"
+	if sbs.Secrets == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "secret data not populated")
+	}
+	sec := &storagebuckets.StorageBucketPersisted{
+		Data: &structpb.Struct{},
+	}
+	if err := proto.Unmarshal(sbs.Secrets, sec.Data); err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter))
+	}
+	return sec, nil
+}
+
 type storageBucketAgg struct {
 	PublicId         string `gorm:"primary_key"`
 	ScopeId          string