Modify storage bucket service to include NormalizeStorageBucketData (#5590)

NormalizeStorageBucketData normalizes a given's storage bucket attributes before persisting.

Co-authored-by: Danielle Miu <29378233+DanielleMiu@users.noreply.github.com>

internal/plugin/loopback/client.go

@@ -71,6 +71,10 @@ func NewWrappingPluginStorageClient(s plgpb.StoragePluginServiceServer) *Wrappin
 	return &WrappingPluginStorageClient{Server: s}
 }
 
+func (tpc *WrappingPluginStorageClient) NormalizeStorageBucketData(ctx context.Context, req *plgpb.NormalizeStorageBucketDataRequest, opts ...grpc.CallOption) (*plgpb.NormalizeStorageBucketDataResponse, error) {
+	return tpc.Server.NormalizeStorageBucketData(ctx, req)
+}
+
 func (tpc *WrappingPluginStorageClient) OnCreateStorageBucket(ctx context.Context, req *plgpb.OnCreateStorageBucketRequest, opts ...grpc.CallOption) (*plgpb.OnCreateStorageBucketResponse, error) {
 	return tpc.Server.OnCreateStorageBucket(ctx, req)
 }

internal/plugin/loopback/loopback.go

@@ -104,6 +104,7 @@ func NewLoopbackPlugin(opt ...TestOption) (*LoopbackPlugin, error) {
 	ret.ListHostsFn = ret.listHosts
 
 	// Set storage methods
+	ret.NormalizeStorageBucketDataFn = ret.normalizeStorageBucketData
 	ret.OnCreateStorageBucketFn = ret.onCreateStorageBucket
 	ret.OnUpdateStorageBucketFn = ret.onUpdateStorageBucket
 	ret.OnDeleteStorageBucketFn = ret.onDeleteStorageBucket

internal/plugin/loopback/storage.go

@@ -27,17 +27,25 @@ var _ plgpb.StoragePluginServiceServer = (*TestPluginStorageServer)(nil)
 
 // TestPluginStorageServer provides a storage plugin service server where each method can be overwritten for tests.
 type TestPluginStorageServer struct {
-	OnCreateStorageBucketFn func(context.Context, *plgpb.OnCreateStorageBucketRequest) (*plgpb.OnCreateStorageBucketResponse, error)
-	OnUpdateStorageBucketFn func(context.Context, *plgpb.OnUpdateStorageBucketRequest) (*plgpb.OnUpdateStorageBucketResponse, error)
-	OnDeleteStorageBucketFn func(context.Context, *plgpb.OnDeleteStorageBucketRequest) (*plgpb.OnDeleteStorageBucketResponse, error)
-	ValidatePermissionsFn   func(context.Context, *plgpb.ValidatePermissionsRequest) (*plgpb.ValidatePermissionsResponse, error)
-	HeadObjectFn            func(context.Context, *plgpb.HeadObjectRequest) (*plgpb.HeadObjectResponse, error)
-	GetObjectFn             func(*plgpb.GetObjectRequest, plgpb.StoragePluginService_GetObjectServer) error
-	PutObjectFn             func(context.Context, *plgpb.PutObjectRequest) (*plgpb.PutObjectResponse, error)
-	DeleteObjectsFn         func(context.Context, *plgpb.DeleteObjectsRequest) (*plgpb.DeleteObjectsResponse, error)
+	NormalizeStorageBucketDataFn func(context.Context, *plgpb.NormalizeStorageBucketDataRequest) (*plgpb.NormalizeStorageBucketDataResponse, error)
+	OnCreateStorageBucketFn      func(context.Context, *plgpb.OnCreateStorageBucketRequest) (*plgpb.OnCreateStorageBucketResponse, error)
+	OnUpdateStorageBucketFn      func(context.Context, *plgpb.OnUpdateStorageBucketRequest) (*plgpb.OnUpdateStorageBucketResponse, error)
+	OnDeleteStorageBucketFn      func(context.Context, *plgpb.OnDeleteStorageBucketRequest) (*plgpb.OnDeleteStorageBucketResponse, error)
+	ValidatePermissionsFn        func(context.Context, *plgpb.ValidatePermissionsRequest) (*plgpb.ValidatePermissionsResponse, error)
+	HeadObjectFn                 func(context.Context, *plgpb.HeadObjectRequest) (*plgpb.HeadObjectResponse, error)
+	GetObjectFn                  func(*plgpb.GetObjectRequest, plgpb.StoragePluginService_GetObjectServer) error
+	PutObjectFn                  func(context.Context, *plgpb.PutObjectRequest) (*plgpb.PutObjectResponse, error)
+	DeleteObjectsFn              func(context.Context, *plgpb.DeleteObjectsRequest) (*plgpb.DeleteObjectsResponse, error)
 	plgpb.UnimplementedStoragePluginServiceServer
 }
 
+func (t TestPluginStorageServer) NormalizeStorageBucketData(ctx context.Context, req *plgpb.NormalizeStorageBucketDataRequest) (*plgpb.NormalizeStorageBucketDataResponse, error) {
+	if t.NormalizeStorageBucketDataFn == nil {
+		return t.UnimplementedStoragePluginServiceServer.NormalizeStorageBucketData(ctx, req)
+	}
+	return t.NormalizeStorageBucketDataFn(ctx, req)
+}
+
 func (t TestPluginStorageServer) OnCreateStorageBucket(ctx context.Context, req *plgpb.OnCreateStorageBucketRequest) (*plgpb.OnCreateStorageBucketResponse, error) {
 	if t.OnCreateStorageBucketFn == nil {
 		return t.UnimplementedStoragePluginServiceServer.OnCreateStorageBucket(ctx, req)
@@ -122,6 +130,33 @@ type LoopbackStorage struct {
 	buckets           map[BucketName]Bucket
 	errs              []PluginMockError
 	putObjectResponse []PluginMockPutObjectResponse
+	normalizations    int
+}
+
+func (l *LoopbackStorage) normalizeStorageBucketData(ctx context.Context, req *plgpb.NormalizeStorageBucketDataRequest) (*plgpb.NormalizeStorageBucketDataResponse, error) {
+	const op = "loopback.(LoopbackStorage).normalizeStorageBucketData"
+	if req == nil {
+		return nil, status.Errorf(codes.InvalidArgument, "%s: request is nil", op)
+	}
+	if req.GetAttributes() == nil {
+		return nil, status.Errorf(codes.InvalidArgument, "%s: missing attributes", op)
+	}
+	l.normalizations++
+	return &plgpb.NormalizeStorageBucketDataResponse{
+		Attributes: req.GetAttributes(),
+	}, nil
+}
+
+// ResetNormalizations sets the number of times that NormalizeStorageBucketData
+// has been called to 0. Useful for unit tests.
+func (l *LoopbackStorage) ResetNormalizations() {
+	l.normalizations = 0
+}
+
+// GetNormalizations returns the number of times that NormalizeStorageBucketData
+// has been called via the loopback plugin. Useful for unit tests.
+func (l *LoopbackStorage) GetNormalizations() int {
+	return l.normalizations
 }
 
 func (l *LoopbackStorage) onCreateStorageBucket(ctx context.Context, req *plgpb.OnCreateStorageBucketRequest) (*plgpb.OnCreateStorageBucketResponse, error) {

internal/proto/plugin/v1/storage_plugin_service.proto

@@ -5,13 +5,34 @@ syntax = "proto3";
 
 package plugin.v1;
 
+import "controller/api/resources/plugins/v1/plugin.proto";
 import "controller/api/resources/storagebuckets/v1/storage_bucket.proto";
+import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 
 option go_package = "github.com/hashicorp/boundary/sdk/pbs/plugin;plugin";
 
 // StoragePluginService describes the service for storage bucket plugins.
 service StoragePluginService {
+  // NormalizeStorageBucketData is a hook that passes attributes to the plugin
+  // and allows those values to be normalized prior to creating or updating
+  // those values within the persisted storage bucket.
+  //
+  // NormalizeStorageBucketData is called before the values of attributes are
+  // persisted. All normalized values will be persisted in Boundary and returned
+  // to all clients.
+  //
+  // NormalizeStorageBucketData could affect other clients. For example, in
+  // Terraform, if data is passed to Boundary and then normalized into a new
+  // data structure, it could cause diffs in Terraform for unchanged values.
+  // This is because, the data structure in Terraform's state will now be
+  // different from the normalized data structure returned from Boundary.
+  //
+  // NormalizeStorageBucketData is called before:
+  // * OnCreateStorageBucket
+  // * OnUpdateStorageBucket
+  rpc NormalizeStorageBucketData(NormalizeStorageBucketDataRequest) returns (NormalizeStorageBucketDataResponse);
+
   // OnCreateStorageBucket is a hook that runs when a
   // storage bucket is created.
   rpc OnCreateStorageBucket(OnCreateStorageBucketRequest) returns (OnCreateStorageBucketResponse);
@@ -43,6 +64,20 @@ service StoragePluginService {
   rpc DeleteObjects(DeleteObjectsRequest) returns (DeleteObjectsResponse);
 }
 
+message NormalizeStorageBucketDataRequest {
+  // The incoming attributes in the create or update request.
+  google.protobuf.Struct attributes = 10;
+
+  // The plugin information for this request.
+  controller.api.resources.plugins.v1.PluginInfo plugin = 20;
+}
+
+message NormalizeStorageBucketDataResponse {
+  // Outgoing attributes. If nil, no changes will be recorded. If non-nil, the
+  // values here will be used in place of the original set of attributes.
+  google.protobuf.Struct attributes = 10;
+}
+
 message OnCreateStorageBucketRequest {
   // Required. The storage bucket to create. The request may contain optional
   // secret data to help authenticate the request against a cloud

sdk/pbs/plugin/storage_plugin_service_grpc.pb.go

@@ -22,20 +22,39 @@ import (
 const _ = grpc.SupportPackageIsVersion7
 
 const (
-	StoragePluginService_OnCreateStorageBucket_FullMethodName = "/plugin.v1.StoragePluginService/OnCreateStorageBucket"
-	StoragePluginService_OnUpdateStorageBucket_FullMethodName = "/plugin.v1.StoragePluginService/OnUpdateStorageBucket"
-	StoragePluginService_OnDeleteStorageBucket_FullMethodName = "/plugin.v1.StoragePluginService/OnDeleteStorageBucket"
-	StoragePluginService_ValidatePermissions_FullMethodName   = "/plugin.v1.StoragePluginService/ValidatePermissions"
-	StoragePluginService_HeadObject_FullMethodName            = "/plugin.v1.StoragePluginService/HeadObject"
-	StoragePluginService_GetObject_FullMethodName             = "/plugin.v1.StoragePluginService/GetObject"
-	StoragePluginService_PutObject_FullMethodName             = "/plugin.v1.StoragePluginService/PutObject"
-	StoragePluginService_DeleteObjects_FullMethodName         = "/plugin.v1.StoragePluginService/DeleteObjects"
+	StoragePluginService_NormalizeStorageBucketData_FullMethodName = "/plugin.v1.StoragePluginService/NormalizeStorageBucketData"
+	StoragePluginService_OnCreateStorageBucket_FullMethodName      = "/plugin.v1.StoragePluginService/OnCreateStorageBucket"
+	StoragePluginService_OnUpdateStorageBucket_FullMethodName      = "/plugin.v1.StoragePluginService/OnUpdateStorageBucket"
+	StoragePluginService_OnDeleteStorageBucket_FullMethodName      = "/plugin.v1.StoragePluginService/OnDeleteStorageBucket"
+	StoragePluginService_ValidatePermissions_FullMethodName        = "/plugin.v1.StoragePluginService/ValidatePermissions"
+	StoragePluginService_HeadObject_FullMethodName                 = "/plugin.v1.StoragePluginService/HeadObject"
+	StoragePluginService_GetObject_FullMethodName                  = "/plugin.v1.StoragePluginService/GetObject"
+	StoragePluginService_PutObject_FullMethodName                  = "/plugin.v1.StoragePluginService/PutObject"
+	StoragePluginService_DeleteObjects_FullMethodName              = "/plugin.v1.StoragePluginService/DeleteObjects"
 )
 
 // StoragePluginServiceClient is the client API for StoragePluginService service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type StoragePluginServiceClient interface {
+	// NormalizeStorageBucketData is a hook that passes attributes to the plugin
+	// and allows those values to be normalized prior to creating or updating
+	// those values within the persisted storage bucket.
+	//
+	// NormalizeStorageBucketData is called before the values of attributes are
+	// persisted. All normalized values will be persisted in Boundary and returned
+	// to all clients.
+	//
+	// NormalizeStorageBucketData could affect other clients. For example, in
+	// Terraform, if data is passed to Boundary and then normalized into a new
+	// data structure, it could cause diffs in Terraform for unchanged values.
+	// This is because, the data structure in Terraform's state will now be
+	// different from the normalized data structure returned from Boundary.
+	//
+	// NormalizeStorageBucketData is called before:
+	// * OnCreateStorageBucket
+	// * OnUpdateStorageBucket
+	NormalizeStorageBucketData(ctx context.Context, in *NormalizeStorageBucketDataRequest, opts ...grpc.CallOption) (*NormalizeStorageBucketDataResponse, error)
 	// OnCreateStorageBucket is a hook that runs when a
 	// storage bucket is created.
 	OnCreateStorageBucket(ctx context.Context, in *OnCreateStorageBucketRequest, opts ...grpc.CallOption) (*OnCreateStorageBucketResponse, error)
@@ -68,6 +87,15 @@ func NewStoragePluginServiceClient(cc grpc.ClientConnInterface) StoragePluginSer
 	return &storagePluginServiceClient{cc}
 }
 
+func (c *storagePluginServiceClient) NormalizeStorageBucketData(ctx context.Context, in *NormalizeStorageBucketDataRequest, opts ...grpc.CallOption) (*NormalizeStorageBucketDataResponse, error) {
+	out := new(NormalizeStorageBucketDataResponse)
+	err := c.cc.Invoke(ctx, StoragePluginService_NormalizeStorageBucketData_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *storagePluginServiceClient) OnCreateStorageBucket(ctx context.Context, in *OnCreateStorageBucketRequest, opts ...grpc.CallOption) (*OnCreateStorageBucketResponse, error) {
 	out := new(OnCreateStorageBucketResponse)
 	err := c.cc.Invoke(ctx, StoragePluginService_OnCreateStorageBucket_FullMethodName, in, out, opts...)
@@ -167,6 +195,24 @@ func (c *storagePluginServiceClient) DeleteObjects(ctx context.Context, in *Dele
 // All implementations must embed UnimplementedStoragePluginServiceServer
 // for forward compatibility
 type StoragePluginServiceServer interface {
+	// NormalizeStorageBucketData is a hook that passes attributes to the plugin
+	// and allows those values to be normalized prior to creating or updating
+	// those values within the persisted storage bucket.
+	//
+	// NormalizeStorageBucketData is called before the values of attributes are
+	// persisted. All normalized values will be persisted in Boundary and returned
+	// to all clients.
+	//
+	// NormalizeStorageBucketData could affect other clients. For example, in
+	// Terraform, if data is passed to Boundary and then normalized into a new
+	// data structure, it could cause diffs in Terraform for unchanged values.
+	// This is because, the data structure in Terraform's state will now be
+	// different from the normalized data structure returned from Boundary.
+	//
+	// NormalizeStorageBucketData is called before:
+	// * OnCreateStorageBucket
+	// * OnUpdateStorageBucket
+	NormalizeStorageBucketData(context.Context, *NormalizeStorageBucketDataRequest) (*NormalizeStorageBucketDataResponse, error)
 	// OnCreateStorageBucket is a hook that runs when a
 	// storage bucket is created.
 	OnCreateStorageBucket(context.Context, *OnCreateStorageBucketRequest) (*OnCreateStorageBucketResponse, error)
@@ -196,6 +242,9 @@ type StoragePluginServiceServer interface {
 type UnimplementedStoragePluginServiceServer struct {
 }
 
+func (UnimplementedStoragePluginServiceServer) NormalizeStorageBucketData(context.Context, *NormalizeStorageBucketDataRequest) (*NormalizeStorageBucketDataResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method NormalizeStorageBucketData not implemented")
+}
 func (UnimplementedStoragePluginServiceServer) OnCreateStorageBucket(context.Context, *OnCreateStorageBucketRequest) (*OnCreateStorageBucketResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method OnCreateStorageBucket not implemented")
 }
@@ -233,6 +282,24 @@ func RegisterStoragePluginServiceServer(s grpc.ServiceRegistrar, srv StoragePlug
 	s.RegisterService(&StoragePluginService_ServiceDesc, srv)
 }
 
+func _StoragePluginService_NormalizeStorageBucketData_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(NormalizeStorageBucketDataRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(StoragePluginServiceServer).NormalizeStorageBucketData(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: StoragePluginService_NormalizeStorageBucketData_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(StoragePluginServiceServer).NormalizeStorageBucketData(ctx, req.(*NormalizeStorageBucketDataRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _StoragePluginService_OnCreateStorageBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(OnCreateStorageBucketRequest)
 	if err := dec(in); err != nil {
@@ -387,6 +454,10 @@ var StoragePluginService_ServiceDesc = grpc.ServiceDesc{
 	ServiceName: "plugin.v1.StoragePluginService",
 	HandlerType: (*StoragePluginServiceServer)(nil),
 	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "NormalizeStorageBucketData",
+			Handler:    _StoragePluginService_NormalizeStorageBucketData_Handler,
+		},
 		{
 			MethodName: "OnCreateStorageBucket",
 			Handler:    _StoragePluginService_OnCreateStorageBucket_Handler,