From cbb76ea7c8774a9bfe66447889db6894fb3ca630 Mon Sep 17 00:00:00 2001 From: Damian Debkowski Date: Thu, 14 May 2026 11:02:32 -0700 Subject: [PATCH] feat: storage plugin list objects (#6723) * feat(proto): add storage plugin proto definitions for session recording recovery Co-authored-by: Damian Debkowski Co-authored-by: Ryan Derr * feat(gen): add generated storage plugin code for session recording recovery Co-authored-by: Damian Debkowski Co-authored-by: Manjeet Singh * feat(plugin): add plugin loopback support for session recording recovery Co-authored-by: Ryan Derr Co-authored-by: aditya2548 --------- Co-authored-by: Ryan Derr Co-authored-by: Manjeet Singh Co-authored-by: aditya2548 --- internal/plugin/loopback/client.go | 4 + internal/plugin/loopback/loopback.go | 1 + internal/plugin/loopback/options.go | 59 +- internal/plugin/loopback/options_test.go | 549 ++++++++++++++++ internal/plugin/loopback/storage.go | 214 +++++++ internal/plugin/loopback/storage_test.go | 605 ++++++++++++++++++ .../plugin/v1/storage_plugin_service.proto | 35 +- sdk/pbs/plugin/storage_plugin_service.pb.go | 351 +++++++--- .../plugin/storage_plugin_service_grpc.pb.go | 56 +- 9 files changed, 1771 insertions(+), 103 deletions(-) create mode 100644 internal/plugin/loopback/options_test.go diff --git a/internal/plugin/loopback/client.go b/internal/plugin/loopback/client.go index 3fa5285c2..eda6c4dae 100644 --- a/internal/plugin/loopback/client.go +++ b/internal/plugin/loopback/client.go @@ -91,6 +91,10 @@ func (tpc *WrappingPluginStorageClient) HeadObject(ctx context.Context, req *plg return tpc.Server.HeadObject(ctx, req) } +func (tpc *WrappingPluginStorageClient) ListObjects(ctx context.Context, req *plgpb.ListObjectsRequest, opts ...grpc.CallOption) (*plgpb.ListObjectsResponse, error) { + return tpc.Server.ListObjects(ctx, req) +} + func (tpc *WrappingPluginStorageClient) ValidatePermissions(ctx context.Context, req *plgpb.ValidatePermissionsRequest, opts ...grpc.CallOption) (*plgpb.ValidatePermissionsResponse, error) { return tpc.Server.ValidatePermissions(ctx, req) } diff --git a/internal/plugin/loopback/loopback.go b/internal/plugin/loopback/loopback.go index ce2087055..f48774525 100644 --- a/internal/plugin/loopback/loopback.go +++ b/internal/plugin/loopback/loopback.go @@ -113,6 +113,7 @@ func NewLoopbackPlugin(opt ...TestOption) (*LoopbackPlugin, error) { ret.GetObjectFn = ret.getObject ret.PutObjectFn = ret.putObject ret.DeleteObjectsFn = ret.deleteObjects + ret.ListObjectsFn = ret.listObjects if len(opts.withMockBuckets) > 0 { ret.buckets = opts.withMockBuckets } diff --git a/internal/plugin/loopback/options.go b/internal/plugin/loopback/options.go index d4b6ddd10..987e3950e 100644 --- a/internal/plugin/loopback/options.go +++ b/internal/plugin/loopback/options.go @@ -4,6 +4,8 @@ package loopback import ( + "strings" + "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets" plgpb "github.com/hashicorp/boundary/sdk/pbs/plugin" "google.golang.org/grpc/codes" @@ -39,6 +41,9 @@ const ( // Will return an error for the DeleteObjects method DeleteObjects + + // Will return an error for the ListObjects method + ListObjects ) const ( @@ -47,12 +52,47 @@ const ( // PluginMockError is used to mock an error when interacting with an external object store. type PluginMockError struct { - BucketName string - BucketPrefix string - ObjectKey string - ErrMsg string - ErrCode codes.Code - ErrMethod Method + // BucketName is the name of the bucket to match on. + BucketName string + + // BucketPrefix is the prefix of the bucket to match on. + BucketPrefix string + + // ObjectKey is the object key to match on. This is optional and may be empty. + // If empty, the error will match any object key, as long as the ErrPath is not set. + // This is because the ErrPath is a less specific match than the ObjectKey. + // The ObjectKey is ignored when the ErrPath is set. + // The ObjectKey is also ignored for the following plugin methods: onCreateStorageBucket, + // onUpdateStorageBucket, onDeleteStorageBucket as these methods do not provide an object key. + ObjectKey string + + // ErrPath is the object key path prefix to match on. This is optional and may be empty. + // If empty, the error will match any object key, as long as the ObjectKey is not set. + // The ErrPath is ignored when the ObjectKey is set. + // The ErrPath is also ignored for the following plugin methods: onCreateStorageBucket, + // onUpdateStorageBucket, onDeleteStorageBucket as these methods do not provide an object key. + // The ErrPath is used to match on a prefix of the object key. For example, if the ErrPath + // is set to "path/to/obj", it will match any object key that starts with "path/to/obj", + // such as "path/to/obj1", "path/to/obj2/subobj", etc. + // This allows for more flexible error mocking based on object key prefixes. + // The ErrPath should always end with a "/" if it is intended to match a directory prefix. + // For example, to match all objects under "path/to/dir/", the ErrPath should be set to "path/to/dir/". + // This ensures that it does not unintentionally match objects like "path/to/dir_obj". + ErrPath string + + // ErrMsg is the error message to return. + ErrMsg string + + // ErrCode is the gRPC error code to return. + ErrCode codes.Code + + // ErrMethod is the plugin method to match on. This is optional and may be set to Any. + // If set to Any, the error will be returned for any plugin method that matches the bucket + // and object key criteria. If set to a specific method, the error will only be returned + // for that method if the bucket and object key criteria also match. + ErrMethod Method + + // StorageBucketCredentialState is the credential state to return. StorageBucketCredentialState *plgpb.StorageBucketCredentialState } @@ -67,7 +107,8 @@ func (e PluginMockError) match(bucket *storagebuckets.StorageBucket, key string, // the object key comparison is ignored when the given key is empty because the following // plugin methods do not provide key values: onCreateStorageBucket, onUpdateStorageBucket, // onDeleteStorageBucket - if key != "" && e.ObjectKey != key { + // if the mocked error object key is empty, it should match any key, as long as the ErrPath is not set. + if key != "" && e.ObjectKey != "" && e.ObjectKey != key { return false } // if the mocked error bucket name does not match the request's bucket name, return false. @@ -87,6 +128,10 @@ func (e PluginMockError) match(bucket *storagebuckets.StorageBucket, key string, if e.ErrMethod != method { return false } + // if the mocked error path is not empty and the key does not have the mocked error path as a prefix, return false. + if e.ErrPath != "" && !strings.HasPrefix(key, e.ErrPath) { + return false + } // all checks comparison checks passed, return true. return true } diff --git a/internal/plugin/loopback/options_test.go b/internal/plugin/loopback/options_test.go new file mode 100644 index 000000000..b2c74a304 --- /dev/null +++ b/internal/plugin/loopback/options_test.go @@ -0,0 +1,549 @@ +// Copyright IBM Corp. 2024, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package loopback + +import ( + "testing" + + "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/storagebuckets" + plgpb "github.com/hashicorp/boundary/sdk/pbs/plugin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +func TestPluginMockError_match(t *testing.T) { + t.Parallel() + + testBucket := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + } + + testBucketNoPrefix := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + } + + tests := []struct { + name string + mockError PluginMockError + bucket *storagebuckets.StorageBucket + key string + method Method + expected bool + }{ + { + name: "exact match with all fields", + mockError: PluginMockError{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + ObjectKey: "test-key", + ErrMethod: PutObject, + }, + bucket: testBucket, + key: "test-key", + method: PutObject, + expected: true, + }, + { + name: "match with Any method", + mockError: PluginMockError{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + ObjectKey: "test-key", + ErrMethod: Any, + }, + bucket: testBucket, + key: "test-key", + method: GetObject, + expected: true, + }, + { + name: "match with empty key (create/update/delete operations)", + mockError: PluginMockError{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + ErrMethod: OnCreateStorageBucket, + }, + bucket: testBucket, + key: "", // Empty key for bucket operations + method: OnCreateStorageBucket, + expected: true, + }, + { + name: "match with path prefix", + mockError: PluginMockError{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + ObjectKey: "logs/session123/data.log", // Must match the key exactly + ErrPath: "logs/", // Path prefix also matches + ErrMethod: PutObject, + }, + bucket: testBucket, + key: "logs/session123/data.log", + method: PutObject, + expected: true, + }, + { + name: "no match - different bucket name", + mockError: PluginMockError{ + BucketName: "different-bucket", + ErrMethod: PutObject, + }, + bucket: testBucket, + key: "test-key", + method: PutObject, + expected: false, + }, + { + name: "no match - different object key", + mockError: PluginMockError{ + BucketName: "test-bucket", + ObjectKey: "different-key", + ErrMethod: PutObject, + }, + bucket: testBucket, + key: "test-key", + method: PutObject, + expected: false, + }, + { + name: "no match - different method", + mockError: PluginMockError{ + BucketName: "test-bucket", + ObjectKey: "test-key", + ErrMethod: GetObject, + }, + bucket: testBucket, + key: "test-key", + method: PutObject, + expected: false, + }, + { + name: "no match - different bucket prefix", + mockError: PluginMockError{ + BucketName: "test-bucket", + BucketPrefix: "different-prefix", + ErrMethod: PutObject, + }, + bucket: testBucket, + key: "test-key", + method: PutObject, + expected: false, + }, + { + name: "no match - path prefix doesn't match", + mockError: PluginMockError{ + BucketName: "test-bucket", + ObjectKey: "", // ObjectKey should be empty when using ErrPath + ErrPath: "logs/", + ErrMethod: PutObject, + }, + bucket: testBucketNoPrefix, + key: "data/session123/info.json", + method: PutObject, + expected: false, + }, + { + name: "match - bucket with no prefix requirement", + mockError: PluginMockError{ + BucketName: "test-bucket", + ObjectKey: "test-key", + ErrMethod: PutObject, + }, + bucket: testBucketNoPrefix, + key: "test-key", + method: PutObject, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.mockError.match(tt.bucket, tt.key, tt.method) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestPluginMockPutObjectResponse_match(t *testing.T) { + t.Parallel() + + testBucket := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + } + + testBucketNoPrefix := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + } + + tests := []struct { + name string + mockResponse PluginMockPutObjectResponse + bucket *storagebuckets.StorageBucket + key string + expected bool + }{ + { + name: "exact match with all fields", + mockResponse: PluginMockPutObjectResponse{ + BucketName: "test-bucket", + BucketPrefix: "test-prefix", + ObjectKey: "test-key", + }, + bucket: testBucket, + key: "test-key", + expected: true, + }, + { + name: "no match - different bucket name", + mockResponse: PluginMockPutObjectResponse{ + BucketName: "different-bucket", + ObjectKey: "test-key", + }, + bucket: testBucket, + key: "test-key", + expected: false, + }, + { + name: "no match - different object key", + mockResponse: PluginMockPutObjectResponse{ + BucketName: "test-bucket", + ObjectKey: "different-key", + }, + bucket: testBucket, + key: "test-key", + expected: false, + }, + { + name: "no match - different bucket prefix", + mockResponse: PluginMockPutObjectResponse{ + BucketName: "test-bucket", + BucketPrefix: "different-prefix", + ObjectKey: "test-key", + }, + bucket: testBucket, + key: "test-key", + expected: false, + }, + { + name: "match - bucket with no prefix requirement", + mockResponse: PluginMockPutObjectResponse{ + BucketName: "test-bucket", + ObjectKey: "test-key", + }, + bucket: testBucketNoPrefix, + key: "test-key", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.mockResponse.match(tt.bucket, tt.key) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGetTestOpts(t *testing.T) { + t.Parallel() + + t.Run("default options", func(t *testing.T) { + opts, err := getTestOpts() + require.NoError(t, err) + + expected := TestOptions{ + withChunkSize: 8, + } + assert.Equal(t, expected.withChunkSize, opts.withChunkSize) + assert.Nil(t, opts.withMockBuckets) + assert.Empty(t, opts.withMockError) + assert.Empty(t, opts.withMockPutObjectResponse) + }) + + t.Run("with mock buckets", func(t *testing.T) { + mockBuckets := map[BucketName]Bucket{ + "test-bucket": make(Bucket), + } + + opts, err := getTestOpts(WithMockBuckets(mockBuckets)) + require.NoError(t, err) + + assert.Equal(t, mockBuckets, opts.withMockBuckets) + }) + + t.Run("with mock error", func(t *testing.T) { + mockError := PluginMockError{ + BucketName: "test-bucket", + ErrMsg: "test error", + ErrCode: codes.Internal, + ErrMethod: PutObject, + } + + opts, err := getTestOpts(WithMockError(mockError)) + require.NoError(t, err) + + require.Len(t, opts.withMockError, 1) + assert.Equal(t, mockError, opts.withMockError[0]) + }) + + t.Run("with multiple mock errors", func(t *testing.T) { + mockError1 := PluginMockError{ + BucketName: "test-bucket-1", + ErrMsg: "test error 1", + ErrCode: codes.Internal, + ErrMethod: PutObject, + } + mockError2 := PluginMockError{ + BucketName: "test-bucket-2", + ErrMsg: "test error 2", + ErrCode: codes.NotFound, + ErrMethod: GetObject, + } + + opts, err := getTestOpts( + WithMockError(mockError1), + WithMockError(mockError2), + ) + require.NoError(t, err) + + require.Len(t, opts.withMockError, 2) + assert.Equal(t, mockError1, opts.withMockError[0]) + assert.Equal(t, mockError2, opts.withMockError[1]) + }) + + t.Run("with mock put object response", func(t *testing.T) { + mockResponse := PluginMockPutObjectResponse{ + BucketName: "test-bucket", + ObjectKey: "test-key", + Response: &plgpb.PutObjectResponse{ + ChecksumSha_256: []byte("test-checksum"), + }, + } + + opts, err := getTestOpts(WithMockPutObjectResponse(mockResponse)) + require.NoError(t, err) + + require.Len(t, opts.withMockPutObjectResponse, 1) + assert.Equal(t, mockResponse, opts.withMockPutObjectResponse[0]) + }) + + t.Run("with custom chunk size", func(t *testing.T) { + customChunkSize := 16 + + opts, err := getTestOpts(WithChunkSize(customChunkSize)) + require.NoError(t, err) + + assert.Equal(t, customChunkSize, opts.withChunkSize) + }) + + t.Run("with all options combined", func(t *testing.T) { + mockBuckets := map[BucketName]Bucket{ + "test-bucket": make(Bucket), + } + mockError := PluginMockError{ + BucketName: "test-bucket", + ErrMsg: "test error", + ErrCode: codes.Internal, + ErrMethod: PutObject, + } + mockResponse := PluginMockPutObjectResponse{ + BucketName: "test-bucket", + ObjectKey: "test-key", + Response: &plgpb.PutObjectResponse{ + ChecksumSha_256: []byte("test-checksum"), + }, + } + customChunkSize := 32 + + opts, err := getTestOpts( + WithMockBuckets(mockBuckets), + WithMockError(mockError), + WithMockPutObjectResponse(mockResponse), + WithChunkSize(customChunkSize), + ) + require.NoError(t, err) + + assert.Equal(t, mockBuckets, opts.withMockBuckets) + require.Len(t, opts.withMockError, 1) + assert.Equal(t, mockError, opts.withMockError[0]) + require.Len(t, opts.withMockPutObjectResponse, 1) + assert.Equal(t, mockResponse, opts.withMockPutObjectResponse[0]) + assert.Equal(t, customChunkSize, opts.withChunkSize) + }) +} + +func TestGetDefaultTestOptions(t *testing.T) { + t.Parallel() + + opts := getDefaultTestOptions() + + assert.Equal(t, 8, opts.withChunkSize) + assert.Nil(t, opts.withMockBuckets) + assert.Empty(t, opts.withMockError) + assert.Empty(t, opts.withMockPutObjectResponse) +} + +func TestMethod_Constants(t *testing.T) { + t.Parallel() + + // Test that Method constants have expected values + assert.Equal(t, Method(0), Any) + assert.Equal(t, Method(1), OnCreateStorageBucket) + assert.Equal(t, Method(2), OnUpdateStorageBucket) + assert.Equal(t, Method(3), OnDeleteStorageBucket) + assert.Equal(t, Method(4), ValidatePermissions) + assert.Equal(t, Method(5), HeadObject) + assert.Equal(t, Method(6), GetObject) + assert.Equal(t, Method(7), PutObject) + assert.Equal(t, Method(8), DeleteObjects) +} + +func TestPluginMockError_ComplexScenarios(t *testing.T) { + t.Parallel() + + t.Run("storage bucket credential state scenarios", func(t *testing.T) { + bucket := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + } + + credentialState := &plgpb.StorageBucketCredentialState{ + State: &plgpb.Permissions{ + Write: &plgpb.Permission{ + State: plgpb.StateType_STATE_TYPE_ERROR, + }, + }, + } + + mockError := PluginMockError{ + BucketName: "test-bucket", + ErrMethod: ValidatePermissions, + StorageBucketCredentialState: credentialState, + } + + result := mockError.match(bucket, "", ValidatePermissions) + assert.True(t, result) + }) + + t.Run("path-based error matching", func(t *testing.T) { + bucket := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + } + + tests := []struct { + name string + errPath string + objectKey string + key string + expected bool + }{ + { + name: "exact path match", + errPath: "sessions/", + objectKey: "sessions/sr_123/data.log", + key: "sessions/sr_123/data.log", + expected: true, + }, + { + name: "nested path match", + errPath: "sessions/sr_123/", + objectKey: "sessions/sr_123/connections/cr_456/data.log", + key: "sessions/sr_123/connections/cr_456/data.log", + expected: true, + }, + { + name: "no path match", + errPath: "sessions/", + objectKey: "recordings/sr_123/data.log", + key: "recordings/sr_123/data.log", + expected: false, + }, + { + name: "path match but different object key", + errPath: "sessions/", + objectKey: "sessions/file1.log", + key: "sessions/file2.log", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockError := PluginMockError{ + BucketName: "test-bucket", + ObjectKey: tt.objectKey, + ErrPath: tt.errPath, + ErrMethod: PutObject, + } + + result := mockError.match(bucket, tt.key, PutObject) + assert.Equal(t, tt.expected, result) + }) + } + }) + + t.Run("path-based matching with empty key (bucket operations)", func(t *testing.T) { + bucket := &storagebuckets.StorageBucket{ + BucketName: "test-bucket", + } + + t.Run("empty path matches bucket operations", func(t *testing.T) { + mockError := PluginMockError{ + BucketName: "test-bucket", + ObjectKey: "", + ErrPath: "", // Empty path works with empty key + ErrMethod: OnCreateStorageBucket, + } + + // Empty key for bucket operations (create/update/delete) + result := mockError.match(bucket, "", OnCreateStorageBucket) + assert.True(t, result) + }) + + t.Run("non-empty path does not match empty key", func(t *testing.T) { + mockError := PluginMockError{ + BucketName: "test-bucket", + ObjectKey: "", + ErrPath: "some/path/", // Non-empty path won't match empty key + ErrMethod: OnCreateStorageBucket, + } + + // Empty key for bucket operations - should not match non-empty path + result := mockError.match(bucket, "", OnCreateStorageBucket) + assert.False(t, result) + }) + }) +} + +func TestTestOptions_EdgeCases(t *testing.T) { + t.Parallel() + + t.Run("zero chunk size", func(t *testing.T) { + opts, err := getTestOpts(WithChunkSize(0)) + require.NoError(t, err) + assert.Equal(t, 0, opts.withChunkSize) + }) + + t.Run("negative chunk size", func(t *testing.T) { + opts, err := getTestOpts(WithChunkSize(-1)) + require.NoError(t, err) + assert.Equal(t, -1, opts.withChunkSize) + }) + + t.Run("empty mock buckets map", func(t *testing.T) { + emptyBuckets := make(map[BucketName]Bucket) + opts, err := getTestOpts(WithMockBuckets(emptyBuckets)) + require.NoError(t, err) + assert.NotNil(t, opts.withMockBuckets) + assert.Empty(t, opts.withMockBuckets) + }) + + t.Run("nil mock buckets map", func(t *testing.T) { + opts, err := getTestOpts(WithMockBuckets(nil)) + require.NoError(t, err) + assert.Nil(t, opts.withMockBuckets) + }) +} diff --git a/internal/plugin/loopback/storage.go b/internal/plugin/loopback/storage.go index d07c1bcaf..b6123a428 100644 --- a/internal/plugin/loopback/storage.go +++ b/internal/plugin/loopback/storage.go @@ -37,6 +37,7 @@ type TestPluginStorageServer struct { GetObjectFn func(*plgpb.GetObjectRequest, plgpb.StoragePluginService_GetObjectServer) error PutObjectFn func(context.Context, *plgpb.PutObjectRequest) (*plgpb.PutObjectResponse, error) DeleteObjectsFn func(context.Context, *plgpb.DeleteObjectsRequest) (*plgpb.DeleteObjectsResponse, error) + ListObjectsFn func(context.Context, *plgpb.ListObjectsRequest) (*plgpb.ListObjectsResponse, error) plgpb.UnimplementedStoragePluginServiceServer } @@ -103,6 +104,13 @@ func (t TestPluginStorageServer) DeleteObjects(ctx context.Context, req *plgpb.D return t.DeleteObjectsFn(ctx, req) } +func (t TestPluginStorageServer) ListObjects(ctx context.Context, req *plgpb.ListObjectsRequest) (*plgpb.ListObjectsResponse, error) { + if t.ListObjectsFn == nil { + return t.UnimplementedStoragePluginServiceServer.ListObjects(ctx, req) + } + return t.ListObjectsFn(ctx, req) +} + type ( BucketName string ObjectName string @@ -157,9 +165,25 @@ func (l *LoopbackStorage) normalizeStorageBucketData(ctx context.Context, req *p // ResetNormalizations sets the number of times that NormalizeStorageBucketData // has been called to 0. Useful for unit tests. func (l *LoopbackStorage) ResetNormalizations() { + l.m.Lock() + defer l.m.Unlock() l.normalizations = 0 } +// ResetMockErrors removes all mock errors. +func (l *LoopbackStorage) ResetMockErrors() { + l.m.Lock() + defer l.m.Unlock() + l.errs = []PluginMockError{} +} + +// SetMockErrors overrides the existing mock errors. +func (l *LoopbackStorage) SetMockErrors(mockErrs ...PluginMockError) { + l.m.Lock() + defer l.m.Unlock() + l.errs = mockErrs +} + // GetNormalizations returns the number of times that NormalizeStorageBucketData // has been called via the loopback plugin. Useful for unit tests. func (l *LoopbackStorage) GetNormalizations() int { @@ -538,6 +562,53 @@ func (l *LoopbackStorage) deleteObjects(ctx context.Context, req *plgpb.DeleteOb }, nil } +func (l *LoopbackStorage) listObjects(ctx context.Context, req *plgpb.ListObjectsRequest) (*plgpb.ListObjectsResponse, error) { + const op = "loopback.(LoopbackStorage).listObjects" + + if req == nil { + return nil, status.Errorf(codes.InvalidArgument, "%s: request is nil", op) + } + if req.GetBucket() == nil { + return nil, status.Errorf(codes.InvalidArgument, "%s: missing storage bucket", op) + } + if req.GetBucket().GetAttributes() == nil { + return nil, status.Errorf(codes.InvalidArgument, "%s: missing bucket attributes", op) + } + l.m.Lock() + defer l.m.Unlock() + bucket, ok := l.buckets[BucketName(req.GetBucket().GetBucketName())] + if !ok { + return nil, status.Errorf(codes.NotFound, "%s: bucket not found", op) + } + + // return an expected mock error if one was provided + for _, err := range l.errs { + if err.match(req.GetBucket(), req.GetKeyPrefix(), ListObjects) { + if err.StorageBucketCredentialState != nil { + return nil, createErrorWithBucketCredentialState(err.ErrCode, fmt.Sprintf("%s: %s", op, err.ErrMsg), err.StorageBucketCredentialState) + } + return nil, status.Errorf(err.ErrCode, "%s: %s", op, err.ErrMsg) + } + } + + searchPrefix := path.Join(req.GetBucket().GetBucketPrefix(), req.GetKeyPrefix()) + // path.Join strips trailing slashes, but we need to preserve them for directory prefixes + // to ensure proper filtering behavior in listObjects functions + if strings.HasSuffix(req.GetKeyPrefix(), "/") || (req.GetKeyPrefix() == "" && strings.HasSuffix(req.GetBucket().GetBucketPrefix(), "/")) { + searchPrefix = searchPrefix + "/" + } + + var objects []*plgpb.Object + if req.GetRecursive() { + objects = listObjectsRecursive(bucket, searchPrefix) + } else { + objects = listObjectsNonRecursive(bucket, searchPrefix) + } + return &plgpb.ListObjectsResponse{ + Objects: objects, + }, nil +} + // CloneBucket returns a clone of the bucket. // returns nil when the bucket is not found. func (l *LoopbackStorage) CloneBucket(name string) Bucket { @@ -612,3 +683,146 @@ func createErrorWithBucketCredentialState(errCode codes.Code, msg string, sbcSta } return st.Err() } + +// listObjectsRecursive returns recursive list of all files and directories found under a prefix from a provided bucket +func listObjectsRecursive(bucket Bucket, searchPrefix string) []*plgpb.Object { + var objects []*plgpb.Object + dirs := make(map[string]struct{}) + seenKeys := make(map[string]struct{}) + + for objectName := range bucket { + key := string(objectName) + // Skip any objects that do not match the search prefix + if !strings.HasPrefix(key, searchPrefix) { + continue + } + + // Skip any objects that have already been seen + if _, exists := seenKeys[key]; exists { + continue + } + + isDirFile := strings.HasSuffix(key, "/") + // If the object is a directory and matches the search prefix, skip it + // This can happen if the prefix itself is a directory + // e.g. prefix = "foo/", object key = "foo/" + // In this case we don't want to return the directory itself as an object + if isDirFile && key == searchPrefix { + continue + } + + dirPaths := buildRecursiveDirectoryPaths(key, searchPrefix) + for _, dirPath := range dirPaths { + dirs[dirPath] = struct{}{} + seenKeys[dirPath] = struct{}{} + } + + // Only add the object if it hasn't already been added as a directory + // This can happen if the object key ends with a '/' and is added as a directory + // in the previous step + if _, exists := seenKeys[key]; !exists { + objects = append(objects, &plgpb.Object{ + Key: key, + IsDir: isDirFile, + }) + } + seenKeys[key] = struct{}{} + } + + for dirKey := range dirs { + objects = append(objects, &plgpb.Object{ + Key: dirKey, + IsDir: true, + }) + } + + return objects +} + +// listObjectsNonRecursive returns only top-level files and directories within the prefix +func listObjectsNonRecursive(bucket Bucket, searchPrefix string) []*plgpb.Object { + var objects []*plgpb.Object + seenKeys := make(map[string]struct{}) + + for objectName := range bucket { + key := string(objectName) + // Skip any objects that do not match the search prefix + if !strings.HasPrefix(key, searchPrefix) { + continue + } + + // Skip any objects that have already been seen + if _, exists := seenKeys[key]; exists { + continue + } + + isDirFile := strings.HasSuffix(key, "/") + // If the object is a directory and matches the search prefix, skip it + // This can happen if the prefix itself is a directory + // e.g. prefix = "foo/", object key = "foo/" + // In this case we don't want to return the directory itself as an object + if isDirFile && key == searchPrefix { + continue + } + + // Trim the prefix and and trailing '/' after which we split the path + // to determine if this object is in a subdirectory or a file at the root prefix + relativeFullPath := strings.TrimSuffix(strings.TrimPrefix(key, searchPrefix), "/") + split := strings.Split(relativeFullPath, "/") + // Adding this to guardrail against the case that the stdlib updates the behavior + // of strings.Split() causing it to not always return at least a 1 length slice + if len(split) == 0 { + continue + } + relativeParentPath := searchPrefix + split[0] + + // If the original object key ends with a '/' we treat it as a directory + // even if it is at the root of the prefix. + // If the split contains more than one entry, it is also a directory as it has sub-paths + if len(split) > 1 || isDirFile { + relativeParentPath += "/" + } + + // Skip any constructed parent paths that have already been seen + if _, exists := seenKeys[relativeParentPath]; exists { + continue + } + + // If the split contains more than one entry, it is a directory as it has sub-paths + // If the original object key ends with a '/' we treat it as a directory + // even if it is at the root of the prefix. + if len(split) > 1 { + objects = append(objects, &plgpb.Object{ + Key: relativeParentPath, + IsDir: true, + }) + } else { + objects = append(objects, &plgpb.Object{ + Key: relativeParentPath, + IsDir: strings.HasSuffix(relativeParentPath, "/"), + }) + } + seenKeys[relativeParentPath] = struct{}{} + } + + return objects +} + +// buildDirectoryPaths collects all parent directory paths from an object key +// that are within the given prefix. This is used to simulate directory +// structures in object storage. +func buildRecursiveDirectoryPaths(objectKey, prefix string) []string { + var dirPaths []string + prefixDir := strings.TrimSuffix(prefix, "/") + + pathParts := strings.Split(objectKey, "/") + // Build all possible parent directory paths from the split parts + for i := 1; i < len(pathParts); i++ { + dirPath := strings.Join(pathParts[:i], "/") + // Only include directories that are deeper than the search prefix and add a trailing slash to denote a directory + if len(dirPath) > len(prefixDir) && strings.HasPrefix(dirPath+"/", prefix) { + dirPaths = append(dirPaths, fmt.Sprintf("%s/", dirPath)) + } + } + return dirPaths +} diff --git a/internal/plugin/loopback/storage_test.go b/internal/plugin/loopback/storage_test.go index adffb477d..bb12fd330 100644 --- a/internal/plugin/loopback/storage_test.go +++ b/internal/plugin/loopback/storage_test.go @@ -1618,6 +1618,611 @@ func TestLoopbackDeleteObjects(t *testing.T) { } } +func TestLoopbackListObjects(t *testing.T) { + require, assert := tr.New(t), ta.New(t) + + mockStorageMapData := map[BucketName]Bucket{ + "aws_s3_mock": { + "test-file-1": MockObject([]Chunk{ + []byte("content of test-file-1"), + }), + "folder1/": MockObject(nil), + "test-file-2": MockObject([]Chunk{ + []byte("content of test-file-2"), + }), + "folder1/file-1": MockObject([]Chunk{ + []byte("content of folder1/file-1"), + }), + "folder1/subfolder/": MockObject(nil), + "folder1/subfolder/file-2": MockObject([]Chunk{ + []byte("content of folder1/subfolder/file-2"), + }), + "folder1/subfolder/nested/file-3": MockObject([]Chunk{ + []byte("content of folder1/subfolder/nested/file-3"), + }), + "folder1/subfolder2/": MockObject(nil), + "folder2/file-4": MockObject([]Chunk{ + []byte("content of folder2/file-4"), + }), + "folder3/": MockObject(nil), + }, + "aws_s3_err": { + "test-file": MockObject([]Chunk{ + []byte("test content"), + }), + }, + } + + plg, err := NewLoopbackPlugin( + WithMockBuckets(mockStorageMapData), + WithMockError(PluginMockError{ + BucketName: "aws_s3_err", + ErrMsg: "invalid credentials", + ErrCode: codes.PermissionDenied, + }), + ) + assert.NoError(err) + + client := NewWrappingPluginStorageClient(plg) + secrets := &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "AWS_ACCESS_KEY_ID": structpb.NewStringValue("access_key_id"), + "AWS_SECRET_ACCESS_KEY": structpb.NewStringValue("secret_access_key"), + }, + } + + tests := []struct { + name string + request *plgpb.ListObjectsRequest + expectedErr codes.Code + expectedObjects []*plgpb.Object + }{ + { + name: "missing request", + expectedErr: codes.InvalidArgument, + }, + { + name: "missing storage bucket", + request: &plgpb.ListObjectsRequest{}, + expectedErr: codes.InvalidArgument, + }, + { + name: "missing attributes", + request: &plgpb.ListObjectsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + }, + }, + expectedErr: codes.InvalidArgument, + }, + { + name: "bucket not found", + request: &plgpb.ListObjectsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "invalid_bucket", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedErr: codes.NotFound, + }, + { + name: "mocked error", + request: &plgpb.ListObjectsRequest{ + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_err", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedErr: codes.PermissionDenied, + }, + { + name: "list all objects without prefix recursively", + request: &plgpb.ListObjectsRequest{ + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "test-file-1", IsDir: false}, + {Key: "test-file-2", IsDir: false}, + {Key: "folder1/", IsDir: true}, + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + {Key: "folder2/", IsDir: true}, + {Key: "folder2/file-4", IsDir: false}, + {Key: "folder3/", IsDir: true}, + }, + }, + { + name: "list all objects without prefix non-recursively", + request: &plgpb.ListObjectsRequest{ + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "test-file-1", IsDir: false}, + {Key: "test-file-2", IsDir: false}, + {Key: "folder1/", IsDir: true}, + {Key: "folder2/", IsDir: true}, + {Key: "folder3/", IsDir: true}, + }, + }, + { + name: "list objects with prefix folder1/ recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + }, + }, + { + name: "list objects with prefix folder1/ non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + }, + }, + { + name: "list objects with prefix folder1/sub recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/sub", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + }, + }, + { + name: "list objects with prefix folder1/sub non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/sub", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + }, + }, + { + name: "list objects with prefix folder3/ recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder3/", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + { + name: "list objects with prefix folder3/ non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder3/", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + { + name: "list objects with prefix folder1/subfolder2 recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/subfolder2", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder2/", IsDir: true}, + }, + }, + { + name: "list objects with prefix folder1/subfolder2 non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/subfolder2", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder2/", IsDir: true}, + }, + }, + { + name: "list objects with prefix folder1/subfolder2/ recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/subfolder2/", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + { + name: "list objects with prefix folder1/subfolder2/ non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/subfolder2/", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + { + name: "list objects with non-existent prefix recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "nonexistent/", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + { + name: "list objects with non-existent prefix non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "nonexistent/", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + { + name: "list objects without secrets non-recursively", + request: &plgpb.ListObjectsRequest{ + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "test-file-1", IsDir: false}, + {Key: "test-file-2", IsDir: false}, + {Key: "folder1/", IsDir: true}, + {Key: "folder2/", IsDir: true}, + {Key: "folder3/", IsDir: true}, + }, + }, + { + name: "list objects without secrets recursively", + request: &plgpb.ListObjectsRequest{ + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "test-file-1", IsDir: false}, + {Key: "test-file-2", IsDir: false}, + {Key: "folder1/", IsDir: true}, + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + {Key: "folder2/", IsDir: true}, + {Key: "folder2/file-4", IsDir: false}, + {Key: "folder3/", IsDir: true}, + }, + }, + { + name: "list objects with bucket prefix and no key prefix recursively", + request: &plgpb.ListObjectsRequest{ + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "folder1/", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + }, + }, + { + name: "list objects with bucket prefix and no key prefix non-recursively", + request: &plgpb.ListObjectsRequest{ + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "folder1/", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + }, + }, + { + name: "list objects with bucket prefix and key prefix recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "subfolder/", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "folder1/", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + }, + }, + { + name: "list objects with bucket prefix and key prefix non-recursively", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "subfolder/", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "folder1/", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + }, + }, + { + name: "list objects with bucket prefix no trailing slash and key prefix with trailing slash", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "subfolder/", + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "folder1", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/subfolder/file-2", IsDir: false}, + {Key: "folder1/subfolder/nested/", IsDir: true}, + {Key: "folder1/subfolder/nested/file-3", IsDir: false}, + }, + }, + { + name: "list objects with bucket prefix empty and key prefix folder1/", + request: &plgpb.ListObjectsRequest{ + KeyPrefix: "folder1/", + Recursive: false, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{ + {Key: "folder1/file-1", IsDir: false}, + {Key: "folder1/subfolder/", IsDir: true}, + {Key: "folder1/subfolder2/", IsDir: true}, + }, + }, + { + name: "list objects with bucket prefix filter matching nothing", + request: &plgpb.ListObjectsRequest{ + Recursive: true, + Bucket: &storagebuckets.StorageBucket{ + BucketName: "aws_s3_mock", + BucketPrefix: "nonexistent/", + Secrets: secrets, + Attributes: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "endpoint": structpb.NewStringValue("0.0.0.0"), + }, + }, + }, + }, + expectedObjects: []*plgpb.Object{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := client.ListObjects(context.Background(), tt.request) + if tt.expectedErr != codes.OK { + assert.Error(err) + assert.Nil(resp) + s, ok := status.FromError(err) + require.True(ok, "invalid error type") + assert.Equal(tt.expectedErr, s.Code()) + return + } + assert.NoError(err) + require.NotNil(resp) + + require.Equal(len(tt.expectedObjects), len(resp.Objects), "object count mismatch") + assert.ElementsMatch(tt.expectedObjects, resp.Objects, "object list mismatch") + }) + } +} + func TestLoopbackStoragePlugin(t *testing.T) { td := t.TempDir() diff --git a/internal/proto/plugin/v1/storage_plugin_service.proto b/internal/proto/plugin/v1/storage_plugin_service.proto index d981ac015..df2c4f11b 100644 --- a/internal/proto/plugin/v1/storage_plugin_service.proto +++ b/internal/proto/plugin/v1/storage_plugin_service.proto @@ -45,17 +45,20 @@ service StoragePluginService { // deleted. rpc OnDeleteStorageBucket(OnDeleteStorageBucketRequest) returns (OnDeleteStorageBucketResponse); - // ValidatePermissions is a hook that checks if the secrets associated with + // ValidatePermissions checks if the secrets associated with // the storage bucket meet the requirements of the plugin. rpc ValidatePermissions(ValidatePermissionsRequest) returns (ValidatePermissionsResponse); - // HeadObject is a hook that retrieves metadata about an object. + // HeadObject returns metadata about an object. rpc HeadObject(HeadObjectRequest) returns (HeadObjectResponse); - // GetObject is a hook that retrieves objects. + // GetObject returns object requested. rpc GetObject(GetObjectRequest) returns (stream GetObjectResponse); - // PutObject is a hook that reads a file stored on local disk and + // ListObjects returns a list of objects within a storage bucket that matches the provided prefix. + rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse); + + // PutObject reads a file stored on local disk and // stores it to an external object store. rpc PutObject(PutObjectRequest) returns (PutObjectResponse); @@ -157,6 +160,30 @@ message GetObjectResponse { bytes file_chunk = 10; } +message ListObjectsRequest { + // Required. The existing state of the storage bucket. + controller.api.resources.storagebuckets.v1.StorageBucket bucket = 10; + + // Optional. The key prefix used to scope the object search. + string key_prefix = 20; + + // Optional. Flag to recursively list all objects under a given prefix. + bool recursive = 30; +} + +message Object { + // The object key defined within the object storage. + string key = 10; + + // Informs whether or not the object is a directory. + bool is_dir = 20; +} + +message ListObjectsResponse { + // A list of objects found under a provided prefix. + repeated Object objects = 10; +} + message PutObjectRequest { // Required. The existing state of the storage bucket. controller.api.resources.storagebuckets.v1.StorageBucket bucket = 10; diff --git a/sdk/pbs/plugin/storage_plugin_service.pb.go b/sdk/pbs/plugin/storage_plugin_service.pb.go index a16c31f7b..cfb0e7cf7 100644 --- a/sdk/pbs/plugin/storage_plugin_service.pb.go +++ b/sdk/pbs/plugin/storage_plugin_service.pb.go @@ -769,6 +769,168 @@ func (x *GetObjectResponse) GetFileChunk() []byte { return nil } +type ListObjectsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Required. The existing state of the storage bucket. + Bucket *storagebuckets.StorageBucket `protobuf:"bytes,10,opt,name=bucket,proto3" json:"bucket,omitempty"` + // Optional. The key prefix used to scope the object search. + KeyPrefix string `protobuf:"bytes,20,opt,name=key_prefix,json=keyPrefix,proto3" json:"key_prefix,omitempty"` + // Optional. Flag to recursively list all objects under a given prefix. + Recursive bool `protobuf:"varint,30,opt,name=recursive,proto3" json:"recursive,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListObjectsRequest) Reset() { + *x = ListObjectsRequest{} + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListObjectsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListObjectsRequest) ProtoMessage() {} + +func (x *ListObjectsRequest) ProtoReflect() protoreflect.Message { + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListObjectsRequest.ProtoReflect.Descriptor instead. +func (*ListObjectsRequest) Descriptor() ([]byte, []int) { + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{14} +} + +func (x *ListObjectsRequest) GetBucket() *storagebuckets.StorageBucket { + if x != nil { + return x.Bucket + } + return nil +} + +func (x *ListObjectsRequest) GetKeyPrefix() string { + if x != nil { + return x.KeyPrefix + } + return "" +} + +func (x *ListObjectsRequest) GetRecursive() bool { + if x != nil { + return x.Recursive + } + return false +} + +type Object struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The object key defined within the object storage. + Key string `protobuf:"bytes,10,opt,name=key,proto3" json:"key,omitempty"` + // Informs whether or not the object is a directory. + IsDir bool `protobuf:"varint,20,opt,name=is_dir,json=isDir,proto3" json:"is_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Object) Reset() { + *x = Object{} + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Object) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Object) ProtoMessage() {} + +func (x *Object) ProtoReflect() protoreflect.Message { + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Object.ProtoReflect.Descriptor instead. +func (*Object) Descriptor() ([]byte, []int) { + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{15} +} + +func (x *Object) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Object) GetIsDir() bool { + if x != nil { + return x.IsDir + } + return false +} + +type ListObjectsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A list of objects found under a provided prefix. + Objects []*Object `protobuf:"bytes,10,rep,name=objects,proto3" json:"objects,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListObjectsResponse) Reset() { + *x = ListObjectsResponse{} + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListObjectsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListObjectsResponse) ProtoMessage() {} + +func (x *ListObjectsResponse) ProtoReflect() protoreflect.Message { + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListObjectsResponse.ProtoReflect.Descriptor instead. +func (*ListObjectsResponse) Descriptor() ([]byte, []int) { + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{16} +} + +func (x *ListObjectsResponse) GetObjects() []*Object { + if x != nil { + return x.Objects + } + return nil +} + type PutObjectRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Required. The existing state of the storage bucket. @@ -783,7 +945,7 @@ type PutObjectRequest struct { func (x *PutObjectRequest) Reset() { *x = PutObjectRequest{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[14] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -795,7 +957,7 @@ func (x *PutObjectRequest) String() string { func (*PutObjectRequest) ProtoMessage() {} func (x *PutObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[14] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -808,7 +970,7 @@ func (x *PutObjectRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PutObjectRequest.ProtoReflect.Descriptor instead. func (*PutObjectRequest) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{14} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{17} } func (x *PutObjectRequest) GetBucket() *storagebuckets.StorageBucket { @@ -842,7 +1004,7 @@ type PutObjectResponse struct { func (x *PutObjectResponse) Reset() { *x = PutObjectResponse{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[15] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -854,7 +1016,7 @@ func (x *PutObjectResponse) String() string { func (*PutObjectResponse) ProtoMessage() {} func (x *PutObjectResponse) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[15] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -867,7 +1029,7 @@ func (x *PutObjectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutObjectResponse.ProtoReflect.Descriptor instead. func (*PutObjectResponse) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{15} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{18} } func (x *PutObjectResponse) GetChecksumSha_256() []byte { @@ -901,7 +1063,7 @@ type DeleteObjectsRequest struct { func (x *DeleteObjectsRequest) Reset() { *x = DeleteObjectsRequest{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[16] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -913,7 +1075,7 @@ func (x *DeleteObjectsRequest) String() string { func (*DeleteObjectsRequest) ProtoMessage() {} func (x *DeleteObjectsRequest) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[16] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -926,7 +1088,7 @@ func (x *DeleteObjectsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteObjectsRequest.ProtoReflect.Descriptor instead. func (*DeleteObjectsRequest) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{16} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{19} } func (x *DeleteObjectsRequest) GetBucket() *storagebuckets.StorageBucket { @@ -964,7 +1126,7 @@ type DeleteObjectsResponse struct { func (x *DeleteObjectsResponse) Reset() { *x = DeleteObjectsResponse{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[17] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -976,7 +1138,7 @@ func (x *DeleteObjectsResponse) String() string { func (*DeleteObjectsResponse) ProtoMessage() {} func (x *DeleteObjectsResponse) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[17] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -989,7 +1151,7 @@ func (x *DeleteObjectsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteObjectsResponse.ProtoReflect.Descriptor instead. func (*DeleteObjectsResponse) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{17} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{20} } func (x *DeleteObjectsResponse) GetObjectsDeleted() uint32 { @@ -1013,7 +1175,7 @@ type Permission struct { func (x *Permission) Reset() { *x = Permission{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[18] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1025,7 +1187,7 @@ func (x *Permission) String() string { func (*Permission) ProtoMessage() {} func (x *Permission) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[18] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1038,7 +1200,7 @@ func (x *Permission) ProtoReflect() protoreflect.Message { // Deprecated: Use Permission.ProtoReflect.Descriptor instead. func (*Permission) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{18} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{21} } func (x *Permission) GetState() StateType { @@ -1076,7 +1238,7 @@ type Permissions struct { func (x *Permissions) Reset() { *x = Permissions{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[19] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1088,7 +1250,7 @@ func (x *Permissions) String() string { func (*Permissions) ProtoMessage() {} func (x *Permissions) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[19] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1101,7 +1263,7 @@ func (x *Permissions) ProtoReflect() protoreflect.Message { // Deprecated: Use Permissions.ProtoReflect.Descriptor instead. func (*Permissions) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{19} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{22} } func (x *Permissions) GetWrite() *Permission { @@ -1137,7 +1299,7 @@ type StorageBucketCredentialState struct { func (x *StorageBucketCredentialState) Reset() { *x = StorageBucketCredentialState{} - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[20] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1149,7 +1311,7 @@ func (x *StorageBucketCredentialState) String() string { func (*StorageBucketCredentialState) ProtoMessage() {} func (x *StorageBucketCredentialState) ProtoReflect() protoreflect.Message { - mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[20] + mi := &file_plugin_v1_storage_plugin_service_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1162,7 +1324,7 @@ func (x *StorageBucketCredentialState) ProtoReflect() protoreflect.Message { // Deprecated: Use StorageBucketCredentialState.ProtoReflect.Descriptor instead. func (*StorageBucketCredentialState) Descriptor() ([]byte, []int) { - return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{20} + return file_plugin_v1_storage_plugin_service_proto_rawDescGZIP(), []int{23} } func (x *StorageBucketCredentialState) GetState() *Permissions { @@ -1236,7 +1398,20 @@ const file_plugin_v1_storage_plugin_service_proto_rawDesc = "" + "\x11GetObjectResponse\x12\x1d\n" + "\n" + "file_chunk\x18\n" + - " \x01(\fR\tfileChunk\"\x8b\x01\n" + + " \x01(\fR\tfileChunk\"\xa4\x01\n" + + "\x12ListObjectsRequest\x12Q\n" + + "\x06bucket\x18\n" + + " \x01(\v29.controller.api.resources.storagebuckets.v1.StorageBucketR\x06bucket\x12\x1d\n" + + "\n" + + "key_prefix\x18\x14 \x01(\tR\tkeyPrefix\x12\x1c\n" + + "\trecursive\x18\x1e \x01(\bR\trecursive\"1\n" + + "\x06Object\x12\x10\n" + + "\x03key\x18\n" + + " \x01(\tR\x03key\x12\x15\n" + + "\x06is_dir\x18\x14 \x01(\bR\x05isDir\"B\n" + + "\x13ListObjectsResponse\x12+\n" + + "\aobjects\x18\n" + + " \x03(\v2\x11.plugin.v1.ObjectR\aobjects\"\x8b\x01\n" + "\x10PutObjectRequest\x12Q\n" + "\x06bucket\x18\n" + " \x01(\v29.controller.api.resources.storagebuckets.v1.StorageBucketR\x06bucket\x12\x10\n" + @@ -1274,7 +1449,7 @@ const file_plugin_v1_storage_plugin_service_proto_rawDesc = "" + "\x16STATE_TYPE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rSTATE_TYPE_OK\x10\x01\x12\x14\n" + "\x10STATE_TYPE_ERROR\x10\x02\x12\x16\n" + - "\x12STATE_TYPE_UNKNOWN\x10\x032\xec\x06\n" + + "\x12STATE_TYPE_UNKNOWN\x10\x032\xba\a\n" + "\x14StoragePluginService\x12y\n" + "\x1aNormalizeStorageBucketData\x12,.plugin.v1.NormalizeStorageBucketDataRequest\x1a-.plugin.v1.NormalizeStorageBucketDataResponse\x12j\n" + "\x15OnCreateStorageBucket\x12'.plugin.v1.OnCreateStorageBucketRequest\x1a(.plugin.v1.OnCreateStorageBucketResponse\x12j\n" + @@ -1283,7 +1458,8 @@ const file_plugin_v1_storage_plugin_service_proto_rawDesc = "" + "\x13ValidatePermissions\x12%.plugin.v1.ValidatePermissionsRequest\x1a&.plugin.v1.ValidatePermissionsResponse\x12I\n" + "\n" + "HeadObject\x12\x1c.plugin.v1.HeadObjectRequest\x1a\x1d.plugin.v1.HeadObjectResponse\x12H\n" + - "\tGetObject\x12\x1b.plugin.v1.GetObjectRequest\x1a\x1c.plugin.v1.GetObjectResponse0\x01\x12F\n" + + "\tGetObject\x12\x1b.plugin.v1.GetObjectRequest\x1a\x1c.plugin.v1.GetObjectResponse0\x01\x12L\n" + + "\vListObjects\x12\x1d.plugin.v1.ListObjectsRequest\x1a\x1e.plugin.v1.ListObjectsResponse\x12F\n" + "\tPutObject\x12\x1b.plugin.v1.PutObjectRequest\x1a\x1c.plugin.v1.PutObjectResponse\x12R\n" + "\rDeleteObjects\x12\x1f.plugin.v1.DeleteObjectsRequest\x1a .plugin.v1.DeleteObjectsResponseB5Z3github.com/hashicorp/boundary/sdk/pbs/plugin;pluginb\x06proto3" @@ -1300,7 +1476,7 @@ func file_plugin_v1_storage_plugin_service_proto_rawDescGZIP() []byte { } var file_plugin_v1_storage_plugin_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_plugin_v1_storage_plugin_service_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_plugin_v1_storage_plugin_service_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_plugin_v1_storage_plugin_service_proto_goTypes = []any{ (StateType)(0), // 0: plugin.v1.StateType (*NormalizeStorageBucketDataRequest)(nil), // 1: plugin.v1.NormalizeStorageBucketDataRequest @@ -1317,66 +1493,73 @@ var file_plugin_v1_storage_plugin_service_proto_goTypes = []any{ (*HeadObjectResponse)(nil), // 12: plugin.v1.HeadObjectResponse (*GetObjectRequest)(nil), // 13: plugin.v1.GetObjectRequest (*GetObjectResponse)(nil), // 14: plugin.v1.GetObjectResponse - (*PutObjectRequest)(nil), // 15: plugin.v1.PutObjectRequest - (*PutObjectResponse)(nil), // 16: plugin.v1.PutObjectResponse - (*DeleteObjectsRequest)(nil), // 17: plugin.v1.DeleteObjectsRequest - (*DeleteObjectsResponse)(nil), // 18: plugin.v1.DeleteObjectsResponse - (*Permission)(nil), // 19: plugin.v1.Permission - (*Permissions)(nil), // 20: plugin.v1.Permissions - (*StorageBucketCredentialState)(nil), // 21: plugin.v1.StorageBucketCredentialState - (*structpb.Struct)(nil), // 22: google.protobuf.Struct - (*plugins.PluginInfo)(nil), // 23: controller.api.resources.plugins.v1.PluginInfo - (*storagebuckets.StorageBucket)(nil), // 24: controller.api.resources.storagebuckets.v1.StorageBucket - (*storagebuckets.StorageBucketPersisted)(nil), // 25: controller.api.resources.storagebuckets.v1.StorageBucketPersisted - (*timestamppb.Timestamp)(nil), // 26: google.protobuf.Timestamp + (*ListObjectsRequest)(nil), // 15: plugin.v1.ListObjectsRequest + (*Object)(nil), // 16: plugin.v1.Object + (*ListObjectsResponse)(nil), // 17: plugin.v1.ListObjectsResponse + (*PutObjectRequest)(nil), // 18: plugin.v1.PutObjectRequest + (*PutObjectResponse)(nil), // 19: plugin.v1.PutObjectResponse + (*DeleteObjectsRequest)(nil), // 20: plugin.v1.DeleteObjectsRequest + (*DeleteObjectsResponse)(nil), // 21: plugin.v1.DeleteObjectsResponse + (*Permission)(nil), // 22: plugin.v1.Permission + (*Permissions)(nil), // 23: plugin.v1.Permissions + (*StorageBucketCredentialState)(nil), // 24: plugin.v1.StorageBucketCredentialState + (*structpb.Struct)(nil), // 25: google.protobuf.Struct + (*plugins.PluginInfo)(nil), // 26: controller.api.resources.plugins.v1.PluginInfo + (*storagebuckets.StorageBucket)(nil), // 27: controller.api.resources.storagebuckets.v1.StorageBucket + (*storagebuckets.StorageBucketPersisted)(nil), // 28: controller.api.resources.storagebuckets.v1.StorageBucketPersisted + (*timestamppb.Timestamp)(nil), // 29: google.protobuf.Timestamp } var file_plugin_v1_storage_plugin_service_proto_depIdxs = []int32{ - 22, // 0: plugin.v1.NormalizeStorageBucketDataRequest.attributes:type_name -> google.protobuf.Struct - 23, // 1: plugin.v1.NormalizeStorageBucketDataRequest.plugin:type_name -> controller.api.resources.plugins.v1.PluginInfo - 22, // 2: plugin.v1.NormalizeStorageBucketDataResponse.attributes:type_name -> google.protobuf.Struct - 24, // 3: plugin.v1.OnCreateStorageBucketRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 25, // 4: plugin.v1.OnCreateStorageBucketResponse.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted - 24, // 5: plugin.v1.OnUpdateStorageBucketRequest.current_bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 24, // 6: plugin.v1.OnUpdateStorageBucketRequest.new_bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 25, // 7: plugin.v1.OnUpdateStorageBucketRequest.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted - 25, // 8: plugin.v1.OnUpdateStorageBucketResponse.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted - 24, // 9: plugin.v1.OnDeleteStorageBucketRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 25, // 10: plugin.v1.OnDeleteStorageBucketRequest.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted - 24, // 11: plugin.v1.ValidatePermissionsRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 24, // 12: plugin.v1.HeadObjectRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 26, // 13: plugin.v1.HeadObjectResponse.last_modified:type_name -> google.protobuf.Timestamp - 24, // 14: plugin.v1.GetObjectRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 24, // 15: plugin.v1.PutObjectRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 24, // 16: plugin.v1.DeleteObjectsRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket - 0, // 17: plugin.v1.Permission.state:type_name -> plugin.v1.StateType - 26, // 18: plugin.v1.Permission.checked_at:type_name -> google.protobuf.Timestamp - 19, // 19: plugin.v1.Permissions.write:type_name -> plugin.v1.Permission - 19, // 20: plugin.v1.Permissions.read:type_name -> plugin.v1.Permission - 19, // 21: plugin.v1.Permissions.delete:type_name -> plugin.v1.Permission - 20, // 22: plugin.v1.StorageBucketCredentialState.state:type_name -> plugin.v1.Permissions - 1, // 23: plugin.v1.StoragePluginService.NormalizeStorageBucketData:input_type -> plugin.v1.NormalizeStorageBucketDataRequest - 3, // 24: plugin.v1.StoragePluginService.OnCreateStorageBucket:input_type -> plugin.v1.OnCreateStorageBucketRequest - 5, // 25: plugin.v1.StoragePluginService.OnUpdateStorageBucket:input_type -> plugin.v1.OnUpdateStorageBucketRequest - 7, // 26: plugin.v1.StoragePluginService.OnDeleteStorageBucket:input_type -> plugin.v1.OnDeleteStorageBucketRequest - 9, // 27: plugin.v1.StoragePluginService.ValidatePermissions:input_type -> plugin.v1.ValidatePermissionsRequest - 11, // 28: plugin.v1.StoragePluginService.HeadObject:input_type -> plugin.v1.HeadObjectRequest - 13, // 29: plugin.v1.StoragePluginService.GetObject:input_type -> plugin.v1.GetObjectRequest - 15, // 30: plugin.v1.StoragePluginService.PutObject:input_type -> plugin.v1.PutObjectRequest - 17, // 31: plugin.v1.StoragePluginService.DeleteObjects:input_type -> plugin.v1.DeleteObjectsRequest - 2, // 32: plugin.v1.StoragePluginService.NormalizeStorageBucketData:output_type -> plugin.v1.NormalizeStorageBucketDataResponse - 4, // 33: plugin.v1.StoragePluginService.OnCreateStorageBucket:output_type -> plugin.v1.OnCreateStorageBucketResponse - 6, // 34: plugin.v1.StoragePluginService.OnUpdateStorageBucket:output_type -> plugin.v1.OnUpdateStorageBucketResponse - 8, // 35: plugin.v1.StoragePluginService.OnDeleteStorageBucket:output_type -> plugin.v1.OnDeleteStorageBucketResponse - 10, // 36: plugin.v1.StoragePluginService.ValidatePermissions:output_type -> plugin.v1.ValidatePermissionsResponse - 12, // 37: plugin.v1.StoragePluginService.HeadObject:output_type -> plugin.v1.HeadObjectResponse - 14, // 38: plugin.v1.StoragePluginService.GetObject:output_type -> plugin.v1.GetObjectResponse - 16, // 39: plugin.v1.StoragePluginService.PutObject:output_type -> plugin.v1.PutObjectResponse - 18, // 40: plugin.v1.StoragePluginService.DeleteObjects:output_type -> plugin.v1.DeleteObjectsResponse - 32, // [32:41] is the sub-list for method output_type - 23, // [23:32] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 25, // 0: plugin.v1.NormalizeStorageBucketDataRequest.attributes:type_name -> google.protobuf.Struct + 26, // 1: plugin.v1.NormalizeStorageBucketDataRequest.plugin:type_name -> controller.api.resources.plugins.v1.PluginInfo + 25, // 2: plugin.v1.NormalizeStorageBucketDataResponse.attributes:type_name -> google.protobuf.Struct + 27, // 3: plugin.v1.OnCreateStorageBucketRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 28, // 4: plugin.v1.OnCreateStorageBucketResponse.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted + 27, // 5: plugin.v1.OnUpdateStorageBucketRequest.current_bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 27, // 6: plugin.v1.OnUpdateStorageBucketRequest.new_bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 28, // 7: plugin.v1.OnUpdateStorageBucketRequest.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted + 28, // 8: plugin.v1.OnUpdateStorageBucketResponse.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted + 27, // 9: plugin.v1.OnDeleteStorageBucketRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 28, // 10: plugin.v1.OnDeleteStorageBucketRequest.persisted:type_name -> controller.api.resources.storagebuckets.v1.StorageBucketPersisted + 27, // 11: plugin.v1.ValidatePermissionsRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 27, // 12: plugin.v1.HeadObjectRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 29, // 13: plugin.v1.HeadObjectResponse.last_modified:type_name -> google.protobuf.Timestamp + 27, // 14: plugin.v1.GetObjectRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 27, // 15: plugin.v1.ListObjectsRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 16, // 16: plugin.v1.ListObjectsResponse.objects:type_name -> plugin.v1.Object + 27, // 17: plugin.v1.PutObjectRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 27, // 18: plugin.v1.DeleteObjectsRequest.bucket:type_name -> controller.api.resources.storagebuckets.v1.StorageBucket + 0, // 19: plugin.v1.Permission.state:type_name -> plugin.v1.StateType + 29, // 20: plugin.v1.Permission.checked_at:type_name -> google.protobuf.Timestamp + 22, // 21: plugin.v1.Permissions.write:type_name -> plugin.v1.Permission + 22, // 22: plugin.v1.Permissions.read:type_name -> plugin.v1.Permission + 22, // 23: plugin.v1.Permissions.delete:type_name -> plugin.v1.Permission + 23, // 24: plugin.v1.StorageBucketCredentialState.state:type_name -> plugin.v1.Permissions + 1, // 25: plugin.v1.StoragePluginService.NormalizeStorageBucketData:input_type -> plugin.v1.NormalizeStorageBucketDataRequest + 3, // 26: plugin.v1.StoragePluginService.OnCreateStorageBucket:input_type -> plugin.v1.OnCreateStorageBucketRequest + 5, // 27: plugin.v1.StoragePluginService.OnUpdateStorageBucket:input_type -> plugin.v1.OnUpdateStorageBucketRequest + 7, // 28: plugin.v1.StoragePluginService.OnDeleteStorageBucket:input_type -> plugin.v1.OnDeleteStorageBucketRequest + 9, // 29: plugin.v1.StoragePluginService.ValidatePermissions:input_type -> plugin.v1.ValidatePermissionsRequest + 11, // 30: plugin.v1.StoragePluginService.HeadObject:input_type -> plugin.v1.HeadObjectRequest + 13, // 31: plugin.v1.StoragePluginService.GetObject:input_type -> plugin.v1.GetObjectRequest + 15, // 32: plugin.v1.StoragePluginService.ListObjects:input_type -> plugin.v1.ListObjectsRequest + 18, // 33: plugin.v1.StoragePluginService.PutObject:input_type -> plugin.v1.PutObjectRequest + 20, // 34: plugin.v1.StoragePluginService.DeleteObjects:input_type -> plugin.v1.DeleteObjectsRequest + 2, // 35: plugin.v1.StoragePluginService.NormalizeStorageBucketData:output_type -> plugin.v1.NormalizeStorageBucketDataResponse + 4, // 36: plugin.v1.StoragePluginService.OnCreateStorageBucket:output_type -> plugin.v1.OnCreateStorageBucketResponse + 6, // 37: plugin.v1.StoragePluginService.OnUpdateStorageBucket:output_type -> plugin.v1.OnUpdateStorageBucketResponse + 8, // 38: plugin.v1.StoragePluginService.OnDeleteStorageBucket:output_type -> plugin.v1.OnDeleteStorageBucketResponse + 10, // 39: plugin.v1.StoragePluginService.ValidatePermissions:output_type -> plugin.v1.ValidatePermissionsResponse + 12, // 40: plugin.v1.StoragePluginService.HeadObject:output_type -> plugin.v1.HeadObjectResponse + 14, // 41: plugin.v1.StoragePluginService.GetObject:output_type -> plugin.v1.GetObjectResponse + 17, // 42: plugin.v1.StoragePluginService.ListObjects:output_type -> plugin.v1.ListObjectsResponse + 19, // 43: plugin.v1.StoragePluginService.PutObject:output_type -> plugin.v1.PutObjectResponse + 21, // 44: plugin.v1.StoragePluginService.DeleteObjects:output_type -> plugin.v1.DeleteObjectsResponse + 35, // [35:45] is the sub-list for method output_type + 25, // [25:35] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_plugin_v1_storage_plugin_service_proto_init() } @@ -1390,7 +1573,7 @@ func file_plugin_v1_storage_plugin_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_plugin_v1_storage_plugin_service_proto_rawDesc), len(file_plugin_v1_storage_plugin_service_proto_rawDesc)), NumEnums: 1, - NumMessages: 21, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/sdk/pbs/plugin/storage_plugin_service_grpc.pb.go b/sdk/pbs/plugin/storage_plugin_service_grpc.pb.go index ae8195410..3e96df95a 100644 --- a/sdk/pbs/plugin/storage_plugin_service_grpc.pb.go +++ b/sdk/pbs/plugin/storage_plugin_service_grpc.pb.go @@ -29,6 +29,7 @@ const ( StoragePluginService_ValidatePermissions_FullMethodName = "/plugin.v1.StoragePluginService/ValidatePermissions" StoragePluginService_HeadObject_FullMethodName = "/plugin.v1.StoragePluginService/HeadObject" StoragePluginService_GetObject_FullMethodName = "/plugin.v1.StoragePluginService/GetObject" + StoragePluginService_ListObjects_FullMethodName = "/plugin.v1.StoragePluginService/ListObjects" StoragePluginService_PutObject_FullMethodName = "/plugin.v1.StoragePluginService/PutObject" StoragePluginService_DeleteObjects_FullMethodName = "/plugin.v1.StoragePluginService/DeleteObjects" ) @@ -66,14 +67,16 @@ type StoragePluginServiceClient interface { // OnDeleteStorageBucket is a hook that runs when a storage bucket is // deleted. OnDeleteStorageBucket(ctx context.Context, in *OnDeleteStorageBucketRequest, opts ...grpc.CallOption) (*OnDeleteStorageBucketResponse, error) - // ValidatePermissions is a hook that checks if the secrets associated with + // ValidatePermissions checks if the secrets associated with // the storage bucket meet the requirements of the plugin. ValidatePermissions(ctx context.Context, in *ValidatePermissionsRequest, opts ...grpc.CallOption) (*ValidatePermissionsResponse, error) - // HeadObject is a hook that retrieves metadata about an object. + // HeadObject returns metadata about an object. HeadObject(ctx context.Context, in *HeadObjectRequest, opts ...grpc.CallOption) (*HeadObjectResponse, error) - // GetObject is a hook that retrieves objects. + // GetObject returns object requested. GetObject(ctx context.Context, in *GetObjectRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[GetObjectResponse], error) - // PutObject is a hook that reads a file stored on local disk and + // ListObjects returns a list of objects within a storage bucket that matches the provided prefix. + ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...grpc.CallOption) (*ListObjectsResponse, error) + // PutObject reads a file stored on local disk and // stores it to an external object store. PutObject(ctx context.Context, in *PutObjectRequest, opts ...grpc.CallOption) (*PutObjectResponse, error) // DeleteObjects deletes one or many files in an external object store @@ -168,6 +171,16 @@ func (c *storagePluginServiceClient) GetObject(ctx context.Context, in *GetObjec // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type StoragePluginService_GetObjectClient = grpc.ServerStreamingClient[GetObjectResponse] +func (c *storagePluginServiceClient) ListObjects(ctx context.Context, in *ListObjectsRequest, opts ...grpc.CallOption) (*ListObjectsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListObjectsResponse) + err := c.cc.Invoke(ctx, StoragePluginService_ListObjects_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *storagePluginServiceClient) PutObject(ctx context.Context, in *PutObjectRequest, opts ...grpc.CallOption) (*PutObjectResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PutObjectResponse) @@ -221,14 +234,16 @@ type StoragePluginServiceServer interface { // OnDeleteStorageBucket is a hook that runs when a storage bucket is // deleted. OnDeleteStorageBucket(context.Context, *OnDeleteStorageBucketRequest) (*OnDeleteStorageBucketResponse, error) - // ValidatePermissions is a hook that checks if the secrets associated with + // ValidatePermissions checks if the secrets associated with // the storage bucket meet the requirements of the plugin. ValidatePermissions(context.Context, *ValidatePermissionsRequest) (*ValidatePermissionsResponse, error) - // HeadObject is a hook that retrieves metadata about an object. + // HeadObject returns metadata about an object. HeadObject(context.Context, *HeadObjectRequest) (*HeadObjectResponse, error) - // GetObject is a hook that retrieves objects. + // GetObject returns object requested. GetObject(*GetObjectRequest, grpc.ServerStreamingServer[GetObjectResponse]) error - // PutObject is a hook that reads a file stored on local disk and + // ListObjects returns a list of objects within a storage bucket that matches the provided prefix. + ListObjects(context.Context, *ListObjectsRequest) (*ListObjectsResponse, error) + // PutObject reads a file stored on local disk and // stores it to an external object store. PutObject(context.Context, *PutObjectRequest) (*PutObjectResponse, error) // DeleteObjects deletes one or many files in an external object store @@ -265,6 +280,9 @@ func (UnimplementedStoragePluginServiceServer) HeadObject(context.Context, *Head func (UnimplementedStoragePluginServiceServer) GetObject(*GetObjectRequest, grpc.ServerStreamingServer[GetObjectResponse]) error { return status.Error(codes.Unimplemented, "method GetObject not implemented") } +func (UnimplementedStoragePluginServiceServer) ListObjects(context.Context, *ListObjectsRequest) (*ListObjectsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListObjects not implemented") +} func (UnimplementedStoragePluginServiceServer) PutObject(context.Context, *PutObjectRequest) (*PutObjectResponse, error) { return nil, status.Error(codes.Unimplemented, "method PutObject not implemented") } @@ -411,6 +429,24 @@ func _StoragePluginService_GetObject_Handler(srv interface{}, stream grpc.Server // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type StoragePluginService_GetObjectServer = grpc.ServerStreamingServer[GetObjectResponse] +func _StoragePluginService_ListObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListObjectsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(StoragePluginServiceServer).ListObjects(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: StoragePluginService_ListObjects_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(StoragePluginServiceServer).ListObjects(ctx, req.(*ListObjectsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _StoragePluginService_PutObject_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PutObjectRequest) if err := dec(in); err != nil { @@ -478,6 +514,10 @@ var StoragePluginService_ServiceDesc = grpc.ServiceDesc{ MethodName: "HeadObject", Handler: _StoragePluginService_HeadObject_Handler, }, + { + MethodName: "ListObjects", + Handler: _StoragePluginService_ListObjects_Handler, + }, { MethodName: "PutObject", Handler: _StoragePluginService_PutObject_Handler,