feat: add ListObjects method & relevant e2e tests

testing/e2e_storage_test.go

@@ -5,6 +5,7 @@ package testing
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"os"
@@ -85,6 +86,12 @@ func TestStoragePlugin(t *testing.T) {
 	iamSecretAccessKeyMissingDeleteObject, err := tf.GetOutputString("iam_secret_access_key_missing_delete_obj")
 	require.NoError(err)
 
+	s3PaginationTestObjectCount, err := tf.GetOutput("s3_pagination_test_object_count")
+	require.NoError(err)
+	objectCountFloat, ok := s3PaginationTestObjectCount.(float64)
+	require.True(ok)
+	objectCount := int(objectCountFloat)
+
 	// Start the workflow now. Set up the storage bucket. Note that this
 	// will cause the state to go out of drift above in the sense that
 	// the access key ID/secret access key will no longer be valid. We
@@ -138,6 +145,11 @@ func TestStoragePlugin(t *testing.T) {
 	keyid, secret = iamAccessKeyIds[5].(string), iamSecretAccessKeys[5].(string)
 	testPluginObjectMethods(ctx, t, p, bucketName, region, keyid, secret)
 
+	// ********************
+	// * ListObjects
+	// ********************
+	testListObjects(ctx, t, p, objectCount, bucketName, region, keyid, secret)
+
 	// ********************
 	// * Validate Permissions
 	// ********************
@@ -514,3 +526,198 @@ func testPluginObjectMethods(ctx context.Context, t *testing.T, p *storage.Stora
 	require.NoError(conn.Close())
 	grpcServer.Stop()
 }
+
+// testListObjects verifies top-level and recursive listing by matching the exact keys and types of returned objects.
+func testListObjects(ctx context.Context, t *testing.T, p *storage.StoragePlugin, objectCount int, bucketName, region, accessKeyId, secretAccessKey string) {
+	t.Helper()
+	t.Logf("testing ListObjects (bucket=%s, region=%s, expectedObjects=%d, creds=%t)",
+		bucketName,
+		region,
+		objectCount,
+		accessKeyId != "" && secretAccessKey != "",
+	)
+	r := require.New(t)
+
+	reqAttrs, err := structpb.NewStruct(map[string]any{
+		credential.ConstRegion:                    region,
+		credential.ConstDisableCredentialRotation: true,
+	})
+	r.NoError(err)
+	reqSecrets, err := structpb.NewStruct(map[string]any{
+		credential.ConstAccessKeyId:     accessKeyId,
+		credential.ConstSecretAccessKey: secretAccessKey,
+	})
+	r.NoError(err)
+
+	bucket := &storagebuckets.StorageBucket{
+		BucketName: bucketName,
+		Attributes: reqAttrs,
+		Secrets:    reqSecrets,
+	}
+
+	// Test Scenario 1: Non-Paginated Listing
+	t.Run("NonPaginatedListing", func(t *testing.T) {
+
+		// Subtest for the non-recursive (top-level) option
+		t.Run("TopLevel", func(t *testing.T) {
+			r := require.New(t)
+			t.Logf("testing ListObjects for a small set with recursive=false")
+			req := &pb.ListObjectsRequest{
+				Bucket:    bucket,
+				KeyPrefix: "list-objects/",
+				Recursive: false,
+			}
+			resp, err := p.ListObjects(ctx, req)
+			r.NoError(err)
+			r.NotNil(resp)
+
+			expectedObjects := []*pb.Object{
+				{Key: "list-objects/file1.txt", IsDir: false},
+				{Key: "list-objects/data.json", IsDir: false},
+				{Key: "list-objects/nested-dir/", IsDir: true},
+				{Key: "list-objects/empty-nested-dir/", IsDir: true},
+			}
+			r.ElementsMatch(resp.Objects, expectedObjects)
+		})
+
+		// Subtest for the recursive option
+		t.Run("Recursive", func(t *testing.T) {
+			r := require.New(t)
+			t.Logf("testing ListObjects for a small set with recursive=true")
+			req := &pb.ListObjectsRequest{
+				Bucket:    bucket,
+				KeyPrefix: "list-objects/",
+				Recursive: true,
+			}
+			resp, err := p.ListObjects(ctx, req)
+			r.NoError(err)
+			r.NotNil(resp)
+
+			expectedObjects := []*pb.Object{
+				{Key: "list-objects/file1.txt", IsDir: false},
+				{Key: "list-objects/data.json", IsDir: false},
+				{Key: "list-objects/nested-dir/file2.txt", IsDir: false},
+				{Key: "list-objects/nested-dir/", IsDir: true},
+				{Key: "list-objects/empty-nested-dir/", IsDir: true},
+			}
+
+			r.ElementsMatch(resp.Objects, expectedObjects)
+		})
+	})
+
+	// Test Scenario 2: Paginated Top-Level Listing
+	t.Run("TopLevelListing", func(t *testing.T) {
+		// Subtest for the root prefix
+		t.Run("AtRoot", func(t *testing.T) {
+			r := require.New(t)
+			t.Logf("testing ListObjects with recursive=false at root prefix")
+			req := &pb.ListObjectsRequest{
+				Bucket:    bucket,
+				KeyPrefix: "list-objects-paginated/",
+				Recursive: false,
+			}
+			resp, err := p.ListObjects(ctx, req)
+			r.NoError(err)
+			r.NotNil(resp)
+
+			var expectedObjects []*pb.Object
+			for i := 0; i < objectCount; i++ {
+				key := fmt.Sprintf("list-objects-paginated/root-file-%d.txt", i)
+				expectedObjects = append(expectedObjects, &pb.Object{Key: key, IsDir: false})
+			}
+			for i := 0; i < objectCount; i++ {
+				keyEmpty := fmt.Sprintf("list-objects-paginated/empty-dir-%d/", i)
+				expectedObjects = append(expectedObjects, &pb.Object{Key: keyEmpty, IsDir: true})
+				keyNested := fmt.Sprintf("list-objects-paginated/nested-dir-%d/", i)
+				expectedObjects = append(expectedObjects, &pb.Object{Key: keyNested, IsDir: true})
+			}
+
+			r.ElementsMatch(resp.Objects, expectedObjects)
+		})
+
+		// Subtest for a nested prefix
+		t.Run("AtNestedPrefix", func(t *testing.T) {
+			r := require.New(t)
+			t.Logf("testing ListObjects with recursive=false at a nested prefix")
+			req := &pb.ListObjectsRequest{
+				Bucket:    bucket,
+				KeyPrefix: "list-objects-paginated/nested-dir-7/",
+				Recursive: false,
+			}
+			resp, err := p.ListObjects(ctx, req)
+			r.NoError(err)
+			r.NotNil(resp)
+
+			expectedObjects := []*pb.Object{
+				{Key: "list-objects-paginated/nested-dir-7/data-file-7.json", IsDir: false},
+			}
+
+			r.ElementsMatch(resp.Objects, expectedObjects)
+		})
+	})
+
+	// Test Scenario 3: Paginated Recursive Listing
+	t.Run("RecursiveListing", func(t *testing.T) {
+		r := require.New(t)
+		t.Logf("testing ListObjects with recursive=true")
+		req := &pb.ListObjectsRequest{
+			Bucket:    bucket,
+			KeyPrefix: "list-objects-paginated/",
+			Recursive: true,
+		}
+		resp, err := p.ListObjects(ctx, req)
+		r.NoError(err)
+		r.NotNil(resp)
+
+		var expectedObjects []*pb.Object
+		for i := 0; i < objectCount; i++ {
+			keyFile := fmt.Sprintf("list-objects-paginated/root-file-%d.txt", i)
+			expectedObjects = append(expectedObjects, &pb.Object{Key: keyFile, IsDir: false})
+			keyNestedFile := fmt.Sprintf("list-objects-paginated/nested-dir-%d/data-file-%d.json", i, i)
+			expectedObjects = append(expectedObjects, &pb.Object{Key: keyNestedFile, IsDir: false})
+		}
+		for i := 0; i < objectCount; i++ {
+			keyNestedDir := fmt.Sprintf("list-objects-paginated/nested-dir-%d/", i)
+			expectedObjects = append(expectedObjects, &pb.Object{Key: keyNestedDir, IsDir: true})
+			keyEmptyDir := fmt.Sprintf("list-objects-paginated/empty-dir-%d/", i)
+			expectedObjects = append(expectedObjects, &pb.Object{Key: keyEmptyDir, IsDir: true})
+			keyNestedEmptyDir := fmt.Sprintf("list-objects-paginated/empty-dir-%d/dir-%d/", i, i)
+			expectedObjects = append(expectedObjects, &pb.Object{Key: keyNestedEmptyDir, IsDir: true})
+		}
+
+		r.ElementsMatch(resp.Objects, expectedObjects)
+	})
+
+	// Test Scenario 4: Edge cases
+	t.Run("EdgeCases", func(t *testing.T) {
+		// Subtest for a prefix that doesn't exist
+		t.Run("NonExistentPrefix", func(t *testing.T) {
+			r := require.New(t)
+			t.Logf("testing ListObjects for a non-existent prefix")
+			req := &pb.ListObjectsRequest{
+				Bucket:    bucket,
+				KeyPrefix: "this/prefix/does/not/exist/",
+				Recursive: false,
+			}
+			resp, err := p.ListObjects(ctx, req)
+			r.NoError(err)
+			r.NotNil(resp)
+			r.Empty(resp.Objects)
+		})
+
+		// Subtest for listing inside a known empty directory
+		t.Run("InsideEmptyDirectory", func(t *testing.T) {
+			r := require.New(t)
+			t.Logf("testing ListObjects inside an empty directory")
+			req := &pb.ListObjectsRequest{
+				Bucket:    bucket,
+				KeyPrefix: "list-objects-paginated/empty-dirs/dir-10/",
+				Recursive: false,
+			}
+			resp, err := p.ListObjects(ctx, req)
+			r.NoError(err)
+			r.NotNil(resp)
+			r.Empty(resp.Objects)
+		})
+	})
+}

testing/testdata/host/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.0"
+      version = "~> 6.0"
     }
   }
 }

testing/testdata/storage/iam.tf

@@ -34,10 +34,18 @@ resource "aws_iam_policy" "valid" {
         "s3:GetObject",
         "s3:DeleteObject",
         "s3:GetObjectAttributes",
-        "s3:ListBucket"
+        "s3:ListBucket",
+        "s3:DeleteObject"
       ],
       "Effect": "Allow",
       "Resource": "${aws_s3_bucket.test.arn}*"
+    },
+    {
+      "Action": [
+        "s3:ListBucket"
+      ],
+      "Effect": "Allow",
+      "Resource": "${aws_s3_bucket.test.arn}"
     }
   ]
 
@@ -196,7 +204,7 @@ resource "aws_iam_role_policy_attachment" "missing_delete_obj" {
 # Static Credential Testing
 resource "random_id" "user_name" {
   count       = var.iam_user_count
-  prefix      = "demo-${local.hashicorp_email}-boundary-iam-user"
+  prefix      = "demo-${local.hashicorp_email}" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
   byte_length = 4
 }
 
@@ -204,10 +212,8 @@ resource "aws_iam_user" "test" {
   count                = var.iam_user_count
   name                 = random_id.user_name[count.index].dec
   force_destroy        = true
-  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
-  tags = {
-    "boundary-demo" = local.hashicorp_email
-  }
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
+  tags                 = local.tags
 }
 
 resource "aws_iam_access_key" "test" {
@@ -277,17 +283,15 @@ resource "aws_iam_user_policy_attachment" "missing_delete_obj" {
 # Edge case for missing GetObject permission
 
 resource "random_id" "missing_get_obj" {
-  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
+  prefix      = "demo-${local.hashicorp_email}" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
   byte_length = 4
 }
 
 resource "aws_iam_user" "missing_get_obj" {
-  name          = random_id.missing_get_obj.dec
-  force_destroy = true
-  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
-  tags = {
-    "boundary-demo" = local.hashicorp_email
-  }
+  name                 = random_id.missing_get_obj.dec
+  force_destroy        = true
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
+  tags                 = local.tags
 }
 
 resource "aws_iam_access_key" "missing_get_obj" {
@@ -302,17 +306,15 @@ resource "aws_iam_user_policy_attachment" "missing_get_obj" {
 # Edge case for missing PutObject permission
 
 resource "random_id" "missing_put_obj" {
-  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
+  prefix      = "demo-${local.hashicorp_email}" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
   byte_length = 4
 }
 
 resource "aws_iam_user" "missing_put_obj" {
-  name          = random_id.missing_put_obj.dec
-  force_destroy = true
-  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
-  tags = {
-    "boundary-demo" = local.hashicorp_email
-  }
+  name                 = random_id.missing_put_obj.dec
+  force_destroy        = true
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
+  tags                 = local.tags
 }
 
 resource "aws_iam_access_key" "missing_put_obj" {

testing/testdata/storage/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.0"
+      version = "~> 6.0"
     }
   }
 }

testing/testdata/storage/output.tf

@@ -6,19 +6,19 @@ output "bucket_name" {
 }
 
 output "valid_policy_arn" {
-    value = aws_iam_role.valid.arn
+  value = aws_iam_role.valid.arn
 }
 
 output "missing_put_obj_policy_arn" {
-    value = aws_iam_role.missing_put_obj.arn
+  value = aws_iam_role.missing_put_obj.arn
 }
 
 output "missing_get_obj_policy_arn" {
-    value = aws_iam_role.missing_get_obj.arn
+  value = aws_iam_role.missing_get_obj.arn
 }
 
 output "missing_delete_obj_policy_arn" {
-    value = aws_iam_role.missing_delete_obj.arn
+  value = aws_iam_role.missing_delete_obj.arn
 }
 
 output "iam_user_names" {
@@ -67,4 +67,9 @@ output "iam_access_key_missing_delete_obj" {
 output "iam_secret_access_key_missing_delete_obj" {
   value     = aws_iam_access_key.missing_delete_obj.secret
   sensitive = true
-}
+}
+
+output "s3_pagination_test_object_count" {
+  description = "The number of S3 objects created for each pagination test scenario."
+  value       = local.s3_pagination_test_object_count
+}

testing/testdata/storage/s3.tf

@@ -9,9 +9,17 @@ resource "aws_s3_bucket" "test" {
 
 resource "aws_s3_bucket_public_access_block" "secure_access" {
   bucket = aws_s3_bucket.test.id
-  
+
   block_public_acls       = true
   block_public_policy     = true
   ignore_public_acls      = true
   restrict_public_buckets = true
-}
+}
+resource "aws_s3_object" "s3_files" {
+  for_each = local.s3_files
+
+  bucket  = aws_s3_bucket.test.id
+  key     = each.key
+  content = each.value
+  tags    = local.tags
+}

testing/testdata/storage/variable.tf

@@ -16,7 +16,47 @@ locals {
   hashicorp_email = split(":", data.aws_caller_identity.current.user_id)[1]
   tags = {
     User        = split(":", data.aws_caller_identity.current.user_id)[1]
-    Repo        = "boundary-plugin-aws" 
+    Repo        = "boundary-plugin-aws"
     Environment = "Demo"
   }
+
+  # total number of objects of each type to create
+  s3_pagination_test_object_count = 350
+  # Dynamically build a flat map of S3 objects:
+  # - root-level text files
+  # - nested JSON files
+  # - placeholder "empty directories"
+  # - some static files in list-objects directory
+  # This map is consumed directly by an aws_s3_object resource
+  s3_files = merge(
+    # Root-level text files
+    {
+      for i in range(local.s3_pagination_test_object_count) :
+      "list-objects-paginated/root-file-${i}.txt" => "This is a root-level file #${i}."
+    },
+
+    # Nested JSON files inside sub-directories
+    {
+      for i in range(local.s3_pagination_test_object_count) :
+      "list-objects-paginated/nested-dir-${i}/data-file-${i}.json" =>
+      jsonencode({
+        id      = i,
+        message = "This is a nested JSON file #${i}."
+      })
+    },
+
+    # "Empty" directories (in S3 these are just placeholder objects ending with `/`)
+    {
+      for i in range(local.s3_pagination_test_object_count) :
+      "list-objects-paginated/empty-dir-${i}/dir-${i}/" => "" # zero-length object to mimic directory
+    },
+
+    # Static example files in list-objects directory
+    {
+      "list-objects/file1.txt"            = "This is the content for file 1."
+      "list-objects/data.json"            = jsonencode({ key = "value", items = [1, 2, 3] })
+      "list-objects/nested-dir/file2.txt" = "This is a nested file."
+      "list-objects/empty-nested-dir/"    = ""
+    }
+  )
 }