feat: add ListObjects method & relevant e2e tests

testing/testdata/host/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.0"
+      version = "~> 6.0"
     }
   }
 }

testing/testdata/storage/iam.tf

@@ -34,10 +34,18 @@ resource "aws_iam_policy" "valid" {
         "s3:GetObject",
         "s3:DeleteObject",
         "s3:GetObjectAttributes",
-        "s3:ListBucket"
+        "s3:ListBucket",
+        "s3:DeleteObject"
       ],
       "Effect": "Allow",
       "Resource": "${aws_s3_bucket.test.arn}*"
+    },
+    {
+      "Action": [
+        "s3:ListBucket"
+      ],
+      "Effect": "Allow",
+      "Resource": "${aws_s3_bucket.test.arn}"
     }
   ]
 
@@ -196,7 +204,7 @@ resource "aws_iam_role_policy_attachment" "missing_delete_obj" {
 # Static Credential Testing
 resource "random_id" "user_name" {
   count       = var.iam_user_count
-  prefix      = "demo-${local.hashicorp_email}-boundary-iam-user"
+  prefix      = "demo-${local.hashicorp_email}" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
   byte_length = 4
 }
 
@@ -204,10 +212,8 @@ resource "aws_iam_user" "test" {
   count                = var.iam_user_count
   name                 = random_id.user_name[count.index].dec
   force_destroy        = true
-  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
-  tags = {
-    "boundary-demo" = local.hashicorp_email
-  }
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
+  tags                 = local.tags
 }
 
 resource "aws_iam_access_key" "test" {
@@ -277,17 +283,15 @@ resource "aws_iam_user_policy_attachment" "missing_delete_obj" {
 # Edge case for missing GetObject permission
 
 resource "random_id" "missing_get_obj" {
-  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
+  prefix      = "demo-${local.hashicorp_email}" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
   byte_length = 4
 }
 
 resource "aws_iam_user" "missing_get_obj" {
-  name          = random_id.missing_get_obj.dec
-  force_destroy = true
-  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
-  tags = {
-    "boundary-demo" = local.hashicorp_email
-  }
+  name                 = random_id.missing_get_obj.dec
+  force_destroy        = true
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
+  tags                 = local.tags
 }
 
 resource "aws_iam_access_key" "missing_get_obj" {
@@ -302,17 +306,15 @@ resource "aws_iam_user_policy_attachment" "missing_get_obj" {
 # Edge case for missing PutObject permission
 
 resource "random_id" "missing_put_obj" {
-  prefix      =  "demo-${local.hashicorp_email}-boundary-iam-user"
+  prefix      = "demo-${local.hashicorp_email}" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
   byte_length = 4
 }
 
 resource "aws_iam_user" "missing_put_obj" {
-  name          = random_id.missing_put_obj.dec
-  force_destroy = true
-  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser"
-  tags = {
-    "boundary-demo" = local.hashicorp_email
-  }
+  name                 = random_id.missing_put_obj.dec
+  force_destroy        = true
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/DemoUser" # Do Not Remove/Edit. This is a requirement to obtain access in creating a iam user in the dev account.
+  tags                 = local.tags
 }
 
 resource "aws_iam_access_key" "missing_put_obj" {

testing/testdata/storage/main.tf

@@ -5,7 +5,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.0"
+      version = "~> 6.0"
     }
   }
 }

testing/testdata/storage/output.tf

@@ -6,19 +6,19 @@ output "bucket_name" {
 }
 
 output "valid_policy_arn" {
-    value = aws_iam_role.valid.arn
+  value = aws_iam_role.valid.arn
 }
 
 output "missing_put_obj_policy_arn" {
-    value = aws_iam_role.missing_put_obj.arn
+  value = aws_iam_role.missing_put_obj.arn
 }
 
 output "missing_get_obj_policy_arn" {
-    value = aws_iam_role.missing_get_obj.arn
+  value = aws_iam_role.missing_get_obj.arn
 }
 
 output "missing_delete_obj_policy_arn" {
-    value = aws_iam_role.missing_delete_obj.arn
+  value = aws_iam_role.missing_delete_obj.arn
 }
 
 output "iam_user_names" {
@@ -67,4 +67,9 @@ output "iam_access_key_missing_delete_obj" {
 output "iam_secret_access_key_missing_delete_obj" {
   value     = aws_iam_access_key.missing_delete_obj.secret
   sensitive = true
-}
+}
+
+output "s3_pagination_test_object_count" {
+  description = "The number of S3 objects created for each pagination test scenario."
+  value       = local.s3_pagination_test_object_count
+}

testing/testdata/storage/s3.tf

@@ -9,9 +9,17 @@ resource "aws_s3_bucket" "test" {
 
 resource "aws_s3_bucket_public_access_block" "secure_access" {
   bucket = aws_s3_bucket.test.id
-  
+
   block_public_acls       = true
   block_public_policy     = true
   ignore_public_acls      = true
   restrict_public_buckets = true
-}
+}
+resource "aws_s3_object" "s3_files" {
+  for_each = local.s3_files
+
+  bucket  = aws_s3_bucket.test.id
+  key     = each.key
+  content = each.value
+  tags    = local.tags
+}

testing/testdata/storage/variable.tf

@@ -16,7 +16,47 @@ locals {
   hashicorp_email = split(":", data.aws_caller_identity.current.user_id)[1]
   tags = {
     User        = split(":", data.aws_caller_identity.current.user_id)[1]
-    Repo        = "boundary-plugin-aws" 
+    Repo        = "boundary-plugin-aws"
     Environment = "Demo"
   }
+
+  # total number of objects of each type to create
+  s3_pagination_test_object_count = 350
+  # Dynamically build a flat map of S3 objects:
+  # - root-level text files
+  # - nested JSON files
+  # - placeholder "empty directories"
+  # - some static files in list-objects directory
+  # This map is consumed directly by an aws_s3_object resource
+  s3_files = merge(
+    # Root-level text files
+    {
+      for i in range(local.s3_pagination_test_object_count) :
+      "list-objects-paginated/root-file-${i}.txt" => "This is a root-level file #${i}."
+    },
+
+    # Nested JSON files inside sub-directories
+    {
+      for i in range(local.s3_pagination_test_object_count) :
+      "list-objects-paginated/nested-dir-${i}/data-file-${i}.json" =>
+      jsonencode({
+        id      = i,
+        message = "This is a nested JSON file #${i}."
+      })
+    },
+
+    # "Empty" directories (in S3 these are just placeholder objects ending with `/`)
+    {
+      for i in range(local.s3_pagination_test_object_count) :
+      "list-objects-paginated/empty-dir-${i}/dir-${i}/" => "" # zero-length object to mimic directory
+    },
+
+    # Static example files in list-objects directory
+    {
+      "list-objects/file1.txt"            = "This is the content for file 1."
+      "list-objects/data.json"            = jsonencode({ key = "value", items = [1, 2, 3] })
+      "list-objects/nested-dir/file2.txt" = "This is a nested file."
+      "list-objects/empty-nested-dir/"    = ""
+    }
+  )
 }