test(e2e): Add dynamic host catalog test (AWS)

This adds the initial set of tests that validate setting up a dynamic host catalog on AWS

Co-authored-by: Josh Brand <jbrand@hashicorp.com>

enos/enos-modules.hcl

@@ -2,6 +2,10 @@ module "az_finder" {
   source = "./modules/az_finder"
 }
 
+module "bats_deps" {
+  source = "./modules/bats_deps"
+}
+
 module "binary_finder" {
   source = "./modules/binary_finder"
 }
@@ -23,10 +27,6 @@ module "boundary" {
   alb_listener_api_port = var.alb_listener_api_port
 }
 
-module "bats_deps" {
-  source = "./modules/bats_deps"
-}
-
 module "build_crt" {
   source = "./modules/build_crt"
 }
@@ -35,6 +35,14 @@ module "build_local" {
   source = "./modules/build_local"
 }
 
+module "generate_aws_host_tag_vars" {
+  source = "./modules/generate_aws_host_tag_vars"
+}
+
+module "iam_setup" {
+  source = "./modules/iam_setup"
+}
+
 module "infra" {
   source  = "app.terraform.io/hashicorp-qti/aws-infra/enos"
   version = ">= 0.3.1"
@@ -77,10 +85,6 @@ module "vault" {
   ssh_aws_keypair = var.aws_ssh_keypair_name
 }
 
-module "test_smoke" {
-  source = "./modules/test_smoke"
-}
-
 module "test_cli_ui" {
   source = "./modules/test_cli_ui"
 }
@@ -88,3 +92,7 @@ module "test_cli_ui" {
 module "test_e2e" {
   source = "./modules/test_e2e"
 }
+
+module "test_smoke" {
+  source = "./modules/test_smoke"
+}

enos/enos-scenario-e2e-credential-vault.hcl

@@ -1,4 +1,4 @@
-scenario "e2e_vault" {
+scenario "e2e_credential_vault" {
   terraform_cli = terraform_cli.default
   terraform     = terraform.default
   providers = [
@@ -18,14 +18,11 @@ scenario "e2e_vault" {
       "local" = "/tmp",
       "crt"   = var.crt_bundle_path == null ? null : abspath(var.crt_bundle_path)
     }
-  }
-
-  step "check_jq_is_installed" {
-    module = module.binary_finder
-
-    variables {
-      name = "jq"
-    }
+    tags = merge({
+      "Project Name" : var.project_name
+      "Project" : "Enos",
+      "Environment" : "ci"
+    }, var.tags)
   }
 
   step "find_azs" {
@@ -56,6 +53,7 @@ scenario "e2e_vault" {
 
     variables {
       availability_zones = step.find_azs.availability_zones
+      common_tags        = local.tags
     }
   }
 
@@ -68,6 +66,7 @@ scenario "e2e_vault" {
 
     variables {
       boundary_install_dir     = local.boundary_install_dir
+      common_tags              = local.tags
       controller_instance_type = var.controller_instance_type
       controller_count         = var.controller_count
       db_pass                  = step.create_db_password.string

enos/enos-scenario-e2e-host-aws.hcl

@@ -0,0 +1,192 @@
+# This scenario requires access to the boundary team's test AWS account
+scenario "e2e_host_aws" {
+  terraform_cli = terraform_cli.default
+  terraform     = terraform.default
+  providers = [
+    provider.aws.default,
+    provider.enos.default
+  ]
+
+  matrix {
+    builder = ["local", "crt"]
+  }
+
+  locals {
+    aws_ssh_private_key_path = abspath(var.aws_ssh_private_key_path)
+    boundary_install_dir     = abspath(var.boundary_install_dir)
+    local_boundary_dir       = abspath(var.local_boundary_dir)
+    build_path = {
+      "local" = "/tmp",
+      "crt"   = var.crt_bundle_path == null ? null : abspath(var.crt_bundle_path)
+    }
+    tags = merge({
+      "Project Name" : var.project_name
+      "Project" : "Enos",
+      "Environment" : "ci"
+    }, var.tags)
+  }
+
+  step "find_azs" {
+    module = module.az_finder
+
+    variables {
+      instance_type = [
+        var.worker_instance_type,
+        var.controller_instance_type
+      ]
+    }
+  }
+
+  step "create_db_password" {
+    module = module.random_stringifier
+  }
+
+  step "build_boundary" {
+    module = matrix.builder == "crt" ? module.build_crt : module.build_local
+
+    variables {
+      path = local.build_path[matrix.builder]
+    }
+  }
+
+  step "create_base_infra" {
+    module = module.infra
+
+    variables {
+      availability_zones = step.find_azs.availability_zones
+      common_tags        = local.tags
+    }
+  }
+
+  step "create_boundary_cluster" {
+    module = module.boundary
+    depends_on = [
+      step.create_base_infra,
+      step.build_boundary
+    ]
+
+    variables {
+      boundary_install_dir     = local.boundary_install_dir
+      common_tags              = local.tags
+      controller_instance_type = var.controller_instance_type
+      controller_count         = var.controller_count
+      db_pass                  = step.create_db_password.string
+      kms_key_arn              = step.create_base_infra.kms_key_arn
+      local_artifact_path      = step.build_boundary.artifact_path
+      ubuntu_ami_id            = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
+      vpc_id                   = step.create_base_infra.vpc_id
+      worker_count             = var.worker_count
+      worker_instance_type     = var.worker_instance_type
+    }
+  }
+
+  step "create_tag1" {
+    module = module.random_stringifier
+  }
+
+  step "create_tag1_inputs" {
+    module     = module.generate_aws_host_tag_vars
+    depends_on = [step.create_tag1]
+
+    variables {
+      tag_name  = step.create_tag1.string
+      tag_value = "true"
+    }
+  }
+
+  step "create_targets_with_tag1" {
+    module     = module.target
+    depends_on = [step.create_base_infra]
+
+    variables {
+      ami_id               = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
+      aws_ssh_keypair_name = var.aws_ssh_keypair_name
+      enos_user            = var.enos_user
+      instance_type        = var.target_instance_type
+      vpc_id               = step.create_base_infra.vpc_id
+      target_count         = 2
+      additional_tags      = step.create_tag1_inputs.tag_map
+    }
+  }
+
+  step "create_tag2" {
+    module = module.random_stringifier
+  }
+
+  step "create_tag2_inputs" {
+    module     = module.generate_aws_host_tag_vars
+    depends_on = [step.create_tag2]
+
+    variables {
+      tag_name  = step.create_tag2.string
+      tag_value = "test"
+    }
+  }
+
+  step "create_targets_with_tag2" {
+    module     = module.target
+    depends_on = [step.create_base_infra]
+
+    variables {
+      ami_id               = step.create_base_infra.ami_ids["ubuntu"]["amd64"]
+      aws_ssh_keypair_name = var.aws_ssh_keypair_name
+      enos_user            = var.enos_user
+      instance_type        = var.target_instance_type
+      vpc_id               = step.create_base_infra.vpc_id
+      target_count         = 1
+      additional_tags      = step.create_tag2_inputs.tag_map
+    }
+  }
+
+  step "create_test_id" {
+    module = module.random_stringifier
+    variables {
+      length = 5
+    }
+  }
+
+  step "iam_setup" {
+    module = module.iam_setup
+    depends_on = [
+      step.create_base_infra,
+      step.create_test_id
+    ]
+
+    variables {
+      test_id    = step.create_test_id.string
+      test_email = var.test_email
+    }
+  }
+
+  step "run_e2e_test" {
+    module = module.test_e2e
+    depends_on = [
+      step.create_boundary_cluster,
+      step.create_targets_with_tag1,
+      step.create_targets_with_tag2,
+      step.iam_setup
+    ]
+
+    variables {
+      test_package             = "github.com/hashicorp/boundary/testing/internal/e2e/host/aws"
+      alb_boundary_api_addr    = step.create_boundary_cluster.alb_boundary_api_addr
+      auth_method_id           = step.create_boundary_cluster.auth_method_id
+      auth_login_name          = step.create_boundary_cluster.auth_login_name
+      auth_password            = step.create_boundary_cluster.auth_password
+      local_boundary_dir       = local.local_boundary_dir
+      aws_ssh_private_key_path = local.aws_ssh_private_key_path
+      target_user              = "ubuntu"
+      aws_access_key_id        = step.iam_setup.access_key_id
+      aws_secret_access_key    = step.iam_setup.secret_access_key
+      aws_host_set_filter1     = step.create_tag1_inputs.tag_string
+      aws_host_set_count1      = 2
+      aws_host_set_ips1        = step.create_targets_with_tag1.target_ips
+      aws_host_set_filter2     = step.create_tag2_inputs.tag_string
+      aws_host_set_count2      = 1
+    }
+  }
+
+  output "test_results" {
+    value = step.run_e2e_test.test_results
+  }
+}

enos/enos-scenario-e2e-host-static.hcl

@@ -1,4 +1,4 @@
-scenario "e2e_target" {
+scenario "e2e_host_static" {
   terraform_cli = terraform_cli.default
   terraform     = terraform.default
   providers = [
@@ -101,7 +101,7 @@ scenario "e2e_target" {
     ]
 
     variables {
-      test_package             = "github.com/hashicorp/boundary/testing/internal/e2e/target"
+      test_package             = "github.com/hashicorp/boundary/testing/internal/e2e/host/static"
       alb_boundary_api_addr    = step.create_boundary_cluster.alb_boundary_api_addr
       auth_method_id           = step.create_boundary_cluster.auth_method_id
       auth_login_name          = step.create_boundary_cluster.auth_login_name

enos/enos-variables.hcl

@@ -109,3 +109,9 @@ variable "vault_instance_type" {
   type        = string
   default     = "t3a.small"
 }
+
+variable "test_email" {
+  description = "Email address for setting up AWS IAM user (module: iam_setup)"
+  type        = string
+  default     = null
+}

enos/modules/generate_aws_host_tag_vars/main.tf

@@ -0,0 +1,19 @@
+variable "tag_name" {
+  type = string
+}
+variable "tag_value" {
+  type = string
+}
+
+locals {
+  tag_map    = { "e2e_${var.tag_name}" : var.tag_value }
+  tag_string = "tag:e2e_${var.tag_name}=${var.tag_value}"
+}
+
+output "tag_map" {
+  value = local.tag_map
+}
+
+output "tag_string" {
+  value = local.tag_string
+}

enos/modules/iam_setup/main.tf

@@ -0,0 +1,45 @@
+data "aws_caller_identity" "current" {}
+
+variable "test_id" {}
+variable "test_email" {}
+
+locals {
+  # Use the AWS provided email if users are running this, override with variable for CI
+  user_email = var.test_email == null ? split(":", data.aws_caller_identity.current.user_id)[1] : var.test_email
+}
+
+resource "aws_iam_user" "boundary" {
+  name                 = "boundary-e2e-${var.test_id}"
+  tags                 = { boundary-demo = local.user_email }
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/BoundaryDemoPermissionsBoundary"
+}
+
+resource "aws_iam_user_policy" "boundary" {
+  name = "boundary_e2e_${var.test_id}"
+  user = aws_iam_user.boundary.name
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Action" : [
+          "ec2:DescribeInstances"
+        ],
+        "Effect" : "Allow",
+        "Resource" : "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_access_key" "boundary" {
+  user = aws_iam_user.boundary.name
+}
+
+output "access_key_id" {
+  value = aws_iam_access_key.boundary.id
+}
+
+output "secret_access_key" {
+  value     = aws_iam_access_key.boundary.secret
+  sensitive = true
+}

enos/modules/random_stringifier/main.tf

@@ -1,5 +1,9 @@
+variable "length" {
+  type    = number
+  default = 10
+}
 resource "random_string" "string" {
-  length  = 10
+  length  = var.length
   lower   = true
   upper   = true
   numeric = true

enos/modules/target/main.tf

@@ -6,6 +6,9 @@ variable "project_name" {}
 variable "instance_type" {}
 variable "aws_ssh_keypair_name" {}
 variable "enos_user" {}
+variable "additional_tags" {
+  default = {}
+}
 
 data "aws_subnets" "infra" {
   filter {
@@ -15,7 +18,7 @@ data "aws_subnets" "infra" {
 }
 
 resource "aws_security_group" "boundary_target" {
-  name        = "boundary-target-sg"
+  name_prefix = "boundary-target-sg"
   description = "SSH and boundary Traffic"
   vpc_id      = var.vpc_id
 
@@ -47,12 +50,12 @@ resource "aws_instance" "target" {
   subnet_id              = tolist(data.aws_subnets.infra.ids)[count.index % length(data.aws_subnets.infra.ids)]
   key_name               = var.aws_ssh_keypair_name
 
-  tags = {
+  tags = merge(var.additional_tags, {
     "Name" : "boundary-target-${count.index}",
     "Type" : "target",
     "Environment" : var.environment
     "Enos User" : var.enos_user,
-  }
+  })
 }
 
 output "target_ips" {

enos/modules/test_e2e/main.tf

@@ -41,6 +41,7 @@ variable "aws_ssh_private_key_path" {
 variable "target_ips" {
   description = "List of IP Addresses of created hosts"
   type        = list(string)
+  default     = [""]
 }
 variable "vault_addr" {
   description = "URL of Vault instance"
@@ -52,10 +53,46 @@ variable "vault_root_token" {
   type        = string
   default     = ""
 }
+variable "aws_access_key_id" {
+  description = "Access Key Id for AWS IAM user used in dynamic host catalogs"
+  type        = string
+  default     = ""
+}
+variable "aws_secret_access_key" {
+  description = "Secret Access Key for AWS IAM user used in dynamic host catalogs"
+  type        = string
+  default     = ""
+}
+variable "aws_host_set_filter1" {
+  description = "Filter tag for host set used in dynamic host catalogs"
+  type        = string
+  default     = ""
+}
+variable "aws_host_set_count1" {
+  description = "Number of hosts in aws_host_set_filter1"
+  type        = number
+  default     = 0
+}
+variable "aws_host_set_ips1" {
+  description = "List of IP addresses in aws_host_set_filter1"
+  type        = list(string)
+  default     = [""]
+}
+variable "aws_host_set_filter2" {
+  description = "Filter tag for host set used in dynamic host catalogs"
+  type        = string
+  default     = ""
+}
+variable "aws_host_set_count2" {
+  description = "Number of hosts in aws_host_set_filter2"
+  type        = number
+  default     = 0
+}
 
 locals {
   aws_ssh_private_key_path = abspath(var.aws_ssh_private_key_path)
   vault_addr               = var.vault_addr != "" ? "http://${var.vault_addr}:8200" : ""
+  aws_host_set_ips1        = jsonencode(var.aws_host_set_ips1)
 }
 
 resource "enos_local_exec" "run_e2e_test" {
@@ -68,7 +105,14 @@ resource "enos_local_exec" "run_e2e_test" {
     E2E_SSH_USER                  = var.target_user,
     E2E_SSH_KEY_PATH              = local.aws_ssh_private_key_path,
     VAULT_ADDR                    = local.vault_addr,
-    VAULT_TOKEN                   = var.vault_root_token
+    VAULT_TOKEN                   = var.vault_root_token,
+    E2E_AWS_ACCESS_KEY_ID         = var.aws_access_key_id,
+    E2E_AWS_SECRET_ACCESS_KEY     = var.aws_secret_access_key,
+    E2E_AWS_HOST_SET_FILTER1      = var.aws_host_set_filter1,
+    E2E_AWS_HOST_SET_COUNT1       = var.aws_host_set_count1,
+    E2E_AWS_HOST_SET_IPS1         = local.aws_host_set_ips1,
+    E2E_AWS_HOST_SET_FILTER2      = var.aws_host_set_filter2,
+    E2E_AWS_HOST_SET_COUNT2       = var.aws_host_set_count2
   }
 
   inline = ["PATH=\"${var.local_boundary_dir}:$PATH\" go test -v ${var.test_package}"]

go.mod

@@ -91,6 +91,7 @@ require github.com/hashicorp/go-dbw v0.0.0-20220725170111-b7cb3aa3d628
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
+	github.com/cenkalti/backoff/v4 v4.1.0
 	github.com/hashicorp/go-kms-wrapping/extras/kms/v2 v2.0.0-20220711120347-32232bae6803
 	github.com/hashicorp/nodeenrollment v0.1.17
 	github.com/kelseyhightower/envconfig v1.4.0
@@ -113,7 +114,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/containerd/continuity v0.1.0 // indirect
 	github.com/coreos/go-oidc/v3 v3.0.0 // indirect

testing/internal/e2e/README.md

@@ -1,24 +1,38 @@
 # boundary-e2e-tests
 
-This test suite tests Boundary in an end-to-end setting using [Enos](https://github.com/hashicorp/Enos-Docs) to spin up the desired infrastructure and [go test](https://pkg.go.dev/testing) to perform user workflows.
+This test suite tests Boundary in an end-to-end setting, utilizing both the Boundary CLI and the
+Boundary Go API to exercise Boundary through various user workflows. It was designed to be run in a
+variety of environments as long as the appropriate environment variables are set. The test suite
+itself uses the standard [go test](https://pkg.go.dev/testing) library.
+
+One method for setting up an environment is utilizing [Enos](https://github.com/hashicorp/Enos-Docs)
+to create the desired infrastructure.
 
 ## Getting Started
-### Usage
-#### Enos
-Setup Enos as described [here](../../enos/README.md)
+### Enos
+Setup Enos as described [here](../../../enos/README.md)
 
+Then, use the following commands to run tests
 ```shell
-enos scenario run e2e_{scenario} builder:local // runs and destroys infra
+cd enos
+enos scenario list
 
-enos scenario launch e2e_{scenario} builder:local  // runs and keeps infra online
-enos scenario output // displays any defined enos output
+# `Run` executes the tests and destroys the associated infrastructure in one command
+enos scenario run e2e_{scenario} builder:local
 
-enos scenario destroy // destroys infra
+# `Launch` executes the tests, but leaves the infrastructure online for debugging purposes
+enos scenario launch e2e_{scenario} builder:local
+enos scenario output  # displays any defined enos output
+enos scenario destroy  # destroys infra
 ```
 
-Enos scenarios set up the infrastructure, set the appropriate environment variables, and run the selected tests. Folders in this directory correspond to an enos scenario (e.g. `enos/enos-scenario-e2e-target.hcl` runs tests in `testing/e2e/target`)
+Enos scenarios set up the infrastructure, set the appropriate environment variables, and run the
+specified tests in its scenario file.
 
-#### Local
+Note: To run the `e2e_host_aws` scenario, you will need access to the boundary team's test AWS
+account.
+
+### Local
 Set the appropriate environment variables...
 ```shell
 export BOUNDARY_ADDR=  # e.g. http://127.0.0.1:9200
@@ -26,11 +40,22 @@ export E2E_PASSWORD_AUTH_METHOD_ID=  # e.g. ampw_1234567890
 export E2E_PASSWORD_ADMIN_LOGIN_NAME=  # e.g. "admin"
 export E2E_PASSWORD_ADMIN_PASSWORD=  # e.g. "password"
 
-# For e2e/target
+# For e2e/host/static
 export E2E_TARGET_IP=  # e.g. 192.168.0.1
 export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
 export E2E_SSH_USER=  # e.g. ubuntu
 
+# For e2e/host/aws
+export E2E_AWS_ACCESS_KEY_ID=
+export E2E_AWS_SECRET_ACCESS_KEY=
+export E2E_AWS_HOST_SET_FILTER1=  # e.g. "tag:testtag=true"
+export E2E_AWS_HOST_SET_COUNT1=  # e.g. 2
+export E2E_AWS_HOST_SET_IPS1=  # e.g. "[\"1.2.3.4\", \"2.3.4.5\"]"
+export E2E_AWS_HOST_SET_FILTER2=  # e.g. "tag:testtagtwo=test"
+export E2E_AWS_HOST_SET_COUNT2=  # e.g. 1
+export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
+export E2E_SSH_USER=  # e.g. ubuntu
+
 # For e2e/credential/vault
 export VAULT_ADDR=  # e.g. http://127.0.0.1:8200
 export VAULT_TOKEN=
@@ -46,3 +71,12 @@ go test ./target/ // run target tests if running from this directory
 go test github.com/hashicorp/boundary/testing/e2e/target -v // verbose
 go test github.com/hashicorp/boundary/testing/e2e/target -v -run '^TestCreateTargetApi$' // run a specific test
 ```
+
+## Adding Tests
+
+Tests live under this directory. Additional tests can be added to an existing go package or a new
+one can be created. If a new package is created, a new enos scenario would also need to be created.
+
+Enos is comprised of scenarios, where a scenario is the environment you want the tests to operate
+in. In one scenario, there may be a boundary cluster and a target. Another scenario might involve a
+boundary cluster and a vault instance. Scenarios can be found in [boundary/enos](../../../enos/)

testing/internal/e2e/boundary/boundary.go

@@ -45,6 +45,11 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
@@ -55,10 +60,6 @@ func NewApiClient() (*api.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = c.validate()
-	if err != nil {
-		return nil, err
-	}
 
 	client, err := api.NewClient(&api.Config{Addr: c.Address})
 	if err != nil {
@@ -85,14 +86,12 @@ func NewApiClient() (*api.Client, error) {
 func AuthenticateCli(t testing.TB) {
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
-	output := e2e.RunCommand("boundary", e2e.WithArgs("authenticate", "password",
+	output := e2e.RunCommand("boundary", "authenticate", "password",
 		"-addr", c.Address,
 		"-auth-method-id", c.AuthMethodId,
 		"-login-name", c.AdminLoginName,
 		"-password", "env://E2E_PASSWORD_ADMIN_PASSWORD",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 }

testing/internal/e2e/boundary/scopes.go

@@ -25,24 +25,29 @@ func CreateNewOrgApi(t testing.TB, ctx context.Context, client *api.Client) stri
 	return newOrgResult.Item.Id
 }
 
+// CreateNewProjectApi creates a new project in boundary using the go api. The project will be created
+// under the provided org id.
+// Returns the id of the new project.
+func CreateNewProjectApi(t testing.TB, ctx context.Context, client *api.Client, orgId string) string {
+	scopeClient := scopes.NewClient(client)
+	newProjResult, err := scopeClient.Create(ctx, orgId, scopes.WithName("e2e Automated Test Project"))
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_, err := scopeClient.Delete(ctx, newProjResult.Item.Id)
+		require.NoError(t, err)
+	})
+
+	return newProjResult.Item.Id
+}
+
 // CreateNewOrgCli creates a new organization in boundary using the cli.
 // Returns the id of the new org.
 func CreateNewOrgCli(t testing.TB) string {
-	return createNewScopeCli(t, "e2e Automated Test Org", "global")
-}
-
-// CreateNewProjectCli creates a new project in boundary using the cli.
-// Returns the id of the new project.
-func CreateNewProjectCli(t testing.TB, scopeId string) string {
-	return createNewScopeCli(t, "e2e Automated Test Project", scopeId)
-}
-
-func createNewScopeCli(t testing.TB, name string, scopeId string) string {
-	output := e2e.RunCommand("boundary", e2e.WithArgs("scopes", "create",
-		"-name", name,
-		"-scope-id", scopeId,
+	output := e2e.RunCommand("boundary", "scopes", "create",
+		"-name", "e2e Automated Test Org",
+		"-scope-id", "global",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	var newOrgResult scopes.ScopeCreateResult
@@ -50,9 +55,32 @@ func createNewScopeCli(t testing.TB, name string, scopeId string) string {
 	require.NoError(t, err)
 
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("scopes", "delete", "-id", newOrgResult.Item.Id))
+		output := e2e.RunCommand("boundary", "scopes", "delete", "-id", newOrgResult.Item.Id)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
 	return newOrgResult.Item.Id
 }
+
+// CreateNewProjectCli creates a new project in boundary using the cli. The project will be created
+// under the provided org id.
+// Returns the id of the new project.
+func CreateNewProjectCli(t testing.TB, orgId string) string {
+	output := e2e.RunCommand("boundary", "scopes", "create",
+		"-name", "e2e Automated Test Project",
+		"-scope-id", orgId,
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+
+	var newProjResult scopes.ScopeCreateResult
+	err := json.Unmarshal(output.Stdout, &newProjResult)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		output := e2e.RunCommand("boundary", "scopes", "delete", "-id", newProjResult.Item.Id)
+		require.NoError(t, output.Err, string(output.Stderr))
+	})
+
+	return newProjResult.Item.Id
+}

testing/internal/e2e/credential/vault/vault_test.go

@@ -1,4 +1,4 @@
-package vaultcredential_test
+package vault_test
 
 import (
 	"context"
@@ -15,7 +15,6 @@ import (
 	"github.com/hashicorp/boundary/api/hostcatalogs"
 	"github.com/hashicorp/boundary/api/hosts"
 	"github.com/hashicorp/boundary/api/hostsets"
-	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/testing/internal/e2e"
 	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
@@ -57,28 +56,36 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
+type createTokenResponse struct {
+	Auth struct {
+		Client_Token string
+	}
+}
+
 // TestCreateVaultCredentialStoreCli uses the boundary and vault clis to add secrets management
 // for a target. The test sets up vault as a credential store, creates a set of credentials
 // in vault to be attached to a target, and attempts to connect to that target using those
 // credentials.
 func TestCreateVaultCredentialStoreCli(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	// Configure vault
 	vaultAddr, boundaryPolicyName := vault.Setup(t)
 
-	output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2"))
+	output := e2e.RunCommand("vault", "secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2")
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "disable", c.VaultSecretPath))
+		output := e2e.RunCommand("vault", "secrets", "disable", c.VaultSecretPath)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
@@ -88,7 +95,7 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 	t.Log("Created Vault Credential")
 
 	// Create vault token for boundary
-	output = e2e.RunCommand("vault", e2e.WithArgs("token", "create",
+	output = e2e.RunCommand("vault", "token", "create",
 		"-no-default-policy=true",
 		"-policy="+boundaryPolicyName,
 		"-policy="+credentialPolicyName,
@@ -96,130 +103,129 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 		"-period=20m",
 		"-renewable=true",
 		"-format=json",
-	),
-		e2e.WithPipe("jq", "-r", ".auth.client_token"),
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
-	credStoreToken := strings.TrimSpace(string(output.Stdout))
+	var tokenCreateResult createTokenResponse
+	err = json.Unmarshal(output.Stdout, &tokenCreateResult)
+	require.NoError(t, err)
+	credStoreToken := tokenCreateResult.Auth.Client_Token
 	t.Log("Created Vault Cred Store Token")
 
 	// Authenticate boundary cli
 	boundary.AuthenticateCli(t)
 
-	// Create an org
+	// Create an org and project
 	newOrgId := boundary.CreateNewOrgCli(t)
 	t.Logf("Created Org Id: %s", newOrgId)
-
-	// Create a project
 	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
 	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a credential store
-	output = e2e.RunCommand("boundary", e2e.WithArgs("credential-stores", "create", "vault",
+	output = e2e.RunCommand("boundary", "credential-stores", "create", "vault",
 		"-scope-id", newProjectId,
 		"-vault-address", vaultAddr,
 		"-vault-token", credStoreToken,
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newCredentialStoreResult credentialstores.CredentialStoreCreateResult
 	err = json.Unmarshal(output.Stdout, &newCredentialStoreResult)
 	require.NoError(t, err)
-	newCredentialStore := newCredentialStoreResult.Item
-	t.Logf("Created Credential Store: %s", newCredentialStore.Id)
+	newCredentialStoreId := newCredentialStoreResult.Item.Id
+	t.Logf("Created Credential Store: %s", newCredentialStoreId)
 
 	// Create a credential library
-	output = e2e.RunCommand("boundary", e2e.WithArgs("credential-libraries", "create", "vault",
-		"-credential-store-id", newCredentialStore.Id,
+	output = e2e.RunCommand("boundary", "credential-libraries", "create", "vault",
+		"-credential-store-id", newCredentialStoreId,
 		"-vault-path", c.VaultSecretPath+"/data/"+secretName,
 		"-name", "e2e Automated Test Vault Credential Library",
 		"-credential-type", "ssh_private_key",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newCredentialLibraryResult credentiallibraries.CredentialLibraryCreateResult
 	err = json.Unmarshal(output.Stdout, &newCredentialLibraryResult)
 	require.NoError(t, err)
-	newCredentialLibrary := newCredentialLibraryResult.Item
-	t.Logf("Created Credential Library: %s", newCredentialLibrary.Id)
+	newCredentialLibraryId := newCredentialLibraryResult.Item.Id
+	t.Logf("Created Credential Library: %s", newCredentialLibraryId)
 
 	// Create a host catalog
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-catalogs", "create", "static",
+	output = e2e.RunCommand("boundary", "host-catalogs", "create", "static",
 		"-scope-id", newProjectId,
 		"-name", "e2e Automated Test Host Catalog",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostCatalogResult hostcatalogs.HostCatalogCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostCatalogResult)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "host-sets", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", "e2e Automated Test Host Set",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostSetResult hostsets.HostSetCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostSetResult)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	newHostSetId := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
-	output = e2e.RunCommand("boundary", e2e.WithArgs("hosts", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "hosts", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", c.TargetIp,
 		"-address", c.TargetIp,
-		"-format", "json"),
+		"-format", "json",
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostResult hosts.HostCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostResult)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
-	t.Logf("Created Host: %s", newHost.Id)
+	newHostId := newHostResult.Item.Id
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "add-hosts",
-		"-id", newHostSet.Id,
-		"-host", newHost.Id,
-	))
+	output = e2e.RunCommand("boundary", "host-sets", "add-hosts",
+		"-id", newHostSetId,
+		"-host", newHostId,
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Create Target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "create", "tcp",
+	output = e2e.RunCommand("boundary", "targets", "create", "tcp",
 		"-scope-id", newProjectId,
 		"-default-port", c.TargetPort,
 		"-name", "e2e Automated Test Target",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newTargetResult targets.TargetCreateResult
 	err = json.Unmarshal(output.Stdout, &newTargetResult)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
-	t.Logf("Created Target: %s", newTarget.Id)
+	newTargetId := newTargetResult.Item.Id
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "add-host-sources",
-		"-id", newTarget.Id,
-		"-host-source", newHostSet.Id,
-	))
+	output = e2e.RunCommand("boundary", "targets", "add-host-sources",
+		"-id", newTargetId,
+		"-host-source", newHostSetId,
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Add brokered credentials to target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "add-credential-sources",
-		"-id", newTarget.Id,
-		"-brokered-credential-source", newCredentialLibrary.Id,
-	))
+	output = e2e.RunCommand("boundary", "targets", "add-credential-sources",
+		"-id", newTargetId,
+		"-brokered-credential-source", newCredentialLibraryId,
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Get credentials for target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "authorize-session", "-id", newTarget.Id, "-format", "json"))
+	output = e2e.RunCommand("boundary", "targets", "authorize-session", "-id", newTargetId, "-format", "json")
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newSessionAuthorizationResult targets.SessionAuthorizationResult
 	err = json.Unmarshal(output.Stdout, &newSessionAuthorizationResult)
@@ -245,8 +251,8 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect to target and print host's IP address using retrieved credentials
-	output = e2e.RunCommand("boundary", e2e.WithArgs("connect",
-		"-target-id", newTarget.Id,
+	output = e2e.RunCommand("boundary", "connect",
+		"-target-id", newTargetId,
 		"-exec", "/usr/bin/ssh", "--",
 		"-l", retrievedUser,
 		"-i", retrievedKeyPath,
@@ -256,7 +262,7 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 		"-p", "{{boundary.port}}", // this is provided by boundary
 		"{{boundary.ip}}",
 		"hostname", "-i",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	parts := strings.Fields(string(output.Stdout))
@@ -270,19 +276,16 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 // a set of credentials in vault that is attached to a target.
 func TestCreateVaultCredentialStoreApi(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	// Configure vault
 	vaultAddr, boundaryPolicyName := vault.Setup(t)
 
-	output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2"))
+	output := e2e.RunCommand("vault", "secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2")
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "disable", c.VaultSecretPath))
+		output := e2e.RunCommand("vault", "secrets", "disable", c.VaultSecretPath)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
@@ -292,7 +295,7 @@ func TestCreateVaultCredentialStoreApi(t *testing.T) {
 	t.Log("Created Vault Credential")
 
 	// Create vault token for boundary
-	output = e2e.RunCommand("vault", e2e.WithArgs("token", "create",
+	output = e2e.RunCommand("vault", "token", "create",
 		"-no-default-policy=true",
 		"-policy="+boundaryPolicyName,
 		"-policy="+credentialPolicyName,
@@ -300,11 +303,11 @@ func TestCreateVaultCredentialStoreApi(t *testing.T) {
 		"-period=20m",
 		"-renewable=true",
 		"-format=json",
-	),
-		e2e.WithPipe("jq", "-r", ".auth.client_token"),
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
-	credStoreToken := strings.TrimSpace(string(output.Stdout))
+	var tokenCreateResult createTokenResponse
+	err = json.Unmarshal(output.Stdout, &tokenCreateResult)
+	credStoreToken := tokenCreateResult.Auth.Client_Token
 	t.Log("Created Vault Cred Store Token")
 
 	// Create boundary api client
@@ -312,108 +315,100 @@ func TestCreateVaultCredentialStoreApi(t *testing.T) {
 	require.NoError(t, err)
 	ctx := context.Background()
 
-	// Create an org
-	scopeClient := scopes.NewClient(client)
-	newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Automated Test Org"))
-	require.NoError(t, err)
-	newOrg := newOrgResult.Item
-	t.Cleanup(func() {
-		_, err := scopeClient.Delete(ctx, newOrg.Id)
-		require.NoError(t, err)
-	})
-	t.Logf("Created Org Id: %s", newOrg.Id)
-
-	// Create a project
-	newProjectResult, err := scopeClient.Create(ctx, newOrg.Id, scopes.WithName("e2e Automated Test Project"))
-	require.NoError(t, err)
-	newProject := newProjectResult.Item
-	t.Logf("Created Project Id: %s", newProject.Id)
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgApi(t, ctx, client)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectApi(t, ctx, client, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a credential store
 	csClient := credentialstores.NewClient(client)
-	newCredentialStoreResult, err := csClient.Create(ctx, "vault", newProject.Id,
+	newCredentialStoreResult, err := csClient.Create(ctx, "vault", newProjectId,
 		credentialstores.WithVaultCredentialStoreAddress(vaultAddr),
 		credentialstores.WithVaultCredentialStoreToken(credStoreToken),
 	)
 	require.NoError(t, err)
-	newCredentialStore := newCredentialStoreResult.Item
-	t.Logf("Created Credential Store: %s", newCredentialStore.Id)
+	newCredentialStoreId := newCredentialStoreResult.Item.Id
+	t.Logf("Created Credential Store: %s", newCredentialStoreId)
 
 	// Create a credential library
 	clClient := credentiallibraries.NewClient(client)
-	newCredentialLibraryResult, err := clClient.Create(ctx, newCredentialStore.Id,
+	newCredentialLibraryResult, err := clClient.Create(ctx, newCredentialStoreId,
 		credentiallibraries.WithVaultCredentialLibraryPath(c.VaultSecretPath+"/data/"+secretName),
 		credentiallibraries.WithCredentialType("ssh_private_key"),
 	)
 	require.NoError(t, err)
-	newCredentialLibrary := newCredentialLibraryResult.Item
-	t.Logf("Created Credential Library: %s", newCredentialLibrary.Id)
+	newCredentialLibraryId := newCredentialLibraryResult.Item.Id
+	t.Logf("Created Credential Library: %s", newCredentialLibraryId)
 
 	// Create a host catalog
 	hcClient := hostcatalogs.NewClient(client)
-	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProject.Id,
+	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProjectId,
 		hostcatalogs.WithName("e2e Automated Test Host Catalog"),
 	)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
 	hsClient := hostsets.NewClient(client)
-	newHostSetResult, err := hsClient.Create(ctx, newHostCatalog.Id)
+	newHostSetResult, err := hsClient.Create(ctx, newHostCatalogId)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	newHostSetId := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
 	hClient := hosts.NewClient(client)
-	newHostResult, err := hClient.Create(ctx, newHostCatalog.Id,
+	newHostResult, err := hClient.Create(ctx, newHostCatalogId,
 		hosts.WithName(c.TargetIp),
 		hosts.WithStaticHostAddress(c.TargetIp),
 	)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
-	t.Logf("Created Host: %s", newHost.Id)
+	newHostId := newHostResult.Item.Id
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	_, err = hsClient.AddHosts(ctx, newHostSet.Id, 0, []string{newHost.Id}, hostsets.WithAutomaticVersioning(true))
+	_, err = hsClient.AddHosts(ctx, newHostSetId, 0, []string{newHostId}, hostsets.WithAutomaticVersioning(true))
 	require.NoError(t, err)
 
 	// Create a target
 	tClient := targets.NewClient(client)
 	targetPort, err := strconv.ParseInt(c.TargetPort, 10, 32)
-	newTargetResult, err := tClient.Create(ctx, "tcp", newProject.Id,
+	newTargetResult, err := tClient.Create(ctx, "tcp", newProjectId,
 		targets.WithName("e2e Automated Test Target"),
 		targets.WithTcpTargetDefaultPort(uint32(targetPort)),
 	)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
-	t.Logf("Created Target: %s", newTarget.Id)
+	newTargetId := newTargetResult.Item.Id
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	_, err = tClient.AddHostSources(ctx, newTarget.Id, 0,
-		[]string{newHostSet.Id},
+	_, err = tClient.AddHostSources(ctx, newTargetId, 0,
+		[]string{newHostSetId},
 		targets.WithAutomaticVersioning(true),
 	)
 	require.NoError(t, err)
 
 	// Add brokered credentials to target
-	_, err = tClient.AddCredentialSources(ctx, newTarget.Id, 0,
-		targets.WithBrokeredCredentialSourceIds([]string{newCredentialLibrary.Id}),
+	_, err = tClient.AddCredentialSources(ctx, newTargetId, 0,
+		targets.WithBrokeredCredentialSourceIds([]string{newCredentialLibraryId}),
 		targets.WithAutomaticVersioning(true),
 	)
 	require.NoError(t, err)
 
 	// Get credentials for target
-	newSessionAuthorizationResult, err := tClient.AuthorizeSession(ctx, newTarget.Id)
+	newSessionAuthorizationResult, err := tClient.AuthorizeSession(ctx, newTargetId)
 	require.NoError(t, err)
 	newSessionAuthorization := newSessionAuthorizationResult.Item
-	retrievedUser := fmt.Sprintf("%s", newSessionAuthorization.Credentials[0].Credential["username"])
-	retrievedKey := fmt.Sprintf("%s", newSessionAuthorization.Credentials[0].Credential["private_key"])
+	retrievedUser, ok := newSessionAuthorization.Credentials[0].Credential["username"].(string)
+	require.True(t, ok)
 	assert.Equal(t, c.TargetSshUser, retrievedUser)
 
+	retrievedKey, ok := newSessionAuthorization.Credentials[0].Credential["private_key"].(string)
+	require.True(t, ok)
 	k, err := os.ReadFile(c.TargetSshKeyPath)
 	require.NoError(t, err)
-	require.Equal(t, string(k), retrievedKey)
+	keysMatch := string(k) == retrievedKey // This is done to prevent printing out key info
+	require.True(t, keysMatch, "Key retrieved from vault does not match expected value")
 	t.Log("Successfully retrieved credentials for target")
 }

testing/internal/e2e/helpers.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"testing"
@@ -49,45 +48,17 @@ const EnvToCheckSkip = "E2E_PASSWORD_AUTH_METHOD_ID"
 // of running the provided command.
 //
 //	RunCommand("ls")
-//	RunCommand("ls", WithArgs("-al", "/path"))
-//	RunCommand("ls", WithArgs("-al", "/path"), WithPipe("grep", "file"))
+//	RunCommand("ls", "-al", "/path")
 //
 // CommandResult is always valid even if there is an error.
-func RunCommand(command string, opt ...Option) *CommandResult {
+func RunCommand(name string, args ...string) *CommandResult {
 	var outbuf, errbuf bytes.Buffer
-	var err error
-	var c1, c2 *exec.Cmd
 
-	opts := getOpts(opt...)
+	cmd := exec.Command(name, args...)
+	cmd.Stdout = &outbuf
+	cmd.Stderr = &errbuf
 
-	if opts.withArgs == nil {
-		c1 = exec.Command(command)
-	} else {
-		c1 = exec.Command(command, opts.withArgs...)
-	}
-
-	if opts.withPipe == nil {
-		c1.Stdout = &outbuf
-		c1.Stderr = &errbuf
-		err = c1.Run()
-	} else {
-		pipeCommand := opts.withPipe[0]
-		pipeArgs := opts.withPipe[1:]
-		c2 = exec.Command(pipeCommand, pipeArgs...)
-
-		r, w := io.Pipe()
-		c1.Stdout = w
-		c2.Stdin = r
-
-		c2.Stdout = &outbuf
-		c2.Stderr = &errbuf
-
-		c1.Start()
-		c2.Start()
-		c1.Wait()
-		w.Close()
-		c2.Wait()
-	}
+	err := cmd.Run()
 
 	var ee *exec.ExitError
 	var exitCode int

testing/internal/e2e/host/aws/dynamichostcatalog_test.go

@@ -0,0 +1,413 @@
+package aws_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/hashicorp/boundary/api/hostcatalogs"
+	"github.com/hashicorp/boundary/api/hosts"
+	"github.com/hashicorp/boundary/api/hostsets"
+	"github.com/hashicorp/boundary/api/targets"
+	"github.com/hashicorp/boundary/testing/internal/e2e"
+	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type config struct {
+	AwsAccessKeyId     string `envconfig:"E2E_AWS_ACCESS_KEY_ID"`
+	AwsSecretAccessKey string `envconfig:"E2E_AWS_SECRET_ACCESS_KEY"`
+	AwsHostSetFilter1  string `envconfig:"E2E_AWS_HOST_SET_FILTER1"`
+	AwsHostSetCount1   string `envconfig:"E2E_AWS_HOST_SET_COUNT1"`
+	AwsHostSetIps1     string `envconfig:"E2E_AWS_HOST_SET_IPS1"`
+	AwsHostSetFilter2  string `envconfig:"E2E_AWS_HOST_SET_FILTER2"`
+	AwsHostSetCount2   string `envconfig:"E2E_AWS_HOST_SET_COUNT2"`
+	TargetSshKeyPath   string `envconfig:"E2E_SSH_KEY_PATH"` // e.g. /Users/username/key.pem
+	TargetSshUser      string `envconfig:"E2E_SSH_USER"`     // e.g. ubuntu
+	TargetPort         string `envconfig:"E2E_SSH_PORT" default:"22"`
+}
+
+func (c *config) validate() error {
+	if c.AwsAccessKeyId == "" {
+		return errors.New("AwsAccessKeyId is empty. Set environment variable: E2E_AWS_ACCESS_KEY_ID")
+	}
+	if c.AwsSecretAccessKey == "" {
+		return errors.New("AwsSecretAccessKey is empty. Set environment variable: E2E_AWS_SECRET_ACCESS_KEY")
+	}
+	if c.AwsHostSetFilter1 == "" {
+		return errors.New("AwsHostSetFilter1 is empty. Set environment variable: E2E_AWS_HOST_SET_FILTER1")
+	}
+	if c.AwsHostSetCount1 == "" {
+		return errors.New("AwsHostSetCount1 is empty. Set environment variable: E2E_AWS_HOST_SET_COUNT1")
+	}
+	if c.AwsHostSetFilter2 == "" {
+		return errors.New("AwsHostSetFilter2 is empty. Set environment variable: E2E_AWS_HOST_SET_FILTER2")
+	}
+	if c.AwsHostSetCount2 == "" {
+		return errors.New("AwsHostSetCount2 is empty. Set environment variable: E2E_AWS_HOST_SET_COUNT2")
+	}
+	if c.TargetSshKeyPath == "" {
+		return errors.New("TargetSshKeyPath is empty. Set environment variable: E2E_SSH_KEY_PATH")
+	}
+	if c.TargetSshUser == "" {
+		return errors.New("TargetSshUser is empty. Set environment variable: E2E_SSH_USER")
+	}
+	if c.TargetPort == "" {
+		return errors.New("TargetPort is empty. Set environment variable: E2E_SSH_PORT")
+	}
+
+	return nil
+}
+
+func loadConfig() (*config, error) {
+	var c config
+	err := envconfig.Process("", &c)
+	if err != nil {
+		return nil, err
+	}
+
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, err
+}
+
+func TestCreateAwsDynamicHostCatalogCli(t *testing.T) {
+	e2e.MaybeSkipTest(t)
+	c, err := loadConfig()
+	require.NoError(t, err)
+
+	boundary.AuthenticateCli(t)
+
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgCli(t)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
+
+	// Create a dynamic host catalog
+	output := e2e.RunCommand("boundary", "host-catalogs", "create", "plugin",
+		"-scope-id", newProjectId,
+		"-plugin-name", "aws",
+		"-attr", "disable_credential_rotation=true",
+		"-attr", "region=us-east-1",
+		"-secret", "access_key_id=env://E2E_AWS_ACCESS_KEY_ID",
+		"-secret", "secret_access_key=env://E2E_AWS_SECRET_ACCESS_KEY",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newHostCatalogResult hostcatalogs.HostCatalogCreateResult
+	err = json.Unmarshal(output.Stdout, &newHostCatalogResult)
+	require.NoError(t, err)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
+
+	// Create a host set
+	output = e2e.RunCommand("boundary", "host-sets", "create", "plugin",
+		"-host-catalog-id", newHostCatalogId,
+		"-attr", "filters="+c.AwsHostSetFilter1,
+		"-name", "e2e Automated Test Host Set",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newHostSetResult hostsets.HostSetCreateResult
+	err = json.Unmarshal(output.Stdout, &newHostSetResult)
+	require.NoError(t, err)
+	newHostSetId1 := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId1)
+
+	// Get list of hosts in host set
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host set...")
+	var actualHostSetCount1 int
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand("boundary", "host-sets", "read",
+				"-id", newHostSetId1,
+				"-format", "json",
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var hostSetsReadResult hostsets.HostSetReadResult
+			err = json.Unmarshal(output.Stdout, &hostSetsReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostSetCount1 = len(hostSetsReadResult.Item.HostIds)
+			if actualHostSetCount1 == 0 {
+				return errors.New("No items are appearing in the host set")
+			}
+
+			t.Logf("Found %d hosts", actualHostSetCount1)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	expectedHostSetCount1, err := strconv.Atoi(c.AwsHostSetCount1)
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostSetCount1, actualHostSetCount1, "Numbers of hosts in host set did not match expected amount")
+
+	// Create another host set
+	output = e2e.RunCommand("boundary", "host-sets", "create", "plugin",
+		"-host-catalog-id", newHostCatalogId,
+		"-attr", "filters="+c.AwsHostSetFilter2,
+		"-name", "e2e Automated Test Host Set2",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newHostSetResult2 hostsets.HostSetCreateResult
+	err = json.Unmarshal(output.Stdout, &newHostSetResult2)
+	require.NoError(t, err)
+	newHostSetId2 := newHostSetResult2.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId2)
+
+	// Get list of hosts in the second host set
+	t.Logf("Looking for items in the second host set...")
+	var actualHostSetCount2 int
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand("boundary", "host-sets", "read",
+				"-id", newHostSetId2,
+				"-format", "json",
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var hostSetsReadResult hostsets.HostSetReadResult
+			err = json.Unmarshal(output.Stdout, &hostSetsReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostSetCount2 = len(hostSetsReadResult.Item.HostIds)
+			if actualHostSetCount2 == 0 {
+				return errors.New("No items are appearing in the host set")
+			}
+
+			t.Logf("Found %d hosts", actualHostSetCount2)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	expectedHostSetCount2, err := strconv.Atoi(c.AwsHostSetCount2)
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostSetCount2, actualHostSetCount2, "Numbers of hosts in host set did not match expected amount")
+
+	// Get list of all hosts from host catalog
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host catalog...")
+	var actualHostCatalogCount int
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand("boundary", "hosts", "list",
+				"-host-catalog-id", newHostCatalogId,
+				"-format", "json",
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var hostCatalogListResult hostcatalogs.HostCatalogListResult
+			err = json.Unmarshal(output.Stdout, &hostCatalogListResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostCatalogCount = len(hostCatalogListResult.Items)
+			if actualHostCatalogCount == 0 {
+				return errors.New("No items are appearing in the host catalog")
+			}
+
+			t.Logf("Found %d hosts", actualHostCatalogCount)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	expectedHostCatalogCount := expectedHostSetCount1 + expectedHostSetCount2
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostCatalogCount, actualHostCatalogCount, "Numbers of hosts in host catalog did not match expected amount")
+
+	// Create target
+	output = e2e.RunCommand("boundary", "targets", "create", "tcp",
+		"-scope-id", newProjectId,
+		"-default-port", c.TargetPort,
+		"-name", "e2e Automated Test Target",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newTargetResult targets.TargetCreateResult
+	err = json.Unmarshal(output.Stdout, &newTargetResult)
+	require.NoError(t, err)
+	newTargetId := newTargetResult.Item.Id
+	t.Logf("Created Target: %s", newTargetId)
+
+	// Add host set to target
+	output = e2e.RunCommand("boundary", "targets", "add-host-sources",
+		"-id", newTargetId,
+		"-host-source", newHostSetId1,
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+
+	// Connect to target
+	output = e2e.RunCommand("boundary", "connect",
+		"-target-id", newTargetId,
+		"-exec", "/usr/bin/ssh", "--",
+		"-l", c.TargetSshUser,
+		"-i", c.TargetSshKeyPath,
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "IdentitiesOnly=yes", // forces the use of the provided key
+		"-p", "{{boundary.port}}", // this is provided by boundary
+		"{{boundary.ip}}",
+		"hostname", "-i",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+
+	parts := strings.Fields(string(output.Stdout))
+	hostIp := parts[len(parts)-1]
+	t.Log("Successfully connected to the target")
+
+	// Check if connected host exists in the host set
+	var targetIps []string
+	err = json.Unmarshal([]byte(c.AwsHostSetIps1), &targetIps)
+	hostIpInList := false
+	for _, v := range targetIps {
+		if v == hostIp {
+			hostIpInList = true
+		}
+	}
+	require.True(t, hostIpInList, fmt.Sprintf("Connected host (%s) is not in expected list (%s)", hostIp, targetIps))
+}
+
+func TestCreateAwsDynamicHostCatalogApi(t *testing.T) {
+	e2e.MaybeSkipTest(t)
+	c, err := loadConfig()
+	require.NoError(t, err)
+
+	// Create boundary api client
+	client, err := boundary.NewApiClient()
+	require.NoError(t, err)
+	ctx := context.Background()
+
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgApi(t, ctx, client)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectApi(t, ctx, client, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
+
+	// Create a dynamic host catalog
+	hcClient := hostcatalogs.NewClient(client)
+	newHostCatalogResult, err := hcClient.Create(ctx, "plugin", newProjectId,
+		hostcatalogs.WithName("e2e Automated Test Host Catalog"),
+		hostcatalogs.WithPluginName("aws"),
+		hostcatalogs.WithAttributes(map[string]interface{}{
+			"disable_credential_rotation": true,
+			"region":                      "us-east-1",
+		}),
+		hostcatalogs.WithSecrets(map[string]interface{}{
+			"access_key_id":     c.AwsAccessKeyId,
+			"secret_access_key": c.AwsSecretAccessKey,
+		}),
+	)
+	require.NoError(t, err)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
+
+	// Create a host set and add to catalog
+	hsClient := hostsets.NewClient(client)
+	newHostSetResult, err := hsClient.Create(ctx, newHostCatalogId,
+		hostsets.WithAttributes(map[string]interface{}{
+			"filters": c.AwsHostSetFilter1,
+		}),
+		hostsets.WithName("e2e Automated Test Host Set"),
+	)
+	require.NoError(t, err)
+	newHostSetId := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId)
+
+	// Get list of hosts in host set
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host set...")
+	var actualHostSetCount int
+	err = backoff.RetryNotify(
+		func() error {
+			hostSetReadResult, err := hsClient.Read(ctx, newHostSetId)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostSetCount = len(hostSetReadResult.Item.HostIds)
+			if actualHostSetCount == 0 {
+				return errors.New("No items are appearing in the host set")
+			}
+
+			t.Logf("Found %d hosts", actualHostSetCount)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Log("Successfully found items in the host set")
+	expectedHostSetCount, err := strconv.Atoi(c.AwsHostSetCount1)
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostSetCount, actualHostSetCount, "Numbers of hosts in host set did not match expected amount")
+
+	// Get list of all hosts from host catalog
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host catalog...")
+	var actualHostCatalogCount int
+	hClient := hosts.NewClient(client)
+	err = backoff.RetryNotify(
+		func() error {
+			hostListResult, err := hClient.List(ctx, newHostCatalogId)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostCatalogCount = len(hostListResult.Items)
+			if actualHostCatalogCount == 0 {
+				return errors.New("No items are appearing in the host catalog")
+			}
+
+			t.Logf("Found %d hosts", actualHostCatalogCount)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Log("Successfully found items in the host catalog")
+	require.NoError(t, err)
+	assert.Equal(t, actualHostCatalogCount, expectedHostSetCount, "Numbers of hosts in host catalog did not match expected amount")
+}

testing/internal/e2e/host/static/statichost_test.go

@@ -1,4 +1,4 @@
-package target_test
+package static_test
 
 import (
 	"context"
@@ -11,7 +11,6 @@ import (
 	"github.com/hashicorp/boundary/api/hostcatalogs"
 	"github.com/hashicorp/boundary/api/hosts"
 	"github.com/hashicorp/boundary/api/hostsets"
-	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/testing/internal/e2e"
 	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
@@ -50,6 +49,11 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
@@ -58,106 +62,101 @@ func loadConfig() (*config, error) {
 // the connection was successful.
 func TestConnectTargetCli(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	boundary.AuthenticateCli(t)
 
-	// Create an org
+	// Create an org and project
 	newOrgId := boundary.CreateNewOrgCli(t)
 	t.Logf("Created Org Id: %s", newOrgId)
-
-	// Create a project
 	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
 	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a host catalog
-	output := e2e.RunCommand("boundary", e2e.WithArgs("host-catalogs", "create", "static",
+	output := e2e.RunCommand("boundary", "host-catalogs", "create", "static",
 		"-scope-id", newProjectId,
 		"-name", "e2e Automated Test Host Catalog",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostCatalogResult hostcatalogs.HostCatalogCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostCatalogResult)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
+	newHostCatalogId := newHostCatalogResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("host-catalogs", "delete", "-id", newHostCatalog.Id))
+		output := e2e.RunCommand("boundary", "host-catalogs", "delete", "-id", newHostCatalogId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "host-sets", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", "e2e Automated Test Host Set",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostSetResult hostsets.HostSetCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostSetResult)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
+	newHostSetId := newHostSetResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "delete", "-id", newHostSet.Id))
+		output := e2e.RunCommand("boundary", "host-sets", "delete", "-id", newHostSetId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
-	output = e2e.RunCommand("boundary", e2e.WithArgs("hosts", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "hosts", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", c.TargetIp,
 		"-address", c.TargetIp,
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostResult hosts.HostCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostResult)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
+	newHostId := newHostResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("hosts", "delete", "-id", newHost.Id))
+		output := e2e.RunCommand("boundary", "hosts", "delete", "-id", newHostId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Host: %s", newHost.Id)
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "add-hosts", "-id", newHostSet.Id, "-host", newHost.Id))
+	output = e2e.RunCommand("boundary", "host-sets", "add-hosts", "-id", newHostSetId, "-host", newHostId)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Create a target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "create", "tcp",
+	output = e2e.RunCommand("boundary", "targets", "create", "tcp",
 		"-scope-id", newProjectId,
 		"-default-port", c.TargetPort,
 		"-name", "e2e Automated Test Target",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newTargetResult targets.TargetCreateResult
 	err = json.Unmarshal(output.Stdout, &newTargetResult)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
+	newTargetId := newTargetResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("targets", "delete", "-id", newTarget.Id))
+		output := e2e.RunCommand("boundary", "targets", "delete", "-id", newTargetId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Target: %s", newTarget.Id)
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "add-host-sources",
-		"-id", newTarget.Id,
-		"-host-source", newHostSet.Id),
+	output = e2e.RunCommand("boundary", "targets", "add-host-sources",
+		"-id", newTargetId,
+		"-host-source", newHostSetId,
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Connect to target and print host's IP address
-	output = e2e.RunCommand("boundary", e2e.WithArgs("connect",
-		"-target-id", newTarget.Id,
+	output = e2e.RunCommand("boundary", "connect",
+		"-target-id", newTargetId,
 		"-exec", "/usr/bin/ssh", "--",
 		"-l", c.TargetSshUser,
 		"-i", c.TargetSshKeyPath,
@@ -167,7 +166,7 @@ func TestConnectTargetCli(t *testing.T) {
 		"-p", "{{boundary.port}}", // this is provided by boundary
 		"{{boundary.ip}}",
 		"hostname", "-i",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	parts := strings.Fields(string(output.Stdout))
@@ -181,98 +180,80 @@ func TestConnectTargetCli(t *testing.T) {
 // when not using the cli.
 func TestCreateTargetApi(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	// Create boundary api client
 	client, err := boundary.NewApiClient()
 	require.NoError(t, err)
 	ctx := context.Background()
 
-	// Create an org
-	scopeClient := scopes.NewClient(client)
-	newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Automated Test Org"))
-	require.NoError(t, err)
-	newOrg := newOrgResult.Item
-	t.Cleanup(func() {
-		_, err := scopeClient.Delete(ctx, newOrg.Id)
-		require.NoError(t, err)
-	})
-	t.Logf("Created Org Id: %s", newOrg.Id)
-
-	// Create a project
-	newProjectResult, err := scopeClient.Create(ctx, newOrg.Id, scopes.WithName("e2e Automated Test Project"))
-	require.NoError(t, err)
-	newProject := newProjectResult.Item
-	t.Cleanup(func() {
-		_, err := scopeClient.Delete(ctx, newProject.Id)
-		require.NoError(t, err)
-	})
-	t.Logf("Created Project Id: %s", newProject.Id)
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgApi(t, ctx, client)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectApi(t, ctx, client, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a host catalog
 	hcClient := hostcatalogs.NewClient(client)
-	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProject.Id,
+	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProjectId,
 		hostcatalogs.WithName("e2e Automated Test Host Catalog"),
 	)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
+	newHostCatalogId := newHostCatalogResult.Item.Id
 	t.Cleanup(func() {
-		_, err := hcClient.Delete(ctx, newHostCatalog.Id)
+		_, err := hcClient.Delete(ctx, newHostCatalogId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
 	hsClient := hostsets.NewClient(client)
-	newHostSetResult, err := hsClient.Create(ctx, newHostCatalog.Id)
+	newHostSetResult, err := hsClient.Create(ctx, newHostCatalogId)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
+	newHostSetId := newHostSetResult.Item.Id
 	t.Cleanup(func() {
-		_, err := hsClient.Delete(ctx, newHostSet.Id)
+		_, err := hsClient.Delete(ctx, newHostSetId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
 	hClient := hosts.NewClient(client)
-	newHostResult, err := hClient.Create(ctx, newHostCatalog.Id,
+	newHostResult, err := hClient.Create(ctx, newHostCatalogId,
 		hosts.WithName(c.TargetIp),
 		hosts.WithStaticHostAddress(c.TargetIp),
 	)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
+	newHostId := newHostResult.Item.Id
 	t.Cleanup(func() {
-		_, err := hClient.Delete(ctx, newHost.Id)
+		_, err := hClient.Delete(ctx, newHostId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Host: %s", newHost.Id)
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	_, err = hsClient.AddHosts(ctx, newHostSet.Id, 0, []string{newHost.Id}, hostsets.WithAutomaticVersioning(true))
+	_, err = hsClient.AddHosts(ctx, newHostSetId, 0, []string{newHostId}, hostsets.WithAutomaticVersioning(true))
 	require.NoError(t, err)
 
 	// Create a target
 	tClient := targets.NewClient(client)
 	targetPort, err := strconv.ParseInt(c.TargetPort, 10, 32)
-	newTargetResult, err := tClient.Create(ctx, "tcp", newProject.Id,
+	newTargetResult, err := tClient.Create(ctx, "tcp", newProjectId,
 		targets.WithName("e2e Automated Test Target"),
 		targets.WithTcpTargetDefaultPort(uint32(targetPort)),
 	)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
+	newTargetId := newTargetResult.Item.Id
 	t.Cleanup(func() {
-		_, err := tClient.Delete(ctx, newTarget.Id)
+		_, err := tClient.Delete(ctx, newTargetId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Target: %s", newTarget.Id)
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	_, err = tClient.AddHostSources(ctx, newTarget.Id, 0,
-		[]string{newHostSet.Id},
+	_, err = tClient.AddHostSources(ctx, newTargetId, 0,
+		[]string{newHostSetId},
 		targets.WithAutomaticVersioning(true),
 	)
 	require.NoError(t, err)

testing/internal/e2e/vault/vault.go

@@ -37,6 +37,11 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
@@ -45,18 +50,16 @@ func loadConfig() (*config, error) {
 func Setup(t testing.TB) (string, string) {
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	_, filename, _, ok := runtime.Caller(0)
 	require.True(t, ok)
 	policyName := "boundary-controller"
-	output := e2e.RunCommand("vault", e2e.WithArgs("policy", "write", policyName,
-		path.Join(path.Dir(filename), "boundary-controller-policy.hcl")),
+	output := e2e.RunCommand("vault", "policy", "write", policyName,
+		path.Join(path.Dir(filename), "boundary-controller-policy.hcl"),
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("policy", "delete", policyName))
+		output := e2e.RunCommand("vault", "policy", "delete", policyName)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
@@ -80,20 +83,20 @@ func CreateKvPrivateKeyCredential(t testing.TB, secretName string, secretPath st
 
 	// Add policy to vault
 	policyName := "kv-read"
-	output := e2e.RunCommand("vault", e2e.WithArgs("policy", "write", policyName, kvPolicyFilePath))
+	output := e2e.RunCommand("vault", "policy", "write", policyName, kvPolicyFilePath)
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("policy", "delete", policyName))
+		output := e2e.RunCommand("vault", "policy", "delete", policyName)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
 	// Create secret
-	output = e2e.RunCommand("vault", e2e.WithArgs("kv", "put",
+	output = e2e.RunCommand("vault", "kv", "put",
 		"-mount", secretPath,
 		secretName,
 		"username="+user,
 		"private_key=@"+keyPath,
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	return policyName