test(e2e): Add dynamic host catalog test (AWS)

This adds the initial set of tests that validate setting up a dynamic host catalog on AWS

Co-authored-by: Josh Brand <jbrand@hashicorp.com>

testing/internal/e2e/README.md

@@ -1,24 +1,38 @@
 # boundary-e2e-tests
 
-This test suite tests Boundary in an end-to-end setting using [Enos](https://github.com/hashicorp/Enos-Docs) to spin up the desired infrastructure and [go test](https://pkg.go.dev/testing) to perform user workflows.
+This test suite tests Boundary in an end-to-end setting, utilizing both the Boundary CLI and the
+Boundary Go API to exercise Boundary through various user workflows. It was designed to be run in a
+variety of environments as long as the appropriate environment variables are set. The test suite
+itself uses the standard [go test](https://pkg.go.dev/testing) library.
+
+One method for setting up an environment is utilizing [Enos](https://github.com/hashicorp/Enos-Docs)
+to create the desired infrastructure.
 
 ## Getting Started
-### Usage
-#### Enos
-Setup Enos as described [here](../../enos/README.md)
+### Enos
+Setup Enos as described [here](../../../enos/README.md)
 
+Then, use the following commands to run tests
 ```shell
-enos scenario run e2e_{scenario} builder:local // runs and destroys infra
+cd enos
+enos scenario list
 
-enos scenario launch e2e_{scenario} builder:local  // runs and keeps infra online
-enos scenario output // displays any defined enos output
+# `Run` executes the tests and destroys the associated infrastructure in one command
+enos scenario run e2e_{scenario} builder:local
 
-enos scenario destroy // destroys infra
+# `Launch` executes the tests, but leaves the infrastructure online for debugging purposes
+enos scenario launch e2e_{scenario} builder:local
+enos scenario output  # displays any defined enos output
+enos scenario destroy  # destroys infra
 ```
 
-Enos scenarios set up the infrastructure, set the appropriate environment variables, and run the selected tests. Folders in this directory correspond to an enos scenario (e.g. `enos/enos-scenario-e2e-target.hcl` runs tests in `testing/e2e/target`)
+Enos scenarios set up the infrastructure, set the appropriate environment variables, and run the
+specified tests in its scenario file.
 
-#### Local
+Note: To run the `e2e_host_aws` scenario, you will need access to the boundary team's test AWS
+account.
+
+### Local
 Set the appropriate environment variables...
 ```shell
 export BOUNDARY_ADDR=  # e.g. http://127.0.0.1:9200
@@ -26,11 +40,22 @@ export E2E_PASSWORD_AUTH_METHOD_ID=  # e.g. ampw_1234567890
 export E2E_PASSWORD_ADMIN_LOGIN_NAME=  # e.g. "admin"
 export E2E_PASSWORD_ADMIN_PASSWORD=  # e.g. "password"
 
-# For e2e/target
+# For e2e/host/static
 export E2E_TARGET_IP=  # e.g. 192.168.0.1
 export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
 export E2E_SSH_USER=  # e.g. ubuntu
 
+# For e2e/host/aws
+export E2E_AWS_ACCESS_KEY_ID=
+export E2E_AWS_SECRET_ACCESS_KEY=
+export E2E_AWS_HOST_SET_FILTER1=  # e.g. "tag:testtag=true"
+export E2E_AWS_HOST_SET_COUNT1=  # e.g. 2
+export E2E_AWS_HOST_SET_IPS1=  # e.g. "[\"1.2.3.4\", \"2.3.4.5\"]"
+export E2E_AWS_HOST_SET_FILTER2=  # e.g. "tag:testtagtwo=test"
+export E2E_AWS_HOST_SET_COUNT2=  # e.g. 1
+export E2E_SSH_KEY_PATH=  # e.g. /Users/username/key.pem
+export E2E_SSH_USER=  # e.g. ubuntu
+
 # For e2e/credential/vault
 export VAULT_ADDR=  # e.g. http://127.0.0.1:8200
 export VAULT_TOKEN=
@@ -46,3 +71,12 @@ go test ./target/ // run target tests if running from this directory
 go test github.com/hashicorp/boundary/testing/e2e/target -v // verbose
 go test github.com/hashicorp/boundary/testing/e2e/target -v -run '^TestCreateTargetApi$' // run a specific test
 ```
+
+## Adding Tests
+
+Tests live under this directory. Additional tests can be added to an existing go package or a new
+one can be created. If a new package is created, a new enos scenario would also need to be created.
+
+Enos is comprised of scenarios, where a scenario is the environment you want the tests to operate
+in. In one scenario, there may be a boundary cluster and a target. Another scenario might involve a
+boundary cluster and a vault instance. Scenarios can be found in [boundary/enos](../../../enos/)

testing/internal/e2e/boundary/boundary.go

@@ -45,6 +45,11 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
@@ -55,10 +60,6 @@ func NewApiClient() (*api.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = c.validate()
-	if err != nil {
-		return nil, err
-	}
 
 	client, err := api.NewClient(&api.Config{Addr: c.Address})
 	if err != nil {
@@ -85,14 +86,12 @@ func NewApiClient() (*api.Client, error) {
 func AuthenticateCli(t testing.TB) {
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
-	output := e2e.RunCommand("boundary", e2e.WithArgs("authenticate", "password",
+	output := e2e.RunCommand("boundary", "authenticate", "password",
 		"-addr", c.Address,
 		"-auth-method-id", c.AuthMethodId,
 		"-login-name", c.AdminLoginName,
 		"-password", "env://E2E_PASSWORD_ADMIN_PASSWORD",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 }

testing/internal/e2e/boundary/scopes.go

@@ -25,24 +25,29 @@ func CreateNewOrgApi(t testing.TB, ctx context.Context, client *api.Client) stri
 	return newOrgResult.Item.Id
 }
 
+// CreateNewProjectApi creates a new project in boundary using the go api. The project will be created
+// under the provided org id.
+// Returns the id of the new project.
+func CreateNewProjectApi(t testing.TB, ctx context.Context, client *api.Client, orgId string) string {
+	scopeClient := scopes.NewClient(client)
+	newProjResult, err := scopeClient.Create(ctx, orgId, scopes.WithName("e2e Automated Test Project"))
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_, err := scopeClient.Delete(ctx, newProjResult.Item.Id)
+		require.NoError(t, err)
+	})
+
+	return newProjResult.Item.Id
+}
+
 // CreateNewOrgCli creates a new organization in boundary using the cli.
 // Returns the id of the new org.
 func CreateNewOrgCli(t testing.TB) string {
-	return createNewScopeCli(t, "e2e Automated Test Org", "global")
-}
-
-// CreateNewProjectCli creates a new project in boundary using the cli.
-// Returns the id of the new project.
-func CreateNewProjectCli(t testing.TB, scopeId string) string {
-	return createNewScopeCli(t, "e2e Automated Test Project", scopeId)
-}
-
-func createNewScopeCli(t testing.TB, name string, scopeId string) string {
-	output := e2e.RunCommand("boundary", e2e.WithArgs("scopes", "create",
-		"-name", name,
-		"-scope-id", scopeId,
+	output := e2e.RunCommand("boundary", "scopes", "create",
+		"-name", "e2e Automated Test Org",
+		"-scope-id", "global",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	var newOrgResult scopes.ScopeCreateResult
@@ -50,9 +55,32 @@ func createNewScopeCli(t testing.TB, name string, scopeId string) string {
 	require.NoError(t, err)
 
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("scopes", "delete", "-id", newOrgResult.Item.Id))
+		output := e2e.RunCommand("boundary", "scopes", "delete", "-id", newOrgResult.Item.Id)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
 	return newOrgResult.Item.Id
 }
+
+// CreateNewProjectCli creates a new project in boundary using the cli. The project will be created
+// under the provided org id.
+// Returns the id of the new project.
+func CreateNewProjectCli(t testing.TB, orgId string) string {
+	output := e2e.RunCommand("boundary", "scopes", "create",
+		"-name", "e2e Automated Test Project",
+		"-scope-id", orgId,
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+
+	var newProjResult scopes.ScopeCreateResult
+	err := json.Unmarshal(output.Stdout, &newProjResult)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		output := e2e.RunCommand("boundary", "scopes", "delete", "-id", newProjResult.Item.Id)
+		require.NoError(t, output.Err, string(output.Stderr))
+	})
+
+	return newProjResult.Item.Id
+}

testing/internal/e2e/credential/vault/vault_test.go

@@ -1,4 +1,4 @@
-package vaultcredential_test
+package vault_test
 
 import (
 	"context"
@@ -15,7 +15,6 @@ import (
 	"github.com/hashicorp/boundary/api/hostcatalogs"
 	"github.com/hashicorp/boundary/api/hosts"
 	"github.com/hashicorp/boundary/api/hostsets"
-	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/testing/internal/e2e"
 	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
@@ -57,28 +56,36 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
+type createTokenResponse struct {
+	Auth struct {
+		Client_Token string
+	}
+}
+
 // TestCreateVaultCredentialStoreCli uses the boundary and vault clis to add secrets management
 // for a target. The test sets up vault as a credential store, creates a set of credentials
 // in vault to be attached to a target, and attempts to connect to that target using those
 // credentials.
 func TestCreateVaultCredentialStoreCli(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	// Configure vault
 	vaultAddr, boundaryPolicyName := vault.Setup(t)
 
-	output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2"))
+	output := e2e.RunCommand("vault", "secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2")
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "disable", c.VaultSecretPath))
+		output := e2e.RunCommand("vault", "secrets", "disable", c.VaultSecretPath)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
@@ -88,7 +95,7 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 	t.Log("Created Vault Credential")
 
 	// Create vault token for boundary
-	output = e2e.RunCommand("vault", e2e.WithArgs("token", "create",
+	output = e2e.RunCommand("vault", "token", "create",
 		"-no-default-policy=true",
 		"-policy="+boundaryPolicyName,
 		"-policy="+credentialPolicyName,
@@ -96,130 +103,129 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 		"-period=20m",
 		"-renewable=true",
 		"-format=json",
-	),
-		e2e.WithPipe("jq", "-r", ".auth.client_token"),
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
-	credStoreToken := strings.TrimSpace(string(output.Stdout))
+	var tokenCreateResult createTokenResponse
+	err = json.Unmarshal(output.Stdout, &tokenCreateResult)
+	require.NoError(t, err)
+	credStoreToken := tokenCreateResult.Auth.Client_Token
 	t.Log("Created Vault Cred Store Token")
 
 	// Authenticate boundary cli
 	boundary.AuthenticateCli(t)
 
-	// Create an org
+	// Create an org and project
 	newOrgId := boundary.CreateNewOrgCli(t)
 	t.Logf("Created Org Id: %s", newOrgId)
-
-	// Create a project
 	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
 	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a credential store
-	output = e2e.RunCommand("boundary", e2e.WithArgs("credential-stores", "create", "vault",
+	output = e2e.RunCommand("boundary", "credential-stores", "create", "vault",
 		"-scope-id", newProjectId,
 		"-vault-address", vaultAddr,
 		"-vault-token", credStoreToken,
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newCredentialStoreResult credentialstores.CredentialStoreCreateResult
 	err = json.Unmarshal(output.Stdout, &newCredentialStoreResult)
 	require.NoError(t, err)
-	newCredentialStore := newCredentialStoreResult.Item
-	t.Logf("Created Credential Store: %s", newCredentialStore.Id)
+	newCredentialStoreId := newCredentialStoreResult.Item.Id
+	t.Logf("Created Credential Store: %s", newCredentialStoreId)
 
 	// Create a credential library
-	output = e2e.RunCommand("boundary", e2e.WithArgs("credential-libraries", "create", "vault",
-		"-credential-store-id", newCredentialStore.Id,
+	output = e2e.RunCommand("boundary", "credential-libraries", "create", "vault",
+		"-credential-store-id", newCredentialStoreId,
 		"-vault-path", c.VaultSecretPath+"/data/"+secretName,
 		"-name", "e2e Automated Test Vault Credential Library",
 		"-credential-type", "ssh_private_key",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newCredentialLibraryResult credentiallibraries.CredentialLibraryCreateResult
 	err = json.Unmarshal(output.Stdout, &newCredentialLibraryResult)
 	require.NoError(t, err)
-	newCredentialLibrary := newCredentialLibraryResult.Item
-	t.Logf("Created Credential Library: %s", newCredentialLibrary.Id)
+	newCredentialLibraryId := newCredentialLibraryResult.Item.Id
+	t.Logf("Created Credential Library: %s", newCredentialLibraryId)
 
 	// Create a host catalog
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-catalogs", "create", "static",
+	output = e2e.RunCommand("boundary", "host-catalogs", "create", "static",
 		"-scope-id", newProjectId,
 		"-name", "e2e Automated Test Host Catalog",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostCatalogResult hostcatalogs.HostCatalogCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostCatalogResult)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "host-sets", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", "e2e Automated Test Host Set",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostSetResult hostsets.HostSetCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostSetResult)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	newHostSetId := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
-	output = e2e.RunCommand("boundary", e2e.WithArgs("hosts", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "hosts", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", c.TargetIp,
 		"-address", c.TargetIp,
-		"-format", "json"),
+		"-format", "json",
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostResult hosts.HostCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostResult)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
-	t.Logf("Created Host: %s", newHost.Id)
+	newHostId := newHostResult.Item.Id
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "add-hosts",
-		"-id", newHostSet.Id,
-		"-host", newHost.Id,
-	))
+	output = e2e.RunCommand("boundary", "host-sets", "add-hosts",
+		"-id", newHostSetId,
+		"-host", newHostId,
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Create Target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "create", "tcp",
+	output = e2e.RunCommand("boundary", "targets", "create", "tcp",
 		"-scope-id", newProjectId,
 		"-default-port", c.TargetPort,
 		"-name", "e2e Automated Test Target",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newTargetResult targets.TargetCreateResult
 	err = json.Unmarshal(output.Stdout, &newTargetResult)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
-	t.Logf("Created Target: %s", newTarget.Id)
+	newTargetId := newTargetResult.Item.Id
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "add-host-sources",
-		"-id", newTarget.Id,
-		"-host-source", newHostSet.Id,
-	))
+	output = e2e.RunCommand("boundary", "targets", "add-host-sources",
+		"-id", newTargetId,
+		"-host-source", newHostSetId,
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Add brokered credentials to target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "add-credential-sources",
-		"-id", newTarget.Id,
-		"-brokered-credential-source", newCredentialLibrary.Id,
-	))
+	output = e2e.RunCommand("boundary", "targets", "add-credential-sources",
+		"-id", newTargetId,
+		"-brokered-credential-source", newCredentialLibraryId,
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Get credentials for target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "authorize-session", "-id", newTarget.Id, "-format", "json"))
+	output = e2e.RunCommand("boundary", "targets", "authorize-session", "-id", newTargetId, "-format", "json")
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newSessionAuthorizationResult targets.SessionAuthorizationResult
 	err = json.Unmarshal(output.Stdout, &newSessionAuthorizationResult)
@@ -245,8 +251,8 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect to target and print host's IP address using retrieved credentials
-	output = e2e.RunCommand("boundary", e2e.WithArgs("connect",
-		"-target-id", newTarget.Id,
+	output = e2e.RunCommand("boundary", "connect",
+		"-target-id", newTargetId,
 		"-exec", "/usr/bin/ssh", "--",
 		"-l", retrievedUser,
 		"-i", retrievedKeyPath,
@@ -256,7 +262,7 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 		"-p", "{{boundary.port}}", // this is provided by boundary
 		"{{boundary.ip}}",
 		"hostname", "-i",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	parts := strings.Fields(string(output.Stdout))
@@ -270,19 +276,16 @@ func TestCreateVaultCredentialStoreCli(t *testing.T) {
 // a set of credentials in vault that is attached to a target.
 func TestCreateVaultCredentialStoreApi(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	// Configure vault
 	vaultAddr, boundaryPolicyName := vault.Setup(t)
 
-	output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2"))
+	output := e2e.RunCommand("vault", "secrets", "enable", "-path="+c.VaultSecretPath, "kv-v2")
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("secrets", "disable", c.VaultSecretPath))
+		output := e2e.RunCommand("vault", "secrets", "disable", c.VaultSecretPath)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
@@ -292,7 +295,7 @@ func TestCreateVaultCredentialStoreApi(t *testing.T) {
 	t.Log("Created Vault Credential")
 
 	// Create vault token for boundary
-	output = e2e.RunCommand("vault", e2e.WithArgs("token", "create",
+	output = e2e.RunCommand("vault", "token", "create",
 		"-no-default-policy=true",
 		"-policy="+boundaryPolicyName,
 		"-policy="+credentialPolicyName,
@@ -300,11 +303,11 @@ func TestCreateVaultCredentialStoreApi(t *testing.T) {
 		"-period=20m",
 		"-renewable=true",
 		"-format=json",
-	),
-		e2e.WithPipe("jq", "-r", ".auth.client_token"),
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
-	credStoreToken := strings.TrimSpace(string(output.Stdout))
+	var tokenCreateResult createTokenResponse
+	err = json.Unmarshal(output.Stdout, &tokenCreateResult)
+	credStoreToken := tokenCreateResult.Auth.Client_Token
 	t.Log("Created Vault Cred Store Token")
 
 	// Create boundary api client
@@ -312,108 +315,100 @@ func TestCreateVaultCredentialStoreApi(t *testing.T) {
 	require.NoError(t, err)
 	ctx := context.Background()
 
-	// Create an org
-	scopeClient := scopes.NewClient(client)
-	newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Automated Test Org"))
-	require.NoError(t, err)
-	newOrg := newOrgResult.Item
-	t.Cleanup(func() {
-		_, err := scopeClient.Delete(ctx, newOrg.Id)
-		require.NoError(t, err)
-	})
-	t.Logf("Created Org Id: %s", newOrg.Id)
-
-	// Create a project
-	newProjectResult, err := scopeClient.Create(ctx, newOrg.Id, scopes.WithName("e2e Automated Test Project"))
-	require.NoError(t, err)
-	newProject := newProjectResult.Item
-	t.Logf("Created Project Id: %s", newProject.Id)
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgApi(t, ctx, client)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectApi(t, ctx, client, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a credential store
 	csClient := credentialstores.NewClient(client)
-	newCredentialStoreResult, err := csClient.Create(ctx, "vault", newProject.Id,
+	newCredentialStoreResult, err := csClient.Create(ctx, "vault", newProjectId,
 		credentialstores.WithVaultCredentialStoreAddress(vaultAddr),
 		credentialstores.WithVaultCredentialStoreToken(credStoreToken),
 	)
 	require.NoError(t, err)
-	newCredentialStore := newCredentialStoreResult.Item
-	t.Logf("Created Credential Store: %s", newCredentialStore.Id)
+	newCredentialStoreId := newCredentialStoreResult.Item.Id
+	t.Logf("Created Credential Store: %s", newCredentialStoreId)
 
 	// Create a credential library
 	clClient := credentiallibraries.NewClient(client)
-	newCredentialLibraryResult, err := clClient.Create(ctx, newCredentialStore.Id,
+	newCredentialLibraryResult, err := clClient.Create(ctx, newCredentialStoreId,
 		credentiallibraries.WithVaultCredentialLibraryPath(c.VaultSecretPath+"/data/"+secretName),
 		credentiallibraries.WithCredentialType("ssh_private_key"),
 	)
 	require.NoError(t, err)
-	newCredentialLibrary := newCredentialLibraryResult.Item
-	t.Logf("Created Credential Library: %s", newCredentialLibrary.Id)
+	newCredentialLibraryId := newCredentialLibraryResult.Item.Id
+	t.Logf("Created Credential Library: %s", newCredentialLibraryId)
 
 	// Create a host catalog
 	hcClient := hostcatalogs.NewClient(client)
-	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProject.Id,
+	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProjectId,
 		hostcatalogs.WithName("e2e Automated Test Host Catalog"),
 	)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
 	hsClient := hostsets.NewClient(client)
-	newHostSetResult, err := hsClient.Create(ctx, newHostCatalog.Id)
+	newHostSetResult, err := hsClient.Create(ctx, newHostCatalogId)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	newHostSetId := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
 	hClient := hosts.NewClient(client)
-	newHostResult, err := hClient.Create(ctx, newHostCatalog.Id,
+	newHostResult, err := hClient.Create(ctx, newHostCatalogId,
 		hosts.WithName(c.TargetIp),
 		hosts.WithStaticHostAddress(c.TargetIp),
 	)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
-	t.Logf("Created Host: %s", newHost.Id)
+	newHostId := newHostResult.Item.Id
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	_, err = hsClient.AddHosts(ctx, newHostSet.Id, 0, []string{newHost.Id}, hostsets.WithAutomaticVersioning(true))
+	_, err = hsClient.AddHosts(ctx, newHostSetId, 0, []string{newHostId}, hostsets.WithAutomaticVersioning(true))
 	require.NoError(t, err)
 
 	// Create a target
 	tClient := targets.NewClient(client)
 	targetPort, err := strconv.ParseInt(c.TargetPort, 10, 32)
-	newTargetResult, err := tClient.Create(ctx, "tcp", newProject.Id,
+	newTargetResult, err := tClient.Create(ctx, "tcp", newProjectId,
 		targets.WithName("e2e Automated Test Target"),
 		targets.WithTcpTargetDefaultPort(uint32(targetPort)),
 	)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
-	t.Logf("Created Target: %s", newTarget.Id)
+	newTargetId := newTargetResult.Item.Id
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	_, err = tClient.AddHostSources(ctx, newTarget.Id, 0,
-		[]string{newHostSet.Id},
+	_, err = tClient.AddHostSources(ctx, newTargetId, 0,
+		[]string{newHostSetId},
 		targets.WithAutomaticVersioning(true),
 	)
 	require.NoError(t, err)
 
 	// Add brokered credentials to target
-	_, err = tClient.AddCredentialSources(ctx, newTarget.Id, 0,
-		targets.WithBrokeredCredentialSourceIds([]string{newCredentialLibrary.Id}),
+	_, err = tClient.AddCredentialSources(ctx, newTargetId, 0,
+		targets.WithBrokeredCredentialSourceIds([]string{newCredentialLibraryId}),
 		targets.WithAutomaticVersioning(true),
 	)
 	require.NoError(t, err)
 
 	// Get credentials for target
-	newSessionAuthorizationResult, err := tClient.AuthorizeSession(ctx, newTarget.Id)
+	newSessionAuthorizationResult, err := tClient.AuthorizeSession(ctx, newTargetId)
 	require.NoError(t, err)
 	newSessionAuthorization := newSessionAuthorizationResult.Item
-	retrievedUser := fmt.Sprintf("%s", newSessionAuthorization.Credentials[0].Credential["username"])
-	retrievedKey := fmt.Sprintf("%s", newSessionAuthorization.Credentials[0].Credential["private_key"])
+	retrievedUser, ok := newSessionAuthorization.Credentials[0].Credential["username"].(string)
+	require.True(t, ok)
 	assert.Equal(t, c.TargetSshUser, retrievedUser)
 
+	retrievedKey, ok := newSessionAuthorization.Credentials[0].Credential["private_key"].(string)
+	require.True(t, ok)
 	k, err := os.ReadFile(c.TargetSshKeyPath)
 	require.NoError(t, err)
-	require.Equal(t, string(k), retrievedKey)
+	keysMatch := string(k) == retrievedKey // This is done to prevent printing out key info
+	require.True(t, keysMatch, "Key retrieved from vault does not match expected value")
 	t.Log("Successfully retrieved credentials for target")
 }

testing/internal/e2e/helpers.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"testing"
@@ -49,45 +48,17 @@ const EnvToCheckSkip = "E2E_PASSWORD_AUTH_METHOD_ID"
 // of running the provided command.
 //
 //	RunCommand("ls")
-//	RunCommand("ls", WithArgs("-al", "/path"))
-//	RunCommand("ls", WithArgs("-al", "/path"), WithPipe("grep", "file"))
+//	RunCommand("ls", "-al", "/path")
 //
 // CommandResult is always valid even if there is an error.
-func RunCommand(command string, opt ...Option) *CommandResult {
+func RunCommand(name string, args ...string) *CommandResult {
 	var outbuf, errbuf bytes.Buffer
-	var err error
-	var c1, c2 *exec.Cmd
 
-	opts := getOpts(opt...)
+	cmd := exec.Command(name, args...)
+	cmd.Stdout = &outbuf
+	cmd.Stderr = &errbuf
 
-	if opts.withArgs == nil {
-		c1 = exec.Command(command)
-	} else {
-		c1 = exec.Command(command, opts.withArgs...)
-	}
-
-	if opts.withPipe == nil {
-		c1.Stdout = &outbuf
-		c1.Stderr = &errbuf
-		err = c1.Run()
-	} else {
-		pipeCommand := opts.withPipe[0]
-		pipeArgs := opts.withPipe[1:]
-		c2 = exec.Command(pipeCommand, pipeArgs...)
-
-		r, w := io.Pipe()
-		c1.Stdout = w
-		c2.Stdin = r
-
-		c2.Stdout = &outbuf
-		c2.Stderr = &errbuf
-
-		c1.Start()
-		c2.Start()
-		c1.Wait()
-		w.Close()
-		c2.Wait()
-	}
+	err := cmd.Run()
 
 	var ee *exec.ExitError
 	var exitCode int

testing/internal/e2e/host/aws/dynamichostcatalog_test.go

@@ -0,0 +1,413 @@
+package aws_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/hashicorp/boundary/api/hostcatalogs"
+	"github.com/hashicorp/boundary/api/hosts"
+	"github.com/hashicorp/boundary/api/hostsets"
+	"github.com/hashicorp/boundary/api/targets"
+	"github.com/hashicorp/boundary/testing/internal/e2e"
+	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type config struct {
+	AwsAccessKeyId     string `envconfig:"E2E_AWS_ACCESS_KEY_ID"`
+	AwsSecretAccessKey string `envconfig:"E2E_AWS_SECRET_ACCESS_KEY"`
+	AwsHostSetFilter1  string `envconfig:"E2E_AWS_HOST_SET_FILTER1"`
+	AwsHostSetCount1   string `envconfig:"E2E_AWS_HOST_SET_COUNT1"`
+	AwsHostSetIps1     string `envconfig:"E2E_AWS_HOST_SET_IPS1"`
+	AwsHostSetFilter2  string `envconfig:"E2E_AWS_HOST_SET_FILTER2"`
+	AwsHostSetCount2   string `envconfig:"E2E_AWS_HOST_SET_COUNT2"`
+	TargetSshKeyPath   string `envconfig:"E2E_SSH_KEY_PATH"` // e.g. /Users/username/key.pem
+	TargetSshUser      string `envconfig:"E2E_SSH_USER"`     // e.g. ubuntu
+	TargetPort         string `envconfig:"E2E_SSH_PORT" default:"22"`
+}
+
+func (c *config) validate() error {
+	if c.AwsAccessKeyId == "" {
+		return errors.New("AwsAccessKeyId is empty. Set environment variable: E2E_AWS_ACCESS_KEY_ID")
+	}
+	if c.AwsSecretAccessKey == "" {
+		return errors.New("AwsSecretAccessKey is empty. Set environment variable: E2E_AWS_SECRET_ACCESS_KEY")
+	}
+	if c.AwsHostSetFilter1 == "" {
+		return errors.New("AwsHostSetFilter1 is empty. Set environment variable: E2E_AWS_HOST_SET_FILTER1")
+	}
+	if c.AwsHostSetCount1 == "" {
+		return errors.New("AwsHostSetCount1 is empty. Set environment variable: E2E_AWS_HOST_SET_COUNT1")
+	}
+	if c.AwsHostSetFilter2 == "" {
+		return errors.New("AwsHostSetFilter2 is empty. Set environment variable: E2E_AWS_HOST_SET_FILTER2")
+	}
+	if c.AwsHostSetCount2 == "" {
+		return errors.New("AwsHostSetCount2 is empty. Set environment variable: E2E_AWS_HOST_SET_COUNT2")
+	}
+	if c.TargetSshKeyPath == "" {
+		return errors.New("TargetSshKeyPath is empty. Set environment variable: E2E_SSH_KEY_PATH")
+	}
+	if c.TargetSshUser == "" {
+		return errors.New("TargetSshUser is empty. Set environment variable: E2E_SSH_USER")
+	}
+	if c.TargetPort == "" {
+		return errors.New("TargetPort is empty. Set environment variable: E2E_SSH_PORT")
+	}
+
+	return nil
+}
+
+func loadConfig() (*config, error) {
+	var c config
+	err := envconfig.Process("", &c)
+	if err != nil {
+		return nil, err
+	}
+
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, err
+}
+
+func TestCreateAwsDynamicHostCatalogCli(t *testing.T) {
+	e2e.MaybeSkipTest(t)
+	c, err := loadConfig()
+	require.NoError(t, err)
+
+	boundary.AuthenticateCli(t)
+
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgCli(t)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
+
+	// Create a dynamic host catalog
+	output := e2e.RunCommand("boundary", "host-catalogs", "create", "plugin",
+		"-scope-id", newProjectId,
+		"-plugin-name", "aws",
+		"-attr", "disable_credential_rotation=true",
+		"-attr", "region=us-east-1",
+		"-secret", "access_key_id=env://E2E_AWS_ACCESS_KEY_ID",
+		"-secret", "secret_access_key=env://E2E_AWS_SECRET_ACCESS_KEY",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newHostCatalogResult hostcatalogs.HostCatalogCreateResult
+	err = json.Unmarshal(output.Stdout, &newHostCatalogResult)
+	require.NoError(t, err)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
+
+	// Create a host set
+	output = e2e.RunCommand("boundary", "host-sets", "create", "plugin",
+		"-host-catalog-id", newHostCatalogId,
+		"-attr", "filters="+c.AwsHostSetFilter1,
+		"-name", "e2e Automated Test Host Set",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newHostSetResult hostsets.HostSetCreateResult
+	err = json.Unmarshal(output.Stdout, &newHostSetResult)
+	require.NoError(t, err)
+	newHostSetId1 := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId1)
+
+	// Get list of hosts in host set
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host set...")
+	var actualHostSetCount1 int
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand("boundary", "host-sets", "read",
+				"-id", newHostSetId1,
+				"-format", "json",
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var hostSetsReadResult hostsets.HostSetReadResult
+			err = json.Unmarshal(output.Stdout, &hostSetsReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostSetCount1 = len(hostSetsReadResult.Item.HostIds)
+			if actualHostSetCount1 == 0 {
+				return errors.New("No items are appearing in the host set")
+			}
+
+			t.Logf("Found %d hosts", actualHostSetCount1)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	expectedHostSetCount1, err := strconv.Atoi(c.AwsHostSetCount1)
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostSetCount1, actualHostSetCount1, "Numbers of hosts in host set did not match expected amount")
+
+	// Create another host set
+	output = e2e.RunCommand("boundary", "host-sets", "create", "plugin",
+		"-host-catalog-id", newHostCatalogId,
+		"-attr", "filters="+c.AwsHostSetFilter2,
+		"-name", "e2e Automated Test Host Set2",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newHostSetResult2 hostsets.HostSetCreateResult
+	err = json.Unmarshal(output.Stdout, &newHostSetResult2)
+	require.NoError(t, err)
+	newHostSetId2 := newHostSetResult2.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId2)
+
+	// Get list of hosts in the second host set
+	t.Logf("Looking for items in the second host set...")
+	var actualHostSetCount2 int
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand("boundary", "host-sets", "read",
+				"-id", newHostSetId2,
+				"-format", "json",
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var hostSetsReadResult hostsets.HostSetReadResult
+			err = json.Unmarshal(output.Stdout, &hostSetsReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostSetCount2 = len(hostSetsReadResult.Item.HostIds)
+			if actualHostSetCount2 == 0 {
+				return errors.New("No items are appearing in the host set")
+			}
+
+			t.Logf("Found %d hosts", actualHostSetCount2)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	expectedHostSetCount2, err := strconv.Atoi(c.AwsHostSetCount2)
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostSetCount2, actualHostSetCount2, "Numbers of hosts in host set did not match expected amount")
+
+	// Get list of all hosts from host catalog
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host catalog...")
+	var actualHostCatalogCount int
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand("boundary", "hosts", "list",
+				"-host-catalog-id", newHostCatalogId,
+				"-format", "json",
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var hostCatalogListResult hostcatalogs.HostCatalogListResult
+			err = json.Unmarshal(output.Stdout, &hostCatalogListResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostCatalogCount = len(hostCatalogListResult.Items)
+			if actualHostCatalogCount == 0 {
+				return errors.New("No items are appearing in the host catalog")
+			}
+
+			t.Logf("Found %d hosts", actualHostCatalogCount)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	expectedHostCatalogCount := expectedHostSetCount1 + expectedHostSetCount2
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostCatalogCount, actualHostCatalogCount, "Numbers of hosts in host catalog did not match expected amount")
+
+	// Create target
+	output = e2e.RunCommand("boundary", "targets", "create", "tcp",
+		"-scope-id", newProjectId,
+		"-default-port", c.TargetPort,
+		"-name", "e2e Automated Test Target",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var newTargetResult targets.TargetCreateResult
+	err = json.Unmarshal(output.Stdout, &newTargetResult)
+	require.NoError(t, err)
+	newTargetId := newTargetResult.Item.Id
+	t.Logf("Created Target: %s", newTargetId)
+
+	// Add host set to target
+	output = e2e.RunCommand("boundary", "targets", "add-host-sources",
+		"-id", newTargetId,
+		"-host-source", newHostSetId1,
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+
+	// Connect to target
+	output = e2e.RunCommand("boundary", "connect",
+		"-target-id", newTargetId,
+		"-exec", "/usr/bin/ssh", "--",
+		"-l", c.TargetSshUser,
+		"-i", c.TargetSshKeyPath,
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "IdentitiesOnly=yes", // forces the use of the provided key
+		"-p", "{{boundary.port}}", // this is provided by boundary
+		"{{boundary.ip}}",
+		"hostname", "-i",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+
+	parts := strings.Fields(string(output.Stdout))
+	hostIp := parts[len(parts)-1]
+	t.Log("Successfully connected to the target")
+
+	// Check if connected host exists in the host set
+	var targetIps []string
+	err = json.Unmarshal([]byte(c.AwsHostSetIps1), &targetIps)
+	hostIpInList := false
+	for _, v := range targetIps {
+		if v == hostIp {
+			hostIpInList = true
+		}
+	}
+	require.True(t, hostIpInList, fmt.Sprintf("Connected host (%s) is not in expected list (%s)", hostIp, targetIps))
+}
+
+func TestCreateAwsDynamicHostCatalogApi(t *testing.T) {
+	e2e.MaybeSkipTest(t)
+	c, err := loadConfig()
+	require.NoError(t, err)
+
+	// Create boundary api client
+	client, err := boundary.NewApiClient()
+	require.NoError(t, err)
+	ctx := context.Background()
+
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgApi(t, ctx, client)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectApi(t, ctx, client, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
+
+	// Create a dynamic host catalog
+	hcClient := hostcatalogs.NewClient(client)
+	newHostCatalogResult, err := hcClient.Create(ctx, "plugin", newProjectId,
+		hostcatalogs.WithName("e2e Automated Test Host Catalog"),
+		hostcatalogs.WithPluginName("aws"),
+		hostcatalogs.WithAttributes(map[string]interface{}{
+			"disable_credential_rotation": true,
+			"region":                      "us-east-1",
+		}),
+		hostcatalogs.WithSecrets(map[string]interface{}{
+			"access_key_id":     c.AwsAccessKeyId,
+			"secret_access_key": c.AwsSecretAccessKey,
+		}),
+	)
+	require.NoError(t, err)
+	newHostCatalogId := newHostCatalogResult.Item.Id
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
+
+	// Create a host set and add to catalog
+	hsClient := hostsets.NewClient(client)
+	newHostSetResult, err := hsClient.Create(ctx, newHostCatalogId,
+		hostsets.WithAttributes(map[string]interface{}{
+			"filters": c.AwsHostSetFilter1,
+		}),
+		hostsets.WithName("e2e Automated Test Host Set"),
+	)
+	require.NoError(t, err)
+	newHostSetId := newHostSetResult.Item.Id
+	t.Logf("Created Host Set: %s", newHostSetId)
+
+	// Get list of hosts in host set
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host set...")
+	var actualHostSetCount int
+	err = backoff.RetryNotify(
+		func() error {
+			hostSetReadResult, err := hsClient.Read(ctx, newHostSetId)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostSetCount = len(hostSetReadResult.Item.HostIds)
+			if actualHostSetCount == 0 {
+				return errors.New("No items are appearing in the host set")
+			}
+
+			t.Logf("Found %d hosts", actualHostSetCount)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Log("Successfully found items in the host set")
+	expectedHostSetCount, err := strconv.Atoi(c.AwsHostSetCount1)
+	require.NoError(t, err)
+	assert.Equal(t, expectedHostSetCount, actualHostSetCount, "Numbers of hosts in host set did not match expected amount")
+
+	// Get list of all hosts from host catalog
+	// Retry is needed here since it can take a few tries before hosts start appearing
+	t.Logf("Looking for items in the host catalog...")
+	var actualHostCatalogCount int
+	hClient := hosts.NewClient(client)
+	err = backoff.RetryNotify(
+		func() error {
+			hostListResult, err := hClient.List(ctx, newHostCatalogId)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			actualHostCatalogCount = len(hostListResult.Items)
+			if actualHostCatalogCount == 0 {
+				return errors.New("No items are appearing in the host catalog")
+			}
+
+			t.Logf("Found %d hosts", actualHostCatalogCount)
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Log("Successfully found items in the host catalog")
+	require.NoError(t, err)
+	assert.Equal(t, actualHostCatalogCount, expectedHostSetCount, "Numbers of hosts in host catalog did not match expected amount")
+}

testing/internal/e2e/host/static/statichost_test.go

@@ -1,4 +1,4 @@
-package target_test
+package static_test
 
 import (
 	"context"
@@ -11,7 +11,6 @@ import (
 	"github.com/hashicorp/boundary/api/hostcatalogs"
 	"github.com/hashicorp/boundary/api/hosts"
 	"github.com/hashicorp/boundary/api/hostsets"
-	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/testing/internal/e2e"
 	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
@@ -50,6 +49,11 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
@@ -58,106 +62,101 @@ func loadConfig() (*config, error) {
 // the connection was successful.
 func TestConnectTargetCli(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	boundary.AuthenticateCli(t)
 
-	// Create an org
+	// Create an org and project
 	newOrgId := boundary.CreateNewOrgCli(t)
 	t.Logf("Created Org Id: %s", newOrgId)
-
-	// Create a project
 	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
 	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a host catalog
-	output := e2e.RunCommand("boundary", e2e.WithArgs("host-catalogs", "create", "static",
+	output := e2e.RunCommand("boundary", "host-catalogs", "create", "static",
 		"-scope-id", newProjectId,
 		"-name", "e2e Automated Test Host Catalog",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostCatalogResult hostcatalogs.HostCatalogCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostCatalogResult)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
+	newHostCatalogId := newHostCatalogResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("host-catalogs", "delete", "-id", newHostCatalog.Id))
+		output := e2e.RunCommand("boundary", "host-catalogs", "delete", "-id", newHostCatalogId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "host-sets", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", "e2e Automated Test Host Set",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostSetResult hostsets.HostSetCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostSetResult)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
+	newHostSetId := newHostSetResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "delete", "-id", newHostSet.Id))
+		output := e2e.RunCommand("boundary", "host-sets", "delete", "-id", newHostSetId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
-	output = e2e.RunCommand("boundary", e2e.WithArgs("hosts", "create", "static",
-		"-host-catalog-id", newHostCatalog.Id,
+	output = e2e.RunCommand("boundary", "hosts", "create", "static",
+		"-host-catalog-id", newHostCatalogId,
 		"-name", c.TargetIp,
 		"-address", c.TargetIp,
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newHostResult hosts.HostCreateResult
 	err = json.Unmarshal(output.Stdout, &newHostResult)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
+	newHostId := newHostResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("hosts", "delete", "-id", newHost.Id))
+		output := e2e.RunCommand("boundary", "hosts", "delete", "-id", newHostId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Host: %s", newHost.Id)
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	output = e2e.RunCommand("boundary", e2e.WithArgs("host-sets", "add-hosts", "-id", newHostSet.Id, "-host", newHost.Id))
+	output = e2e.RunCommand("boundary", "host-sets", "add-hosts", "-id", newHostSetId, "-host", newHostId)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Create a target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "create", "tcp",
+	output = e2e.RunCommand("boundary", "targets", "create", "tcp",
 		"-scope-id", newProjectId,
 		"-default-port", c.TargetPort,
 		"-name", "e2e Automated Test Target",
 		"-format", "json",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	var newTargetResult targets.TargetCreateResult
 	err = json.Unmarshal(output.Stdout, &newTargetResult)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
+	newTargetId := newTargetResult.Item.Id
 	t.Cleanup(func() {
-		output := e2e.RunCommand("boundary", e2e.WithArgs("targets", "delete", "-id", newTarget.Id))
+		output := e2e.RunCommand("boundary", "targets", "delete", "-id", newTargetId)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
-	t.Logf("Created Target: %s", newTarget.Id)
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	output = e2e.RunCommand("boundary", e2e.WithArgs("targets", "add-host-sources",
-		"-id", newTarget.Id,
-		"-host-source", newHostSet.Id),
+	output = e2e.RunCommand("boundary", "targets", "add-host-sources",
+		"-id", newTargetId,
+		"-host-source", newHostSetId,
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	// Connect to target and print host's IP address
-	output = e2e.RunCommand("boundary", e2e.WithArgs("connect",
-		"-target-id", newTarget.Id,
+	output = e2e.RunCommand("boundary", "connect",
+		"-target-id", newTargetId,
 		"-exec", "/usr/bin/ssh", "--",
 		"-l", c.TargetSshUser,
 		"-i", c.TargetSshKeyPath,
@@ -167,7 +166,7 @@ func TestConnectTargetCli(t *testing.T) {
 		"-p", "{{boundary.port}}", // this is provided by boundary
 		"{{boundary.ip}}",
 		"hostname", "-i",
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	parts := strings.Fields(string(output.Stdout))
@@ -181,98 +180,80 @@ func TestConnectTargetCli(t *testing.T) {
 // when not using the cli.
 func TestCreateTargetApi(t *testing.T) {
 	e2e.MaybeSkipTest(t)
-
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	// Create boundary api client
 	client, err := boundary.NewApiClient()
 	require.NoError(t, err)
 	ctx := context.Background()
 
-	// Create an org
-	scopeClient := scopes.NewClient(client)
-	newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Automated Test Org"))
-	require.NoError(t, err)
-	newOrg := newOrgResult.Item
-	t.Cleanup(func() {
-		_, err := scopeClient.Delete(ctx, newOrg.Id)
-		require.NoError(t, err)
-	})
-	t.Logf("Created Org Id: %s", newOrg.Id)
-
-	// Create a project
-	newProjectResult, err := scopeClient.Create(ctx, newOrg.Id, scopes.WithName("e2e Automated Test Project"))
-	require.NoError(t, err)
-	newProject := newProjectResult.Item
-	t.Cleanup(func() {
-		_, err := scopeClient.Delete(ctx, newProject.Id)
-		require.NoError(t, err)
-	})
-	t.Logf("Created Project Id: %s", newProject.Id)
+	// Create an org and project
+	newOrgId := boundary.CreateNewOrgApi(t, ctx, client)
+	t.Logf("Created Org Id: %s", newOrgId)
+	newProjectId := boundary.CreateNewProjectApi(t, ctx, client, newOrgId)
+	t.Logf("Created Project Id: %s", newProjectId)
 
 	// Create a host catalog
 	hcClient := hostcatalogs.NewClient(client)
-	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProject.Id,
+	newHostCatalogResult, err := hcClient.Create(ctx, "static", newProjectId,
 		hostcatalogs.WithName("e2e Automated Test Host Catalog"),
 	)
 	require.NoError(t, err)
-	newHostCatalog := newHostCatalogResult.Item
+	newHostCatalogId := newHostCatalogResult.Item.Id
 	t.Cleanup(func() {
-		_, err := hcClient.Delete(ctx, newHostCatalog.Id)
+		_, err := hcClient.Delete(ctx, newHostCatalogId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Host Catalog: %s", newHostCatalog.Id)
+	t.Logf("Created Host Catalog: %s", newHostCatalogId)
 
 	// Create a host set and add to catalog
 	hsClient := hostsets.NewClient(client)
-	newHostSetResult, err := hsClient.Create(ctx, newHostCatalog.Id)
+	newHostSetResult, err := hsClient.Create(ctx, newHostCatalogId)
 	require.NoError(t, err)
-	newHostSet := newHostSetResult.Item
+	newHostSetId := newHostSetResult.Item.Id
 	t.Cleanup(func() {
-		_, err := hsClient.Delete(ctx, newHostSet.Id)
+		_, err := hsClient.Delete(ctx, newHostSetId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Host Set: %s", newHostSet.Id)
+	t.Logf("Created Host Set: %s", newHostSetId)
 
 	// Create a host
 	hClient := hosts.NewClient(client)
-	newHostResult, err := hClient.Create(ctx, newHostCatalog.Id,
+	newHostResult, err := hClient.Create(ctx, newHostCatalogId,
 		hosts.WithName(c.TargetIp),
 		hosts.WithStaticHostAddress(c.TargetIp),
 	)
 	require.NoError(t, err)
-	newHost := newHostResult.Item
+	newHostId := newHostResult.Item.Id
 	t.Cleanup(func() {
-		_, err := hClient.Delete(ctx, newHost.Id)
+		_, err := hClient.Delete(ctx, newHostId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Host: %s", newHost.Id)
+	t.Logf("Created Host: %s", newHostId)
 
 	// Add host to host set
-	_, err = hsClient.AddHosts(ctx, newHostSet.Id, 0, []string{newHost.Id}, hostsets.WithAutomaticVersioning(true))
+	_, err = hsClient.AddHosts(ctx, newHostSetId, 0, []string{newHostId}, hostsets.WithAutomaticVersioning(true))
 	require.NoError(t, err)
 
 	// Create a target
 	tClient := targets.NewClient(client)
 	targetPort, err := strconv.ParseInt(c.TargetPort, 10, 32)
-	newTargetResult, err := tClient.Create(ctx, "tcp", newProject.Id,
+	newTargetResult, err := tClient.Create(ctx, "tcp", newProjectId,
 		targets.WithName("e2e Automated Test Target"),
 		targets.WithTcpTargetDefaultPort(uint32(targetPort)),
 	)
 	require.NoError(t, err)
-	newTarget := newTargetResult.Item
+	newTargetId := newTargetResult.Item.Id
 	t.Cleanup(func() {
-		_, err := tClient.Delete(ctx, newTarget.Id)
+		_, err := tClient.Delete(ctx, newTargetId)
 		require.NoError(t, err)
 	})
-	t.Logf("Created Target: %s", newTarget.Id)
+	t.Logf("Created Target: %s", newTargetId)
 
 	// Add host set to target
-	_, err = tClient.AddHostSources(ctx, newTarget.Id, 0,
-		[]string{newHostSet.Id},
+	_, err = tClient.AddHostSources(ctx, newTargetId, 0,
+		[]string{newHostSetId},
 		targets.WithAutomaticVersioning(true),
 	)
 	require.NoError(t, err)

testing/internal/e2e/vault/vault.go

@@ -37,6 +37,11 @@ func loadConfig() (*config, error) {
 		return nil, err
 	}
 
+	err = c.validate()
+	if err != nil {
+		return nil, err
+	}
+
 	return &c, err
 }
 
@@ -45,18 +50,16 @@ func loadConfig() (*config, error) {
 func Setup(t testing.TB) (string, string) {
 	c, err := loadConfig()
 	require.NoError(t, err)
-	err = c.validate()
-	require.NoError(t, err)
 
 	_, filename, _, ok := runtime.Caller(0)
 	require.True(t, ok)
 	policyName := "boundary-controller"
-	output := e2e.RunCommand("vault", e2e.WithArgs("policy", "write", policyName,
-		path.Join(path.Dir(filename), "boundary-controller-policy.hcl")),
+	output := e2e.RunCommand("vault", "policy", "write", policyName,
+		path.Join(path.Dir(filename), "boundary-controller-policy.hcl"),
 	)
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("policy", "delete", policyName))
+		output := e2e.RunCommand("vault", "policy", "delete", policyName)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
@@ -80,20 +83,20 @@ func CreateKvPrivateKeyCredential(t testing.TB, secretName string, secretPath st
 
 	// Add policy to vault
 	policyName := "kv-read"
-	output := e2e.RunCommand("vault", e2e.WithArgs("policy", "write", policyName, kvPolicyFilePath))
+	output := e2e.RunCommand("vault", "policy", "write", policyName, kvPolicyFilePath)
 	require.NoError(t, output.Err, string(output.Stderr))
 	t.Cleanup(func() {
-		output := e2e.RunCommand("vault", e2e.WithArgs("policy", "delete", policyName))
+		output := e2e.RunCommand("vault", "policy", "delete", policyName)
 		require.NoError(t, output.Err, string(output.Stderr))
 	})
 
 	// Create secret
-	output = e2e.RunCommand("vault", e2e.WithArgs("kv", "put",
+	output = e2e.RunCommand("vault", "kv", "put",
 		"-mount", secretPath,
 		secretName,
 		"username="+user,
 		"private_key=@"+keyPath,
-	))
+	)
 	require.NoError(t, output.Err, string(output.Stderr))
 
 	return policyName