From 56ed25a88c2919e2322684deb38014ff9baa97c3 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Wed, 19 Oct 2022 14:36:10 -0400 Subject: [PATCH 1/4] test(e2e): Rename file for clarity --- .../host/static/{session_test.go => session_cancel_test.go} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename testing/internal/e2e/host/static/{session_test.go => session_cancel_test.go} (96%) diff --git a/testing/internal/e2e/host/static/session_test.go b/testing/internal/e2e/host/static/session_cancel_test.go similarity index 96% rename from testing/internal/e2e/host/static/session_test.go rename to testing/internal/e2e/host/static/session_cancel_test.go index 51a13ae91..249d45718 100644 --- a/testing/internal/e2e/host/static/session_test.go +++ b/testing/internal/e2e/host/static/session_cancel_test.go @@ -15,7 +15,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestSessionCancelingCli(t *testing.T) { +// TestSessionCancelCli uses the boundary cli to start and then cancel a session +func TestSessionCancelCli(t *testing.T) { e2e.MaybeSkipTest(t) c, err := loadConfig() require.NoError(t, err) From ab223493a394c2ab17c03190c88a612bff3d351c Mon Sep 17 00:00:00 2001 From: Michael Li Date: Thu, 20 Oct 2022 10:36:08 -0400 Subject: [PATCH 2/4] fix(e2e): Remove unused code --- testing/internal/e2e/helpers.go | 43 +-------------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/testing/internal/e2e/helpers.go b/testing/internal/e2e/helpers.go index f713a29dc..acc15ced7 100644 --- a/testing/internal/e2e/helpers.go +++ b/testing/internal/e2e/helpers.go @@ -10,32 +10,7 @@ import ( "testing" ) -// Option is a func that sets optional attributes for a call. This does not need -// to be used directly, but instead option arguments are built from the -// functions in this package. WithX options set a value to that given in the -// argument; DefaultX options indicate that the value should be set to its -// default. When an API call is made options are processed in ther order they -// appear in the function call, so for a given argument X, a succession of WithX -// or DefaultX calls will result in the last call taking effect. -type Option func(*options) - -type options struct { - withArgs []string - withPipe []string -} - -func getOpts(opt ...Option) options { - opts := options{} - for _, o := range opt { - if o != nil { - o(&opts) - } - } - - return opts -} - -// CommandResult encapsulates the output from running an external command +// CommandResult captures the output from running an external command type CommandResult struct { Stdout []byte Stderr []byte @@ -75,22 +50,6 @@ func RunCommand(ctx context.Context, name string, args ...string) *CommandResult } } -// WithArgs is an option to RunCommand that allows the user to specify arguments -// for the provided command -func WithArgs(args ...string) Option { - return func(o *options) { - o.withArgs = args - } -} - -// WithPipe is an option to RunCommand that allows the user to specify a command+arguments -// to pipe to -func WithPipe(command ...string) Option { - return func(o *options) { - o.withPipe = command - } -} - // MaybeSkipTest is a check used at the start of the test to determine if the test should run func MaybeSkipTest(t testing.TB) { if _, ok := os.LookupEnv(EnvToCheckSkip); !ok { From 79bc3d48784b4d133d159f07372eafdea4d03d73 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Thu, 20 Oct 2022 10:40:29 -0400 Subject: [PATCH 3/4] refact(e2e): Create credential store function --- testing/internal/e2e/boundary/credential.go | 41 +++++++++++++++++++ testing/internal/e2e/boundary/host.go | 10 ++--- testing/internal/e2e/boundary/scope.go | 8 ++-- testing/internal/e2e/boundary/target.go | 4 +- .../host/static/connect_authz_token_test.go | 18 ++------ .../e2e/host/static/connect_ssh_test.go | 26 ++---------- 6 files changed, 58 insertions(+), 49 deletions(-) create mode 100644 testing/internal/e2e/boundary/credential.go diff --git a/testing/internal/e2e/boundary/credential.go b/testing/internal/e2e/boundary/credential.go new file mode 100644 index 000000000..c9b9b64be --- /dev/null +++ b/testing/internal/e2e/boundary/credential.go @@ -0,0 +1,41 @@ +package boundary + +import ( + "context" + "encoding/json" + "testing" + + "github.com/hashicorp/boundary/api" + "github.com/hashicorp/boundary/api/credentialstores" + "github.com/hashicorp/boundary/testing/internal/e2e" + "github.com/stretchr/testify/require" +) + +// CreateNewCredentialStoreStaticApi creates a new static credential store using the go api. +// Returns the id of the new credential store +func CreateNewCredentialStoreStaticApi(t testing.TB, ctx context.Context, client *api.Client, projectId string) string { + csClient := credentialstores.NewClient(client) + newCredentialStoreResult, err := csClient.Create(ctx, "static", projectId, credentialstores.WithName("e2e Credential Store")) + require.NoError(t, err) + newCredentialStoreId := newCredentialStoreResult.Item.Id + t.Logf("Created Credential Store: %s", newCredentialStoreId) + + return newCredentialStoreId +} + +// CreateNewCredentialStoreStaticCli creates a new static credential store using the cli. +// Returns the id of the new credential store +func CreateNewCredentialStoreStaticCli(t testing.TB, projectId string) string { + output := e2e.RunCommand(context.Background(), "boundary", "credential-stores", "create", "static", + "-scope-id", projectId, + "-format", "json", + ) + require.NoError(t, output.Err, string(output.Stderr)) + var newCredentialStoreResult credentialstores.CredentialStoreCreateResult + err := json.Unmarshal(output.Stdout, &newCredentialStoreResult) + require.NoError(t, err) + newCredentialStoreId := newCredentialStoreResult.Item.Id + t.Logf("Created Credential Store: %s", newCredentialStoreId) + + return newCredentialStoreId +} diff --git a/testing/internal/e2e/boundary/host.go b/testing/internal/e2e/boundary/host.go index 5788d2a12..786561bff 100644 --- a/testing/internal/e2e/boundary/host.go +++ b/testing/internal/e2e/boundary/host.go @@ -17,9 +17,7 @@ import ( // Returns the id of the new host catalog. func CreateNewHostCatalogApi(t testing.TB, ctx context.Context, client *api.Client, projectId string) string { hcClient := hostcatalogs.NewClient(client) - newHostCatalogResult, err := hcClient.Create(ctx, "static", projectId, - hostcatalogs.WithName("e2e Automated Test Host Catalog"), - ) + newHostCatalogResult, err := hcClient.Create(ctx, "static", projectId, hostcatalogs.WithName("e2e Host Catalog")) require.NoError(t, err) newHostCatalogId := newHostCatalogResult.Item.Id t.Logf("Created Host Catalog: %s", newHostCatalogId) @@ -31,7 +29,7 @@ func CreateNewHostCatalogApi(t testing.TB, ctx context.Context, client *api.Clie // Returns the id of the new host set. func CreateNewHostSetApi(t testing.TB, ctx context.Context, client *api.Client, hostCatalogId string) string { hsClient := hostsets.NewClient(client) - newHostSetResult, err := hsClient.Create(ctx, hostCatalogId) + newHostSetResult, err := hsClient.Create(ctx, hostCatalogId, hostsets.WithName("e2e Host")) require.NoError(t, err) newHostSetId := newHostSetResult.Item.Id t.Logf("Created Host Set: %s", newHostSetId) @@ -66,7 +64,7 @@ func AddHostToHostSetApi(t testing.TB, ctx context.Context, client *api.Client, func CreateNewHostCatalogCli(t testing.TB, projectId string) string { output := e2e.RunCommand(context.Background(), "boundary", "host-catalogs", "create", "static", "-scope-id", projectId, - "-name", "e2e Automated Test Host Catalog", + "-name", "e2e Host Catalog", "-format", "json", ) require.NoError(t, output.Err, string(output.Stderr)) @@ -84,7 +82,7 @@ func CreateNewHostCatalogCli(t testing.TB, projectId string) string { func CreateNewHostSetCli(t testing.TB, hostCatalogId string) string { output := e2e.RunCommand(context.Background(), "boundary", "host-sets", "create", "static", "-host-catalog-id", hostCatalogId, - "-name", "e2e Automated Test Host Set", + "-name", "e2e Host Set", "-format", "json", ) require.NoError(t, output.Err, string(output.Stderr)) diff --git a/testing/internal/e2e/boundary/scope.go b/testing/internal/e2e/boundary/scope.go index 6d39f246f..ec2188409 100644 --- a/testing/internal/e2e/boundary/scope.go +++ b/testing/internal/e2e/boundary/scope.go @@ -15,7 +15,7 @@ import ( // Returns the id of the new org. func CreateNewOrgApi(t testing.TB, ctx context.Context, client *api.Client) string { scopeClient := scopes.NewClient(client) - newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Automated Test Org")) + newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Org")) require.NoError(t, err) newOrgId := newOrgResult.Item.Id @@ -33,7 +33,7 @@ func CreateNewOrgApi(t testing.TB, ctx context.Context, client *api.Client) stri // Returns the id of the new project. func CreateNewProjectApi(t testing.TB, ctx context.Context, client *api.Client, orgId string) string { scopeClient := scopes.NewClient(client) - newProjResult, err := scopeClient.Create(ctx, orgId, scopes.WithName("e2e Automated Test Project")) + newProjResult, err := scopeClient.Create(ctx, orgId, scopes.WithName("e2e Project")) require.NoError(t, err) newProjectId := newProjResult.Item.Id @@ -46,7 +46,7 @@ func CreateNewProjectApi(t testing.TB, ctx context.Context, client *api.Client, func CreateNewOrgCli(t testing.TB) string { ctx := context.Background() output := e2e.RunCommand(ctx, "boundary", "scopes", "create", - "-name", "e2e Automated Test Org", + "-name", "e2e Org", "-scope-id", "global", "-format", "json", ) @@ -72,7 +72,7 @@ func CreateNewOrgCli(t testing.TB) string { func CreateNewProjectCli(t testing.TB, orgId string) string { ctx := context.Background() output := e2e.RunCommand(ctx, "boundary", "scopes", "create", - "-name", "e2e Automated Test Project", + "-name", "e2e Project", "-scope-id", orgId, "-format", "json", ) diff --git a/testing/internal/e2e/boundary/target.go b/testing/internal/e2e/boundary/target.go index 54a3deecb..1a2ab415a 100644 --- a/testing/internal/e2e/boundary/target.go +++ b/testing/internal/e2e/boundary/target.go @@ -19,7 +19,7 @@ func CreateNewTargetApi(t testing.TB, ctx context.Context, client *api.Client, p targetPort, err := strconv.ParseInt(defaultPort, 10, 32) require.NoError(t, err) newTargetResult, err := tClient.Create(ctx, "tcp", projectId, - targets.WithName("e2e Automated Test Target"), + targets.WithName("e2e Target"), targets.WithTcpTargetDefaultPort(uint32(targetPort)), ) require.NoError(t, err) @@ -45,7 +45,7 @@ func CreateNewTargetCli(t testing.TB, projectId string, defaultPort string) stri output := e2e.RunCommand(context.Background(), "boundary", "targets", "create", "tcp", "-scope-id", projectId, "-default-port", defaultPort, - "-name", "e2e Automated Test Target", + "-name", "e2e Target", "-format", "json", ) require.NoError(t, output.Err, string(output.Stderr)) diff --git a/testing/internal/e2e/host/static/connect_authz_token_test.go b/testing/internal/e2e/host/static/connect_authz_token_test.go index 01798aba0..3b91e264d 100644 --- a/testing/internal/e2e/host/static/connect_authz_token_test.go +++ b/testing/internal/e2e/host/static/connect_authz_token_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/hashicorp/boundary/api/credentials" - "github.com/hashicorp/boundary/api/credentialstores" "github.com/hashicorp/boundary/api/targets" "github.com/hashicorp/boundary/testing/internal/e2e" "github.com/hashicorp/boundary/testing/internal/e2e/boundary" @@ -33,22 +32,11 @@ func TestConnectTargetWithAuthzTokenCli(t *testing.T) { boundary.AddHostToHostSetCli(t, newHostSetId, newHostId) newTargetId := boundary.CreateNewTargetCli(t, newProjectId, c.TargetPort) boundary.AddHostSourceToTargetCli(t, newTargetId, newHostSetId) - - // Create a credential store - ctx := context.Background() - output := e2e.RunCommand(ctx, "boundary", "credential-stores", "create", "static", - "-scope-id", newProjectId, - "-format", "json", - ) - require.NoError(t, output.Err, string(output.Stderr)) - var newCredentialStoreResult credentialstores.CredentialStoreCreateResult - err = json.Unmarshal(output.Stdout, &newCredentialStoreResult) - require.NoError(t, err) - newCredentialStoreId := newCredentialStoreResult.Item.Id - t.Logf("Created Credential Store: %s", newCredentialStoreId) + newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId) // Create credentials - output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key", + ctx := context.Background() + output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key", "-credential-store-id", newCredentialStoreId, "-username", c.TargetSshUser, "-private-key", "file://"+c.TargetSshKeyPath, diff --git a/testing/internal/e2e/host/static/connect_ssh_test.go b/testing/internal/e2e/host/static/connect_ssh_test.go index df0d3e9a0..d74a6cd57 100644 --- a/testing/internal/e2e/host/static/connect_ssh_test.go +++ b/testing/internal/e2e/host/static/connect_ssh_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/hashicorp/boundary/api/credentials" - "github.com/hashicorp/boundary/api/credentialstores" "github.com/hashicorp/boundary/api/targets" "github.com/hashicorp/boundary/testing/internal/e2e" "github.com/hashicorp/boundary/testing/internal/e2e/boundary" @@ -33,22 +32,11 @@ func TestConnectTargetWithSshCli(t *testing.T) { boundary.AddHostToHostSetCli(t, newHostSetId, newHostId) newTargetId := boundary.CreateNewTargetCli(t, newProjectId, c.TargetPort) boundary.AddHostSourceToTargetCli(t, newTargetId, newHostSetId) - - // Create a credential store - ctx := context.Background() - output := e2e.RunCommand(ctx, "boundary", "credential-stores", "create", "static", - "-scope-id", newProjectId, - "-format", "json", - ) - require.NoError(t, output.Err, string(output.Stderr)) - var newCredentialStoreResult credentialstores.CredentialStoreCreateResult - err = json.Unmarshal(output.Stdout, &newCredentialStoreResult) - require.NoError(t, err) - newCredentialStoreId := newCredentialStoreResult.Item.Id - t.Logf("Created Credential Store: %s", newCredentialStoreId) + newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId) // Create credentials - output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key", + ctx := context.Background() + output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key", "-credential-store-id", newCredentialStoreId, "-username", c.TargetSshUser, "-private-key", "file://"+c.TargetSshKeyPath, @@ -115,13 +103,7 @@ func TestCreateTargetWithStaticCredentialStoreApi(t *testing.T) { boundary.AddHostToHostSetApi(t, ctx, client, newHostSetId, newHostId) newTargetId := boundary.CreateNewTargetApi(t, ctx, client, newProjectId, c.TargetPort) boundary.AddHostSourceToTargetApi(t, ctx, client, newTargetId, newHostSetId) - - // Create a credential store - csClient := credentialstores.NewClient(client) - newCredentialStoreResult, err := csClient.Create(ctx, "static", newProjectId) - require.NoError(t, err) - newCredentialStoreId := newCredentialStoreResult.Item.Id - t.Logf("Created Credential Store: %s", newCredentialStoreId) + newCredentialStoreId := boundary.CreateNewCredentialStoreStaticApi(t, ctx, client, newProjectId) // Create credentials cClient := credentials.NewClient(client) From 5b0ff4f058361656fae85b58ad55a43b93750187 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Tue, 11 Oct 2022 14:22:48 -0400 Subject: [PATCH 4/4] test(e2e): Port "credential stores deletion check" from bats test to e2e test suite The "can not read deleted $NEW_STORE static store" bats test was demonstrating flaky behavior where it would fail due to the credential store read still returning something valid. This check has been moved to an e2e test in order to add some retry logic to the check. --- .../tests/cli/boundary/credential_stores.bats | 20 ++--- testing/internal/e2e/helpers.go | 5 ++ .../static/static_credential_store_test.go | 88 +++++++++++++++++++ 3 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 testing/internal/e2e/host/static/static_credential_store_test.go diff --git a/internal/tests/cli/boundary/credential_stores.bats b/internal/tests/cli/boundary/credential_stores.bats index a2dbde63a..a58388ade 100644 --- a/internal/tests/cli/boundary/credential_stores.bats +++ b/internal/tests/cli/boundary/credential_stores.bats @@ -13,35 +13,29 @@ export NEW_STORE='test' } @test "boundary/credential-stores: can create $NEW_STORE static store in default project" { - run create_static_credential_store $NEW_STORE $DEFAULT_P_ID + run create_static_credential_store $NEW_STORE $DEFAULT_P_ID echo "$output" - [ "$status" -eq 0 ] + [ "$status" -eq 0 ] } @test "boundary/credential-stores: can not create already created $NEW_STORE static store" { - run create_static_credential_store $NEW_STORE $DEFAULT_P_ID + run create_static_credential_store $NEW_STORE $DEFAULT_P_ID echo "$output" - [ "$status" -eq 1 ] + [ "$status" -eq 1 ] } @test "boundary/credential-stores: can read $NEW_STORE static store" { local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID) - run read_credential_store $csid + run read_credential_store $csid echo "$output" - [ "$status" -eq 0 ] + [ "$status" -eq 0 ] } @test "boundary/credential-stores: can delete $NEW_STORE static store" { local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID) run delete_credential_store $csid echo "$output" + [ "$status" -eq 0 ] run has_status_code "$output" "204" [ "$status" -eq 0 ] } - -@test "boundary/credential-stores: can not read deleted $NEW_STORE static store" { - local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID) - run read_credential_store $csid - echo "$output" - [ "$status" -eq 1 ] -} diff --git a/testing/internal/e2e/helpers.go b/testing/internal/e2e/helpers.go index acc15ced7..a4f885401 100644 --- a/testing/internal/e2e/helpers.go +++ b/testing/internal/e2e/helpers.go @@ -18,6 +18,11 @@ type CommandResult struct { Err error } +// CliError parses the Stderr from running a boundary command +type CliError struct { + Status int `json:"status"` +} + const EnvToCheckSkip = "E2E_PASSWORD_AUTH_METHOD_ID" // RunCommand executes external commands on the system. Returns the results diff --git a/testing/internal/e2e/host/static/static_credential_store_test.go b/testing/internal/e2e/host/static/static_credential_store_test.go new file mode 100644 index 000000000..c00e9d3fe --- /dev/null +++ b/testing/internal/e2e/host/static/static_credential_store_test.go @@ -0,0 +1,88 @@ +package static_test + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/hashicorp/boundary/api/credentials" + "github.com/hashicorp/boundary/testing/internal/e2e" + "github.com/hashicorp/boundary/testing/internal/e2e/boundary" + "github.com/stretchr/testify/require" +) + +// TestStaticCredentialStoreCli validates various credential-store operations using the cli +func TestStaticCredentialStoreCli(t *testing.T) { + e2e.MaybeSkipTest(t) + c, err := loadConfig() + require.NoError(t, err) + + boundary.AuthenticateCli(t) + newOrgId := boundary.CreateNewOrgCli(t) + newProjectId := boundary.CreateNewProjectCli(t, newOrgId) + newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId) + + // Create ssh key credentials + ctx := context.Background() + output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key", + "-credential-store-id", newCredentialStoreId, + "-username", c.TargetSshUser, + "-private-key", "file://"+c.TargetSshKeyPath, + "-format", "json", + ) + require.NoError(t, output.Err, string(output.Stderr)) + var keyCredentialsResult credentials.CredentialCreateResult + err = json.Unmarshal(output.Stdout, &keyCredentialsResult) + require.NoError(t, err) + keyCredentialsId := keyCredentialsResult.Item.Id + t.Logf("Created SSH Private Key Credentials: %s", keyCredentialsId) + + // Create username/password credentials + os.Setenv("E2E_CREDENTIALS_PASSWORD", "password") + output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "username-password", + "-credential-store-id", newCredentialStoreId, + "-username", c.TargetSshUser, + "-password", "env://E2E_CREDENTIALS_PASSWORD", + "-format", "json", + ) + require.NoError(t, output.Err, string(output.Stderr)) + var pwCredentialsResult credentials.CredentialCreateResult + err = json.Unmarshal(output.Stdout, &pwCredentialsResult) + require.NoError(t, err) + pwCredentialsId := pwCredentialsResult.Item.Id + t.Logf("Created Username/Password Credentials: %s", pwCredentialsId) + + // Delete credential store + output = e2e.RunCommand(ctx, "boundary", "credential-stores", "delete", "-id", newCredentialStoreId) + require.NoError(t, output.Err, string(output.Stderr)) + err = backoff.RetryNotify( + func() error { + output = e2e.RunCommand(ctx, "boundary", "credential-stores", "read", "-id", newCredentialStoreId, "-format", "json") + if output.Err == nil { + return fmt.Errorf("Deleted credential can still be read: '%s'", output.Stdout) + } + + var response e2e.CliError + err = json.Unmarshal(output.Stderr, &response) + require.NoError(t, err) + statusCode := response.Status + if statusCode != 404 { + return backoff.Permanent( + fmt.Errorf("Command did not return expected status code. Expected: 404, Actual: %d", statusCode), + ) + } + + return nil + }, + backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5), + func(err error, td time.Duration) { + t.Logf("%s. Retrying...", err.Error()) + }, + ) + require.NoError(t, err) + t.Logf("Successfully deleted credential store") +}