CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2549 from hashicorp/moduli-flaky-bats-credential-store-ICU-6336

Browse Source 
test(e2e): Fix flaky test in `credential_stores.bats`
This commit is contained in:
Michael Li authored and GitHub committed 2022-10-20 15:00:47 -04:00
2 parents d2572a343c 5b0ff4f058
commit 8af92997c8
10 files changed
+161 -105
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/tests/cli/boundary/credential_stores.bats
+7 -13
View File
@@ -13,35 +13,29 @@ export NEW_STORE='test'
}
@test "boundary/credential-stores: can create $NEW_STORE static store in default project" {
run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
echo "$output"
[ "$status" -eq 0 ]
[ "$status" -eq 0 ]
}
@test "boundary/credential-stores: can not create already created $NEW_STORE static store" {
run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
echo "$output"
[ "$status" -eq 1 ]
[ "$status" -eq 1 ]
}
@test "boundary/credential-stores: can read $NEW_STORE static store" {
local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID)
run read_credential_store $csid
run read_credential_store $csid
echo "$output"
[ "$status" -eq 0 ]
[ "$status" -eq 0 ]
}
@test "boundary/credential-stores: can delete $NEW_STORE static store" {
local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID)
run delete_credential_store $csid
echo "$output"
[ "$status" -eq 0 ]
run has_status_code "$output" "204"
[ "$status" -eq 0 ]
}
@test "boundary/credential-stores: can not read deleted $NEW_STORE static store" {
local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID)
run read_credential_store $csid
echo "$output"
[ "$status" -eq 1 ]
}
testing/internal/e2e/boundary/credential.go
+41
View File
@@ -0,0 +1,41 @@
package boundary
import (
"context"
"encoding/json"
"testing"
"github.com/hashicorp/boundary/api"
"github.com/hashicorp/boundary/api/credentialstores"
"github.com/hashicorp/boundary/testing/internal/e2e"
"github.com/stretchr/testify/require"
)
// CreateNewCredentialStoreStaticApi creates a new static credential store using the go api.
// Returns the id of the new credential store
func CreateNewCredentialStoreStaticApi(t testing.TB, ctx context.Context, client *api.Client, projectId string) string {
csClient := credentialstores.NewClient(client)
newCredentialStoreResult, err := csClient.Create(ctx, "static", projectId, credentialstores.WithName("e2e Credential Store"))
require.NoError(t, err)
newCredentialStoreId := newCredentialStoreResult.Item.Id
t.Logf("Created Credential Store: %s", newCredentialStoreId)
return newCredentialStoreId
}
// CreateNewCredentialStoreStaticCli creates a new static credential store using the cli.
// Returns the id of the new credential store
func CreateNewCredentialStoreStaticCli(t testing.TB, projectId string) string {
output := e2e.RunCommand(context.Background(), "boundary", "credential-stores", "create", "static",
"-scope-id", projectId,
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
var newCredentialStoreResult credentialstores.CredentialStoreCreateResult
err := json.Unmarshal(output.Stdout, &newCredentialStoreResult)
require.NoError(t, err)
newCredentialStoreId := newCredentialStoreResult.Item.Id
t.Logf("Created Credential Store: %s", newCredentialStoreId)
return newCredentialStoreId
}
testing/internal/e2e/boundary/host.go
+4 -6
View File
@@ -17,9 +17,7 @@ import (
// Returns the id of the new host catalog.
func CreateNewHostCatalogApi(t testing.TB, ctx context.Context, client *api.Client, projectId string) string {
hcClient := hostcatalogs.NewClient(client)
newHostCatalogResult, err := hcClient.Create(ctx, "static", projectId,
hostcatalogs.WithName("e2e Automated Test Host Catalog"),
)
newHostCatalogResult, err := hcClient.Create(ctx, "static", projectId, hostcatalogs.WithName("e2e Host Catalog"))
require.NoError(t, err)
newHostCatalogId := newHostCatalogResult.Item.Id
t.Logf("Created Host Catalog: %s", newHostCatalogId)
@@ -31,7 +29,7 @@ func CreateNewHostCatalogApi(t testing.TB, ctx context.Context, client *api.Clie
// Returns the id of the new host set.
func CreateNewHostSetApi(t testing.TB, ctx context.Context, client *api.Client, hostCatalogId string) string {
hsClient := hostsets.NewClient(client)
newHostSetResult, err := hsClient.Create(ctx, hostCatalogId)
newHostSetResult, err := hsClient.Create(ctx, hostCatalogId, hostsets.WithName("e2e Host"))
require.NoError(t, err)
newHostSetId := newHostSetResult.Item.Id
t.Logf("Created Host Set: %s", newHostSetId)
@@ -66,7 +64,7 @@ func AddHostToHostSetApi(t testing.TB, ctx context.Context, client *api.Client,
func CreateNewHostCatalogCli(t testing.TB, projectId string) string {
output := e2e.RunCommand(context.Background(), "boundary", "host-catalogs", "create", "static",
"-scope-id", projectId,
"-name", "e2e Automated Test Host Catalog",
"-name", "e2e Host Catalog",
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
@@ -84,7 +82,7 @@ func CreateNewHostCatalogCli(t testing.TB, projectId string) string {
func CreateNewHostSetCli(t testing.TB, hostCatalogId string) string {
output := e2e.RunCommand(context.Background(), "boundary", "host-sets", "create", "static",
"-host-catalog-id", hostCatalogId,
"-name", "e2e Automated Test Host Set",
"-name", "e2e Host Set",
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
testing/internal/e2e/boundary/scope.go
+4 -4
View File
@@ -15,7 +15,7 @@ import (
// Returns the id of the new org.
func CreateNewOrgApi(t testing.TB, ctx context.Context, client *api.Client) string {
scopeClient := scopes.NewClient(client)
newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Automated Test Org"))
newOrgResult, err := scopeClient.Create(ctx, "global", scopes.WithName("e2e Org"))
require.NoError(t, err)
newOrgId := newOrgResult.Item.Id
@@ -33,7 +33,7 @@ func CreateNewOrgApi(t testing.TB, ctx context.Context, client *api.Client) stri
// Returns the id of the new project.
func CreateNewProjectApi(t testing.TB, ctx context.Context, client *api.Client, orgId string) string {
scopeClient := scopes.NewClient(client)
newProjResult, err := scopeClient.Create(ctx, orgId, scopes.WithName("e2e Automated Test Project"))
newProjResult, err := scopeClient.Create(ctx, orgId, scopes.WithName("e2e Project"))
require.NoError(t, err)
newProjectId := newProjResult.Item.Id
@@ -46,7 +46,7 @@ func CreateNewProjectApi(t testing.TB, ctx context.Context, client *api.Client,
func CreateNewOrgCli(t testing.TB) string {
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "scopes", "create",
"-name", "e2e Automated Test Org",
"-name", "e2e Org",
"-scope-id", "global",
"-format", "json",
)
@@ -72,7 +72,7 @@ func CreateNewOrgCli(t testing.TB) string {
func CreateNewProjectCli(t testing.TB, orgId string) string {
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "scopes", "create",
"-name", "e2e Automated Test Project",
"-name", "e2e Project",
"-scope-id", orgId,
"-format", "json",
)
testing/internal/e2e/boundary/target.go
+2 -2
View File
@@ -19,7 +19,7 @@ func CreateNewTargetApi(t testing.TB, ctx context.Context, client *api.Client, p
targetPort, err := strconv.ParseInt(defaultPort, 10, 32)
require.NoError(t, err)
newTargetResult, err := tClient.Create(ctx, "tcp", projectId,
targets.WithName("e2e Automated Test Target"),
targets.WithName("e2e Target"),
targets.WithTcpTargetDefaultPort(uint32(targetPort)),
)
require.NoError(t, err)
@@ -45,7 +45,7 @@ func CreateNewTargetCli(t testing.TB, projectId string, defaultPort string) stri
output := e2e.RunCommand(context.Background(), "boundary", "targets", "create", "tcp",
"-scope-id", projectId,
"-default-port", defaultPort,
"-name", "e2e Automated Test Target",
"-name", "e2e Target",
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
testing/internal/e2e/helpers.go
+6 -42
View File
@@ -10,32 +10,7 @@ import (
"testing"
)
// Option is a func that sets optional attributes for a call. This does not need
// to be used directly, but instead option arguments are built from the
// functions in this package. WithX options set a value to that given in the
// argument; DefaultX options indicate that the value should be set to its
// default. When an API call is made options are processed in ther order they
// appear in the function call, so for a given argument X, a succession of WithX
// or DefaultX calls will result in the last call taking effect.
type Option func(*options)
type options struct {
withArgs []string
withPipe []string
}
func getOpts(opt ...Option) options {
opts := options{}
for _, o := range opt {
if o != nil {
o(&opts)
}
}
return opts
}
// CommandResult encapsulates the output from running an external command
// CommandResult captures the output from running an external command
type CommandResult struct {
Stdout []byte
Stderr []byte
@@ -43,6 +18,11 @@ type CommandResult struct {
Err error
}
// CliError parses the Stderr from running a boundary command
type CliError struct {
Status int `json:"status"`
}
const EnvToCheckSkip = "E2E_PASSWORD_AUTH_METHOD_ID"
// RunCommand executes external commands on the system. Returns the results
@@ -75,22 +55,6 @@ func RunCommand(ctx context.Context, name string, args ...string) *CommandResult
}
}
// WithArgs is an option to RunCommand that allows the user to specify arguments
// for the provided command
func WithArgs(args ...string) Option {
return func(o *options) {
o.withArgs = args
}
}
// WithPipe is an option to RunCommand that allows the user to specify a command+arguments
// to pipe to
func WithPipe(command ...string) Option {
return func(o *options) {
o.withPipe = command
}
}
// MaybeSkipTest is a check used at the start of the test to determine if the test should run
func MaybeSkipTest(t testing.TB) {
if _, ok := os.LookupEnv(EnvToCheckSkip); !ok {
testing/internal/e2e/host/static/connect_authz_token_test.go
+3 -15
View File
@@ -9,7 +9,6 @@ import (
"testing"
"github.com/hashicorp/boundary/api/credentials"
"github.com/hashicorp/boundary/api/credentialstores"
"github.com/hashicorp/boundary/api/targets"
"github.com/hashicorp/boundary/testing/internal/e2e"
"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
@@ -33,22 +32,11 @@ func TestConnectTargetWithAuthzTokenCli(t *testing.T) {
boundary.AddHostToHostSetCli(t, newHostSetId, newHostId)
newTargetId := boundary.CreateNewTargetCli(t, newProjectId, c.TargetPort)
boundary.AddHostSourceToTargetCli(t, newTargetId, newHostSetId)
// Create a credential store
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "credential-stores", "create", "static",
"-scope-id", newProjectId,
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
var newCredentialStoreResult credentialstores.CredentialStoreCreateResult
err = json.Unmarshal(output.Stdout, &newCredentialStoreResult)
require.NoError(t, err)
newCredentialStoreId := newCredentialStoreResult.Item.Id
t.Logf("Created Credential Store: %s", newCredentialStoreId)
newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId)
// Create credentials
output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key",
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key",
"-credential-store-id", newCredentialStoreId,
"-username", c.TargetSshUser,
"-private-key", "file://"+c.TargetSshKeyPath,
testing/internal/e2e/host/static/connect_ssh_test.go
+4 -22
View File
@@ -8,7 +8,6 @@ import (
"testing"
"github.com/hashicorp/boundary/api/credentials"
"github.com/hashicorp/boundary/api/credentialstores"
"github.com/hashicorp/boundary/api/targets"
"github.com/hashicorp/boundary/testing/internal/e2e"
"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
@@ -33,22 +32,11 @@ func TestConnectTargetWithSshCli(t *testing.T) {
boundary.AddHostToHostSetCli(t, newHostSetId, newHostId)
newTargetId := boundary.CreateNewTargetCli(t, newProjectId, c.TargetPort)
boundary.AddHostSourceToTargetCli(t, newTargetId, newHostSetId)
// Create a credential store
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "credential-stores", "create", "static",
"-scope-id", newProjectId,
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
var newCredentialStoreResult credentialstores.CredentialStoreCreateResult
err = json.Unmarshal(output.Stdout, &newCredentialStoreResult)
require.NoError(t, err)
newCredentialStoreId := newCredentialStoreResult.Item.Id
t.Logf("Created Credential Store: %s", newCredentialStoreId)
newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId)
// Create credentials
output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key",
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key",
"-credential-store-id", newCredentialStoreId,
"-username", c.TargetSshUser,
"-private-key", "file://"+c.TargetSshKeyPath,
@@ -115,13 +103,7 @@ func TestCreateTargetWithStaticCredentialStoreApi(t *testing.T) {
boundary.AddHostToHostSetApi(t, ctx, client, newHostSetId, newHostId)
newTargetId := boundary.CreateNewTargetApi(t, ctx, client, newProjectId, c.TargetPort)
boundary.AddHostSourceToTargetApi(t, ctx, client, newTargetId, newHostSetId)
// Create a credential store
csClient := credentialstores.NewClient(client)
newCredentialStoreResult, err := csClient.Create(ctx, "static", newProjectId)
require.NoError(t, err)
newCredentialStoreId := newCredentialStoreResult.Item.Id
t.Logf("Created Credential Store: %s", newCredentialStoreId)
newCredentialStoreId := boundary.CreateNewCredentialStoreStaticApi(t, ctx, client, newProjectId)
// Create credentials
cClient := credentials.NewClient(client)
testing/internal/e2e/host/static/session_test.go → testing/internal/e2e/host/static/session_cancel_test.go
+2 -1
View File
@@ -15,7 +15,8 @@ import (
"github.com/stretchr/testify/require"
)
func TestSessionCancelingCli(t *testing.T) {
// TestSessionCancelCli uses the boundary cli to start and then cancel a session
func TestSessionCancelCli(t *testing.T) {
e2e.MaybeSkipTest(t)
c, err := loadConfig()
require.NoError(t, err)
testing/internal/e2e/host/static/static_credential_store_test.go
+88
View File
@@ -0,0 +1,88 @@
package static_test
import (
"context"
"encoding/json"
"fmt"
"os"
"testing"
"time"
"github.com/cenkalti/backoff/v4"
"github.com/hashicorp/boundary/api/credentials"
"github.com/hashicorp/boundary/testing/internal/e2e"
"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
"github.com/stretchr/testify/require"
)
// TestStaticCredentialStoreCli validates various credential-store operations using the cli
func TestStaticCredentialStoreCli(t *testing.T) {
e2e.MaybeSkipTest(t)
c, err := loadConfig()
require.NoError(t, err)
boundary.AuthenticateCli(t)
newOrgId := boundary.CreateNewOrgCli(t)
newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId)
// Create ssh key credentials
ctx := context.Background()
output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key",
"-credential-store-id", newCredentialStoreId,
"-username", c.TargetSshUser,
"-private-key", "file://"+c.TargetSshKeyPath,
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
var keyCredentialsResult credentials.CredentialCreateResult
err = json.Unmarshal(output.Stdout, &keyCredentialsResult)
require.NoError(t, err)
keyCredentialsId := keyCredentialsResult.Item.Id
t.Logf("Created SSH Private Key Credentials: %s", keyCredentialsId)
// Create username/password credentials
os.Setenv("E2E_CREDENTIALS_PASSWORD", "password")
output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "username-password",
"-credential-store-id", newCredentialStoreId,
"-username", c.TargetSshUser,
"-password", "env://E2E_CREDENTIALS_PASSWORD",
"-format", "json",
)
require.NoError(t, output.Err, string(output.Stderr))
var pwCredentialsResult credentials.CredentialCreateResult
err = json.Unmarshal(output.Stdout, &pwCredentialsResult)
require.NoError(t, err)
pwCredentialsId := pwCredentialsResult.Item.Id
t.Logf("Created Username/Password Credentials: %s", pwCredentialsId)
// Delete credential store
output = e2e.RunCommand(ctx, "boundary", "credential-stores", "delete", "-id", newCredentialStoreId)
require.NoError(t, output.Err, string(output.Stderr))
err = backoff.RetryNotify(
func() error {
output = e2e.RunCommand(ctx, "boundary", "credential-stores", "read", "-id", newCredentialStoreId, "-format", "json")
if output.Err == nil {
return fmt.Errorf("Deleted credential can still be read: '%s'", output.Stdout)
}
var response e2e.CliError
err = json.Unmarshal(output.Stderr, &response)
require.NoError(t, err)
statusCode := response.Status
if statusCode != 404 {
return backoff.Permanent(
fmt.Errorf("Command did not return expected status code. Expected: 404, Actual: %d", statusCode),
)
}
return nil
},
backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
func(err error, td time.Duration) {
t.Logf("%s. Retrying...", err.Error())
},
)
require.NoError(t, err)
t.Logf("Successfully deleted credential store")
}