test(e2e): Port "credential stores deletion check" from bats test to e2e test suite

The "can not read deleted $NEW_STORE static store" bats test was demonstrating flaky behavior where it would fail due to the credential store read still returning something valid. This check has been moved to an e2e test in order to add some retry logic to the check.

internal/tests/cli/boundary/credential_stores.bats

@@ -13,35 +13,29 @@ export NEW_STORE='test'
 }
 
 @test "boundary/credential-stores: can create $NEW_STORE static store in default project" {
-	run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
+  run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
   echo "$output"
-	[ "$status" -eq 0 ]
+  [ "$status" -eq 0 ]
 }
 
 @test "boundary/credential-stores: can not create already created $NEW_STORE static store" {
-	run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
+  run create_static_credential_store $NEW_STORE $DEFAULT_P_ID
   echo "$output"
-	[ "$status" -eq 1 ]
+  [ "$status" -eq 1 ]
 }
 
 @test "boundary/credential-stores: can read $NEW_STORE static store" {
   local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID)
-	run read_credential_store $csid
+  run read_credential_store $csid
   echo "$output"
-	[ "$status" -eq 0 ]
+  [ "$status" -eq 0 ]
 }
 
 @test "boundary/credential-stores: can delete $NEW_STORE static store" {
   local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID)
   run delete_credential_store $csid
   echo "$output"
+  [ "$status" -eq 0 ]
   run has_status_code "$output" "204"
   [ "$status" -eq 0 ]
 }
-
-@test "boundary/credential-stores: can not read deleted $NEW_STORE static store" {
-  local csid=$(credential_store_id $NEW_STORE $DEFAULT_P_ID)
-	run read_credential_store $csid
-  echo "$output"
-	[ "$status" -eq 1 ]
-}

testing/internal/e2e/helpers.go

@@ -18,6 +18,11 @@ type CommandResult struct {
 	Err      error
 }
 
+// CliError parses the Stderr from running a boundary command
+type CliError struct {
+	Status int `json:"status"`
+}
+
 const EnvToCheckSkip = "E2E_PASSWORD_AUTH_METHOD_ID"
 
 // RunCommand executes external commands on the system. Returns the results

testing/internal/e2e/host/static/static_credential_store_test.go

@@ -0,0 +1,88 @@
+package static_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/hashicorp/boundary/api/credentials"
+	"github.com/hashicorp/boundary/testing/internal/e2e"
+	"github.com/hashicorp/boundary/testing/internal/e2e/boundary"
+	"github.com/stretchr/testify/require"
+)
+
+// TestStaticCredentialStoreCli validates various credential-store operations using the cli
+func TestStaticCredentialStoreCli(t *testing.T) {
+	e2e.MaybeSkipTest(t)
+	c, err := loadConfig()
+	require.NoError(t, err)
+
+	boundary.AuthenticateCli(t)
+	newOrgId := boundary.CreateNewOrgCli(t)
+	newProjectId := boundary.CreateNewProjectCli(t, newOrgId)
+	newCredentialStoreId := boundary.CreateNewCredentialStoreStaticCli(t, newProjectId)
+
+	// Create ssh key credentials
+	ctx := context.Background()
+	output := e2e.RunCommand(ctx, "boundary", "credentials", "create", "ssh-private-key",
+		"-credential-store-id", newCredentialStoreId,
+		"-username", c.TargetSshUser,
+		"-private-key", "file://"+c.TargetSshKeyPath,
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var keyCredentialsResult credentials.CredentialCreateResult
+	err = json.Unmarshal(output.Stdout, &keyCredentialsResult)
+	require.NoError(t, err)
+	keyCredentialsId := keyCredentialsResult.Item.Id
+	t.Logf("Created SSH Private Key Credentials: %s", keyCredentialsId)
+
+	// Create username/password credentials
+	os.Setenv("E2E_CREDENTIALS_PASSWORD", "password")
+	output = e2e.RunCommand(ctx, "boundary", "credentials", "create", "username-password",
+		"-credential-store-id", newCredentialStoreId,
+		"-username", c.TargetSshUser,
+		"-password", "env://E2E_CREDENTIALS_PASSWORD",
+		"-format", "json",
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	var pwCredentialsResult credentials.CredentialCreateResult
+	err = json.Unmarshal(output.Stdout, &pwCredentialsResult)
+	require.NoError(t, err)
+	pwCredentialsId := pwCredentialsResult.Item.Id
+	t.Logf("Created Username/Password Credentials: %s", pwCredentialsId)
+
+	// Delete credential store
+	output = e2e.RunCommand(ctx, "boundary", "credential-stores", "delete", "-id", newCredentialStoreId)
+	require.NoError(t, output.Err, string(output.Stderr))
+	err = backoff.RetryNotify(
+		func() error {
+			output = e2e.RunCommand(ctx, "boundary", "credential-stores", "read", "-id", newCredentialStoreId, "-format", "json")
+			if output.Err == nil {
+				return fmt.Errorf("Deleted credential can still be read: '%s'", output.Stdout)
+			}
+
+			var response e2e.CliError
+			err = json.Unmarshal(output.Stderr, &response)
+			require.NoError(t, err)
+			statusCode := response.Status
+			if statusCode != 404 {
+				return backoff.Permanent(
+					fmt.Errorf("Command did not return expected status code. Expected: 404, Actual: %d", statusCode),
+				)
+			}
+
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	t.Logf("Successfully deleted credential store")
+}