fix(e2e): Add retry when checking if session is active (#2586)

Observed some flakiness here. A test reported that the session was still "pending". Adding additional retries to ensure that it will eventually go active.

testing/internal/e2e/tests/aws/dynamichostcatalog_test.go

@@ -98,7 +98,7 @@ func TestCliCreateAwsDynamicHostCatalog(t *testing.T) {
 	var actualHostSetCount1 int
 	err = backoff.RetryNotify(
 		func() error {
-			output = e2e.RunCommand(ctx, "boundary",
+			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs(
 					"host-sets", "read",
 					"-id", newHostSetId1,
@@ -110,7 +110,7 @@ func TestCliCreateAwsDynamicHostCatalog(t *testing.T) {
 			}
 
 			var hostSetsReadResult hostsets.HostSetReadResult
-			err = json.Unmarshal(output.Stdout, &hostSetsReadResult)
+			err := json.Unmarshal(output.Stdout, &hostSetsReadResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}
@@ -158,7 +158,7 @@ func TestCliCreateAwsDynamicHostCatalog(t *testing.T) {
 	var actualHostSetCount2 int
 	err = backoff.RetryNotify(
 		func() error {
-			output = e2e.RunCommand(ctx, "boundary",
+			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs("host-sets", "read", "-id", newHostSetId2, "-format", "json"),
 			)
 			if output.Err != nil {
@@ -166,7 +166,7 @@ func TestCliCreateAwsDynamicHostCatalog(t *testing.T) {
 			}
 
 			var hostSetsReadResult hostsets.HostSetReadResult
-			err = json.Unmarshal(output.Stdout, &hostSetsReadResult)
+			err := json.Unmarshal(output.Stdout, &hostSetsReadResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}
@@ -196,7 +196,7 @@ func TestCliCreateAwsDynamicHostCatalog(t *testing.T) {
 	var actualHostCatalogCount int
 	err = backoff.RetryNotify(
 		func() error {
-			output = e2e.RunCommand(ctx, "boundary",
+			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs("hosts", "list", "-host-catalog-id", newHostCatalogId, "-format", "json"),
 			)
 			if output.Err != nil {
@@ -204,7 +204,7 @@ func TestCliCreateAwsDynamicHostCatalog(t *testing.T) {
 			}
 
 			var hostCatalogListResult hostcatalogs.HostCatalogListResult
-			err = json.Unmarshal(output.Stdout, &hostCatalogListResult)
+			err := json.Unmarshal(output.Stdout, &hostCatalogListResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}

testing/internal/e2e/tests/static/connect_localhost_test.go

@@ -60,7 +60,7 @@ func TestCliConnectTargetWithLocalhost(t *testing.T) {
 			}
 
 			var sessionListResult sessions.SessionListResult
-			err = json.Unmarshal(output.Stdout, &sessionListResult)
+			err := json.Unmarshal(output.Stdout, &sessionListResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}

testing/internal/e2e/tests/static/credential_store_test.go

@@ -111,7 +111,7 @@ func TestCliStaticCredentialStore(t *testing.T) {
 	require.NoError(t, output.Err, string(output.Stderr))
 	err = backoff.RetryNotify(
 		func() error {
-			output = e2e.RunCommand(ctx, "boundary",
+			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs("credential-stores", "read", "-id", newCredentialStoreId, "-format", "json"),
 			)
 			if output.Err == nil {
@@ -119,7 +119,7 @@ func TestCliStaticCredentialStore(t *testing.T) {
 			}
 
 			var response e2e.CliError
-			err = json.Unmarshal(output.Stderr, &response)
+			err := json.Unmarshal(output.Stderr, &response)
 			require.NoError(t, err)
 			statusCode := response.Status
 			if statusCode != 404 {

testing/internal/e2e/tests/static/session_cancel_admin_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
@@ -55,34 +56,53 @@ func TestCliSessionCancelAdmin(t *testing.T) {
 	}()
 	t.Cleanup(cancel)
 
-	// Get list of sessions
+	// Wait for session to be active
 	var session *sessions.Session
 	err = backoff.RetryNotify(
 		func() error {
+			// List sessions
 			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs("sessions", "list", "-scope-id", newProjectId, "-format", "json"),
 			)
 			if output.Err != nil {
 				return backoff.Permanent(errors.New(string(output.Stderr)))
 			}
-
 			var sessionListResult sessions.SessionListResult
-			err = json.Unmarshal(output.Stdout, &sessionListResult)
+			err := json.Unmarshal(output.Stdout, &sessionListResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}
 
+			// Check if there is one session
 			sessionCount := len(sessionListResult.Items)
 			if sessionCount == 0 {
 				return errors.New("No items are appearing in the session list")
 			}
-
 			t.Logf("Found %d session(s)", sessionCount)
 			if sessionCount != 1 {
 				return backoff.Permanent(errors.New("Only one session was expected to be found"))
 			}
 
+			// Check if session is active
 			session = sessionListResult.Items[0]
+			output = e2e.RunCommand(ctx, "boundary",
+				e2e.WithArgs("sessions", "read", "-id", session.Id, "-format", "json"),
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+			var sessionReadResult sessions.SessionReadResult
+			err = json.Unmarshal(output.Stdout, &sessionReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+			if sessionReadResult.Item.Status != "active" {
+				return errors.New(fmt.Sprintf("Waiting for session to be active... Expected: %s, Actual: %s",
+					"active",
+					sessionReadResult.Item.Status,
+				))
+			}
+
 			return nil
 		},
 		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
@@ -93,7 +113,6 @@ func TestCliSessionCancelAdmin(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, newTargetId, session.TargetId)
 	assert.Equal(t, newHostId, session.HostId)
-	require.Equal(t, "active", session.Status)
 
 	// Cancel session
 	output := e2e.RunCommand(ctx, "boundary",

testing/internal/e2e/tests/static/session_cancel_group_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
@@ -123,34 +124,55 @@ func TestCliSessionCancelGroup(t *testing.T) {
 	}()
 	t.Cleanup(cancel)
 
-	// Get list of sessions
+	// Wait for session to be active
 	var session *sessions.Session
 	err = backoff.RetryNotify(
 		func() error {
+			// List sessions
 			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs("sessions", "list", "-scope-id", newProjectId, "-format", "json"),
 			)
 			if output.Err != nil {
 				return backoff.Permanent(errors.New(string(output.Stderr)))
 			}
-
 			var sessionListResult sessions.SessionListResult
-			err = json.Unmarshal(output.Stdout, &sessionListResult)
+			err := json.Unmarshal(output.Stdout, &sessionListResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}
 
+			// Check if there is one session
 			sessionCount := len(sessionListResult.Items)
 			if sessionCount == 0 {
 				return errors.New("No items are appearing in the session list")
 			}
-
 			t.Logf("Found %d session(s)", sessionCount)
 			if sessionCount != 1 {
 				return backoff.Permanent(errors.New("Only one session was expected to be found"))
 			}
 
+			// Check if session is active
 			session = sessionListResult.Items[0]
+			output = e2e.RunCommand(ctx, "boundary",
+				e2e.WithArgs("sessions", "read", "-id", session.Id, "-format", "json"),
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var sessionReadResult sessions.SessionReadResult
+			err = json.Unmarshal(output.Stdout, &sessionReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			if sessionReadResult.Item.Status != "active" {
+				return errors.New(fmt.Sprintf("Waiting for session to be active... Expected: %s, Actual: %s",
+					"active",
+					sessionReadResult.Item.Status,
+				))
+			}
+
 			return nil
 		},
 		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
@@ -161,7 +183,6 @@ func TestCliSessionCancelGroup(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, newTargetId, session.TargetId)
 	assert.Equal(t, newHostId, session.HostId)
-	require.Equal(t, "active", session.Status)
 
 	// Cancel session
 	output = e2e.RunCommand(ctx, "boundary",

testing/internal/e2e/tests/static/session_cancel_user_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
@@ -97,34 +98,55 @@ func TestCliSessionCancelUser(t *testing.T) {
 	}()
 	t.Cleanup(cancel)
 
-	// Get list of sessions
+	// Wait for session to be active
 	var session *sessions.Session
 	err = backoff.RetryNotify(
 		func() error {
+			// List sessions
 			output := e2e.RunCommand(ctx, "boundary",
 				e2e.WithArgs("sessions", "list", "-scope-id", newProjectId, "-format", "json"),
 			)
 			if output.Err != nil {
 				return backoff.Permanent(errors.New(string(output.Stderr)))
 			}
-
 			var sessionListResult sessions.SessionListResult
-			err = json.Unmarshal(output.Stdout, &sessionListResult)
+			err := json.Unmarshal(output.Stdout, &sessionListResult)
 			if err != nil {
 				return backoff.Permanent(err)
 			}
 
+			// Check if there is one session
 			sessionCount := len(sessionListResult.Items)
 			if sessionCount == 0 {
 				return errors.New("No items are appearing in the session list")
 			}
-
 			t.Logf("Found %d session(s)", sessionCount)
 			if sessionCount != 1 {
 				return backoff.Permanent(errors.New("Only one session was expected to be found"))
 			}
 
+			// Check if session is active
 			session = sessionListResult.Items[0]
+			output = e2e.RunCommand(ctx, "boundary",
+				e2e.WithArgs("sessions", "read", "-id", session.Id, "-format", "json"),
+			)
+			if output.Err != nil {
+				return backoff.Permanent(errors.New(string(output.Stderr)))
+			}
+
+			var sessionReadResult sessions.SessionReadResult
+			err = json.Unmarshal(output.Stdout, &sessionReadResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			if sessionReadResult.Item.Status != "active" {
+				return errors.New(fmt.Sprintf("Waiting for session to be active... Expected: %s, Actual: %s",
+					"active",
+					sessionReadResult.Item.Status,
+				))
+			}
+
 			return nil
 		},
 		backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 5),
@@ -135,7 +157,6 @@ func TestCliSessionCancelUser(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, newTargetId, session.TargetId)
 	assert.Equal(t, newHostId, session.HostId)
-	require.Equal(t, "active", session.Status)
 
 	// Cancel session
 	output = e2e.RunCommand(ctx, "boundary",