From 147dcaafb7b650426c9de51076590b956e990a0b Mon Sep 17 00:00:00 2001 From: Michael Li Date: Mon, 23 Mar 2026 15:03:53 -0400 Subject: [PATCH] fix(e2e): Address TestCliSearch flakiness (#6496) * refact(e2e): Create start/stop cache methods * fix(e2e): Address potential flake * CR * CR: Improve check * CR --- .../internal/e2e/tests/base/search_test.go | 260 +++++++++--------- 1 file changed, 126 insertions(+), 134 deletions(-) diff --git a/testing/internal/e2e/tests/base/search_test.go b/testing/internal/e2e/tests/base/search_test.go index e11798665..6c7b00c52 100644 --- a/testing/internal/e2e/tests/base/search_test.go +++ b/testing/internal/e2e/tests/base/search_test.go @@ -8,6 +8,8 @@ import ( "encoding/json" "errors" "fmt" + "os" + "slices" "strconv" "strings" "testing" @@ -22,7 +24,6 @@ import ( "github.com/hashicorp/boundary/testing/internal/e2e/boundary" "github.com/hashicorp/boundary/version" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" ) // TestCliSearch asserts that the CLI can search for targets @@ -37,49 +38,18 @@ func TestCliSearch(t *testing.T) { // shorter refresh interval output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) if output.Err == nil { - t.Log("Stopping cache...") - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop")) - require.NoError(t, output.Err, string(output.Stderr)) + err := stopCacheAndWait(t, ctx) + require.NoError(t, err) } - output = e2e.RunCommand(ctx, "boundary", - e2e.WithArgs( - "cache", "start", - "-refresh-interval", "5s", - "-background", - ), - ) - require.NoError(t, output.Err, string(output.Stderr)) - t.Cleanup(func() { - ctx := context.Background() - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop")) - require.NoError(t, output.Err, string(output.Stderr)) - }) - // Wait for cache to be up and running - t.Log("Waiting for cache to start...") - var statusResult clientcache.StatusResult - err = backoff.RetryNotify( - func() error { - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) - if output.Err != nil { - return errors.New(strings.TrimSpace(string(output.Stderr))) - } - - err = json.Unmarshal(output.Stdout, &statusResult) - if err != nil { - return backoff.Permanent(err) - } - - return nil - }, - backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), - func(err error, td time.Duration) { - t.Logf("%s. Retrying...", err.Error()) - }, - ) + statusResult, err := startCacheAndWait(t, ctx) require.NoError(t, err) - require.Equal(t, statusResult.StatusCode, 200) require.GreaterOrEqual(t, statusResult.Item.Uptime, 0*time.Second) + require.Equal(t, 200, statusResult.StatusCode) + t.Cleanup(func() { + output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop")) + require.NoError(t, err) + }) // Confirm cache version matches CLI version output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("version", "-format", "json")) @@ -212,7 +182,6 @@ func TestCliSearch(t *testing.T) { t.Log(string(output.Stdout)) return errors.New("No users are appearing in the status") } - idx := slices.IndexFunc( statusResult.Item.Users[0].Resources, func(r clientcache.ResourceStatus) bool { @@ -368,56 +337,12 @@ func TestCliSearch(t *testing.T) { ) // Restart cache and confirm search works - t.Log("Stopping cache...") - output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop")) - require.NoError(t, output.Err, string(output.Stderr)) - err = backoff.RetryNotify( - func() error { - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) - - if strings.Contains(string(output.Stderr), "The cache process is not running") { - return nil - } - - return fmt.Errorf("Waiting for cache to stop: %s", string(output.Stdout)) - }, - backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), - func(err error, td time.Duration) { - t.Logf("%s. Retrying...", err.Error()) - }, - ) + err = stopCacheAndWait(t, ctx) require.NoError(t, err) - t.Log("Starting cache...") - output = e2e.RunCommand(ctx, "boundary", - e2e.WithArgs( - "cache", "start", - "-refresh-interval", "5s", - "-background", - ), - ) - require.NoError(t, output.Err, string(output.Stderr)) - err = backoff.RetryNotify( - func() error { - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) - if output.Err != nil { - return errors.New(strings.TrimSpace(string(output.Stderr))) - } - - err = json.Unmarshal(output.Stdout, &statusResult) - if err != nil { - return backoff.Permanent(err) - } - - return nil - }, - backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), - func(err error, td time.Duration) { - t.Logf("%s. Retrying...", err.Error()) - }, - ) + statusResult, err = startCacheAndWait(t, ctx) require.NoError(t, err) - require.Equal(t, statusResult.StatusCode, 200) require.GreaterOrEqual(t, statusResult.Item.Uptime, 0*time.Second) + require.Equal(t, 200, statusResult.StatusCode) output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs( "search", @@ -434,56 +359,13 @@ func TestCliSearch(t *testing.T) { // Log out and restart cache. Log in and confirm search works output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("logout")) - t.Log("Stopping cache...") - output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop")) require.NoError(t, output.Err, string(output.Stderr)) - err = backoff.RetryNotify( - func() error { - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) - - if strings.Contains(string(output.Stderr), "The cache process is not running") { - return nil - } - - return fmt.Errorf("Waiting for cache to stop: %s", string(output.Stdout)) - }, - backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), - func(err error, td time.Duration) { - t.Logf("%s. Retrying...", err.Error()) - }, - ) + err = stopCacheAndWait(t, ctx) require.NoError(t, err) - t.Log("Starting cache...") - output = e2e.RunCommand(ctx, "boundary", - e2e.WithArgs( - "cache", "start", - "-refresh-interval", "5s", - "-background", - ), - ) - require.NoError(t, output.Err, string(output.Stderr)) - err = backoff.RetryNotify( - func() error { - output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) - if output.Err != nil { - return errors.New(strings.TrimSpace(string(output.Stderr))) - } - - err = json.Unmarshal(output.Stdout, &statusResult) - if err != nil { - return backoff.Permanent(err) - } - - return nil - }, - backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), - func(err error, td time.Duration) { - t.Logf("%s. Retrying...", err.Error()) - }, - ) + statusResult, err = startCacheAndWait(t, ctx) require.NoError(t, err) - require.Equal(t, statusResult.StatusCode, 200) require.GreaterOrEqual(t, statusResult.Item.Uptime, 0*time.Second) + require.Equal(t, 200, statusResult.StatusCode) boundary.AuthenticateAdminCli(t, ctx) output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs( @@ -499,3 +381,113 @@ func TestCliSearch(t *testing.T) { require.NoError(t, err) require.Len(t, searchResult.Item.Targets, len(targetIds)) } + +func startCacheAndWait(t *testing.T, ctx context.Context) (clientcache.StatusResult, error) { + t.Helper() + t.Log("Starting cache...") + output := e2e.RunCommand(ctx, "boundary", + e2e.WithArgs( + "cache", "start", + "-refresh-interval", "5s", + "-background", + ), + ) + if output.Err != nil { + return clientcache.StatusResult{}, errors.New(string(output.Stderr)) + } + + var statusResult clientcache.StatusResult + err := backoff.RetryNotify( + func() error { + output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) + if output.Err != nil { + return errors.New(strings.TrimSpace(string(output.Stderr))) + } + + statusResult = clientcache.StatusResult{} + err := json.Unmarshal(output.Stdout, &statusResult) + if err != nil { + return backoff.Permanent(err) + } + + return nil + }, + backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), + func(err error, td time.Duration) { + t.Logf("%s. Retrying...", err.Error()) + }, + ) + + return statusResult, err +} + +func stopCacheAndWait(t *testing.T, ctx context.Context) error { + t.Helper() + + // Get latest status if cache is already running. + var statusResult clientcache.StatusResult + output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) + if output.Err == nil { + err := json.Unmarshal(output.Stdout, &statusResult) + if err != nil { + return err + } + } + + t.Log("Stopping cache...") + output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop")) + if output.Err != nil { + return errors.New(string(output.Stderr)) + } + err := backoff.RetryNotify( + func() error { + output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json")) + // When the cache is not running, `boundary cache status` exits non-zero + // and prints "The cache process is not running." to stderr. Treat that + // condition as success, even if output.Err is non-nil. + if strings.Contains(string(output.Stderr), "The cache process is not running.") { + return nil + } + + // For other non-zero exits, surface the error and retry. + if output.Err != nil { + return errors.New(strings.TrimSpace(string(output.Stderr))) + } + + // Cache is still running; keep retrying until it stops. + return fmt.Errorf("waiting for cache to stop: %s", strings.TrimSpace(string(output.Stdout))) + }, + backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), + func(err error, td time.Duration) { + t.Logf("%s. Retrying...", err.Error()) + }, + ) + if err != nil { + return err + } + + // check domain socket + if statusResult.Item.SocketAddress != "" { + t.Log("Checking if cache socket is cleaned up...") + err = backoff.RetryNotify( + func() error { + _, statErr := os.Stat(statusResult.Item.SocketAddress) + if statErr == nil { + return errors.New("cache socket still exists") + } + + if os.IsNotExist(statErr) { + return nil + } + + return backoff.Permanent(fmt.Errorf("failed to stat cache socket: %w", statErr)) + }, + backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5), + func(err error, td time.Duration) { + t.Logf("%s. Retrying...", err.Error()) + }, + ) + } + + return err +}