From e3f19279c7061a28179c812beb9d03991c319d7f Mon Sep 17 00:00:00 2001 From: Michael Li Date: Fri, 10 Apr 2026 09:19:24 -0400 Subject: [PATCH] fix(e2e): Add debug for flaky bytes up down test (#6572) * fix(e2e): Increase number of tries in bytes up down test * test(e2e): Additional debug --- .../base_connect/bytes_up_down_empty_test.go | 18 ++++++++--- .../tests/base_connect/bytes_up_down_test.go | 31 +++++++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/testing/internal/e2e/tests/base_connect/bytes_up_down_empty_test.go b/testing/internal/e2e/tests/base_connect/bytes_up_down_empty_test.go index 29a18c36d..52fb55a83 100644 --- a/testing/internal/e2e/tests/base_connect/bytes_up_down_empty_test.go +++ b/testing/internal/e2e/tests/base_connect/bytes_up_down_empty_test.go @@ -95,8 +95,18 @@ func TestCliBytesUpDownEmpty(t *testing.T) { return fmt.Errorf("no connections found in session") } - bytesUp = int(newSessionReadResult.Item.Connections[0].BytesUp) - bytesDown = int(newSessionReadResult.Item.Connections[0].BytesDown) + conn := newSessionReadResult.Item.Connections[0] + bytesUp = int(conn.BytesUp) + bytesDown = int(conn.BytesDown) + + // Log connection details for debugging rare failures + t.Logf("Connection - bytes_up: %d, bytes_down: %d, closed_reason: %q, client: %s:%d, endpoint: %s:%d, session status: %s", + bytesUp, bytesDown, conn.ClosedReason, + conn.ClientTcpAddress, conn.ClientTcpPort, + conn.EndpointTcpAddress, conn.EndpointTcpPort, + newSessionReadResult.Item.Status, + ) + if bytesUp <= 0 || bytesDown <= 0 { return fmt.Errorf( "bytes_up: %d, bytes_down: %d, bytes_up or bytes_down is not greater than 0", @@ -108,7 +118,7 @@ func TestCliBytesUpDownEmpty(t *testing.T) { t.Logf("bytes_up: %d, bytes_down: %d", bytesUp, bytesDown) return nil }, - backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 6), + backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 20), func(err error, td time.Duration) { t.Logf("%s. Retrying...", err.Error()) }, @@ -117,7 +127,7 @@ func TestCliBytesUpDownEmpty(t *testing.T) { // Confirm that bytesUp and bytesDown do not change t.Log("Verifying bytes_up/bytes_down values do not change...") - for i := 0; i < 5; i++ { + for i := 0; i < 10; i++ { time.Sleep(3 * time.Second) output := e2e.RunCommand(ctx, "boundary", diff --git a/testing/internal/e2e/tests/base_connect/bytes_up_down_test.go b/testing/internal/e2e/tests/base_connect/bytes_up_down_test.go index 399810aa4..f41a72b30 100644 --- a/testing/internal/e2e/tests/base_connect/bytes_up_down_test.go +++ b/testing/internal/e2e/tests/base_connect/bytes_up_down_test.go @@ -99,8 +99,34 @@ func TestCliBytesUpDownTransferData(t *testing.T) { return fmt.Errorf("no connections found in session") } - bytesUp = int(newSessionReadResult.Item.Connections[0].BytesUp) - bytesDown = int(newSessionReadResult.Item.Connections[0].BytesDown) + conn := newSessionReadResult.Item.Connections[0] + bytesUp = int(conn.BytesUp) + bytesDown = int(conn.BytesDown) + + // Log connection details for debugging rare failures + t.Logf("Connection - bytes_up: %d, bytes_down: %d, closed_reason: %q, client: %s:%d, endpoint: %s:%d, session status: %s", + bytesUp, bytesDown, conn.ClosedReason, + conn.ClientTcpAddress, conn.ClientTcpPort, + conn.EndpointTcpAddress, conn.EndpointTcpPort, + newSessionReadResult.Item.Status, + ) + + // Check if the SSH command failed early + select { + case result := <-errChan: + if result.Err != nil { + return backoff.Permanent(fmt.Errorf("SSH command failed, stdout: %s, stderr: %s", string(result.Stdout), string(result.Stderr))) + } + return backoff.Permanent(fmt.Errorf("SSH command completed unexpectedly early, stdout: %s, stderr: %s", string(result.Stdout), string(result.Stderr))) + default: + // Command still running, continue checking bytes + } + + // Check if connection was closed prematurely + if conn.ClosedReason != "" { + return backoff.Permanent(fmt.Errorf("connection closed before data transmission: %s", conn.ClosedReason)) + } + if bytesUp <= 0 || bytesDown <= 0 { return fmt.Errorf( "bytes_up: %d, bytes_down: %d, bytes_up or bytes_down is not greater than 0", @@ -109,7 +135,6 @@ func TestCliBytesUpDownTransferData(t *testing.T) { ) } - t.Logf("bytes_up: %d, bytes_down: %d", bytesUp, bytesDown) return nil }, backoff.WithMaxRetries(backoff.NewConstantBackOff(3*time.Second), 20),