From 94c32d217364e6bd467d677da67be0713625012e Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 17 Nov 2025 12:38:34 -0500 Subject: [PATCH 1/2] ce: refact(e2e): Update WaitForSessionCli to accept e2e options (cherry picked from commit ab3b5fb361e6f2f854f24306979565a28664fc0a) --- testing/internal/e2e/boundary/session.go | 15 ++++++++++++--- .../tests/aws/dynamichostcatalog_host_set_test.go | 2 +- .../e2e/tests/base/bytes_up_down_empty_test.go | 2 +- .../internal/e2e/tests/base/bytes_up_down_test.go | 2 +- .../e2e/tests/base/session_cancel_admin_test.go | 2 +- .../e2e/tests/base/session_cancel_group_test.go | 2 +- .../e2e/tests/base/session_cancel_user_test.go | 2 +- .../base/session_end_delete_host_set_test.go | 2 +- .../tests/base/session_end_delete_host_test.go | 2 +- .../tests/base/session_end_delete_project_test.go | 2 +- .../tests/base/session_end_delete_target_test.go | 2 +- .../tests/base/session_end_delete_user_test.go | 2 +- .../target_tcp_connect_connection_limits_test.go | 2 +- ...t_tcp_connect_exec_long_lasting_script_test.go | 2 +- .../tests/base/target_tcp_connect_go_ssh_test.go | 2 +- .../tests/base/target_tcp_connect_http_test.go | 2 +- .../base/target_tcp_connect_listen_port_test.go | 4 ++-- .../base/target_tcp_connect_localhost_test.go | 2 +- .../e2e/tests/base/target_tcp_connect_scp_test.go | 2 +- ...target_tcp_connect_session_max_seconds_test.go | 4 ++-- 20 files changed, 33 insertions(+), 24 deletions(-) diff --git a/testing/internal/e2e/boundary/session.go b/testing/internal/e2e/boundary/session.go index ef62254c1..6ddb62873 100644 --- a/testing/internal/e2e/boundary/session.go +++ b/testing/internal/e2e/boundary/session.go @@ -20,19 +20,28 @@ import ( // WaitForSessionCli waits for sessions to appear in the session list and returns the most recent session // information. If the WithExpectedSessionsCount option is provided, expects to find specified amount of sessions // or only 1 otherwise. -func WaitForSessionCli(t testing.TB, ctx context.Context, projectId string, opt ...SessionOption) *sessions.Session { - opts := getSessionOpts(opt...) +func WaitForSessionCli(t testing.TB, ctx context.Context, projectId string, sessionOpts []SessionOption, e2eOpts ...e2e.Option) *sessions.Session { + opts := getSessionOpts(sessionOpts...) + + var options []e2e.Option + options = append(options, + e2e.WithArgs("sessions", "list", "-scope-id", projectId, "-include-terminated", "-format", "json"), + ) + options = append(options, e2eOpts...) + t.Log("Waiting for session to appear...") var session *sessions.Session + err := backoff.RetryNotify( func() error { // List sessions output := e2e.RunCommand(ctx, "boundary", - e2e.WithArgs("sessions", "list", "-scope-id", projectId, "-include-terminated", "-format", "json"), + options..., ) if output.Err != nil { return backoff.Permanent(errors.New(string(output.Stderr))) } + var sessionListResult sessions.SessionListResult err := json.Unmarshal(output.Stdout, &sessionListResult) if err != nil { diff --git a/testing/internal/e2e/tests/aws/dynamichostcatalog_host_set_test.go b/testing/internal/e2e/tests/aws/dynamichostcatalog_host_set_test.go index 91b5eedb5..c131391d7 100644 --- a/testing/internal/e2e/tests/aws/dynamichostcatalog_host_set_test.go +++ b/testing/internal/e2e/tests/aws/dynamichostcatalog_host_set_test.go @@ -156,7 +156,7 @@ func TestCliCreateAwsDynamicHostCatalogWithHostSet(t *testing.T) { }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) output = e2e.RunCommand(ctx, "boundary", diff --git a/testing/internal/e2e/tests/base/bytes_up_down_empty_test.go b/testing/internal/e2e/tests/base/bytes_up_down_empty_test.go index a205dd9d8..527436581 100644 --- a/testing/internal/e2e/tests/base/bytes_up_down_empty_test.go +++ b/testing/internal/e2e/tests/base/bytes_up_down_empty_test.go @@ -70,7 +70,7 @@ func TestCliBytesUpDownEmpty(t *testing.T) { }() t.Cleanup(cancel) - session := boundary.WaitForSessionCli(t, ctx, projectId) + session := boundary.WaitForSessionCli(t, ctx, projectId, nil) assert.Equal(t, targetId, session.TargetId) // Wait until bytes up and down is greater than 0 diff --git a/testing/internal/e2e/tests/base/bytes_up_down_test.go b/testing/internal/e2e/tests/base/bytes_up_down_test.go index df19a963b..731bb3f6d 100644 --- a/testing/internal/e2e/tests/base/bytes_up_down_test.go +++ b/testing/internal/e2e/tests/base/bytes_up_down_test.go @@ -73,7 +73,7 @@ func TestCliBytesUpDownTransferData(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) diff --git a/testing/internal/e2e/tests/base/session_cancel_admin_test.go b/testing/internal/e2e/tests/base/session_cancel_admin_test.go index de642d703..bdaa22a35 100644 --- a/testing/internal/e2e/tests/base/session_cancel_admin_test.go +++ b/testing/internal/e2e/tests/base/session_cancel_admin_test.go @@ -68,7 +68,7 @@ func TestCliSessionCancelAdmin(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/session_cancel_group_test.go b/testing/internal/e2e/tests/base/session_cancel_group_test.go index eae54c98d..b49aff16b 100644 --- a/testing/internal/e2e/tests/base/session_cancel_group_test.go +++ b/testing/internal/e2e/tests/base/session_cancel_group_test.go @@ -145,7 +145,7 @@ func TestCliSessionCancelGroup(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/session_cancel_user_test.go b/testing/internal/e2e/tests/base/session_cancel_user_test.go index 9334aca05..caee20f06 100644 --- a/testing/internal/e2e/tests/base/session_cancel_user_test.go +++ b/testing/internal/e2e/tests/base/session_cancel_user_test.go @@ -138,7 +138,7 @@ func TestCliSessionCancelUser(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/session_end_delete_host_set_test.go b/testing/internal/e2e/tests/base/session_end_delete_host_set_test.go index 603f84b6a..d0b796bdc 100644 --- a/testing/internal/e2e/tests/base/session_end_delete_host_set_test.go +++ b/testing/internal/e2e/tests/base/session_end_delete_host_set_test.go @@ -101,7 +101,7 @@ func TestCliSessionEndWhenHostSetIsDeleted(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/session_end_delete_host_test.go b/testing/internal/e2e/tests/base/session_end_delete_host_test.go index 0b90cc5d2..392edb5d3 100644 --- a/testing/internal/e2e/tests/base/session_end_delete_host_test.go +++ b/testing/internal/e2e/tests/base/session_end_delete_host_test.go @@ -101,7 +101,7 @@ func TestCliSessionEndWhenHostIsDeleted(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/session_end_delete_project_test.go b/testing/internal/e2e/tests/base/session_end_delete_project_test.go index ccabc2604..be21ff9d4 100644 --- a/testing/internal/e2e/tests/base/session_end_delete_project_test.go +++ b/testing/internal/e2e/tests/base/session_end_delete_project_test.go @@ -92,7 +92,7 @@ func TestCliSessionEndWhenProjectIsDeleted(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) diff --git a/testing/internal/e2e/tests/base/session_end_delete_target_test.go b/testing/internal/e2e/tests/base/session_end_delete_target_test.go index d46bddf28..6546cf848 100644 --- a/testing/internal/e2e/tests/base/session_end_delete_target_test.go +++ b/testing/internal/e2e/tests/base/session_end_delete_target_test.go @@ -101,7 +101,7 @@ func TestCliSessionEndWhenTargetIsDeleted(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/session_end_delete_user_test.go b/testing/internal/e2e/tests/base/session_end_delete_user_test.go index d3964dc18..92874d1b1 100644 --- a/testing/internal/e2e/tests/base/session_end_delete_user_test.go +++ b/testing/internal/e2e/tests/base/session_end_delete_user_test.go @@ -106,7 +106,7 @@ func TestCliSessionEndWhenUserIsDeleted(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) assert.Equal(t, targetId, s.TargetId) assert.Equal(t, hostId, s.HostId) diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_connection_limits_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_connection_limits_test.go index 1cc942c49..417953c7b 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_connection_limits_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_connection_limits_test.go @@ -67,7 +67,7 @@ func TestCliTcpTargetConnectTargetWithConnectionLimits(t *testing.T) { }() t.Cleanup(cancel) - boundary.WaitForSessionCli(t, ctx, projectId) + boundary.WaitForSessionCli(t, ctx, projectId, nil) // Start connections. Expect an error once the limit is reached for i := 0; i <= sessionConnectionLimit; i++ { diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_exec_long_lasting_script_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_exec_long_lasting_script_test.go index d9511ca32..e5f1d9a28 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_exec_long_lasting_script_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_exec_long_lasting_script_test.go @@ -75,7 +75,7 @@ func TestCliTcpTargetConnectExecLongLastingScript(t *testing.T) { ) }() t.Cleanup(cancel) - boundary.WaitForSessionCli(t, ctx, projectId) + boundary.WaitForSessionCli(t, ctx, projectId, nil) t.Log("Copying script to host...") output := e2e.RunCommand(ctx, "scp", diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_go_ssh_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_go_ssh_test.go index 37ff2c4ee..fb260b087 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_go_ssh_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_go_ssh_test.go @@ -55,7 +55,7 @@ func TestCliTcpTargetConnectGoSsh(t *testing.T) { }() t.Cleanup(cancel) - boundary.WaitForSessionCli(t, ctx, projectId) + boundary.WaitForSessionCli(t, ctx, projectId, nil) // Connect to the target using the go experimental ssh library privateKeyRaw, err := os.ReadFile(c.TargetSshKeyPath) diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_http_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_http_test.go index 44b273efa..7b2b9a1a8 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_http_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_http_test.go @@ -79,7 +79,7 @@ Server: netcat-can-you-believe-it _, _ = io.Copy(io.Discard, f) // Not checking error here since it will return an error on session close }() - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) // Create http target and connect to it diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_listen_port_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_listen_port_test.go index 1bc15b903..05dc66b53 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_listen_port_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_listen_port_test.go @@ -57,7 +57,7 @@ func TestCliTcpTargetConnectListenPort(t *testing.T) { }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusPending.String()) t.Logf("Successfully connected to target with listen port %s", ListenPort) @@ -118,7 +118,7 @@ func TestCliTcpTargetConnectDefaultClientPort(t *testing.T) { }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusPending.String()) t.Logf("Successfully connected to target with client port %d", DefaultClientPort) diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_localhost_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_localhost_test.go index 17de377a9..7b26deab9 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_localhost_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_localhost_test.go @@ -65,7 +65,7 @@ func TestCliTcpTargetConnectTargetWithLocalhost(t *testing.T) { }() t.Cleanup(cancel) - boundary.WaitForSessionCli(t, ctx, projectId) + boundary.WaitForSessionCli(t, ctx, projectId, nil) // Connect to target and print host's IP address output := e2e.RunCommand(ctx, "ssh", diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_scp_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_scp_test.go index 0167b1e90..2e3ec4e28 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_scp_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_scp_test.go @@ -64,7 +64,7 @@ func TestCliTcpTargetConnectTargetAndScp(t *testing.T) { }() t.Cleanup(cancel) - boundary.WaitForSessionCli(t, ctx, projectId) + boundary.WaitForSessionCli(t, ctx, projectId, nil) // Create file to scp testDir := t.TempDir() diff --git a/testing/internal/e2e/tests/base/target_tcp_connect_session_max_seconds_test.go b/testing/internal/e2e/tests/base/target_tcp_connect_session_max_seconds_test.go index c47034069..7fb60168b 100644 --- a/testing/internal/e2e/tests/base/target_tcp_connect_session_max_seconds_test.go +++ b/testing/internal/e2e/tests/base/target_tcp_connect_session_max_seconds_test.go @@ -71,7 +71,7 @@ func TestCliTcpTargetConnectTargetWithSessionMaxSecondsTearDown(t *testing.T) { ) }() t.Cleanup(cancel) - s := boundary.WaitForSessionCli(t, ctx, projectId) + s := boundary.WaitForSessionCli(t, ctx, projectId, nil) boundary.WaitForSessionStatusCli(t, ctx, s.Id, session.StatusActive.String()) // Check that session was closed once time limit is reached @@ -140,7 +140,7 @@ func TestCliTcpTargetConnectTargetWithSessionMaxSecondsRejectNew(t *testing.T) { ) }() t.Cleanup(cancel) - boundary.WaitForSessionCli(t, ctx, projectId) + boundary.WaitForSessionCli(t, ctx, projectId, nil) // Start connections. Expect an error once the time limit is reached t.Log("Creating connections...") From 6a0de4337b5467265c2310a917b6d9a705065288 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Tue, 18 Nov 2025 14:28:26 -0500 Subject: [PATCH 2/2] ce: test(e2e): Add connect method that can work in windows (cherry picked from commit 6aa5469160e81e5bb8bf38c8cf6ed0c34d6d81f5) --- testing/internal/e2e/boundary/connect.go | 44 ------------- testing/internal/e2e/helpers.go | 83 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 44 deletions(-) diff --git a/testing/internal/e2e/boundary/connect.go b/testing/internal/e2e/boundary/connect.go index fb437d4f2..ce96f45a5 100644 --- a/testing/internal/e2e/boundary/connect.go +++ b/testing/internal/e2e/boundary/connect.go @@ -7,7 +7,6 @@ import ( "bufio" "context" "encoding/json" - "fmt" "os/exec" "testing" @@ -44,46 +43,3 @@ func ConnectCli(t testing.TB, ctx context.Context, targetId string) ConnectCliOu return connectCliOutput } - -// ConnectCliStdoutPipe uses the boundary CLI to establish connection to the target. -// It captures stdout via a pipe, parses proxy details from the command output, and returns them. -// The connection must be closed separately via the `boundary sessions cancel` command -// This implementation works on cross platforms (Windows, Unix, Linux, macOS) - unlike the pty-based version above (Unix, Linux, macOS). -// StdoutPipe only captures stdout, as opposed to pty which is an interactive pseudo-terminal -func ConnectCliStdoutPipe(t testing.TB, ctx context.Context, targetId string) ConnectCliOutput { - cmd := exec.CommandContext(ctx, - "boundary", "connect", - "-target-id", targetId, - "-format", "json", - ) - - // Capture stdout - stdoutPipe, err := cmd.StdoutPipe() - require.NoError(t, err) - - // Start the command - err = cmd.Start() - require.NoError(t, err) - - // Register cleanup to kill the process - t.Cleanup(func() { - if cmd.Process != nil { - _ = cmd.Process.Kill() - } - }) - - // Read the first line of output (JSON with connection info) - scanner := bufio.NewScanner(stdoutPipe) - var outputLine string - if scanner.Scan() { - outputLine = scanner.Text() - } - require.NoError(t, scanner.Err()) - - // Parse the JSON output - var connectOutput ConnectCliOutput - err = json.Unmarshal([]byte(outputLine), &connectOutput) - require.NoError(t, err, fmt.Sprintf("received output: %s", outputLine)) - - return connectOutput -} diff --git a/testing/internal/e2e/helpers.go b/testing/internal/e2e/helpers.go index 837e38976..a85f77da9 100644 --- a/testing/internal/e2e/helpers.go +++ b/testing/internal/e2e/helpers.go @@ -4,9 +4,12 @@ package e2e import ( + "bufio" "bytes" "context" "errors" + "fmt" + "io" "os" "os/exec" "testing" @@ -101,6 +104,86 @@ func RunCommand(ctx context.Context, command string, opt ...Option) *CommandResu } } +// RunCommandWithPipe runs a command and captures the first line of output from its stdout pipe. +// This implementation works on cross platforms (Windows, Unix, Linux, macOS) - unlike the pty-based version above (Unix, Linux, macOS). +// StdoutPipe only captures stdout, as opposed to pty which is an interactive pseudo-terminal +func RunCommandWithPipe(t testing.TB, ctx context.Context, command string, opt ...Option) (string, error) { + ctxCancel, cancel := context.WithCancel(ctx) + // channels to capture output and errors from goroutine + outputChan := make(chan string, 1) + errorChan := make(chan error, 1) + + // Process options + opts := getOpts(opt...) + + // Build command with args + var cmd *exec.Cmd + if opts.withArgs == nil { + cmd = exec.CommandContext(ctxCancel, command) + } else { + cmd = exec.CommandContext(ctxCancel, command, opts.withArgs...) + } + + // Apply environment variables + if opts.withEnv != nil { + cmd.Env = os.Environ() + for k, v := range opts.withEnv { + cmd.Env = append(cmd.Env, k+"="+v) + } + } + + // Run goroutine in background + go func() { + // Capture stdout via pipe + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + errorChan <- fmt.Errorf("failed to create standard out pipe: %w", err) + return + } + + // Start the command (don't wait for it to finish) + err = cmd.Start() + if err != nil { + errorChan <- fmt.Errorf("failed to start command: %w", err) + return + } + + // Read first line of output and send to channel + scanner := bufio.NewScanner(stdoutPipe) + if scanner.Scan() { + outputChan <- scanner.Text() + } else { + if err := scanner.Err(); err != nil { + errorChan <- fmt.Errorf("failed read output from pipe: %w", err) + } else { + errorChan <- errors.New("no output from command") + } + } + // Command continues running after this as proxy connection is intended to stay open + // We only need the first line of output + + // Continuously drains the stdout pipe in the background to prevent the command from blocking + go func() { + _, _ = io.Copy(io.Discard, stdoutPipe) + }() + }() + + // Cleanup kills the process + t.Cleanup(func() { + cancel() + }) + + // Return result of goroutine + select { + case output := <-outputChan: + return output, nil + case err := <-errorChan: + return "", err + case <-ctxCancel.Done(): + return "", ctxCancel.Err() + } +} + // WithArgs is an option to RunCommand that allows the user to specify arguments // for the provided command. This option can be used multiple times in one command. //