package worker import ( "context" "testing" "time" "github.com/hashicorp/boundary/internal/cmd/config" "github.com/hashicorp/boundary/internal/daemon/controller" "github.com/hashicorp/boundary/internal/server" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/connectivity" ) func TestDeleteConnectedWorkers(t *testing.T) { ctx := context.Background() logger := hclog.New(&hclog.LoggerOptions{ Level: hclog.Trace, }) conf, err := config.DevController() require.NoError(t, err) c := controller.NewTestController(t, &controller.TestControllerOpts{ Config: conf, Logger: logger.Named("controller"), }) t.Cleanup(c.Shutdown) _, directPkiWorker, multiHoppedPkiWorker := NewTestMultihopWorkers(t, logger, c.Context(), c.ClusterAddrs(), c.Config().WorkerAuthKms, c.Controller().ServersRepoFn, nil, nil) serverRepo, err := c.Controller().ServersRepoFn() require.NoError(t, err) workers, err := serverRepo.ListWorkers(ctx, []string{"global"}, server.WithLiveness(-1)) require.NoError(t, err) var childWorker *server.Worker var pkiWorker *server.Worker for _, w := range workers { if w.Type == "pki" && w.GetAddress() == multiHoppedPkiWorker.ProxyAddrs()[0] { childWorker = w } if w.Type == "pki" && w.GetAddress() == directPkiWorker.ProxyAddrs()[0] { pkiWorker = w } } require.NotNil(t, childWorker) require.NotNil(t, pkiWorker) cases := []struct { name string workerId string testWorker *TestWorker }{ { name: "multi hop worker", workerId: childWorker.GetPublicId(), testWorker: multiHoppedPkiWorker, }, { name: "directly connected worker", workerId: pkiWorker.GetPublicId(), testWorker: directPkiWorker, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { prevState := tc.testWorker.Worker().GrpcClientConn.GetState() require.NotEqual(t, connectivity.TransientFailure, prevState) require.NotEqual(t, connectivity.Shutdown, prevState) _, err = serverRepo.DeleteWorker(ctx, tc.workerId) require.NoError(t, err) stateChangeCtx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() for { tc.testWorker.Worker().GrpcClientConn.ResetConnectBackoff() if !tc.testWorker.Worker().GrpcClientConn.WaitForStateChange(stateChangeCtx, prevState) { assert.Fail(t, "State didn't change before context timed out") break } newState := tc.testWorker.Worker().GrpcClientConn.GetState() t.Logf("Changed from previous state: %s to new state: %s", prevState, newState) if newState == connectivity.Shutdown || newState == connectivity.TransientFailure { break } prevState = newState } }) } }