feat(workers): worker operational state (#2427)

* feat(workers): worker operational state

internal/daemon/cluster/handlers/worker_service.go

@@ -92,12 +92,23 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 		})
 	}
 
+	if wStat.OperationalState == "" {
+		// If this is an older worker (pre 0.11), it will not have ReleaseVersion and we'll default to active.
+		// Otherwise, default to Uknown.
+		if wStat.ReleaseVersion == "" {
+			wStat.OperationalState = server.ActiveOperationalState.String()
+		} else {
+			wStat.OperationalState = server.UnknownOperationalState.String()
+		}
+	}
+
 	wConf := server.NewWorker(scope.Global.String(),
 		server.WithName(wStat.GetName()),
 		server.WithDescription(wStat.GetDescription()),
 		server.WithAddress(wStat.GetAddress()),
 		server.WithWorkerTags(workerTags...),
-		server.WithReleaseVersion(wStat.ReleaseVersion))
+		server.WithReleaseVersion(wStat.ReleaseVersion),
+		server.WithOperationalState(wStat.OperationalState))
 	opts := []server.Option{server.WithUpdateTags(req.GetUpdateTags())}
 	if wStat.GetPublicId() != "" {
 		opts = append(opts, server.WithPublicId(wStat.GetPublicId()))

internal/daemon/cluster/handlers/worker_service_status_test.go

@@ -723,3 +723,108 @@ func TestStatusWorkerWithKeyId(t *testing.T) {
 		})
 	}
 }
+
+func TestWorkerOperationalStatus(t *testing.T) {
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kms := kms.TestKms(t, conn, wrapper)
+
+	serverRepo, _ := server.NewRepository(rw, rw, kms)
+	serverRepo.UpsertController(ctx, &store.Controller{
+		PrivateId: "test_controller1",
+		Address:   "127.0.0.1",
+	})
+	serversRepoFn := func() (*server.Repository, error) {
+		return serverRepo, nil
+	}
+	sessionRepoFn := func(opt ...session.Option) (*session.Repository, error) {
+		return session.NewRepository(ctx, rw, rw, kms)
+	}
+	connRepoFn := func() (*session.ConnectionRepository, error) {
+		return session.NewConnectionRepository(ctx, rw, rw, kms)
+	}
+
+	worker1 := server.TestKmsWorker(t, conn, wrapper)
+
+	s := handlers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
+	require.NotNil(t, s)
+
+	cases := []struct {
+		name       string
+		wantErr    bool
+		wantErrMsg string
+		req        *pbs.StatusRequest
+		wantState  string
+	}{
+		{
+			name:    "Active worker",
+			wantErr: false,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &pb.ServerWorkerStatus{
+					PublicId:         worker1.GetPublicId(),
+					Name:             worker1.GetName(),
+					Address:          worker1.GetAddress(),
+					OperationalState: "active",
+				},
+			},
+			wantState: "active",
+		},
+		{
+			name:    "Worker in shutdown",
+			wantErr: false,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &pb.ServerWorkerStatus{
+					PublicId:         worker1.GetPublicId(),
+					Name:             worker1.GetName(),
+					Address:          worker1.GetAddress(),
+					OperationalState: "shutdown",
+				},
+			},
+			wantState: "shutdown",
+		},
+		{
+			name:    "No operational state- default to unknown",
+			wantErr: false,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &pb.ServerWorkerStatus{
+					PublicId:       worker1.GetPublicId(),
+					Name:           worker1.GetName(),
+					Address:        worker1.GetAddress(),
+					ReleaseVersion: "Boundary v0.11.0",
+				},
+			},
+			wantState: "unknown",
+		},
+		{
+			name:    "Old worker (empty release version) and  no operational state- default to active",
+			wantErr: false,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &pb.ServerWorkerStatus{
+					PublicId: worker1.GetPublicId(),
+					Name:     worker1.GetName(),
+					Address:  worker1.GetAddress(),
+				},
+			},
+			wantState: "active",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			got, err := s.Status(ctx, tc.req)
+			if tc.wantErr {
+				require.Error(err)
+				assert.Equal(tc.wantErrMsg, err.Error())
+				return
+			}
+			require.NoError(err)
+			require.NotNil(got)
+			repoWorker, err := serverRepo.LookupWorkerByName(ctx, worker1.Name)
+			require.NoError(err)
+			assert.Equal(tc.wantState, repoWorker.OperationalState)
+		})
+	}
+}

internal/daemon/controller/handlers/targets/target_service.go

@@ -813,7 +813,7 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession
 	// worker IDs below is used to contain their IDs in the same order. This is
 	// used to fetch tags for filtering. But we avoid allocation unless we
 	// actually need it.
-	selectedWorkers, err := serversRepo.ListWorkers(ctx, []string{scope.Global.String()})
+	selectedWorkers, err := serversRepo.ListWorkers(ctx, []string{scope.Global.String()}, server.WithExcludeShutdownWorkers(true))
 	if err != nil {
 		return nil, err
 	}

internal/daemon/worker/status.go

@@ -7,13 +7,13 @@ import (
 	"math/rand"
 	"time"
 
-	"github.com/hashicorp/boundary/version"
-
 	"github.com/hashicorp/boundary/internal/daemon/worker/common"
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
 	pb "github.com/hashicorp/boundary/internal/gen/controller/servers"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/version"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 )
 
@@ -165,12 +165,13 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager sess
 	result, err := client.Status(statusCtx, &pbs.StatusRequest{
 		Jobs: activeJobs,
 		WorkerStatus: &pb.ServerWorkerStatus{
-			Name:           w.conf.RawConfig.Worker.Name,
-			Description:    w.conf.RawConfig.Worker.Description,
-			Address:        w.conf.RawConfig.Worker.PublicAddr,
-			Tags:           tags,
-			KeyId:          keyId,
-			ReleaseVersion: versionInfo.FullVersionNumber(false),
+			Name:             w.conf.RawConfig.Worker.Name,
+			Description:      w.conf.RawConfig.Worker.Description,
+			Address:          w.conf.RawConfig.Worker.PublicAddr,
+			Tags:             tags,
+			KeyId:            keyId,
+			ReleaseVersion:   versionInfo.FullVersionNumber(false),
+			OperationalState: w.operationalState.Load().(server.OperationalState).String(),
 		},
 		UpdateTags: w.updateTags.Load(),
 	})

internal/daemon/worker/worker.go

@@ -13,28 +13,28 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/hashicorp/boundary/internal/errors"
-	pb "github.com/hashicorp/boundary/internal/gen/controller/servers"
-	"github.com/hashicorp/nodeenrollment"
-	nodeenet "github.com/hashicorp/nodeenrollment/net"
-	"github.com/mr-tron/base58"
-	"google.golang.org/grpc/resolver/manual"
-
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/daemon/worker/internal/metric"
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	"github.com/hashicorp/boundary/internal/errors"
+	pb "github.com/hashicorp/boundary/internal/gen/controller/servers"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/boundary/internal/server"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/hashicorp/go-secure-stdlib/mlock"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
+	"github.com/hashicorp/nodeenrollment"
+	nodeenet "github.com/hashicorp/nodeenrollment/net"
 	nodeefile "github.com/hashicorp/nodeenrollment/storage/file"
 	"github.com/hashicorp/nodeenrollment/types"
+	"github.com/mr-tron/base58"
 	ua "go.uber.org/atomic"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/resolver/manual"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -90,6 +90,7 @@ type Worker struct {
 	everAuthenticated    *ua.Uint32
 	lastStatusSuccess    *atomic.Value
 	workerStartTime      time.Time
+	operationalState     *atomic.Value
 
 	controllerMultihopConn *atomic.Value
 
@@ -142,6 +143,7 @@ func New(conf *Config) (*Worker, error) {
 		updateTags:             ua.NewBool(false),
 		nonceFn:                base62.Random,
 		WorkerAuthCurrentKeyId: new(ua.String),
+		operationalState:       new(atomic.Value),
 	}
 
 	if downstreamRouterFactory != nil {
@@ -382,6 +384,8 @@ func (w *Worker) Start() error {
 		return errors.Wrap(w.baseContext, err, op, errors.WithMsg("error starting worker listeners"))
 	}
 
+	w.operationalState.Store(server.ActiveOperationalState)
+
 	// Rather than deal with some of the potential error conditions for Add on
 	// the waitgroup vs. Done (in case a function exits immediately), we will
 	// always start rotation and simply exit early if we're using KMS
@@ -430,6 +434,8 @@ func (w *Worker) Shutdown() error {
 		return nil
 	}
 
+	// Set state to shutdown
+	w.operationalState.Store(server.ShutdownOperationalState)
 	// Stop listeners first to prevent new connections to the
 	// controller.
 	defer w.started.Store(false)