Merge remote-tracking branch 'origin/llb-byow'

internal/daemon/cluster/handlers/multihop_service.go

@@ -0,0 +1,113 @@
+package handlers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/multihop"
+	"github.com/hashicorp/nodeenrollment/registration"
+	"github.com/hashicorp/nodeenrollment/rotation"
+	"github.com/hashicorp/nodeenrollment/tls"
+	"github.com/hashicorp/nodeenrollment/types"
+	"github.com/hashicorp/nodeenrollment/util/temperror"
+)
+
+type multihopServiceServer struct {
+	multihop.UnimplementedMultihopServiceServer
+
+	storage nodeenrollment.Storage
+	direct  bool
+	client  *atomic.Value
+	options []nodeenrollment.Option
+}
+
+// NewMultihopServiceServer creates a new service implementing
+// MultihopServiceServer, storing values used for the implementing functions.
+func NewMultihopServiceServer(storage nodeenrollment.Storage, direct bool, client *atomic.Value, opt ...nodeenrollment.Option) (*multihopServiceServer, error) {
+	const op = "cluster.handlers.NewMultihopServiceServer"
+
+	switch {
+	case direct && nodeenrollment.IsNil(storage):
+		return nil, fmt.Errorf("%s: running on controller and nil storage provided", op)
+	case !direct && client == nil:
+		return nil, fmt.Errorf("%s: running on worker and nil client provided", op)
+	}
+
+	return &multihopServiceServer{
+		storage: storage,
+		direct:  direct,
+		client:  client,
+		options: opt,
+	}, nil
+}
+
+var _ multihop.MultihopServiceServer = (*multihopServiceServer)(nil)
+
+// FetchNodeCredentials implements the MultihopServiceServer interface. If it's
+// direct (e.g. running on a controller) it handles the request directly,
+// otherwise sends it to its next hop.
+func (m *multihopServiceServer) FetchNodeCredentials(ctx context.Context, req *types.FetchNodeCredentialsRequest) (*types.FetchNodeCredentialsResponse, error) {
+	const op = "cluster.handlers.(multihopServiceServer).FetchNodeCredentials"
+	switch m.direct {
+	case true:
+		return registration.FetchNodeCredentials(ctx, m.storage, req, m.options...)
+
+	default:
+		client := m.client.Load()
+		if client == nil {
+			return nil, fmt.Errorf("%s: error fetching multihop connection, client is nil", op)
+		}
+		multihopClient, ok := client.(multihop.MultihopServiceClient)
+		if !ok {
+			return nil, temperror.New(errors.New("client could not be understood as a multihop service client"))
+		}
+		return multihopClient.FetchNodeCredentials(ctx, req)
+	}
+}
+
+// GenerateServerCertificates implements the MultihopServiceServer interface. If
+// it's direct (e.g. running on a controller) it handles the request directly,
+// otherwise sends it to its next hop.
+func (m *multihopServiceServer) GenerateServerCertificates(ctx context.Context, req *types.GenerateServerCertificatesRequest) (*types.GenerateServerCertificatesResponse, error) {
+	const op = "cluster.handlers.(multihopServiceServer).GenerateServerCertificates"
+	switch m.direct {
+	case true:
+		return tls.GenerateServerCertificates(ctx, m.storage, req, m.options...)
+
+	default:
+		client := m.client.Load()
+		if client == nil {
+			return nil, fmt.Errorf("%s: error fetching multihop connection, client is nil", op)
+		}
+		multihopClient, ok := client.(multihop.MultihopServiceClient)
+		if !ok {
+			return nil, temperror.New(errors.New("client could not be understood as a multihop service client"))
+		}
+		return multihopClient.GenerateServerCertificates(ctx, req)
+	}
+}
+
+// RotateNodeCredentials implements the MultihopServiceServer interface. If it's
+// direct (e.g. running on a controller) it handles the request directly,
+// otherwise sends it to its next hop.
+func (m *multihopServiceServer) RotateNodeCredentials(ctx context.Context, req *types.RotateNodeCredentialsRequest) (*types.RotateNodeCredentialsResponse, error) {
+	const op = "cluster.handlers.(multihopServiceServer).RotateNodeCredentials"
+	switch m.direct {
+	case true:
+		return rotation.RotateNodeCredentials(ctx, m.storage, req, m.options...)
+
+	default:
+		client := m.client.Load()
+		if client == nil {
+			return nil, fmt.Errorf("%s: error fetching multihop connection, client is nil", op)
+		}
+		multihopClient, ok := client.(multihop.MultihopServiceClient)
+		if !ok {
+			return nil, temperror.New(errors.New("client could not be understood as a multihop service client"))
+		}
+		return multihopClient.RotateNodeCredentials(ctx, req)
+	}
+}

internal/daemon/cluster/handlers/worker_service.go

@@ -0,0 +1,507 @@
+package handlers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/daemon/controller/common"
+	"github.com/hashicorp/boundary/internal/daemon/controller/handlers"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/session"
+	"github.com/hashicorp/boundary/internal/types/scope"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/targets"
+	"github.com/hashicorp/go-bexpr"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+type workerServiceServer struct {
+	pbs.UnimplementedServerCoordinationServiceServer
+	pbs.UnimplementedSessionServiceServer
+
+	serversRepoFn    common.ServersRepoFactory
+	sessionRepoFn    common.SessionRepoFactory
+	connectionRepoFn common.ConnectionRepoFactory
+	updateTimes      *sync.Map
+	kms              *kms.Kms
+}
+
+func NewWorkerServiceServer(
+	serversRepoFn common.ServersRepoFactory,
+	sessionRepoFn common.SessionRepoFactory,
+	connectionRepoFn common.ConnectionRepoFactory,
+	updateTimes *sync.Map,
+	kms *kms.Kms,
+) *workerServiceServer {
+	return &workerServiceServer{
+		serversRepoFn:    serversRepoFn,
+		sessionRepoFn:    sessionRepoFn,
+		connectionRepoFn: connectionRepoFn,
+		updateTimes:      updateTimes,
+		kms:              kms,
+	}
+}
+
+var (
+	_ pbs.SessionServiceServer            = &workerServiceServer{}
+	_ pbs.ServerCoordinationServiceServer = &workerServiceServer{}
+)
+
+func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
+	const op = "workers.(workerServiceServer).Status"
+	// TODO: on the worker, if we get errors back from this repeatedly, do we
+	// terminate all sessions since we can't know if they were canceled?
+
+	wStat := req.GetWorkerStatus()
+	if wStat == nil {
+		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Worker sent nil status.")
+	}
+	switch {
+	case wStat.GetName() == "" && wStat.GetKeyId() == "":
+		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Name and keyId are not set in the request; one is required.")
+	case wStat.GetName() != "" && wStat.GetKeyId() != "":
+		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Name and keyId are both set in the request; only one is allowed.")
+	case wStat.GetAddress() == "":
+		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Address is not set but is required.")
+	}
+	// This Store call is currently only for testing purposes
+	ws.updateTimes.Store(wStat.GetName(), time.Now())
+
+	serverRepo, err := ws.serversRepoFn()
+	if err != nil {
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting servers repo"))
+		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to store worker status: %v", err)
+	}
+
+	// Convert API tags to storage tags
+	wTags := wStat.GetTags()
+	workerTags := make([]*servers.Tag, 0, len(wTags))
+	for _, v := range wTags {
+		workerTags = append(workerTags, &servers.Tag{
+			Key:   v.GetKey(),
+			Value: v.GetValue(),
+		})
+	}
+
+	wConf := servers.NewWorker(scope.Global.String(),
+		servers.WithName(wStat.GetName()),
+		servers.WithDescription(wStat.GetDescription()),
+		servers.WithAddress(wStat.GetAddress()),
+		servers.WithWorkerTags(workerTags...))
+	opts := []servers.Option{servers.WithUpdateTags(req.GetUpdateTags())}
+	if wStat.GetPublicId() != "" {
+		opts = append(opts, servers.WithPublicId(wStat.GetPublicId()))
+	}
+	if wStat.GetKeyId() != "" {
+		opts = append(opts, servers.WithKeyId(wStat.GetKeyId()))
+	}
+	wrk, err := serverRepo.UpsertWorkerStatus(ctx, wConf, opts...)
+	if err != nil {
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error storing worker status"))
+		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error storing worker status: %v", err)
+	}
+	controllers, err := serverRepo.ListControllers(ctx)
+	if err != nil {
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting current controllers"))
+		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting current controllers: %v", err)
+	}
+
+	responseControllers := []*pbs.UpstreamServer{}
+	for _, c := range controllers {
+		thisController := &pbs.UpstreamServer{
+			Address: c.Address,
+			Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+		}
+		responseControllers = append(responseControllers, thisController)
+	}
+	ret := &pbs.StatusResponse{
+		CalculatedUpstreams: responseControllers,
+		WorkerId:            wrk.GetPublicId(),
+	}
+
+	stateReport := make([]session.StateReport, 0, len(req.GetJobs()))
+
+	for _, jobStatus := range req.GetJobs() {
+		switch jobStatus.Job.GetType() {
+		case pbs.JOBTYPE_JOBTYPE_SESSION:
+			si := jobStatus.GetJob().GetSessionInfo()
+			if si == nil {
+				return nil, status.Error(codes.Internal, "Error getting session info at status time")
+			}
+
+			switch si.Status {
+			case pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
+				pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED:
+				// No need to see about canceling anything
+				continue
+			}
+
+			sr := session.StateReport{
+				SessionId:     si.GetSessionId(),
+				ConnectionIds: make([]string, 0, len(si.GetConnections())),
+			}
+			for _, conn := range si.GetConnections() {
+				switch conn.Status {
+				case pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED,
+					pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED:
+					sr.ConnectionIds = append(sr.ConnectionIds, conn.GetConnectionId())
+				}
+			}
+			stateReport = append(stateReport, sr)
+		}
+	}
+
+	sessRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting sessions repo"))
+		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
+	}
+	connectionRepo, err := ws.connectionRepoFn()
+	if err != nil {
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting connection repo"))
+		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
+	}
+
+	notActive, err := session.WorkerStatusReport(ctx, sessRepo, connectionRepo, wrk.GetPublicId(), stateReport)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal,
+			"Error comparing state of sessions for worker with public id %q: %v",
+			wrk.GetPublicId(), err)
+	}
+	for _, na := range notActive {
+		var connChanges []*pbs.Connection
+		for _, connId := range na.ConnectionIds {
+			connChanges = append(connChanges, &pbs.Connection{
+				ConnectionId: connId,
+				Status:       session.StatusClosed.ProtoVal(),
+			})
+		}
+		ret.JobsRequests = append(ret.JobsRequests, &pbs.JobChangeRequest{
+			Job: &pbs.Job{
+				Type: pbs.JOBTYPE_JOBTYPE_SESSION,
+				JobInfo: &pbs.Job_SessionInfo{
+					SessionInfo: &pbs.SessionJobInfo{
+						SessionId:   na.SessionId,
+						Status:      na.Status.ProtoVal(),
+						Connections: connChanges,
+					},
+				},
+			},
+			RequestType: pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE,
+		})
+	}
+
+	return ret, nil
+}
+
+func (ws *workerServiceServer) LookupSession(ctx context.Context, req *pbs.LookupSessionRequest) (*pbs.LookupSessionResponse, error) {
+	const op = "workers.(workerServiceServer).LookupSession"
+
+	sessRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error getting session repo: %v", err)
+	}
+
+	sessionInfo, authzSummary, err := sessRepo.LookupSession(ctx, req.GetSessionId())
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error looking up session: %v", err)
+	}
+	if sessionInfo == nil {
+		return nil, status.Error(codes.PermissionDenied, "Unknown session ID.")
+	}
+	if len(sessionInfo.States) == 0 {
+		return nil, status.Error(codes.Internal, "Empty session states during lookup.")
+	}
+
+	if sessionInfo.WorkerFilter != "" {
+		if req.ServerId == "" {
+			event.WriteError(ctx, op, errors.New("worker filter enabled for session but got no server ID from worker"))
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Did not receive server ID when looking up session but filtering is enabled: %v", err)
+		}
+		serversRepo, err := ws.serversRepoFn()
+		if err != nil {
+			event.WriteError(ctx, op, err, event.WithInfoMsg("error getting servers repo"))
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error acquiring server repo when looking up session: %v", err)
+		}
+		var w *servers.Worker
+		if req.WorkerKeyId != "" {
+			workerId, err := serversRepo.LookupWorkerIdByKeyId(ctx, req.WorkerKeyId)
+			if err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error looking up worker by keyId", "keyId", req.WorkerKeyId))
+				return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error looking up tags for server: %v", err)
+			}
+			w, err = serversRepo.LookupWorker(ctx, workerId)
+			if err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error looking up worker by workerId", "workerId", workerId))
+				return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error looking up tags for server: %v", err)
+			}
+		} else {
+			w, err = serversRepo.LookupWorkerByName(ctx, req.GetServerId())
+			if err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error looking up worker by name", "name", req.ServerId))
+				return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error looking up tags for server: %v", err)
+			}
+		}
+		// Build the map for filtering.
+		tagMap := w.CanonicalTags()
+
+		// Create the evaluator
+		eval, err := bexpr.CreateEvaluator(sessionInfo.WorkerFilter)
+		if err != nil {
+			event.WriteError(ctx, op, err, event.WithInfoMsg("error creating worker filter evaluator", "server_id", req.ServerId))
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error creating worker filter evaluator: %v", err)
+		}
+		filterInput := map[string]interface{}{
+			"name": w.GetName(),
+			"tags": tagMap,
+		}
+		ok, err := eval.Evaluate(filterInput)
+		if err != nil {
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
+				fmt.Sprintf("Worker filter expression evaluation resulted in error: %s", err))
+		}
+		if !ok {
+			return nil, handlers.ApiErrorWithCodeAndMessage(
+				codes.FailedPrecondition,
+				"Worker filter expression precludes this worker from serving this session")
+		}
+	}
+
+	creds, err := sessRepo.ListSessionCredentials(ctx, sessionInfo.ScopeId, sessionInfo.PublicId)
+	if err != nil {
+		return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
+			fmt.Sprintf("Error retrieving session credentials: %s", err))
+	}
+	var workerCreds []*pbs.Credential
+	for _, c := range creds {
+		m := &pbs.Credential{}
+		err = proto.Unmarshal(c, m)
+		if err != nil {
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
+				fmt.Sprintf("Error unmarshaling credentials: %s", err))
+		}
+		workerCreds = append(workerCreds, m)
+	}
+
+	resp := &pbs.LookupSessionResponse{
+		Authorization: &targets.SessionAuthorizationData{
+			SessionId:   sessionInfo.GetPublicId(),
+			Certificate: sessionInfo.Certificate,
+		},
+		Status:          sessionInfo.States[0].Status.ProtoVal(),
+		Version:         sessionInfo.Version,
+		TofuToken:       string(sessionInfo.TofuToken),
+		Endpoint:        sessionInfo.Endpoint,
+		Expiration:      sessionInfo.ExpirationTime.Timestamp,
+		ConnectionLimit: sessionInfo.ConnectionLimit,
+		ConnectionsLeft: authzSummary.ConnectionLimit,
+		HostId:          sessionInfo.HostId,
+		HostSetId:       sessionInfo.HostSetId,
+		TargetId:        sessionInfo.TargetId,
+		UserId:          sessionInfo.UserId,
+		Credentials:     workerCreds,
+	}
+	if resp.ConnectionsLeft != -1 {
+		resp.ConnectionsLeft -= int32(authzSummary.CurrentConnectionCount)
+	}
+
+	wrapper, err := ws.kms.GetWrapper(ctx, sessionInfo.ScopeId, kms.KeyPurposeSessions, kms.WithKeyId(sessionInfo.KeyId))
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error getting sessions wrapper: %v", err)
+	}
+
+	// Derive the private key, which should match. Deriving on both ends allows
+	// us to not store it in the DB.
+	_, resp.Authorization.PrivateKey, err = session.DeriveED25519Key(ctx, wrapper, sessionInfo.UserId, sessionInfo.GetPublicId())
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error deriving session key: %v", err)
+	}
+
+	return resp, nil
+}
+
+func (ws *workerServiceServer) CancelSession(ctx context.Context, req *pbs.CancelSessionRequest) (*pbs.CancelSessionResponse, error) {
+	const op = "workers.(workerServiceServer).CancelSession"
+
+	sessRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+	}
+
+	ses, _, err := sessRepo.LookupSession(ctx, req.GetSessionId())
+	if err != nil {
+		return nil, err
+	}
+
+	ses, err = sessRepo.CancelSession(ctx, req.GetSessionId(), ses.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pbs.CancelSessionResponse{
+		Status: ses.States[0].Status.ProtoVal(),
+	}, nil
+}
+
+func (ws *workerServiceServer) ActivateSession(ctx context.Context, req *pbs.ActivateSessionRequest) (*pbs.ActivateSessionResponse, error) {
+	const op = "workers.(workerServiceServer).ActivateSession"
+
+	sessRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+	}
+
+	sessionInfo, sessionStates, err := sessRepo.ActivateSession(ctx, req.GetSessionId(), req.GetVersion(), []byte(req.GetTofuToken()))
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error looking up session: %v", err)
+	}
+	if sessionInfo == nil {
+		return nil, status.Error(codes.PermissionDenied, "Unknown session ID.")
+	}
+	if len(sessionStates) == 0 {
+		return nil, status.Error(codes.Internal, "Invalid session state in activate response.")
+	}
+
+	return &pbs.ActivateSessionResponse{
+		Status: sessionStates[0].Status.ProtoVal(),
+	}, nil
+}
+
+func (ws *workerServiceServer) AuthorizeConnection(ctx context.Context, req *pbs.AuthorizeConnectionRequest) (*pbs.AuthorizeConnectionResponse, error) {
+	const op = "workers.(workerServiceServer).AuthorizeConnection"
+	connectionRepo, err := ws.connectionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+	}
+
+	sessionRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+	}
+
+	serversRepo, err := ws.serversRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting servers repo: %v", err)
+	}
+	w, err := serversRepo.LookupWorkerByName(ctx, req.GetWorkerId())
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error looking up worker: %v", err)
+	}
+	if w == nil {
+		return nil, status.Errorf(codes.NotFound, "worker not found with name %q", req.GetWorkerId())
+	}
+
+	connectionInfo, connStates, authzSummary, err := session.AuthorizeConnection(ctx, sessionRepo, connectionRepo, req.GetSessionId(), w.GetPublicId())
+	if err != nil {
+		return nil, err
+	}
+	if connectionInfo == nil {
+		return nil, status.Error(codes.Internal, "Invalid authorize connection response.")
+	}
+	if len(connStates) == 0 {
+		return nil, status.Error(codes.Internal, "Invalid connection state in authorize response.")
+	}
+
+	ret := &pbs.AuthorizeConnectionResponse{
+		ConnectionId:    connectionInfo.GetPublicId(),
+		Status:          connStates[0].Status.ProtoVal(),
+		ConnectionsLeft: authzSummary.ConnectionLimit,
+	}
+	if ret.ConnectionsLeft != -1 {
+		ret.ConnectionsLeft -= int32(authzSummary.CurrentConnectionCount)
+	}
+
+	return ret, nil
+}
+
+func (ws *workerServiceServer) ConnectConnection(ctx context.Context, req *pbs.ConnectConnectionRequest) (*pbs.ConnectConnectionResponse, error) {
+	const op = "workers.(workerServiceServer).ConnectConnection"
+	connRepo, err := ws.connectionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+	}
+
+	connectionInfo, connStates, err := connRepo.ConnectConnection(ctx, session.ConnectWith{
+		ConnectionId:       req.GetConnectionId(),
+		ClientTcpAddress:   req.GetClientTcpAddress(),
+		ClientTcpPort:      req.GetClientTcpPort(),
+		EndpointTcpAddress: req.GetEndpointTcpAddress(),
+		EndpointTcpPort:    req.GetEndpointTcpPort(),
+		UserClientIp:       req.GetUserClientIp(),
+	})
+	if err != nil {
+		return nil, err
+	}
+	if connectionInfo == nil {
+		return nil, status.Error(codes.Internal, "Invalid connect connection response.")
+	}
+
+	return &pbs.ConnectConnectionResponse{
+		Status: connStates[0].Status.ProtoVal(),
+	}, nil
+}
+
+func (ws *workerServiceServer) CloseConnection(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) {
+	const op = "workers.(workerServiceServer).CloseConnection"
+	numCloses := len(req.GetCloseRequestData())
+	if numCloses == 0 {
+		return &pbs.CloseConnectionResponse{}, nil
+	}
+
+	closeWiths := make([]session.CloseWith, 0, numCloses)
+	closeIds := make([]string, 0, numCloses)
+
+	for _, v := range req.GetCloseRequestData() {
+		closeIds = append(closeIds, v.GetConnectionId())
+		closeWiths = append(closeWiths, session.CloseWith{
+			ConnectionId: v.GetConnectionId(),
+			BytesUp:      v.GetBytesUp(),
+			BytesDown:    v.GetBytesDown(),
+			ClosedReason: session.ClosedReason(v.GetReason()),
+		})
+	}
+	connRepo, err := ws.connectionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting connection repo: %v", err)
+	}
+
+	sessRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+	}
+
+	closeInfos, err := session.CloseConnections(ctx, sessRepo, connRepo, closeWiths)
+	if err != nil {
+		return nil, err
+	}
+	if closeInfos == nil {
+		return nil, status.Error(codes.Internal, "Invalid close connection response.")
+	}
+
+	closeData := make([]*pbs.CloseConnectionResponseData, 0, numCloses)
+	for _, v := range closeInfos {
+		if v.Connection == nil {
+			return nil, status.Errorf(codes.Internal, "No connection found while closing one of the connection IDs: %v", closeIds)
+		}
+		if len(v.ConnectionStates) == 0 {
+			return nil, status.Errorf(codes.Internal, "No connection states found while closing one of the connection IDs: %v", closeIds)
+		}
+		closeData = append(closeData, &pbs.CloseConnectionResponseData{
+			ConnectionId: v.Connection.GetPublicId(),
+			Status:       v.ConnectionStates[0].Status.ProtoVal(),
+		})
+	}
+
+	ret := &pbs.CloseConnectionResponse{
+		CloseResponseData: closeData,
+	}
+
+	return ret, nil
+}

internal/daemon/cluster/handlers/worker_service_status_test.go

@@ -1,4 +1,4 @@
-package workers_test
+package handlers_test
 
 import (
 	"context"
@@ -8,18 +8,26 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/hashicorp/boundary/internal/authtoken"
-	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/workers"
+	"github.com/hashicorp/boundary/internal/daemon/cluster/handlers"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/servers/store"
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/boundary/internal/target"
 	"github.com/hashicorp/boundary/internal/target/tcp"
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/registration"
+	"github.com/hashicorp/nodeenrollment/rotation"
+	"github.com/hashicorp/nodeenrollment/storage/file"
+	"github.com/hashicorp/nodeenrollment/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func TestStatus(t *testing.T) {
@@ -31,17 +39,10 @@ func TestStatus(t *testing.T) {
 	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 
 	serverRepo, _ := servers.NewRepository(rw, rw, kms)
-	serverRepo.UpsertServer(ctx, &servers.Server{
+	serverRepo.UpsertController(ctx, &store.Controller{
 		PrivateId: "test_controller1",
-		Type:      "controller",
 		Address:   "127.0.0.1",
 	})
-	serverRepo.UpsertServer(ctx, &servers.Server{
-		PrivateId: "test_worker1",
-		Type:      "worker",
-		Address:   "127.0.0.1",
-	})
-
 	serversRepoFn := func() (*servers.Repository, error) {
 		return serverRepo, nil
 	}
@@ -70,7 +71,7 @@ func TestStatus(t *testing.T) {
 		target.WithSessionConnectionLimit(-1),
 	)
 
-	worker1 := session.TestWorker(t, conn, wrapper)
+	worker1 := servers.TestKmsWorker(t, conn, wrapper)
 
 	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
 		UserId:          uId,
@@ -83,14 +84,14 @@ func TestStatus(t *testing.T) {
 		ConnectionLimit: 10,
 	})
 	tofu := session.TestTofu(t)
-	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, worker1.PrivateId, worker1.Type, tofu)
+	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, tofu)
 	require.NoError(t, err)
 	require.NoError(t, err)
 
-	s := workers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
+	s := handlers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
 	require.NotNil(t, s)
 
-	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PrivateId)
+	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PublicId)
 	require.NoError(t, err)
 
 	cases := []struct {
@@ -104,23 +105,31 @@ func TestStatus(t *testing.T) {
 			name:    "No Sessions",
 			wantErr: false,
 			req: &pbs.StatusRequest{
-				Worker: worker1,
+				WorkerStatus: &servers.ServerWorkerStatus{
+					PublicId: worker1.GetPublicId(),
+					Name:     worker1.GetName(),
+					Address:  worker1.GetAddress(),
+				},
 			},
 			want: &pbs.StatusResponse{
-				Controllers: []*servers.Server{
+				CalculatedUpstreams: []*pbs.UpstreamServer{
 					{
-						PrivateId: "test_controller1",
-						Type:      "controller",
-						Address:   "127.0.0.1",
+						Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+						Address: "127.0.0.1",
 					},
 				},
+				WorkerId: worker1.PublicId,
 			},
 		},
 		{
 			name:    "Still Active",
 			wantErr: false,
 			req: &pbs.StatusRequest{
-				Worker: worker1,
+				WorkerStatus: &servers.ServerWorkerStatus{
+					PublicId: worker1.GetPublicId(),
+					Name:     worker1.GetName(),
+					Address:  worker1.GetAddress(),
+				},
 				Jobs: []*pbs.JobStatus{
 					{
 						Job: &pbs.Job{
@@ -142,15 +151,37 @@ func TestStatus(t *testing.T) {
 				},
 			},
 			want: &pbs.StatusResponse{
-				Controllers: []*servers.Server{
+				CalculatedUpstreams: []*pbs.UpstreamServer{
 					{
-						PrivateId: "test_controller1",
-						Type:      "controller",
-						Address:   "127.0.0.1",
+						Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+						Address: "127.0.0.1",
 					},
 				},
+				WorkerId: worker1.PublicId,
 			},
 		},
+		{
+			name:    "No Name or keyId",
+			wantErr: true,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &servers.ServerWorkerStatus{
+					PublicId: worker1.GetPublicId(),
+					Address:  worker1.GetAddress(),
+				},
+			},
+			wantErrMsg: status.Error(codes.InvalidArgument, "Name and keyId are not set in the request; one is required.").Error(),
+		},
+		{
+			name:    "No Address",
+			wantErr: true,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &servers.ServerWorkerStatus{
+					PublicId: worker1.GetPublicId(),
+					Name:     worker1.GetName(),
+				},
+			},
+			wantErrMsg: status.Error(codes.InvalidArgument, "Address is not set but is required.").Error(),
+		},
 	}
 
 	for _, tc := range cases {
@@ -160,7 +191,7 @@ func TestStatus(t *testing.T) {
 			got, err := s.Status(ctx, tc.req)
 			if tc.wantErr {
 				require.Error(err)
-				assert.Nil(got)
+				assert.Equal(got, &pbs.StatusResponse{})
 				assert.Equal(tc.wantErrMsg, err.Error())
 				return
 			}
@@ -170,14 +201,15 @@ func TestStatus(t *testing.T) {
 					got,
 					cmpopts.IgnoreUnexported(
 						pbs.StatusResponse{},
-						servers.Server{},
+						servers.ServerWorkerStatus{},
+						pbs.UpstreamServer{},
 						pbs.JobChangeRequest{},
 						pbs.Job{},
 						pbs.Job_SessionInfo{},
 						pbs.SessionJobInfo{},
 						pbs.Connection{},
 					),
-					cmpopts.IgnoreFields(servers.Server{}, "CreateTime", "UpdateTime"),
+					cmpopts.IgnoreFields(servers.ServerWorkerStatus{}, "Tags"),
 				),
 			)
 		})
@@ -193,17 +225,10 @@ func TestStatusSessionClosed(t *testing.T) {
 	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 
 	serverRepo, _ := servers.NewRepository(rw, rw, kms)
-	serverRepo.UpsertServer(ctx, &servers.Server{
+	serverRepo.UpsertController(ctx, &store.Controller{
 		PrivateId: "test_controller1",
-		Type:      "controller",
 		Address:   "127.0.0.1",
 	})
-	serverRepo.UpsertServer(ctx, &servers.Server{
-		PrivateId: "test_worker1",
-		Type:      "worker",
-		Address:   "127.0.0.1",
-	})
-
 	serversRepoFn := func() (*servers.Repository, error) {
 		return serverRepo, nil
 	}
@@ -232,7 +257,7 @@ func TestStatusSessionClosed(t *testing.T) {
 		target.WithSessionConnectionLimit(-1),
 	)
 
-	worker1 := session.TestWorker(t, conn, wrapper)
+	worker1 := servers.TestKmsWorker(t, conn, wrapper)
 
 	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
 		UserId:          uId,
@@ -245,7 +270,7 @@ func TestStatusSessionClosed(t *testing.T) {
 		ConnectionLimit: 10,
 	})
 	tofu := session.TestTofu(t)
-	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, worker1.PrivateId, worker1.Type, tofu)
+	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, tofu)
 	require.NoError(t, err)
 	sess2 := session.TestSession(t, conn, wrapper, session.ComposedOf{
 		UserId:          uId,
@@ -258,13 +283,13 @@ func TestStatusSessionClosed(t *testing.T) {
 		ConnectionLimit: 10,
 	})
 	tofu2 := session.TestTofu(t)
-	sess2, _, err = repo.ActivateSession(ctx, sess2.PublicId, sess2.Version, worker1.PrivateId, worker1.Type, tofu2)
+	sess2, _, err = repo.ActivateSession(ctx, sess2.PublicId, sess2.Version, tofu2)
 	require.NoError(t, err)
 
-	s := workers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
+	s := handlers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
 	require.NotNil(t, s)
 
-	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PrivateId)
+	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PublicId)
 	require.NoError(t, err)
 
 	cases := []struct {
@@ -283,7 +308,11 @@ func TestStatusSessionClosed(t *testing.T) {
 				require.NoError(t, err)
 			},
 			req: &pbs.StatusRequest{
-				Worker: worker1,
+				WorkerStatus: &servers.ServerWorkerStatus{
+					PublicId: worker1.GetPublicId(),
+					Name:     worker1.GetName(),
+					Address:  worker1.GetAddress(),
+				},
 				Jobs: []*pbs.JobStatus{
 					{
 						Job: &pbs.Job{
@@ -305,11 +334,10 @@ func TestStatusSessionClosed(t *testing.T) {
 				},
 			},
 			want: &pbs.StatusResponse{
-				Controllers: []*servers.Server{
+				CalculatedUpstreams: []*pbs.UpstreamServer{
 					{
-						PrivateId: "test_controller1",
-						Type:      "controller",
-						Address:   "127.0.0.1",
+						Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+						Address: "127.0.0.1",
 					},
 				},
 				JobsRequests: []*pbs.JobChangeRequest{
@@ -326,6 +354,7 @@ func TestStatusSessionClosed(t *testing.T) {
 						RequestType: pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE,
 					},
 				},
+				WorkerId: worker1.PublicId,
 			},
 		},
 	}
@@ -350,14 +379,15 @@ func TestStatusSessionClosed(t *testing.T) {
 					got,
 					cmpopts.IgnoreUnexported(
 						pbs.StatusResponse{},
-						servers.Server{},
+						servers.ServerWorkerStatus{},
+						pbs.UpstreamServer{},
 						pbs.JobChangeRequest{},
 						pbs.Job{},
 						pbs.Job_SessionInfo{},
 						pbs.SessionJobInfo{},
 						pbs.Connection{},
 					),
-					cmpopts.IgnoreFields(servers.Server{}, "CreateTime", "UpdateTime"),
+					cmpopts.IgnoreFields(servers.ServerWorkerStatus{}, "Tags"),
 				),
 			)
 		})
@@ -373,17 +403,13 @@ func TestStatusDeadConnection(t *testing.T) {
 	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 
 	serverRepo, _ := servers.NewRepository(rw, rw, kms)
-	serverRepo.UpsertServer(ctx, &servers.Server{
+	serverRepo.UpsertController(ctx, &store.Controller{
 		PrivateId: "test_controller1",
-		Type:      "controller",
-		Address:   "127.0.0.1",
-	})
-	serverRepo.UpsertServer(ctx, &servers.Server{
-		PrivateId: "test_worker1",
-		Type:      "worker",
 		Address:   "127.0.0.1",
 	})
 
+	worker1 := servers.TestKmsWorker(t, conn, wrapper)
+
 	serversRepoFn := func() (*servers.Repository, error) {
 		return serverRepo, nil
 	}
@@ -412,8 +438,6 @@ func TestStatusDeadConnection(t *testing.T) {
 		target.WithSessionConnectionLimit(-1),
 	)
 
-	worker1 := session.TestWorker(t, conn, wrapper)
-
 	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
 		UserId:          uId,
 		HostId:          h.GetPublicId(),
@@ -425,7 +449,7 @@ func TestStatusDeadConnection(t *testing.T) {
 		ConnectionLimit: 10,
 	})
 	tofu := session.TestTofu(t)
-	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, worker1.PrivateId, worker1.Type, tofu)
+	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, tofu)
 	require.NoError(t, err)
 	sess2 := session.TestSession(t, conn, wrapper, session.ComposedOf{
 		UserId:          uId,
@@ -438,20 +462,24 @@ func TestStatusDeadConnection(t *testing.T) {
 		ConnectionLimit: 10,
 	})
 	tofu2 := session.TestTofu(t)
-	sess2, _, err = repo.ActivateSession(ctx, sess2.PublicId, sess2.Version, worker1.PrivateId, worker1.Type, tofu2)
+	sess2, _, err = repo.ActivateSession(ctx, sess2.PublicId, sess2.Version, tofu2)
 	require.NoError(t, err)
 
-	s := workers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
+	s := handlers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
 	require.NotNil(t, s)
 
-	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PrivateId)
+	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PublicId)
 	require.NoError(t, err)
-	deadConn, _, err := connRepo.AuthorizeConnection(ctx, sess2.PublicId, worker1.PrivateId)
+	deadConn, _, err := connRepo.AuthorizeConnection(ctx, sess2.PublicId, worker1.PublicId)
 	require.NoError(t, err)
 	require.NotEqual(t, deadConn.PublicId, connection.PublicId)
 
 	req := &pbs.StatusRequest{
-		Worker: worker1,
+		WorkerStatus: &servers.ServerWorkerStatus{
+			PublicId: worker1.GetPublicId(),
+			Name:     worker1.GetName(),
+			Address:  worker1.GetAddress(),
+		},
 		Jobs: []*pbs.JobStatus{
 			{
 				Job: &pbs.Job{
@@ -473,13 +501,13 @@ func TestStatusDeadConnection(t *testing.T) {
 		},
 	}
 	want := &pbs.StatusResponse{
-		Controllers: []*servers.Server{
+		CalculatedUpstreams: []*pbs.UpstreamServer{
 			{
-				PrivateId: "test_controller1",
-				Type:      "controller",
-				Address:   "127.0.0.1",
+				Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+				Address: "127.0.0.1",
 			},
 		},
+		WorkerId: worker1.PublicId,
 	}
 
 	got, err := s.Status(ctx, req)
@@ -489,14 +517,15 @@ func TestStatusDeadConnection(t *testing.T) {
 			got,
 			cmpopts.IgnoreUnexported(
 				pbs.StatusResponse{},
-				servers.Server{},
+				servers.ServerWorkerStatus{},
+				pbs.UpstreamServer{},
 				pbs.JobChangeRequest{},
 				pbs.Job{},
 				pbs.Job_SessionInfo{},
 				pbs.SessionJobInfo{},
 				pbs.Connection{},
 			),
-			cmpopts.IgnoreFields(servers.Server{}, "CreateTime", "UpdateTime"),
+			cmpopts.IgnoreFields(servers.ServerWorkerStatus{}, "Tags"),
 		),
 	)
 
@@ -507,3 +536,189 @@ func TestStatusDeadConnection(t *testing.T) {
 	assert.Nil(t, states[0].EndTime)
 	assert.Equal(t, session.StatusClosed, states[0].Status)
 }
+
+func TestStatusWorkerWithKeyId(t *testing.T) {
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kms := kms.TestKms(t, conn, wrapper)
+	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+
+	serverRepo, _ := servers.NewRepository(rw, rw, kms)
+	serverRepo.UpsertController(ctx, &store.Controller{
+		PrivateId: "test_controller1",
+		Address:   "127.0.0.1",
+	})
+	serversRepoFn := func() (*servers.Repository, error) {
+		return serverRepo, nil
+	}
+	sessionRepoFn := func() (*session.Repository, error) {
+		return session.NewRepository(rw, rw, kms)
+	}
+	connRepoFn := func() (*session.ConnectionRepository, error) {
+		return session.NewConnectionRepository(ctx, rw, rw, kms)
+	}
+
+	repo, err := sessionRepoFn()
+	require.NoError(t, err)
+	connRepo, err := connRepoFn()
+	require.NoError(t, err)
+
+	at := authtoken.TestAuthToken(t, conn, kms, org.GetPublicId())
+	uId := at.GetIamUserId()
+	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
+	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
+	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
+	tar := tcp.TestTarget(
+		ctx,
+		t, conn, prj.GetPublicId(), "test",
+		target.WithHostSources([]string{hs.GetPublicId()}),
+		target.WithSessionConnectionLimit(-1),
+	)
+
+	worker1 := servers.TestPkiWorker(t, conn, wrapper)
+
+	rootStorage, err := servers.NewRepositoryStorage(ctx, rw, rw, kms)
+	require.NoError(t, err)
+
+	_, err = rotation.RotateRootCertificates(ctx, rootStorage)
+	require.NoError(t, err)
+
+	// Create struct to pass in with workerId that will be passed along to storage
+	state, err := servers.AttachWorkerIdToState(ctx, worker1.PublicId)
+	require.NoError(t, err)
+
+	// This happens on the worker
+	fileStorage, err := file.New(ctx)
+	require.NoError(t, err)
+	nodeCreds, err := types.NewNodeCredentials(ctx, fileStorage)
+	require.NoError(t, err)
+	// Create request using worker id
+	fetchReq, err := nodeCreds.CreateFetchNodeCredentialsRequest(ctx)
+	require.NoError(t, err)
+
+	// The AuthorizeNode request will result in a WorkerAuth record being stored under the workerId
+	nodeInfo, err := registration.AuthorizeNode(ctx, rootStorage, fetchReq, nodeenrollment.WithState(state))
+	require.NoError(t, err)
+
+	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
+		UserId:          uId,
+		HostId:          h.GetPublicId(),
+		TargetId:        tar.GetPublicId(),
+		HostSetId:       hs.GetPublicId(),
+		AuthTokenId:     at.GetPublicId(),
+		ScopeId:         prj.GetPublicId(),
+		Endpoint:        "tcp://127.0.0.1:22",
+		ConnectionLimit: 10,
+	})
+	tofu := session.TestTofu(t)
+	sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, tofu)
+	require.NoError(t, err)
+	require.NoError(t, err)
+
+	s := handlers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connRepoFn, new(sync.Map), kms)
+	require.NotNil(t, s)
+
+	connection, _, err := connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PublicId)
+	require.NoError(t, err)
+
+	cases := []struct {
+		name       string
+		wantErr    bool
+		wantErrMsg string
+		req        *pbs.StatusRequest
+		want       *pbs.StatusResponse
+	}{
+		{
+			name:    "Identify workerID based on keyId in status",
+			wantErr: false,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &servers.ServerWorkerStatus{
+					Address: "someaddress",
+					KeyId:   nodeInfo.Id,
+				},
+			},
+			want: &pbs.StatusResponse{
+				CalculatedUpstreams: []*pbs.UpstreamServer{
+					{
+						Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+						Address: "127.0.0.1",
+					},
+				},
+				WorkerId: worker1.PublicId,
+			},
+		},
+		{
+			name:    "Active keyId Worker",
+			wantErr: false,
+			req: &pbs.StatusRequest{
+				WorkerStatus: &servers.ServerWorkerStatus{
+					KeyId:   nodeInfo.Id,
+					Address: "someaddress",
+				},
+				Jobs: []*pbs.JobStatus{
+					{
+						Job: &pbs.Job{
+							Type: pbs.JOBTYPE_JOBTYPE_SESSION,
+							JobInfo: &pbs.Job_SessionInfo{
+								SessionInfo: &pbs.SessionJobInfo{
+									SessionId: sess.PublicId,
+									Status:    pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
+									Connections: []*pbs.Connection{
+										{
+											ConnectionId: connection.PublicId,
+											Status:       pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			want: &pbs.StatusResponse{
+				CalculatedUpstreams: []*pbs.UpstreamServer{
+					{
+						Type:    pbs.UpstreamServer_TYPE_CONTROLLER,
+						Address: "127.0.0.1",
+					},
+				},
+				WorkerId: worker1.PublicId,
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+
+			got, err := s.Status(ctx, tc.req)
+			if tc.wantErr {
+				require.Error(err)
+				assert.Equal(got, &pbs.StatusResponse{})
+				assert.Equal(tc.wantErrMsg, err.Error())
+				return
+			}
+			require.NoError(err)
+			assert.Empty(
+				cmp.Diff(
+					tc.want,
+					got,
+					cmpopts.IgnoreUnexported(
+						pbs.StatusResponse{},
+						servers.ServerWorkerStatus{},
+						pbs.UpstreamServer{},
+						pbs.JobChangeRequest{},
+						pbs.Job{},
+						pbs.Job_SessionInfo{},
+						pbs.SessionJobInfo{},
+						pbs.Connection{},
+					),
+					cmpopts.IgnoreFields(servers.ServerWorkerStatus{}, "Tags"),
+				),
+			)
+		})
+	}
+}

internal/daemon/cluster/handlers/worker_service_test.go

@@ -0,0 +1,176 @@
+package handlers_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/hashicorp/boundary/internal/authtoken"
+	"github.com/hashicorp/boundary/internal/daemon/cluster/handlers"
+	"github.com/hashicorp/boundary/internal/db"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	"github.com/hashicorp/boundary/internal/host/static"
+	"github.com/hashicorp/boundary/internal/iam"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/session"
+	"github.com/hashicorp/boundary/internal/target"
+	"github.com/hashicorp/boundary/internal/target/tcp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+)
+
+func TestLookupSession(t *testing.T) {
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kms := kms.TestKms(t, conn, wrapper)
+	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+
+	serversRepoFn := func() (*servers.Repository, error) {
+		return servers.NewRepository(rw, rw, kms)
+	}
+	sessionRepoFn := func() (*session.Repository, error) {
+		return session.NewRepository(rw, rw, kms)
+	}
+	connectionRepoFn := func() (*session.ConnectionRepository, error) {
+		return session.NewConnectionRepository(ctx, rw, rw, kms)
+	}
+
+	at := authtoken.TestAuthToken(t, conn, kms, org.GetPublicId())
+	uId := at.GetIamUserId()
+	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
+	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
+	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
+	tar := tcp.TestTarget(ctx, t, conn, prj.GetPublicId(), "test", target.WithHostSources([]string{hs.GetPublicId()}))
+
+	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
+		UserId:      uId,
+		HostId:      h.GetPublicId(),
+		TargetId:    tar.GetPublicId(),
+		HostSetId:   hs.GetPublicId(),
+		AuthTokenId: at.GetPublicId(),
+		ScopeId:     prj.GetPublicId(),
+		Endpoint:    "tcp://127.0.0.1:22",
+	})
+
+	egressSess := session.TestSession(t, conn, wrapper, session.ComposedOf{
+		UserId:      uId,
+		HostId:      h.GetPublicId(),
+		TargetId:    tar.GetPublicId(),
+		HostSetId:   hs.GetPublicId(),
+		AuthTokenId: at.GetPublicId(),
+		ScopeId:     prj.GetPublicId(),
+		Endpoint:    "tcp://127.0.0.1:22",
+	})
+
+	repo, err := sessionRepoFn()
+	require.NoError(t, err)
+
+	creds := []*pbs.Credential{
+		{
+			Credential: &pbs.Credential_UserPassword{
+				UserPassword: &pbs.UserPassword{
+					Username: "username",
+					Password: "password",
+				},
+			},
+		},
+		{
+			Credential: &pbs.Credential_UserPassword{
+				UserPassword: &pbs.UserPassword{
+					Username: "another-username",
+					Password: "a different password",
+				},
+			},
+		},
+	}
+
+	workerCreds := make([]session.Credential, 0, len(creds))
+	for _, c := range creds {
+		data, err := proto.Marshal(c)
+		require.NoError(t, err)
+		workerCreds = append(workerCreds, data)
+	}
+	err = repo.AddSessionCredentials(ctx, egressSess.ScopeId, egressSess.GetPublicId(), workerCreds)
+	require.NoError(t, err)
+
+	s := handlers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connectionRepoFn, new(sync.Map), kms)
+	require.NotNil(t, s)
+
+	cases := []struct {
+		name       string
+		wantErr    bool
+		wantErrMsg string
+		want       *pbs.LookupSessionResponse
+		sessionId  string
+	}{
+		{
+			name:       "Invalid session id",
+			sessionId:  "s_fakesession",
+			wantErr:    true,
+			wantErrMsg: "rpc error: code = PermissionDenied desc = Unknown session ID.",
+		},
+		{
+			name:      "Valid",
+			sessionId: sess.PublicId,
+			want: &pbs.LookupSessionResponse{
+				ConnectionLimit: 1,
+				ConnectionsLeft: 1,
+				Version:         1,
+				Endpoint:        sess.Endpoint,
+				HostId:          sess.HostId,
+				HostSetId:       sess.HostSetId,
+				TargetId:        sess.TargetId,
+				UserId:          sess.UserId,
+				Status:          pbs.SESSIONSTATUS_SESSIONSTATUS_PENDING,
+			},
+		},
+		{
+			name:      "Valid-with-egress-creds",
+			sessionId: egressSess.PublicId,
+			want: &pbs.LookupSessionResponse{
+				ConnectionLimit: 1,
+				ConnectionsLeft: 1,
+				Version:         1,
+				Endpoint:        egressSess.Endpoint,
+				HostId:          egressSess.HostId,
+				HostSetId:       egressSess.HostSetId,
+				TargetId:        egressSess.TargetId,
+				UserId:          egressSess.UserId,
+				Status:          pbs.SESSIONSTATUS_SESSIONSTATUS_PENDING,
+				Credentials:     creds,
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			req := &pbs.LookupSessionRequest{
+				SessionId: tc.sessionId,
+			}
+
+			got, err := s.LookupSession(ctx, req)
+			if tc.wantErr {
+				require.Error(err)
+				assert.Nil(got)
+				assert.Equal(tc.wantErrMsg, err.Error())
+				return
+			}
+			assert.Empty(
+				cmp.Diff(
+					tc.want,
+					got,
+					cmpopts.IgnoreUnexported(pbs.LookupSessionResponse{}, pbs.Credential{}, pbs.UserPassword{}),
+					cmpopts.IgnoreFields(pbs.LookupSessionResponse{}, "Expiration", "Authorization"),
+				),
+			)
+		})
+	}
+}

internal/daemon/controller/common/common.go

@@ -15,17 +15,18 @@ import (
 )
 
 type (
-	AuthTokenRepoFactory        = oidc.AuthTokenRepoFactory
-	VaultCredentialRepoFactory  = func() (*vault.Repository, error)
-	StaticCredentialRepoFactory = func() (*credstatic.Repository, error)
-	IamRepoFactory              func() (*iam.Repository, error)
-	OidcAuthRepoFactory         = oidc.OidcRepoFactory
-	PasswordAuthRepoFactory     func() (*password.Repository, error)
-	ServersRepoFactory          func() (*servers.Repository, error)
-	StaticRepoFactory           func() (*static.Repository, error)
-	PluginHostRepoFactory       func() (*pluginhost.Repository, error)
-	HostPluginRepoFactory       func() (*hostplugin.Repository, error)
-	SessionRepoFactory          func() (*session.Repository, error)
-	ConnectionRepoFactory       func() (*session.ConnectionRepository, error)
-	TargetRepoFactory           func() (*target.Repository, error)
+	AuthTokenRepoFactory         = oidc.AuthTokenRepoFactory
+	VaultCredentialRepoFactory   = func() (*vault.Repository, error)
+	StaticCredentialRepoFactory  = func() (*credstatic.Repository, error)
+	IamRepoFactory               func() (*iam.Repository, error)
+	OidcAuthRepoFactory          = oidc.OidcRepoFactory
+	PasswordAuthRepoFactory      func() (*password.Repository, error)
+	ServersRepoFactory           func() (*servers.Repository, error)
+	StaticRepoFactory            func() (*static.Repository, error)
+	PluginHostRepoFactory        func() (*pluginhost.Repository, error)
+	HostPluginRepoFactory        func() (*hostplugin.Repository, error)
+	SessionRepoFactory           func() (*session.Repository, error)
+	ConnectionRepoFactory        func() (*session.ConnectionRepository, error)
+	TargetRepoFactory            func() (*target.Repository, error)
+	WorkerAuthRepoStorageFactory func() (*servers.WorkerAuthRepositoryStorage, error)
 )

internal/daemon/controller/controller.go

@@ -28,6 +28,7 @@ import (
 	"github.com/hashicorp/boundary/internal/scheduler"
 	"github.com/hashicorp/boundary/internal/scheduler/job"
 	"github.com/hashicorp/boundary/internal/servers"
+	serversjob "github.com/hashicorp/boundary/internal/servers/job"
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/boundary/internal/target"
 	"github.com/hashicorp/boundary/internal/types/scope"
@@ -65,19 +66,20 @@ type Controller struct {
 	apiGrpcGatewayTicket  string
 
 	// Repo factory methods
-	AuthTokenRepoFn        common.AuthTokenRepoFactory
-	VaultCredentialRepoFn  common.VaultCredentialRepoFactory
-	StaticCredentialRepoFn common.StaticCredentialRepoFactory
-	IamRepoFn              common.IamRepoFactory
-	OidcRepoFn             common.OidcAuthRepoFactory
-	PasswordAuthRepoFn     common.PasswordAuthRepoFactory
-	ServersRepoFn          common.ServersRepoFactory
-	SessionRepoFn          common.SessionRepoFactory
-	ConnectionRepoFn       common.ConnectionRepoFactory
-	StaticHostRepoFn       common.StaticRepoFactory
-	PluginHostRepoFn       common.PluginHostRepoFactory
-	HostPluginRepoFn       common.HostPluginRepoFactory
-	TargetRepoFn           common.TargetRepoFactory
+	AuthTokenRepoFn         common.AuthTokenRepoFactory
+	VaultCredentialRepoFn   common.VaultCredentialRepoFactory
+	StaticCredentialRepoFn  common.StaticCredentialRepoFactory
+	IamRepoFn               common.IamRepoFactory
+	OidcRepoFn              common.OidcAuthRepoFactory
+	PasswordAuthRepoFn      common.PasswordAuthRepoFactory
+	ServersRepoFn           common.ServersRepoFactory
+	SessionRepoFn           common.SessionRepoFactory
+	ConnectionRepoFn        common.ConnectionRepoFactory
+	StaticHostRepoFn        common.StaticRepoFactory
+	PluginHostRepoFn        common.PluginHostRepoFactory
+	HostPluginRepoFn        common.HostPluginRepoFactory
+	TargetRepoFn            common.TargetRepoFactory
+	WorkerAuthRepoStorageFn common.WorkerAuthRepoStorageFactory
 
 	scheduler *scheduler.Scheduler
 
@@ -299,6 +301,10 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 	c.ConnectionRepoFn = func() (*session.ConnectionRepository, error) {
 		return session.NewConnectionRepository(ctx, dbase, dbase, c.kms)
 	}
+	c.WorkerAuthRepoStorageFn = func() (*servers.WorkerAuthRepositoryStorage, error) {
+		return servers.NewRepositoryStorage(ctx, dbase, dbase, c.kms)
+	}
+
 	return c, nil
 }
 
@@ -308,7 +314,9 @@ func (c *Controller) Start() error {
 		event.WriteSysEvent(context.TODO(), op, "already started, skipping")
 		return nil
 	}
+
 	c.baseContext, c.baseCancel = context.WithCancel(context.Background())
+
 	if err := c.registerJobs(); err != nil {
 		return fmt.Errorf("error registering jobs: %w", err)
 	}
@@ -316,9 +324,9 @@ func (c *Controller) Start() error {
 		return fmt.Errorf("error starting controller listeners: %w", err)
 	}
 
-	// Upsert server before starting tickers and scheduler to ensure the server exists
-	if err := c.upsertServer(c.baseContext); err != nil {
-		return fmt.Errorf("error upserting server: %w", err)
+	// Upsert controller before starting tickers and scheduler to ensure the controller exists
+	if err := c.upsertController(c.baseContext); err != nil {
+		return fmt.Errorf("error upserting controller: %w", err)
 	}
 	if err := c.scheduler.Start(c.baseContext, c.schedulerWg); err != nil {
 		return fmt.Errorf("error starting scheduler: %w", err)
@@ -360,6 +368,9 @@ func (c *Controller) registerJobs() error {
 	if err := session.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.conf.StatusGracePeriodDuration); err != nil {
 		return err
 	}
+	if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms); err != nil {
+		return err
+	}
 
 	return nil
 }

internal/daemon/controller/handler.go

@@ -35,6 +35,7 @@ import (
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/sessions"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/targets"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/users"
+	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/workers"
 	"github.com/hashicorp/boundary/internal/daemon/controller/internal/metric"
 	"github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
@@ -224,6 +225,13 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 		}
 		services.RegisterCredentialLibraryServiceServer(s, cl)
 	}
+	if _, ok := currentServices[services.WorkerService_ServiceDesc.ServiceName]; !ok {
+		ws, err := workers.NewService(c.baseContext, c.ServersRepoFn, c.IamRepoFn)
+		if err != nil {
+			return fmt.Errorf("failed to create worker handler service: %w", err)
+		}
+		services.RegisterWorkerServiceServer(s, ws)
+	}
 	if _, ok := currentServices[services.CredentialService_ServiceDesc.ServiceName]; !ok {
 		c, err := credentials.NewService(c.StaticCredentialRepoFn, c.IamRepoFn)
 		if err != nil {
@@ -288,6 +296,9 @@ func registerGrpcGatewayEndpoints(ctx context.Context, gwMux *runtime.ServeMux,
 	if err := services.RegisterCredentialLibraryServiceHandlerFromEndpoint(ctx, gwMux, gatewayTarget, dialOptions); err != nil {
 		return fmt.Errorf("failed to register credential library service handler: %w", err)
 	}
+	if err := services.RegisterWorkerServiceHandlerFromEndpoint(ctx, gwMux, gatewayTarget, dialOptions); err != nil {
+		return fmt.Errorf("failed to register worker service handler: %w", err)
+	}
 	if err := services.RegisterCredentialServiceHandlerFromEndpoint(ctx, gwMux, gatewayTarget, dialOptions); err != nil {
 		return fmt.Errorf("failed to register credential service handler: %w", err)
 	}

internal/daemon/controller/handlers/scopes/scope_service.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/sessions"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/targets"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/users"
+	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/workers"
 	"github.com/hashicorp/boundary/internal/errors"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	"github.com/hashicorp/boundary/internal/iam"
@@ -63,6 +64,7 @@ var (
 			resource.Role:       roles.CollectionActions,
 			resource.Scope:      CollectionActions,
 			resource.User:       users.CollectionActions,
+			resource.Worker:     workers.CollectionActions,
 		},
 
 		scope.Org.String(): {

internal/daemon/controller/handlers/scopes/scope_service_test.go

@@ -95,6 +95,12 @@ var globalAuthorizedCollectionActions = map[string]*structpb.ListValue{
 			structpb.NewStringValue("list"),
 		},
 	},
+	"workers": {
+		Values: []*structpb.Value{
+			structpb.NewStringValue("create:worker-led"),
+			structpb.NewStringValue("list"),
+		},
+	},
 }
 
 var orgAuthorizedCollectionActions = map[string]*structpb.ListValue{

internal/daemon/controller/handlers/targets/target_service.go

@@ -799,77 +799,27 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession
 	// worker IDs below is used to contain their IDs in the same order. This is
 	// used to fetch tags for filtering. But we avoid allocation unless we
 	// actually need it.
-	var workers []*pb.WorkerInfo
-	var workerIds []string
-	hasWorkerFilter := len(t.GetWorkerFilter()) > 0
-	servers, err := serversRepo.ListServers(ctx, servers.ServerTypeWorker)
+	selectedWorkers, err := serversRepo.ListWorkers(ctx, []string{scope.Global.String()})
 	if err != nil {
 		return nil, err
 	}
-	for _, v := range servers {
-		if hasWorkerFilter {
-			workerIds = append(workerIds, v.GetPrivateId())
-		}
-		workers = append(workers, &pb.WorkerInfo{Address: v.Address})
-	}
 
-	if hasWorkerFilter && len(workerIds) > 0 {
-		finalWorkers := make([]*pb.WorkerInfo, 0, len(workers))
-		// Fetch the tags for the given worker IDs
-		tags, err := serversRepo.ListTagsForServers(ctx, workerIds)
-		if err != nil {
-			return nil, err
-		}
-		// Build the map for filtering. This is similar to the filter map we
-		// built from the worker config, but with one extra level: a map of the
-		// worker's ID to its filter map.
-		tagMap := make(map[string]map[string][]string)
-		for _, tag := range tags {
-			currWorkerMap := tagMap[tag.ServerId]
-			if currWorkerMap == nil {
-				currWorkerMap = make(map[string][]string)
-				tagMap[tag.ServerId] = currWorkerMap
-			}
-			currWorkerMap[tag.Key] = append(currWorkerMap[tag.Key], tag.Value)
-			// We don't need to reinsert after the fact because maps are
-			// reference types, so we don't need to re-insert into tagMap
-		}
-
-		// Create the evaluator
+	if len(t.GetWorkerFilter()) > 0 && len(selectedWorkers) > 0 {
 		eval, err := bexpr.CreateEvaluator(t.GetWorkerFilter())
 		if err != nil {
 			return nil, err
 		}
-
-		// Iterate through the known worker IDs, and evaluate. If evaluation
-		// returns true, add to the final worker slice, which is assigned back
-		// to workers after this.
-		for i, worker := range workerIds {
-			filterInput := map[string]interface{}{
-				"name": worker,
-				"tags": tagMap[worker],
-			}
-			ok, err := eval.Evaluate(filterInput)
-			if err != nil && !stderrors.Is(err, pointerstructure.ErrNotFound) {
-				return nil, handlers.ApiErrorWithCodeAndMessage(
-					codes.FailedPrecondition,
-					fmt.Sprintf("Worker filter expression evaluation resulted in error: %s", err))
-			}
-			if ok {
-				finalWorkers = append(finalWorkers, workers[i])
-			}
+		selectedWorkers, err = workerList(selectedWorkers).filtered(eval)
+		if err != nil {
+			return nil, err
 		}
-		workers = finalWorkers
 	}
-	if len(workers) == 0 {
+
+	if len(selectedWorkers) == 0 {
 		return nil, handlers.ApiErrorWithCodeAndMessage(
 			codes.FailedPrecondition,
 			"No workers are available to handle this session, or all have been filtered.")
 	}
-	workerAddresses := make([]string, 0, len(workers))
-	for _, worker := range workers {
-		workerAddresses = append(workerAddresses, worker.GetAddress())
-	}
 
 	requestedId := req.GetHostId()
 	staticHostRepo, err := s.staticHostRepoFn()
@@ -992,7 +942,7 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession
 	if err != nil {
 		return nil, err
 	}
-	sess, privKey, err := sessionRepo.CreateSession(ctx, wrapper, sess, workerAddresses)
+	sess, privKey, err := sessionRepo.CreateSession(ctx, wrapper, sess, workerList(selectedWorkers).addresses())
 	if err != nil {
 		return nil, err
 	}
@@ -1091,7 +1041,7 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession
 		PrivateKey:      privKey,
 		HostId:          chosenEndpoint.HostId,
 		Endpoint:        endpointUrl.String(),
-		WorkerInfo:      workers,
+		WorkerInfo:      workerList(selectedWorkers).workerInfos(),
 		ConnectionLimit: t.GetSessionConnectionLimit(),
 	}
 	marshaledSad, err := proto.Marshal(sad)
@@ -1966,3 +1916,46 @@ func validateAuthorizeSessionRequest(req *pbs.AuthorizeSessionRequest) error {
 	}
 	return nil
 }
+
+// workerList is a helper type to make the selection of workers clearer and more declarative.
+type workerList []*servers.Worker
+
+// addresses converts the slice of workers to a slice of their addresses
+func (w workerList) addresses() []string {
+	ret := make([]string, 0, len(w))
+	for _, worker := range w {
+		ret = append(ret, worker.GetAddress())
+	}
+	return ret
+}
+
+// workerInfos converts the slice of workers to a slice of their workerInfo protos
+func (w workerList) workerInfos() []*pb.WorkerInfo {
+	ret := make([]*pb.WorkerInfo, 0, len(w))
+	for _, worker := range w {
+		ret = append(ret, &pb.WorkerInfo{Address: worker.GetAddress()})
+	}
+	return ret
+}
+
+// filtered returns a new workerList where all elements contained in it are the
+// ones which from the original workerList that pass the evaluator's evaluation.
+func (w workerList) filtered(eval *bexpr.Evaluator) (workerList, error) {
+	var ret []*servers.Worker
+	for _, worker := range w {
+		filterInput := map[string]interface{}{
+			"name": worker.GetName(),
+			"tags": worker.CanonicalTags(),
+		}
+		ok, err := eval.Evaluate(filterInput)
+		if err != nil && !stderrors.Is(err, pointerstructure.ErrNotFound) {
+			return nil, handlers.ApiErrorWithCodeAndMessage(
+				codes.FailedPrecondition,
+				fmt.Sprintf("Worker filter expression evaluation resulted in error: %s", err))
+		}
+		if ok {
+			ret = append(ret, worker)
+		}
+	}
+	return ret, nil
+}

internal/daemon/controller/handlers/targets/target_service_test.go

@@ -0,0 +1,79 @@
+package targets
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/types/scope"
+	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/targets"
+	"github.com/hashicorp/go-bexpr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWorkerList_Addresses(t *testing.T) {
+	addresses := []string{
+		"test1",
+		"test2",
+		"test3",
+		"test4",
+	}
+	var workerInfos []*pb.WorkerInfo
+	var tested workerList
+	for _, a := range addresses {
+		workerInfos = append(workerInfos, &pb.WorkerInfo{Address: a})
+		tested = append(tested, servers.NewWorker(scope.Global.String(),
+			servers.WithName(a),
+			servers.WithAddress(a)))
+	}
+	assert.Equal(t, addresses, tested.addresses())
+	assert.Equal(t, workerInfos, tested.workerInfos())
+}
+
+func TestWorkerList_Filter(t *testing.T) {
+	conn, _ := db.TestSetup(t, "postgres")
+	wrapper := db.TestWrapper(t)
+	var workers []*servers.Worker
+	for i := 0; i < 5; i++ {
+		switch {
+		case i%2 == 0:
+			workers = append(workers, servers.TestKmsWorker(t, conn, wrapper,
+				servers.WithName(fmt.Sprintf("test_worker_%d", i)),
+				servers.WithWorkerTags(&servers.Tag{
+					Key:   fmt.Sprintf("key%d", i),
+					Value: fmt.Sprintf("value%d", i),
+				})))
+		default:
+			workers = append(workers, servers.TestPkiWorker(t, conn, wrapper,
+				servers.WithName(fmt.Sprintf("test_worker_%d", i)),
+				servers.WithWorkerTags(&servers.Tag{
+					Key:   "key",
+					Value: "configvalue",
+				})))
+		}
+	}
+
+	{
+		ev, err := bexpr.CreateEvaluator(`"/name" matches "test_worker_[13]" and "configvalue2" in "/tags/key"`)
+		require.NoError(t, err)
+		got, err := workerList(workers).filtered(ev)
+		require.NoError(t, err)
+		assert.Len(t, got, 0)
+	}
+	{
+		ev, err := bexpr.CreateEvaluator(`"/name" matches "test_worker_[12]" and "configvalue" in "/tags/key"`)
+		require.NoError(t, err)
+		got, err := workerList(workers).filtered(ev)
+		require.NoError(t, err)
+		assert.Len(t, got, 1)
+	}
+	{
+		ev, err := bexpr.CreateEvaluator(`"configvalue" in "/tags/key"`)
+		require.NoError(t, err)
+		got, err := workerList(workers).filtered(ev)
+		require.NoError(t, err)
+		assert.Len(t, got, 2)
+	}
+}

internal/daemon/controller/handlers/targets/tcp/target_service_test.go

@@ -16,12 +16,12 @@ import (
 	"github.com/hashicorp/boundary/internal/credential"
 	credstatic "github.com/hashicorp/boundary/internal/credential/static"
 	"github.com/hashicorp/boundary/internal/credential/vault"
+	cluster "github.com/hashicorp/boundary/internal/daemon/cluster/handlers"
 	"github.com/hashicorp/boundary/internal/daemon/controller/auth"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/credentiallibraries"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/credentials"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/targets"
-	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/workers"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
@@ -2371,9 +2371,6 @@ func TestAuthorizeSession(t *testing.T) {
 	sessionRepoFn := func() (*session.Repository, error) {
 		return session.NewRepository(rw, rw, kms)
 	}
-	connectionRepoFn := func() (*session.ConnectionRepository, error) {
-		return session.NewConnectionRepository(ctx, rw, rw, kms)
-	}
 	staticRepo, err := static.NewRepository(rw, rw, kms)
 	require.NoError(t, err)
 	staticHostRepoFn := func() (*static.Repository, error) {
@@ -2532,14 +2529,7 @@ func TestAuthorizeSession(t *testing.T) {
 			require.NoError(t, err)
 
 			// Tell our DB that there is a worker ready to serve the data
-			workerService := workers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connectionRepoFn, &sync.Map{}, kms)
-			_, err = workerService.Status(ctx, &spbs.StatusRequest{
-				Worker: &spb.Server{
-					PrivateId: "testworker",
-					Address:   "localhost:8457",
-				},
-			})
-			require.NoError(t, err)
+			servers.TestKmsWorker(t, conn, wrapper)
 
 			asRes1, err := s.AuthorizeSession(ctx, &pbs.AuthorizeSessionRequest{
 				Id: tar.GetPublicId(),
@@ -2641,9 +2631,6 @@ func TestAuthorizeSessionTypedCredentials(t *testing.T) {
 	sessionRepoFn := func() (*session.Repository, error) {
 		return session.NewRepository(rw, rw, kms)
 	}
-	connectionRepoFn := func() (*session.ConnectionRepository, error) {
-		return session.NewConnectionRepository(ctx, rw, rw, kms)
-	}
 	staticHostRepoFn := func() (*static.Repository, error) {
 		return static.NewRepository(rw, rw, kms)
 	}
@@ -2887,14 +2874,7 @@ func TestAuthorizeSessionTypedCredentials(t *testing.T) {
 			require.NoError(t, err)
 
 			// Tell our DB that there is a worker ready to serve the data
-			workerService := workers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connectionRepoFn, &sync.Map{}, kms)
-			_, err = workerService.Status(ctx, &spbs.StatusRequest{
-				Worker: &spb.Server{
-					PrivateId: "testworker",
-					Address:   "localhost:8457",
-				},
-			})
-			require.NoError(t, err)
+			servers.TestKmsWorker(t, conn, wrapper)
 
 			asRes, err := s.AuthorizeSession(ctx, &pbs.AuthorizeSessionRequest{
 				Id: tar.GetPublicId(),
@@ -3002,11 +2982,12 @@ func TestAuthorizeSession_Errors(t *testing.T) {
 	store := vault.TestCredentialStore(t, conn, wrapper, proj.GetPublicId(), v.Addr, tok, sec.Auth.Accessor)
 
 	workerExists := func(tar target.Target) (version uint32) {
-		workerService := workers.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connectionRepoFn, &sync.Map{}, kms)
+		workerService := cluster.NewWorkerServiceServer(serversRepoFn, sessionRepoFn, connectionRepoFn, &sync.Map{}, kms)
 		_, err := workerService.Status(context.Background(), &spbs.StatusRequest{
-			Worker: &spb.Server{
-				PrivateId: "testworker",
-				Address:   "localhost:123",
+			WorkerStatus: &spb.ServerWorkerStatus{
+				PublicId: "w_1234567890",
+				Name:     "w_1234567890",
+				Address:  "localhost:123",
 			},
 		})
 		require.NoError(t, err)

internal/daemon/controller/handlers/workers/worker_service.go

@@ -2,432 +2,616 @@ package workers
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"sync"
-	"time"
+	"strings"
 
+	"github.com/hashicorp/boundary/globals"
+	"github.com/hashicorp/boundary/internal/daemon/controller/auth"
 	"github.com/hashicorp/boundary/internal/daemon/controller/common"
+	"github.com/hashicorp/boundary/internal/daemon/controller/common/scopeids"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers"
-	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
-	"github.com/hashicorp/boundary/internal/kms"
-	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/boundary/internal/errors"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	"github.com/hashicorp/boundary/internal/perms"
+	"github.com/hashicorp/boundary/internal/requests"
 	"github.com/hashicorp/boundary/internal/servers"
-	"github.com/hashicorp/boundary/internal/session"
+	"github.com/hashicorp/boundary/internal/servers/store"
+	"github.com/hashicorp/boundary/internal/types/action"
 	"github.com/hashicorp/boundary/internal/types/resource"
-	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/targets"
-	"github.com/hashicorp/go-bexpr"
+	"github.com/hashicorp/boundary/internal/types/scope"
+	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/workers"
+	"github.com/hashicorp/go-secure-stdlib/strutil"
+	"github.com/hashicorp/nodeenrollment/types"
+	"github.com/mr-tron/base58"
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
-type workerServiceServer struct {
-	pbs.UnimplementedServerCoordinationServiceServer
-	pbs.UnimplementedSessionServiceServer
-
-	serversRepoFn    common.ServersRepoFactory
-	sessionRepoFn    common.SessionRepoFactory
-	connectionRepoFn common.ConnectionRepoFactory
-	updateTimes      *sync.Map
-	kms              *kms.Kms
-}
-
-func NewWorkerServiceServer(
-	serversRepoFn common.ServersRepoFactory,
-	sessionRepoFn common.SessionRepoFactory,
-	connectionRepoFn common.ConnectionRepoFactory,
-	updateTimes *sync.Map,
-	kms *kms.Kms,
-) *workerServiceServer {
-	return &workerServiceServer{
-		serversRepoFn:    serversRepoFn,
-		sessionRepoFn:    sessionRepoFn,
-		connectionRepoFn: connectionRepoFn,
-		updateTimes:      updateTimes,
-		kms:              kms,
-	}
-}
+const (
+	PkiWorkerType = "pki"
+	KmsWorkerType = "kms"
+)
 
 var (
-	_ pbs.SessionServiceServer            = &workerServiceServer{}
-	_ pbs.ServerCoordinationServiceServer = &workerServiceServer{}
+	maskManager handlers.MaskManager
+
+	// IdActions contains the set of actions that can be performed on
+	// individual resources
+	IdActions = action.ActionSet{
+		action.NoOp,
+		action.Read,
+		action.Update,
+		action.Delete,
+	}
+
+	// CollectionActions contains the set of actions that can be performed on
+	// this collection
+	CollectionActions = action.ActionSet{
+		action.CreateWorkerLed,
+		action.List,
+	}
 )
 
-func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
-	const op = "workers.(workerServiceServer).Status"
-	// TODO: on the worker, if we get errors back from this repeatedly, do we
-	// terminate all sessions since we can't know if they were canceled?
-	ws.updateTimes.Store(req.Worker.PrivateId, time.Now())
-	serverRepo, err := ws.serversRepoFn()
+func init() {
+	var err error
+	if maskManager, err = handlers.NewMaskManager(handlers.MaskDestination{&store.Worker{}}, handlers.MaskSource{&pb.Worker{}}); err != nil {
+		panic(err)
+	}
+}
+
+// Service handles request as described by the pbs.WorkerServiceServer interface.
+type Service struct {
+	pbs.UnimplementedWorkerServiceServer
+
+	repoFn    common.ServersRepoFactory
+	iamRepoFn common.IamRepoFactory
+}
+
+// NewService returns a worker service which handles worker related requests to boundary.
+func NewService(ctx context.Context, repo common.ServersRepoFactory, iamRepoFn common.IamRepoFactory) (Service, error) {
+	const op = "workers.NewService"
+	if repo == nil {
+		return Service{}, errors.New(ctx, errors.InvalidParameter, op, "missing servers repository")
+	}
+	if iamRepoFn == nil {
+		return Service{}, errors.New(ctx, errors.InvalidParameter, op, "missing iam repository")
+	}
+	return Service{repoFn: repo, iamRepoFn: iamRepoFn}, nil
+}
+
+var _ pbs.WorkerServiceServer = Service{}
+
+// ListWorkers implements the interface pbs.WorkerServiceServer.
+func (s Service) ListWorkers(ctx context.Context, req *pbs.ListWorkersRequest) (*pbs.ListWorkersResponse, error) {
+	if err := validateListRequest(req); err != nil {
+		return nil, err
+	}
+	authResults := s.authResult(ctx, req.GetScopeId(), action.List)
+	if authResults.Error != nil {
+		// If it's forbidden, and it's a recursive request, and they're
+		// successfully authenticated but just not authorized, keep going as we
+		// may have authorization on downstream scopes. Or, if they've not
+		// authenticated, still process in case u_anon has permissions.
+		if (authResults.Error == handlers.ForbiddenError() || authResults.Error == handlers.UnauthenticatedError()) &&
+			req.GetRecursive() &&
+			authResults.AuthenticationFinished {
+		} else {
+			return nil, authResults.Error
+		}
+	}
+
+	scopeIds, scopeInfoMap, err := scopeids.GetListingScopeIds(
+		ctx, s.iamRepoFn, authResults, req.GetScopeId(), resource.Worker, req.GetRecursive())
 	if err != nil {
-		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting servers repo"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to store worker status: %v", err)
+		return nil, err
 	}
-	sessRepo, err := ws.sessionRepoFn()
-	connectionRepo, err := ws.connectionRepoFn()
+	// If no scopes match, return an empty response
+	if len(scopeIds) == 0 {
+		return &pbs.ListWorkersResponse{}, nil
+	}
+
+	ul, err := s.listFromRepo(ctx, scopeIds)
 	if err != nil {
-		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting sessions repo"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
+		return nil, err
 	}
-	req.Worker.Type = resource.Worker.String()
-	controllers, _, err := serverRepo.UpsertServer(ctx, req.Worker, servers.WithUpdateTags(req.GetUpdateTags()))
+	if len(ul) == 0 {
+		return &pbs.ListWorkersResponse{}, nil
+	}
+
+	filter, err := handlers.NewFilter(req.GetFilter())
 	if err != nil {
-		event.WriteError(ctx, op, err, event.WithInfoMsg("error storing worker status"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error storing worker status: %v", err)
+		return nil, err
 	}
-	ret := &pbs.StatusResponse{
-		Controllers: controllers,
+	finalItems := make([]*pb.Worker, 0, len(ul))
+	res := perms.Resource{
+		Type: resource.Worker,
+	}
+	for _, item := range ul {
+		res.Id = item.GetPublicId()
+		res.ScopeId = item.GetScopeId()
+		authorizedActions := authResults.FetchActionSetForId(ctx, item.GetPublicId(), IdActions, auth.WithResource(&res)).Strings()
+		if len(authorizedActions) == 0 {
+			continue
+		}
+
+		outputFields := authResults.FetchOutputFields(res, action.List).SelfOrDefaults(authResults.UserId)
+		outputOpts := make([]handlers.Option, 0, 3)
+		outputOpts = append(outputOpts, handlers.WithOutputFields(&outputFields))
+		if outputFields.Has(globals.ScopeField) {
+			outputOpts = append(outputOpts, handlers.WithScope(scopeInfoMap[item.GetScopeId()]))
+		}
+		if outputFields.Has(globals.AuthorizedActionsField) {
+			outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
+		}
+
+		item, err := toProto(ctx, item, outputOpts...)
+		if err != nil {
+			return nil, err
+		}
+
+		if filter.Match(item) {
+			finalItems = append(finalItems, item)
+		}
+	}
+	return &pbs.ListWorkersResponse{Items: finalItems}, nil
+}
+
+// GetWorker implements the interface pbs.WorkerServiceServer.
+func (s Service) GetWorker(ctx context.Context, req *pbs.GetWorkerRequest) (*pbs.GetWorkerResponse, error) {
+	const op = "workers.(Service).GetWorker"
+
+	if err := validateGetRequest(req); err != nil {
+		return nil, err
+	}
+	authResults := s.authResult(ctx, req.GetId(), action.Read)
+	if authResults.Error != nil {
+		return nil, authResults.Error
+	}
+	w, err := s.getFromRepo(ctx, req.GetId())
+	if err != nil {
+		return nil, err
 	}
 
-	stateReport := make([]session.StateReport, 0, len(req.GetJobs()))
+	outputFields, ok := requests.OutputFields(ctx)
+	if !ok {
+		return nil, errors.New(ctx, errors.Internal, op, "no request context found")
+	}
 
-	for _, jobStatus := range req.GetJobs() {
-		switch jobStatus.Job.GetType() {
-		case pbs.JOBTYPE_JOBTYPE_SESSION:
-			si := jobStatus.GetJob().GetSessionInfo()
-			if si == nil {
-				return nil, status.Error(codes.Internal, "Error getting session info at status time")
-			}
+	outputOpts := make([]handlers.Option, 0, 3)
+	outputOpts = append(outputOpts, handlers.WithOutputFields(&outputFields))
+	if outputFields.Has(globals.ScopeField) {
+		outputOpts = append(outputOpts, handlers.WithScope(authResults.Scope))
+	}
+	if outputFields.Has(globals.AuthorizedActionsField) {
+		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, w.GetPublicId(), IdActions).Strings()))
+	}
 
-			switch si.Status {
-			case pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
-				pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED:
-				// No need to see about canceling anything
-				continue
-			}
+	item, err := toProto(ctx, w, outputOpts...)
+	if err != nil {
+		return nil, err
+	}
 
-			sr := session.StateReport{
-				SessionId:     si.GetSessionId(),
-				ConnectionIds: make([]string, 0, len(si.GetConnections())),
-			}
-			for _, conn := range si.GetConnections() {
-				switch conn.Status {
-				case pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED,
-					pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED:
-					sr.ConnectionIds = append(sr.ConnectionIds, conn.GetConnectionId())
+	return &pbs.GetWorkerResponse{Item: item}, nil
+}
+
+// CreateWorker implements the interface pbs.WorkerServiceServer.
+func (s Service) CreateWorkerLed(ctx context.Context, req *pbs.CreateWorkerLedRequest) (*pbs.CreateWorkerLedResponse, error) {
+	const op = "workers.(Service).CreateWorkerLed"
+
+	if err := validateCreateRequest(req); err != nil {
+		return nil, err
+	}
+	authResults := s.authResult(ctx, req.GetItem().GetScopeId(), action.CreateWorkerLed)
+	if authResults.Error != nil {
+		return nil, authResults.Error
+	}
+	reqBytes, err := base58.FastBase58Decoding(req.GetItem().WorkerGeneratedAuthToken.GetValue())
+	if err != nil {
+		return nil, fmt.Errorf("%s: error decoding node_credentials_token: %w", op, err)
+	}
+	// Decode the proto into the request
+	creds := new(types.FetchNodeCredentialsRequest)
+	if err := proto.Unmarshal(reqBytes, creds); err != nil {
+		return nil, fmt.Errorf("%s: error unmarshaling node_credentials_token: %w", op, err)
+	}
+	created, err := s.createInRepo(ctx, req.GetItem(), creds)
+	if err != nil {
+		return nil, fmt.Errorf("%s: error creating worker: %w", op, err)
+	}
+
+	outputFields, ok := requests.OutputFields(ctx)
+	if !ok {
+		return nil, errors.New(ctx, errors.Internal, op, "no request context found")
+	}
+
+	outputOpts := make([]handlers.Option, 0, 3)
+	outputOpts = append(outputOpts, handlers.WithOutputFields(&outputFields))
+	if outputFields.Has(globals.ScopeField) {
+		outputOpts = append(outputOpts, handlers.WithScope(authResults.Scope))
+	}
+	if outputFields.Has(globals.AuthorizedActionsField) {
+		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, created.GetPublicId(), IdActions).Strings()))
+	}
+
+	item, err := toProto(ctx, created, outputOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pbs.CreateWorkerLedResponse{Item: item}, nil
+}
+
+// DeleteWorker implements the interface pbs.WorkerServiceServer.
+func (s Service) DeleteWorker(ctx context.Context, req *pbs.DeleteWorkerRequest) (*pbs.DeleteWorkerResponse, error) {
+	if err := validateDeleteRequest(req); err != nil {
+		return nil, err
+	}
+	authResults := s.authResult(ctx, req.GetId(), action.Delete)
+	if authResults.Error != nil {
+		return nil, authResults.Error
+	}
+	_, err := s.deleteFromRepo(ctx, req.GetId())
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+// UpdateWorker implements the interface pbs.WorkerServiceServer.
+func (s Service) UpdateWorker(ctx context.Context, req *pbs.UpdateWorkerRequest) (*pbs.UpdateWorkerResponse, error) {
+	const op = "workers.(Service).UpdateWorker"
+
+	if err := validateUpdateRequest(req); err != nil {
+		return nil, err
+	}
+	authResults := s.authResult(ctx, req.GetId(), action.Update)
+	if authResults.Error != nil {
+		return nil, authResults.Error
+	}
+	w, err := s.updateInRepo(ctx, authResults.Scope.GetId(), req.GetId(), req.GetUpdateMask().GetPaths(), req.GetItem())
+	if err != nil {
+		return nil, err
+	}
+
+	outputFields, ok := requests.OutputFields(ctx)
+	if !ok {
+		return nil, errors.New(ctx, errors.Internal, op, "no request context found")
+	}
+
+	outputOpts := make([]handlers.Option, 0, 3)
+	outputOpts = append(outputOpts, handlers.WithOutputFields(&outputFields))
+	if outputFields.Has(globals.ScopeField) {
+		outputOpts = append(outputOpts, handlers.WithScope(authResults.Scope))
+	}
+	if outputFields.Has(globals.AuthorizedActionsField) {
+		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, w.GetPublicId(), IdActions).Strings()))
+	}
+
+	item, err := toProto(ctx, w, outputOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pbs.UpdateWorkerResponse{Item: item}, nil
+}
+
+func (s Service) listFromRepo(ctx context.Context, scopeIds []string) ([]*servers.Worker, error) {
+	repo, err := s.repoFn()
+	if err != nil {
+		return nil, err
+	}
+	wl, err := repo.ListWorkers(ctx, scopeIds, servers.WithLiveness(-1))
+	if err != nil {
+		return nil, err
+	}
+	return wl, nil
+}
+
+func (s Service) getFromRepo(ctx context.Context, id string) (*servers.Worker, error) {
+	repo, err := s.repoFn()
+	if err != nil {
+		return nil, err
+	}
+	w, err := repo.LookupWorker(ctx, id)
+	if err != nil {
+		if errors.IsNotFoundError(err) {
+			return nil, handlers.NotFoundErrorf("Worker %q doesn't exist.", id)
+		}
+		return nil, err
+	}
+	if w == nil {
+		return nil, handlers.NotFoundErrorf("Worker %q doesn't exist.", id)
+	}
+	return w, nil
+}
+
+func (s Service) createInRepo(ctx context.Context, worker *pb.Worker, creds *types.FetchNodeCredentialsRequest) (*servers.Worker, error) {
+	const op = "workers.(Service).createInRepo"
+	repo, err := s.repoFn()
+	if err != nil {
+		return nil, err
+	}
+	newWorker := servers.NewWorker(
+		worker.GetScopeId(),
+		servers.WithName(worker.GetName().GetValue()),
+		servers.WithDescription(worker.GetDescription().GetValue()),
+	)
+	retWorker, err := repo.CreateWorker(ctx, newWorker, servers.WithFetchNodeCredentialsRequest(creds))
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("unable to create worker"))
+	}
+	return retWorker, nil
+}
+
+func (s Service) deleteFromRepo(ctx context.Context, id string) (bool, error) {
+	const op = "workers.(Service).deleteFromRepo"
+	repo, err := s.repoFn()
+	if err != nil {
+		return false, err
+	}
+	rows, err := repo.DeleteWorker(ctx, id)
+	if err != nil {
+		if errors.IsNotFoundError(err) {
+			return false, nil
+		}
+		return false, errors.Wrap(ctx, err, op, errors.WithMsg("unable to delete worker"))
+	}
+	return rows > 0, nil
+}
+
+func (s Service) updateInRepo(ctx context.Context, scopeId, id string, mask []string, item *pb.Worker) (*servers.Worker, error) {
+	const op = "workers.(Service).updateInRepo"
+	var opts []servers.Option
+	if desc := item.GetDescription(); desc != nil {
+		opts = append(opts, servers.WithDescription(desc.GetValue()))
+	}
+	if name := item.GetName(); name != nil {
+		opts = append(opts, servers.WithName(name.GetValue()))
+	}
+	w := servers.NewWorker(scopeId, opts...)
+	w.PublicId = id
+	dbMask := maskManager.Translate(mask)
+	if len(dbMask) == 0 {
+		return nil, handlers.InvalidArgumentErrorf("No valid fields included in the update mask.", map[string]string{"update_mask": "No valid fields provided in the update mask."})
+	}
+	repo, err := s.repoFn()
+	if err != nil {
+		return nil, err
+	}
+	out, rowsUpdated, err := repo.UpdateWorker(ctx, w, item.GetVersion(), dbMask)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("unable to update worker"))
+	}
+	if rowsUpdated == 0 {
+		return nil, handlers.NotFoundErrorf("Worker %q doesn't exist or incorrect version provided.", id)
+	}
+	return out, nil
+}
+
+func (s Service) authResult(ctx context.Context, id string, a action.Type) auth.VerifyResults {
+	res := auth.VerifyResults{}
+	repo, err := s.repoFn()
+	if err != nil {
+		res.Error = err
+		return res
+	}
+
+	var parentId string
+	opts := []auth.Option{auth.WithType(resource.Worker), auth.WithAction(a)}
+	switch a {
+	case action.List, action.CreateWorkerLed:
+		parentId = id
+	default:
+		w, err := repo.LookupWorker(ctx, id)
+		if err != nil {
+			res.Error = err
+			return res
+		}
+		if w == nil {
+			res.Error = handlers.NotFoundError()
+			return res
+		}
+		parentId = w.GetScopeId()
+		opts = append(opts, auth.WithId(id))
+	}
+	opts = append(opts, auth.WithScopeId(parentId))
+	return auth.Verify(ctx, opts...)
+}
+
+func toProto(ctx context.Context, in *servers.Worker, opt ...handlers.Option) (*pb.Worker, error) {
+	const op = "workers.toProto"
+	opts := handlers.GetOpts(opt...)
+	if opts.WithOutputFields == nil {
+		return nil, handlers.ApiErrorWithCodeAndMessage(codes.Internal, "output fields not found when building worker proto")
+	}
+	outputFields := *opts.WithOutputFields
+
+	out := pb.Worker{}
+	if outputFields.Has(globals.IdField) {
+		out.Id = in.GetPublicId()
+	}
+	if outputFields.Has(globals.ScopeIdField) {
+		out.ScopeId = in.GetScopeId()
+	}
+	if outputFields.Has(globals.DescriptionField) && in.GetDescription() != "" {
+		out.Description = wrapperspb.String(in.GetDescription())
+	}
+	if outputFields.Has(globals.NameField) && in.GetName() != "" {
+		out.Name = wrapperspb.String(in.GetName())
+	}
+	if outputFields.Has(globals.CreatedTimeField) {
+		out.CreatedTime = in.GetCreateTime().GetTimestamp()
+	}
+	if outputFields.Has(globals.UpdatedTimeField) {
+		out.UpdatedTime = in.GetUpdateTime().GetTimestamp()
+	}
+	if outputFields.Has(globals.VersionField) {
+		out.Version = in.GetVersion()
+	}
+	if outputFields.Has(globals.ScopeField) {
+		out.Scope = opts.WithScope
+	}
+	if outputFields.Has(globals.AuthorizedActionsField) {
+		out.AuthorizedActions = opts.WithAuthorizedActions
+		if in.Type == KmsWorkerType && out.AuthorizedActions != nil {
+			// KMS workers cannot be updated through the API
+			allActions := out.AuthorizedActions
+			out.AuthorizedActions = make([]string, 0, len(allActions))
+			for _, act := range allActions {
+				if act != action.Update.String() {
+					out.AuthorizedActions = append(out.AuthorizedActions, act)
 				}
 			}
-			stateReport = append(stateReport, sr)
+		}
+	}
+	if outputFields.Has(globals.AddressField) && in.GetAddress() != "" {
+		out.Address = in.GetAddress()
+	}
+	if outputFields.Has(globals.TypeField) && in.GetType() != "" {
+		out.Type = in.GetType()
+	}
+	if outputFields.Has(globals.LastStatusTimeField) {
+		out.LastStatusTime = in.GetLastStatusTime().GetTimestamp()
+	}
+	if outputFields.Has(globals.ActiveConnectionCountField) {
+		out.ActiveConnectionCount = &wrapperspb.UInt32Value{Value: in.ActiveConnectionCount()}
+	}
+	if outputFields.Has(globals.ConfigTagsField) && len(in.GetConfigTags()) > 0 {
+		var err error
+		out.ConfigTags, err = tagsToMapProto(in.GetConfigTags())
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error preparing config tags proto"))
+		}
+	}
+	if outputFields.Has(globals.CanonicalTagsField) && len(in.CanonicalTags()) > 0 {
+		var err error
+		out.CanonicalTags, err = tagsToMapProto(in.CanonicalTags())
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error preparing canonical tags proto"))
 		}
 	}
 
-	notActive, err := session.WorkerStatusReport(ctx, sessRepo, connectionRepo, req.Worker.PrivateId, stateReport)
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Error comparing state of sessions for worker: %s: %v", req.Worker.PrivateId, err)
-	}
-	for _, na := range notActive {
-		var connChanges []*pbs.Connection
-		for _, connId := range na.ConnectionIds {
-			connChanges = append(connChanges, &pbs.Connection{
-				ConnectionId: connId,
-				Status:       session.StatusClosed.ProtoVal(),
-			})
-		}
-		ret.JobsRequests = append(ret.JobsRequests, &pbs.JobChangeRequest{
-			Job: &pbs.Job{
-				Type: pbs.JOBTYPE_JOBTYPE_SESSION,
-				JobInfo: &pbs.Job_SessionInfo{
-					SessionInfo: &pbs.SessionJobInfo{
-						SessionId:   na.SessionId,
-						Status:      na.Status.ProtoVal(),
-						Connections: connChanges,
-					},
-				},
-			},
-			RequestType: pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE,
-		})
-	}
+	return &out, nil
+}
 
+func tagsToMapProto(in map[string][]string) (map[string]*structpb.ListValue, error) {
+	b := make(map[string][]interface{})
+	for k, v := range in {
+		result := make([]interface{}, 0, len(v))
+		for _, t := range v {
+			result = append(result, t)
+		}
+		b[k] = result
+	}
+	ret := make(map[string]*structpb.ListValue)
+	var err error
+	for k, v := range b {
+		ret[k], err = structpb.NewList(v)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return ret, nil
 }
 
-func (ws *workerServiceServer) LookupSession(ctx context.Context, req *pbs.LookupSessionRequest) (*pbs.LookupSessionResponse, error) {
-	const op = "workers.(workerServiceServer).LookupSession"
-
-	sessRepo, err := ws.sessionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Error getting session repo: %v", err)
-	}
-
-	sessionInfo, authzSummary, err := sessRepo.LookupSession(ctx, req.GetSessionId())
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Error looking up session: %v", err)
-	}
-	if sessionInfo == nil {
-		return nil, status.Error(codes.PermissionDenied, "Unknown session ID.")
-	}
-	if len(sessionInfo.States) == 0 {
-		return nil, status.Error(codes.Internal, "Empty session states during lookup.")
-	}
-
-	if sessionInfo.WorkerFilter != "" {
-		if req.ServerId == "" {
-			event.WriteError(ctx, op, errors.New("worker filter enabled for session but got no server ID from worker"))
-			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Did not receive server ID when looking up session but filtering is enabled: %v", err)
-		}
-		serversRepo, err := ws.serversRepoFn()
-		if err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error getting servers repo"))
-			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error acquiring server repo when looking up session: %v", err)
-		}
-		tags, err := serversRepo.ListTagsForServers(ctx, []string{req.ServerId})
-		if err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error looking up tags for server", "server_id", req.ServerId))
-			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error looking up tags for server: %v", err)
-		}
-		// Build the map for filtering.
-		tagMap := make(map[string][]string)
-		for _, tag := range tags {
-			tagMap[tag.Key] = append(tagMap[tag.Key], tag.Value)
-			// We don't need to reinsert after the fact because maps are
-			// reference types, so we don't need to re-insert into tagMap
-		}
-
-		// Create the evaluator
-		eval, err := bexpr.CreateEvaluator(sessionInfo.WorkerFilter)
-		if err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error creating worker filter evaluator", "server_id", req.ServerId))
-			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error creating worker filter evaluator: %v", err)
-		}
-		filterInput := map[string]interface{}{
-			"name": req.ServerId,
-			"tags": tagMap,
-		}
-		ok, err := eval.Evaluate(filterInput)
-		if err != nil {
-			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
-				fmt.Sprintf("Worker filter expression evaluation resulted in error: %s", err))
-		}
-		if !ok {
-			return nil, handlers.ApiErrorWithCodeAndMessage(
-				codes.FailedPrecondition,
-				"Worker filter expression precludes this worker from serving this session")
-		}
-	}
-
-	creds, err := sessRepo.ListSessionCredentials(ctx, sessionInfo.ScopeId, sessionInfo.PublicId)
-	if err != nil {
-		return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
-			fmt.Sprintf("Error retrieving session credentials: %s", err))
-	}
-	var workerCreds []*pbs.Credential
-	for _, c := range creds {
-		m := &pbs.Credential{}
-		err = proto.Unmarshal(c, m)
-		if err != nil {
-			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
-				fmt.Sprintf("Error unmarshaling credentials: %s", err))
-		}
-		workerCreds = append(workerCreds, m)
-	}
-
-	resp := &pbs.LookupSessionResponse{
-		Authorization: &targets.SessionAuthorizationData{
-			SessionId:   sessionInfo.GetPublicId(),
-			Certificate: sessionInfo.Certificate,
-		},
-		Status:          sessionInfo.States[0].Status.ProtoVal(),
-		Version:         sessionInfo.Version,
-		TofuToken:       string(sessionInfo.TofuToken),
-		Endpoint:        sessionInfo.Endpoint,
-		Expiration:      sessionInfo.ExpirationTime.Timestamp,
-		ConnectionLimit: sessionInfo.ConnectionLimit,
-		ConnectionsLeft: authzSummary.ConnectionLimit,
-		HostId:          sessionInfo.HostId,
-		HostSetId:       sessionInfo.HostSetId,
-		TargetId:        sessionInfo.TargetId,
-		UserId:          sessionInfo.UserId,
-		Credentials:     workerCreds,
-	}
-	if resp.ConnectionsLeft != -1 {
-		resp.ConnectionsLeft -= int32(authzSummary.CurrentConnectionCount)
-	}
-
-	wrapper, err := ws.kms.GetWrapper(ctx, sessionInfo.ScopeId, kms.KeyPurposeSessions, kms.WithKeyId(sessionInfo.KeyId))
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Error getting sessions wrapper: %v", err)
-	}
-
-	// Derive the private key, which should match. Deriving on both ends allows
-	// us to not store it in the DB.
-	_, resp.Authorization.PrivateKey, err = session.DeriveED25519Key(ctx, wrapper, sessionInfo.UserId, sessionInfo.GetPublicId())
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Error deriving session key: %v", err)
-	}
-
-	return resp, nil
+// A validateX method should exist for each method above.  These methods do not make calls to any backing service but enforce
+// requirements on the structure of the request.  They verify that:
+//  * The path passed in is correctly formatted
+//  * All required parameters are set
+//  * There are no conflicting parameters provided
+func validateGetRequest(req *pbs.GetWorkerRequest) error {
+	return handlers.ValidateGetRequest(handlers.NoopValidatorFn, req, servers.WorkerPrefix)
 }
 
-func (ws *workerServiceServer) CancelSession(ctx context.Context, req *pbs.CancelSessionRequest) (*pbs.CancelSessionResponse, error) {
-	const op = "workers.(workerServiceServer).CancelSession"
-
-	sessRepo, err := ws.sessionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
+func validateListRequest(req *pbs.ListWorkersRequest) error {
+	badFields := map[string]string{}
+	if req.GetScopeId() != scope.Global.String() {
+		badFields["scope_id"] = "Must be 'global' when listing."
 	}
-
-	ses, _, err := sessRepo.LookupSession(ctx, req.GetSessionId())
-	if err != nil {
-		return nil, err
+	if _, err := handlers.NewFilter(req.GetFilter()); err != nil {
+		badFields["filter"] = fmt.Sprintf("This field could not be parsed. %v", err)
 	}
-
-	ses, err = sessRepo.CancelSession(ctx, req.GetSessionId(), ses.Version)
-	if err != nil {
-		return nil, err
+	if len(badFields) > 0 {
+		return handlers.InvalidArgumentErrorf("Error in provided request.", badFields)
 	}
-
-	return &pbs.CancelSessionResponse{
-		Status: ses.States[0].Status.ProtoVal(),
-	}, nil
+	return nil
 }
 
-func (ws *workerServiceServer) ActivateSession(ctx context.Context, req *pbs.ActivateSessionRequest) (*pbs.ActivateSessionResponse, error) {
-	const op = "workers.(workerServiceServer).ActivateSession"
-
-	sessRepo, err := ws.sessionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
-	}
-
-	sessionInfo, sessionStates, err := sessRepo.ActivateSession(
-		ctx,
-		req.GetSessionId(),
-		req.GetVersion(),
-		req.GetWorkerId(),
-		resource.Worker.String(),
-		[]byte(req.GetTofuToken()))
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error looking up session: %v", err)
-	}
-	if sessionInfo == nil {
-		return nil, status.Error(codes.PermissionDenied, "Unknown session ID.")
-	}
-	if len(sessionStates) == 0 {
-		return nil, status.Error(codes.Internal, "Invalid session state in activate response.")
-	}
-
-	return &pbs.ActivateSessionResponse{
-		Status: sessionStates[0].Status.ProtoVal(),
-	}, nil
+func validateDeleteRequest(req *pbs.DeleteWorkerRequest) error {
+	return handlers.ValidateDeleteRequest(handlers.NoopValidatorFn, req, servers.WorkerPrefix)
 }
 
-func (ws *workerServiceServer) AuthorizeConnection(ctx context.Context, req *pbs.AuthorizeConnectionRequest) (*pbs.AuthorizeConnectionResponse, error) {
-	const op = "workers.(workerServiceServer).AuthorizeConnection"
-	connectionRepo, err := ws.connectionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
-	}
-
-	sessionRepo, err := ws.sessionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
-	}
-
-	connectionInfo, connStates, authzSummary, err := session.AuthorizeConnection(ctx, sessionRepo, connectionRepo, req.GetSessionId(), req.GetWorkerId())
-	if err != nil {
-		return nil, err
-	}
-	if connectionInfo == nil {
-		return nil, status.Error(codes.Internal, "Invalid authorize connection response.")
-	}
-	if len(connStates) == 0 {
-		return nil, status.Error(codes.Internal, "Invalid connection state in authorize response.")
-	}
-
-	ret := &pbs.AuthorizeConnectionResponse{
-		ConnectionId:    connectionInfo.GetPublicId(),
-		Status:          connStates[0].Status.ProtoVal(),
-		ConnectionsLeft: authzSummary.ConnectionLimit,
-	}
-	if ret.ConnectionsLeft != -1 {
-		ret.ConnectionsLeft -= int32(authzSummary.CurrentConnectionCount)
-	}
-
-	return ret, nil
+func validateUpdateRequest(req *pbs.UpdateWorkerRequest) error {
+	return handlers.ValidateUpdateRequest(req, req.GetItem(), func() map[string]string {
+		badFields := map[string]string{}
+		if req.GetItem().GetAddress() != "" {
+			badFields[globals.AddressField] = "This is a read only field."
+		}
+		if req.GetItem().GetCanonicalTags() != nil {
+			badFields[globals.CanonicalAddressField] = "This is a read only field."
+		}
+		if req.GetItem().GetConfigTags() != nil {
+			badFields[globals.TagsField] = "This is a read only field."
+		}
+		nameString := req.GetItem().GetName().String()
+		if !strutil.Printable(nameString) {
+			badFields[globals.NameField] = "Contains non-printable characters"
+		}
+		if strings.ToLower(nameString) != nameString {
+			badFields[globals.NameField] = "Must be all lowercase."
+		}
+		descriptionString := req.GetItem().GetDescription().String()
+		if !strutil.Printable(descriptionString) {
+			badFields[globals.DescriptionField] = "Contains non-printable characters."
+		}
+		return badFields
+	}, servers.WorkerPrefix)
 }
 
-func (ws *workerServiceServer) ConnectConnection(ctx context.Context, req *pbs.ConnectConnectionRequest) (*pbs.ConnectConnectionResponse, error) {
-	const op = "workers.(workerServiceServer).ConnectConnection"
-	connRepo, err := ws.connectionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
-	}
-
-	connectionInfo, connStates, err := connRepo.ConnectConnection(ctx, session.ConnectWith{
-		ConnectionId:       req.GetConnectionId(),
-		ClientTcpAddress:   req.GetClientTcpAddress(),
-		ClientTcpPort:      req.GetClientTcpPort(),
-		EndpointTcpAddress: req.GetEndpointTcpAddress(),
-		EndpointTcpPort:    req.GetEndpointTcpPort(),
-		UserClientIp:       req.GetUserClientIp(),
+func validateCreateRequest(req *pbs.CreateWorkerLedRequest) error {
+	return handlers.ValidateCreateRequest(req.GetItem(), func() map[string]string {
+		const (
+			mustBeGlobalMsg  = "Must be 'global'"
+			cannotBeEmptyMsg = "Cannot be empty."
+			readOnlyFieldMsg = "This is a read only field."
+		)
+		badFields := map[string]string{}
+		if scope.Global.String() != req.GetItem().GetScopeId() {
+			badFields[globals.ScopeIdField] = mustBeGlobalMsg
+		}
+		// FIXME: in the future, we won't require this token since we'll support
+		// the server led flow where a token is returned when a worker is created.
+		if req.GetItem().WorkerGeneratedAuthToken == nil {
+			badFields[globals.WorkerGeneratedAuthTokenField] = cannotBeEmptyMsg
+		}
+		if req.GetItem().Address != "" {
+			badFields[globals.CanonicalAddressField] = readOnlyFieldMsg
+		}
+		if req.GetItem().CanonicalTags != nil {
+			badFields[globals.CanonicalTagsField] = readOnlyFieldMsg
+		}
+		if req.GetItem().ConfigTags != nil {
+			badFields[globals.ConfigTagsField] = readOnlyFieldMsg
+		}
+		if req.GetItem().LastStatusTime != nil {
+			badFields[globals.LastStatusTimeField] = readOnlyFieldMsg
+		}
+		if req.GetItem().AuthorizedActions != nil {
+			badFields[globals.AuthorizedActionsField] = readOnlyFieldMsg
+		}
+		nameString := req.GetItem().GetName().String()
+		if !strutil.Printable(nameString) {
+			badFields[globals.NameField] = "Name contains non-printable characters."
+		}
+		if strings.ToLower(nameString) != nameString {
+			badFields[globals.NameField] = "Name must be all lowercase."
+		}
+		descriptionString := req.GetItem().GetDescription().String()
+		if !strutil.Printable(descriptionString) {
+			badFields[globals.DescriptionField] = "Description contains non-printable characters."
+		}
+		return badFields
 	})
-	if err != nil {
-		return nil, err
-	}
-	if connectionInfo == nil {
-		return nil, status.Error(codes.Internal, "Invalid connect connection response.")
-	}
-
-	return &pbs.ConnectConnectionResponse{
-		Status: connStates[0].Status.ProtoVal(),
-	}, nil
-}
-
-func (ws *workerServiceServer) CloseConnection(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) {
-	const op = "workers.(workerServiceServer).CloseConnection"
-	numCloses := len(req.GetCloseRequestData())
-	if numCloses == 0 {
-		return &pbs.CloseConnectionResponse{}, nil
-	}
-
-	closeWiths := make([]session.CloseWith, 0, numCloses)
-	closeIds := make([]string, 0, numCloses)
-
-	for _, v := range req.GetCloseRequestData() {
-		closeIds = append(closeIds, v.GetConnectionId())
-		closeWiths = append(closeWiths, session.CloseWith{
-			ConnectionId: v.GetConnectionId(),
-			BytesUp:      v.GetBytesUp(),
-			BytesDown:    v.GetBytesDown(),
-			ClosedReason: session.ClosedReason(v.GetReason()),
-		})
-	}
-	connRepo, err := ws.connectionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting connection repo: %v", err)
-	}
-
-	sessRepo, err := ws.sessionRepoFn()
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "error getting session repo: %v", err)
-	}
-
-	closeInfos, err := session.CloseConnections(ctx, sessRepo, connRepo, closeWiths)
-	if err != nil {
-		return nil, err
-	}
-	if closeInfos == nil {
-		return nil, status.Error(codes.Internal, "Invalid close connection response.")
-	}
-
-	closeData := make([]*pbs.CloseConnectionResponseData, 0, numCloses)
-	for _, v := range closeInfos {
-		if v.Connection == nil {
-			return nil, status.Errorf(codes.Internal, "No connection found while closing one of the connection IDs: %v", closeIds)
-		}
-		if len(v.ConnectionStates) == 0 {
-			return nil, status.Errorf(codes.Internal, "No connection states found while closing one of the connection IDs: %v", closeIds)
-		}
-		closeData = append(closeData, &pbs.CloseConnectionResponseData{
-			ConnectionId: v.Connection.GetPublicId(),
-			Status:       v.ConnectionStates[0].Status.ProtoVal(),
-		})
-	}
-
-	ret := &pbs.CloseConnectionResponse{
-		CloseResponseData: closeData,
-	}
-
-	return ret, nil
 }

internal/daemon/controller/intercepting_listener.go

@@ -1,12 +1,14 @@
 package controller
 
 import (
-	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/hashicorp/boundary/internal/observability/event"
+	nodee "github.com/hashicorp/nodeenrollment"
 )
 
 // tempError is an error that satisfies the temporary error interface that is
@@ -62,42 +64,61 @@ func newInterceptingListener(c *Controller, baseLn net.Listener) *interceptingLi
 
 func (m *interceptingListener) Accept() (net.Conn, error) {
 	const op = "controller.(interceptingListener).Accept"
-	ctx := context.TODO()
+	ctx := m.c.baseContext
 	conn, err := m.baseLn.Accept()
 	if err != nil {
+		// We may already be closed but can't be sure, so just try anyways
 		if conn != nil {
 			if err := conn.Close(); err != nil {
-				event.WriteError(context.TODO(), op, err, event.WithInfoMsg("error closing worker connection"))
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
 			}
 		}
 		return nil, newTempError(err)
 	}
 
-	nonce := make([]byte, 20)
-	read, err := conn.Read(nonce)
-	if err != nil {
-		if err := conn.Close(); err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+	tlsConn := conn.(*tls.Conn)
+	switch {
+	case nodee.ContainsKnownAlpnProto(tlsConn.ConnectionState().NegotiatedProtocol):
+		keyId, err := nodee.KeyIdFromPkix(tlsConn.ConnectionState().PeerCertificates[0].SubjectKeyId)
+		if err != nil {
+			if err := conn.Close(); err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+			}
+			return nil, newTempError(fmt.Errorf("error deriving key id from newly-authenticated node: %w", err))
 		}
-		return nil, newTempError(fmt.Errorf("error reading nonce from connection: %w", err))
-	}
-	if read != len(nonce) {
-		if err := conn.Close(); err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+		event.WriteSysEvent(ctx, op, "worker successfully authenticated", "key_id", keyId)
+		return conn, nil
+
+	case strings.HasPrefix(tlsConn.ConnectionState().NegotiatedProtocol, "v1workerauth"):
+		nonce := make([]byte, 20)
+		read, err := conn.Read(nonce)
+		if err != nil {
+			if err := conn.Close(); err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+			}
+			return nil, newTempError(fmt.Errorf("error reading nonce from connection: %w", err))
 		}
-		return nil, newTempError(fmt.Errorf("error reading nonce from worker, expected %d bytes, got %d", 20, read))
-	}
-	workerInfoRaw, found := m.c.workerAuthCache.Load(string(nonce))
-	if !found {
-		if err := conn.Close(); err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+		if read != len(nonce) {
+			if err := conn.Close(); err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+			}
+			return nil, newTempError(fmt.Errorf("error reading nonce from worker, expected %d bytes, got %d", 20, read))
 		}
-		return nil, newTempError(errors.New("did not find valid nonce for incoming worker"))
+		workerInfoRaw, found := m.c.workerAuthCache.Load(string(nonce))
+		if !found {
+			if err := conn.Close(); err != nil {
+				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+			}
+			return nil, newTempError(errors.New("did not find valid nonce for incoming worker"))
+		}
+		workerInfo := workerInfoRaw.(*workerAuthEntry)
+		workerInfo.conn = tlsConn
+		event.WriteSysEvent(ctx, op, "worker successfully authed", "name", workerInfo.Name, "description", workerInfo.Description, "proxy_address", workerInfo.ProxyAddress)
+		return tlsConn, nil
+
+	default:
+		return nil, newTempError(errors.New("unable to authenticate incoming connection"))
 	}
-	workerInfo := workerInfoRaw.(*workerAuthEntry)
-	workerInfo.conn = conn
-	event.WriteSysEvent(ctx, op, "worker successfully authed", "name", workerInfo.Name)
-	return conn, nil
 }
 
 func (m *interceptingListener) Close() error {

internal/daemon/controller/listeners.go

@@ -13,10 +13,12 @@ import (
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/hashicorp/boundary/internal/cmd/base"
-	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/workers"
+	"github.com/hashicorp/boundary/internal/daemon/cluster/handlers"
 	"github.com/hashicorp/boundary/internal/daemon/controller/internal/metric"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/nodeenrollment/multihop"
+	"github.com/hashicorp/nodeenrollment/protocol"
 	"google.golang.org/grpc"
 )
 
@@ -103,9 +105,25 @@ func (c *Controller) configureForApi(ln *base.ServerListener) ([]func(), error)
 }
 
 func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error) {
-	l := tls.NewListener(ln.ClusterListener, &tls.Config{
-		GetConfigForClient: c.validateWorkerTls,
-	})
+	const op = "controller.configureForCluster"
+
+	workerAuthStorage, err := c.WorkerAuthRepoStorageFn()
+	if err != nil {
+		return nil, fmt.Errorf("error fetching worker auth storage: %w", err)
+	}
+
+	l, err := protocol.NewInterceptingListener(
+		&protocol.InterceptingListenerConfiguration{
+			Context:      c.baseContext,
+			Storage:      workerAuthStorage,
+			BaseListener: ln.ClusterListener,
+			BaseTlsConfiguration: &tls.Config{
+				GetConfigForClient: c.validateWorkerTls,
+			},
+		})
+	if err != nil {
+		return nil, fmt.Errorf("error instantiating node auth listener: %w", err)
+	}
 
 	workerReqInterceptor, err := workerRequestInfoInterceptor(c.baseContext, c.conf.Eventer)
 	if err != nil {
@@ -125,10 +143,21 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 		),
 	)
 
-	workerService := workers.NewWorkerServiceServer(c.ServersRepoFn, c.SessionRepoFn, c.ConnectionRepoFn,
+	workerService := handlers.NewWorkerServiceServer(c.ServersRepoFn, c.SessionRepoFn, c.ConnectionRepoFn,
 		c.workerStatusUpdateTimes, c.kms)
 	pbs.RegisterServerCoordinationServiceServer(workerServer, workerService)
 	pbs.RegisterSessionServiceServer(workerServer, workerService)
+
+	multihopService, err := handlers.NewMultihopServiceServer(
+		workerAuthStorage,
+		true,
+		nil,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("%s: error creating multihop service handler: %w", op, err)
+	}
+	multihop.RegisterMultihopServiceServer(workerServer, multihopService)
+
 	metric.InitializeClusterCollectors(c.conf.PrometheusRegisterer, workerServer)
 
 	ln.GrpcServer = workerServer

internal/daemon/controller/testing.go

@@ -735,10 +735,10 @@ func (tc *TestController) AddClusterControllerMember(t testing.TB, opts *TestCon
 // WaitForNextWorkerStatusUpdate waits for the next status check from a worker to
 // come in. If it does not come in within the default status grace
 // period, this function returns an error.
-func (tc *TestController) WaitForNextWorkerStatusUpdate(workerId string) error {
+func (tc *TestController) WaitForNextWorkerStatusUpdate(workerStatusName string) error {
 	const op = "controller.(TestController).WaitForNextWorkerStatusUpdate"
 	ctx := context.TODO()
-	event.WriteSysEvent(ctx, op, "waiting for next status report from worker", "worker", workerId)
+	event.WriteSysEvent(ctx, op, "waiting for next status report from worker", "worker", workerStatusName)
 	waitStatusStart := time.Now()
 	ctx, cancel := context.WithTimeout(tc.ctx, tc.b.StatusGracePeriodDuration)
 	defer cancel()
@@ -777,7 +777,7 @@ func (tc *TestController) WaitForNextWorkerStatusUpdate(workerId string) error {
 				return false
 			}
 
-			if workerStatusUpdateId == workerId {
+			if workerStatusUpdateId == workerStatusName {
 				waitStatusCurrent = workerStatusUpdateTime
 				return false
 			}
@@ -795,10 +795,10 @@ func (tc *TestController) WaitForNextWorkerStatusUpdate(workerId string) error {
 	}
 
 	if err != nil {
-		event.WriteError(ctx, op, err, event.WithInfoMsg("error waiting for next status report from worker", "worker", workerId))
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error waiting for next status report from worker", "worker", workerStatusName))
 		return err
 	}
-	event.WriteSysEvent(ctx, op, "waiting for next status report from worker received successfully", "worker", workerId)
+	event.WriteSysEvent(ctx, op, "waiting for next status report from worker received successfully", "worker", workerStatusName)
 	return nil
 }
 

internal/daemon/controller/tickers.go

@@ -5,10 +5,10 @@ import (
 	"math/rand"
 	"time"
 
+	"github.com/hashicorp/boundary/internal/servers/store"
+
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/observability/event"
-	"github.com/hashicorp/boundary/internal/servers"
-	"github.com/hashicorp/boundary/internal/types/resource"
 )
 
 // In the future we could make this configurable
@@ -30,7 +30,7 @@ func (c *Controller) startStatusTicking(cancelCtx context.Context) {
 			return
 
 		case <-timer.C:
-			if err := c.upsertServer(cancelCtx); err != nil {
+			if err := c.upsertController(cancelCtx); err != nil {
 				event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error fetching repository for status update"))
 			}
 			timer.Reset(statusInterval)
@@ -38,21 +38,18 @@ func (c *Controller) startStatusTicking(cancelCtx context.Context) {
 	}
 }
 
-func (c *Controller) upsertServer(ctx context.Context) error {
-	const op = "controller.(Controller).upsertServer"
-	server := &servers.Server{
-		PrivateId:   c.conf.RawConfig.Controller.Name,
-		Name:        c.conf.RawConfig.Controller.Name,
-		Type:        resource.Controller.String(),
-		Description: c.conf.RawConfig.Controller.Description,
-		Address:     c.conf.RawConfig.Controller.PublicClusterAddr,
+func (c *Controller) upsertController(ctx context.Context) error {
+	const op = "controller.(Controller).upsertController"
+	controller := &store.Controller{
+		PrivateId: c.conf.RawConfig.Controller.Name,
+		Address:   c.conf.RawConfig.Controller.PublicClusterAddr,
 	}
 	repo, err := c.ServersRepoFn()
 	if err != nil {
 		return errors.Wrap(ctx, err, op, errors.WithMsg("error fetching repository for status update"))
 	}
 
-	_, _, err = repo.UpsertServer(ctx, server)
+	_, err = repo.UpsertController(ctx, controller)
 	if err != nil {
 		return errors.Wrap(ctx, err, op, errors.WithMsg("error performing status update"))
 	}

internal/daemon/controller/worker_tls_config.go

@@ -27,6 +27,7 @@ type workerAuthEntry struct {
 // been replayed, then stores connection information into the auth cache, which
 // is used by the intercepting listener to log information.
 func (c Controller) validateWorkerTls(hello *tls.ClientHelloInfo) (*tls.Config, error) {
+	const op = "controller.(Controller).validateWorkerTls"
 	for _, p := range hello.SupportedProtos {
 		switch {
 		case strings.HasPrefix(p, "v1workerauth-"):
@@ -50,7 +51,7 @@ func (c Controller) validateWorkerTls(hello *tls.ClientHelloInfo) (*tls.Config,
 			return tlsConf, err
 		}
 	}
-	return nil, nil
+	return nil, errors.New("no supported protos found in hello")
 }
 
 // v1WorkerAuthConfig:

internal/daemon/worker/auth_rotation.go

@@ -0,0 +1,202 @@
+package worker
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	berrors "github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/multihop"
+	"github.com/hashicorp/nodeenrollment/types"
+)
+
+func (w *Worker) startAuthRotationTicking(cancelCtx context.Context) {
+	const op = "worker.(Worker).startAuthRotationTicking"
+	if w.conf.WorkerAuthKms != nil {
+		// Nothing to do here, just immediately exit
+		// FIXME: Write a system event
+		return
+	}
+
+	// The default time to use when we encounter an error or some other reason
+	// we can't get a better reset time
+	const defaultResetDuration = time.Hour
+	timer := time.NewTimer(defaultResetDuration)
+	// You're not supposed to call reset on timers that haven't been stopped or
+	// expired, so we stop it here so it resets in the loop below. That way if
+	// we want to adjust time, e.g. for tests, we can set the resetDuration
+	// value from within the loop
+	timer.Stop()
+	var resetDuration time.Duration // will start at 0, so we run immediately
+	for {
+		if w.TestOverrideAuthRotationPeriod != 0 {
+			resetDuration = w.TestOverrideAuthRotationPeriod
+		}
+		timer.Reset(resetDuration)
+
+		select {
+		case <-cancelCtx.Done():
+			event.WriteSysEvent(cancelCtx, op, "auth rotation ticking shutting down")
+			return
+
+		case <-timer.C:
+			resetDuration = defaultResetDuration
+
+			// Check if it's time to rotate and if not don't do anything
+			currentNodeCreds, err := types.LoadNodeCredentials(cancelCtx, w.WorkerAuthStorage, nodeenrollment.CurrentId, nodeenrollment.WithWrapper(w.conf.WorkerAuthStorageKms))
+			if err != nil {
+				if errors.Is(err, nodeenrollment.ErrNotFound) {
+					// Be silent
+					continue
+				}
+				event.WriteError(cancelCtx, op, err)
+				continue
+			}
+
+			if currentNodeCreds == nil {
+				event.WriteSysEvent(cancelCtx, op, "no error loading worker auth creds but nil creds, skipping rotation")
+				continue
+			}
+
+			// Check the certificates to see if it's time
+			if len(currentNodeCreds.CertificateBundles) != 2 {
+				// Likely this means we haven't been authorized yet, and
+				// unclear what to do in any other situation
+				continue
+			}
+
+			var earliestValid, latestValid time.Time
+			// Go through the bundles and find the full range of time that we
+			// have valid certificates
+			for _, bundle := range currentNodeCreds.CertificateBundles {
+				if curr := bundle.CertificateNotBefore.AsTime(); earliestValid.IsZero() || curr.Before(earliestValid) {
+					earliestValid = curr
+				}
+				if curr := bundle.CertificateNotAfter.AsTime(); latestValid.IsZero() || curr.After(latestValid) {
+					latestValid = curr
+				}
+			}
+			// Ensure that our time periods aren't somehow quite messed up
+			now := time.Now()
+			if earliestValid.After(now) || latestValid.Before(now) || earliestValid.After(latestValid) {
+				// We basically have no valid creds so we can't rotate.
+				// TODO (maybe): Have this trigger a new set of creds and request?
+				event.WriteSysEvent(cancelCtx, op, "not within node creds validity period, unsure what to do, not rotating")
+				continue
+			}
+
+			// Figure out the midpoint; if we're after it, try to rotate
+			var shouldRotate bool
+			switch {
+			case w.TestOverrideAuthRotationPeriod != 0:
+				shouldRotate = true
+			default:
+				delta := latestValid.Sub(earliestValid)
+				shouldRotate = now.Before(earliestValid.Add(delta / 2))
+			}
+
+			if !shouldRotate {
+				continue
+			}
+
+			if err := rotateWorkerAuth(cancelCtx, w, currentNodeCreds); err != nil {
+				event.WriteError(cancelCtx, op, err)
+				continue
+			}
+
+			// TODO (maybe): Calculate new delta and set a custom retry time on the timer?
+		}
+	}
+}
+
+// rotateWorkerAuth is a one-stop shop to perform a rotation, given a valid set
+// of current node credentials. Note that it doesn't really sanity check inputs;
+// it is meant to be called from other functions that have validated these
+// inputs and is separated out purely for (a) cleanliness/readability of the
+// ticking function and (b) possible future re-use.
+func rotateWorkerAuth(ctx context.Context, w *Worker, currentNodeCreds *types.NodeCredentials) error {
+	const op = "worker.rotateWorkerAuth"
+
+	randReaderOpt := nodeenrollment.WithRandomReader(w.conf.SecureRandomReader)
+
+	// Ensure we can get some needed values prior to actually doing generation
+	currentKeyId, err := nodeenrollment.KeyIdFromPkix(currentNodeCreds.CertificatePublicKeyPkix)
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+	client := w.controllerMultihopConn.Load()
+	if client == nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+	multihopClient, ok := client.(multihop.MultihopServiceClient)
+	if !ok {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	// Generate a new set of credentials but don't persist them yet
+	newNodeCreds, err := types.NewNodeCredentials(
+		ctx,
+		w.WorkerAuthStorage,
+		randReaderOpt,
+		nodeenrollment.WithWrapper(w.conf.WorkerAuthKms),
+		nodeenrollment.WithSkipStorage(true),
+	)
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	// Get a signed request from the new credentials
+	fetchReq, err := newNodeCreds.CreateFetchNodeCredentialsRequest(ctx, randReaderOpt)
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	// Encrypt the values to the server
+	encFetchReq, err := nodeenrollment.EncryptMessage(ctx, currentKeyId, fetchReq, currentNodeCreds, randReaderOpt)
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	resp, err := multihopClient.RotateNodeCredentials(ctx, &types.RotateNodeCredentialsRequest{
+		CertificatePublicKeyPkix:             currentNodeCreds.CertificatePublicKeyPkix,
+		EncryptedFetchNodeCredentialsRequest: encFetchReq,
+	})
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	// Decrypt the response using current credentials
+	fetchResp := new(types.FetchNodeCredentialsResponse)
+	if err := nodeenrollment.DecryptMessage(
+		ctx,
+		currentKeyId,
+		resp.EncryptedFetchNodeCredentialsResponse,
+		currentNodeCreds,
+		fetchResp,
+	); err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	// Call handle on the inner response, which will be encrypted to the
+	// new node credentials. This will also store the result,
+	// overwriting the current credentials.
+	newNodeCreds, err = newNodeCreds.HandleFetchNodeCredentialsResponse(
+		ctx,
+		w.WorkerAuthStorage,
+		fetchResp,
+		nodeenrollment.WithWrapper(w.conf.WorkerAuthKms),
+	)
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+
+	newKeyId, err := nodeenrollment.KeyIdFromPkix(newNodeCreds.CertificatePublicKeyPkix)
+	if err != nil {
+		return berrors.Wrap(ctx, err, op)
+	}
+	w.WorkerAuthCurrentKeyId.Store(newKeyId)
+
+	return nil
+}

internal/daemon/worker/auth_rotation_test.go

@@ -0,0 +1,118 @@
+package worker_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/cmd/config"
+	"github.com/hashicorp/boundary/internal/daemon/controller"
+	"github.com/hashicorp/boundary/internal/daemon/worker"
+	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/servers/store"
+	"github.com/hashicorp/boundary/internal/types/scope"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/types"
+	"github.com/mr-tron/base58"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/protobuf/proto"
+)
+
+// TestRotationTicking ensures that we see new credential information for a
+// worker on both the controller side and worker side in a periodic fashion
+func TestRotationTicking(t *testing.T) {
+	t.Parallel()
+
+	require, assert := require.New(t), assert.New(t)
+	logger := hclog.New(&hclog.LoggerOptions{
+		Level: hclog.Trace,
+	})
+
+	conf, err := config.DevController()
+	require.NoError(err)
+
+	c := controller.NewTestController(t, &controller.TestControllerOpts{
+		Config: conf,
+		Logger: logger.Named("controller"),
+	})
+	t.Cleanup(c.Shutdown)
+
+	const rotationPeriod = 20 * time.Second
+
+	w := worker.NewTestWorker(t, &worker.TestWorkerOpts{
+		InitialUpstreams:   c.ClusterAddrs(),
+		Logger:             logger.Named("worker"),
+		AuthRotationPeriod: rotationPeriod,
+	})
+	t.Cleanup(w.Shutdown)
+
+	// Get a servers repo and worker auth repo
+	serversRepo, err := c.Controller().ServersRepoFn()
+	require.NoError(err)
+	workerAuthRepo, err := c.Controller().WorkerAuthRepoStorageFn()
+	require.NoError(err)
+
+	// Give time for the job to ensure that roots are created and offset us from
+	// the rotation period
+	time.Sleep(rotationPeriod / 2)
+
+	// Verify that there are no nodes authorized yet
+	auths, err := workerAuthRepo.List(c.Context(), (*types.NodeInformation)(nil))
+	require.NoError(err)
+	assert.Len(auths, 0)
+
+	// Perform initial authentication of worker to controller
+	reqBytes, err := base58.FastBase58Decoding(w.Worker().WorkerAuthRegistrationRequest)
+	require.NoError(err)
+
+	// Decode the proto into the request and create the worker
+	req := new(types.FetchNodeCredentialsRequest)
+	require.NoError(proto.Unmarshal(reqBytes, req))
+	_, err = serversRepo.CreateWorker(c.Context(), &servers.Worker{
+		Worker: &store.Worker{
+			ScopeId: scope.Global.String(),
+		},
+	}, servers.WithFetchNodeCredentialsRequest(req))
+	require.NoError(err)
+
+	// Verify we see one authorized set of credentials now
+	auths, err = workerAuthRepo.List(c.Context(), (*types.NodeInformation)(nil))
+	require.NoError(err)
+	assert.Len(auths, 1)
+	// Fetch creds and store current key
+	currNodeCreds, err := types.LoadNodeCredentials(w.Context(), w.Worker().WorkerAuthStorage, nodeenrollment.CurrentId)
+	require.NoError(err)
+	currKey := currNodeCreds.CertificatePublicKeyPkix
+
+	// Now we wait and check that we see new values in the DB and different
+	// creds on the worker after each rotation period
+	for i := 2; i < 5; i++ {
+		time.Sleep(rotationPeriod)
+
+		// Verify we see the expected number, since we aren't expiring any it
+		// should be equal to the number of times we did rotations
+		auths, err = workerAuthRepo.List(c.Context(), (*types.NodeInformation)(nil))
+		require.NoError(err)
+		assert.Len(auths, i)
+
+		// Fetch creds and compare current key
+		currNodeCreds, err := types.LoadNodeCredentials(w.Context(), w.Worker().WorkerAuthStorage, nodeenrollment.CurrentId)
+		require.NoError(err)
+		assert.NotEqual(currKey, currNodeCreds.CertificatePublicKeyPkix)
+		currKey = currNodeCreds.CertificatePublicKeyPkix
+		currKeyId, err := nodeenrollment.KeyIdFromPkix(currNodeCreds.CertificatePublicKeyPkix)
+		require.NoError(err)
+		assert.Equal(currKeyId, w.Worker().WorkerAuthCurrentKeyId.Load())
+
+		// Stop and start the client connections to ensure the new credentials
+		// are valid; if not, we won't establish a new connection and rotation
+		// will fail
+		w.Worker().Resolver().UpdateState(resolver.State{Addresses: []resolver.Address{}})
+		cc, err := w.Worker().GrpcClientConn()
+		require.NoError(err)
+		require.NoError(cc.Close())
+		require.NoError(w.Worker().StartControllerConnections())
+	}
+}

internal/daemon/worker/controller_connection.go

@@ -14,6 +14,8 @@ import (
 	"math/big"
 	mathrand "math/rand"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -23,15 +25,24 @@ import (
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/go-secure-stdlib/base62"
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/multihop"
+	"github.com/hashicorp/nodeenrollment/protocol"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/protobuf/proto"
 )
 
-func (w *Worker) startControllerConnections() error {
-	const op = "worker.(Worker).startControllerConnections"
-	initialAddrs := make([]resolver.Address, 0, len(w.conf.RawConfig.Worker.Controllers))
-	for _, addr := range w.conf.RawConfig.Worker.Controllers {
+const hcpbUrlSuffix = ".proxy.boundary.hashicorp.cloud:9202"
+
+// StartControllerConnections starts up the resolver and initiates controller
+// connection client creation
+func (w *Worker) StartControllerConnections() error {
+	const op = "worker.(Worker).StartControllerConnections"
+	initialAddrs := make([]resolver.Address, 0, len(w.conf.RawConfig.Worker.InitialUpstreams))
+	for _, addr := range w.conf.RawConfig.Worker.InitialUpstreams {
 		switch {
 		case strings.HasPrefix(addr, "/"):
 			initialAddrs = append(initialAddrs, resolver.Address{Addr: addr})
@@ -41,21 +52,27 @@ func (w *Worker) startControllerConnections() error {
 				host, port, err = net.SplitHostPort(net.JoinHostPort(addr, "9201"))
 			}
 			if err != nil {
-				return fmt.Errorf("error parsing controller address: %w", err)
+				return fmt.Errorf("error parsing upstream address: %w", err)
 			}
 			initialAddrs = append(initialAddrs, resolver.Address{Addr: net.JoinHostPort(host, port)})
 		}
 	}
 
 	if len(initialAddrs) == 0 {
-		return errors.New("no initial controller addresses found")
+		if w.conf.RawConfig.HCPBClusterId != "" {
+			clusterAddress := fmt.Sprintf("%s%s", w.conf.RawConfig.HCPBClusterId, hcpbUrlSuffix)
+			initialAddrs = append(initialAddrs, resolver.Address{Addr: clusterAddress})
+			event.WriteSysEvent(w.baseContext, op, fmt.Sprintf("Setting HCPB Cluster address %s as upstream address", clusterAddress))
+		} else {
+			return errors.New("no initial upstream addresses found")
+		}
 	}
 
 	w.Resolver().InitialState(resolver.State{
 		Addresses: initialAddrs,
 	})
 	if err := w.createClientConn(initialAddrs[0].Addr); err != nil {
-		return fmt.Errorf("error making client connection to controller: %w", err)
+		return fmt.Errorf("error making client connection to upstream address: %w", err)
 	}
 
 	return nil
@@ -64,39 +81,79 @@ func (w *Worker) startControllerConnections() error {
 func (w *Worker) controllerDialerFunc() func(context.Context, string) (net.Conn, error) {
 	const op = "worker.(Worker).controllerDialerFunc"
 	return func(ctx context.Context, addr string) (net.Conn, error) {
-		tlsConf, authInfo, err := w.workerAuthTLSConfig()
-		if err != nil {
-			return nil, fmt.Errorf("error creating tls config for worker auth: %w", err)
-		}
-		dialer := &net.Dialer{}
-		var nonTlsConn net.Conn
+		var conn net.Conn
+		var err error
 		switch {
-		case strings.HasPrefix(addr, "/"):
-			nonTlsConn, err = dialer.DialContext(ctx, "unix", addr)
+		case w.conf.WorkerAuthKms != nil:
+			conn, err = w.v1KmsAuthDialFn(ctx, addr)
 		default:
-			nonTlsConn, err = dialer.DialContext(ctx, "tcp", addr)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("unable to dial to controller: %w", err)
-		}
-		tlsConn := tls.Client(nonTlsConn, tlsConf)
-		written, err := tlsConn.Write([]byte(authInfo.ConnectionNonce))
-		if err != nil {
-			if err := nonTlsConn.Close(); err != nil {
-				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing connection after writing failure"))
+			conn, err = protocol.Dial(ctx, w.WorkerAuthStorage, addr, nodeenrollment.WithWrapper(w.conf.WorkerAuthStorageKms))
+			// No error and a valid connection means the WorkerAuthRegistrationRequest was populated
+			// We can remove the stored workerAuthRequest file
+			if err == nil && conn != nil {
+				if w.WorkerAuthStorage.BaseDir() != "" {
+					workerAuthReqFilePath := filepath.Join(w.WorkerAuthStorage.BaseDir(), base.WorkerAuthReqFile)
+					// Intentionally ignoring any error removing this file
+					_ = os.Remove(workerAuthReqFilePath)
+				}
 			}
-			return nil, fmt.Errorf("unable to write connection nonce: %w", err)
 		}
-		if written != len(authInfo.ConnectionNonce) {
-			if err := nonTlsConn.Close(); err != nil {
-				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing connection after writing failure"))
+		switch err {
+		case nil:
+		case nodeenrollment.ErrNotAuthorized:
+			// We don't event in this case, because the function retries often
+			// and will spam the logs. The status function will event indicating
+			// that it can't send status because it's not authorized, so that
+			// will be a fine hint to the user as to the issue.
+		default:
+			event.WriteError(ctx, op, err)
+		}
+
+		if err == nil && conn != nil {
+			if !w.everAuthenticated.Load() {
+				w.everAuthenticated.Store(true)
 			}
-			return nil, fmt.Errorf("expected to write %d bytes of connection nonce, wrote %d", len(authInfo.ConnectionNonce), written)
+			event.WriteSysEvent(ctx, op, "worker has successfully authenticated")
 		}
-		return tlsConn, nil
+
+		return conn, err
 	}
 }
 
+func (w *Worker) v1KmsAuthDialFn(ctx context.Context, addr string) (net.Conn, error) {
+	const op = "worker.(Worker).v1KmsAuthDialFn"
+	tlsConf, authInfo, err := w.workerAuthTLSConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error creating tls config for worker auth: %w", err)
+	}
+	dialer := &net.Dialer{}
+	var nonTlsConn net.Conn
+	switch {
+	case strings.HasPrefix(addr, "/"):
+		nonTlsConn, err = dialer.DialContext(ctx, "unix", addr)
+	default:
+		nonTlsConn, err = dialer.DialContext(ctx, "tcp", addr)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("unable to dial to upstream: %w", err)
+	}
+	tlsConn := tls.Client(nonTlsConn, tlsConf)
+	written, err := tlsConn.Write([]byte(authInfo.ConnectionNonce))
+	if err != nil {
+		if err := nonTlsConn.Close(); err != nil {
+			event.WriteError(ctx, op, err, event.WithInfoMsg("error closing connection after writing failure"))
+		}
+		return nil, fmt.Errorf("unable to write connection nonce: %w", err)
+	}
+	if written != len(authInfo.ConnectionNonce) {
+		if err := nonTlsConn.Close(); err != nil {
+			event.WriteError(ctx, op, err, event.WithInfoMsg("error closing connection after writing failure"))
+		}
+		return nil, fmt.Errorf("expected to write %d bytes of connection nonce, wrote %d", len(authInfo.ConnectionNonce), written)
+	}
+	return tlsConn, nil
+}
+
 func (w *Worker) createClientConn(addr string) error {
 	const op = "worker.(Worker).createClientConn"
 	defaultTimeout := (time.Second + time.Nanosecond).String()
@@ -113,35 +170,47 @@ func (w *Worker) createClientConn(addr string) error {
 	  }
 	  `, defaultTimeout)
 	res := w.Resolver()
-	cc, err := grpc.DialContext(w.baseContext,
-		fmt.Sprintf("%s:///%s", res.Scheme(), addr),
+	dialOpts := []grpc.DialOption{
 		grpc.WithResolvers(res),
 		grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
 		grpc.WithContextDialer(w.controllerDialerFunc()),
-		grpc.WithInsecure(),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.WithDefaultServiceConfig(defServiceConfig),
 		// Don't have the resolver reach out for a service config from the
 		// resolver, use the one specified as default
 		grpc.WithDisableServiceConfig(),
+		grpc.WithConnectParams(grpc.ConnectParams{
+			Backoff: backoff.Config{
+				BaseDelay:  time.Second,
+				Multiplier: 1.2,
+				Jitter:     0.2,
+				MaxDelay:   3 * time.Second,
+			},
+		}),
+	}
+	cc, err := grpc.DialContext(w.baseContext,
+		fmt.Sprintf("%s:///%s", res.Scheme(), addr),
+		dialOpts...,
 	)
 	if err != nil {
 		return fmt.Errorf("error dialing controller for worker auth: %w", err)
 	}
+	w.grpcClientConn.Store(cc)
 
 	w.controllerStatusConn.Store(pbs.NewServerCoordinationServiceClient(cc))
 	w.controllerSessionConn.Store(pbs.NewSessionServiceClient(cc))
-
-	event.WriteSysEvent(w.baseContext, op, "connected to controller", "address", addr)
+	w.controllerMultihopConn.Store(multihop.NewMultihopServiceClient(cc))
 	return nil
 }
 
 func (w *Worker) workerAuthTLSConfig() (*tls.Config, *base.WorkerAuthInfo, error) {
 	var err error
 	info := &base.WorkerAuthInfo{
-		Name:        w.conf.RawConfig.Worker.Name,
-		Description: w.conf.RawConfig.Worker.Description,
+		Name:         w.conf.RawConfig.Worker.Name,
+		Description:  w.conf.RawConfig.Worker.Description,
+		ProxyAddress: w.conf.RawConfig.Worker.PublicAddr,
 	}
 
 	info.ConnectionNonce, err = w.nonceFn(20)

internal/daemon/worker/listeners.go

@@ -6,14 +6,24 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"math"
 	"net"
 	"net/http"
 	"os"
 	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
+	"github.com/hashicorp/boundary/internal/daemon/cluster/handlers"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/go-multierror"
+	nodee "github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/multihop"
+	nodeenet "github.com/hashicorp/nodeenrollment/net"
+	"github.com/hashicorp/nodeenrollment/protocol"
+	"github.com/hashicorp/nodeenrollment/types"
+	"github.com/hashicorp/nodeenrollment/util/temperror"
+	"google.golang.org/grpc"
 )
 
 func (w *Worker) startListeners() error {
@@ -27,32 +37,30 @@ func (w *Worker) startListeners() error {
 	if err != nil {
 		return fmt.Errorf("%s: unable to initialize std logger: %w", op, err)
 	}
-
-	servers := make([]func(), 0, len(w.listeners))
-	for i := range w.listeners {
-		ln := w.listeners[i]
-		workerServer, err := w.configureForWorker(ln, logger)
-		if err != nil {
-			return fmt.Errorf("%s: failed to configure for worker: %w", op, err)
-		}
-		servers = append(servers, workerServer)
+	if w.proxyListener == nil {
+		return fmt.Errorf("%s: nil proxy listener", op)
 	}
 
-	for _, s := range servers {
-		s()
+	workerServer, err := w.configureForWorker(w.proxyListener, logger)
+	if err != nil {
+		return fmt.Errorf("%s: failed to configure for worker: %w", op, err)
 	}
 
+	workerServer()
+
 	return nil
 }
 
 func (w *Worker) configureForWorker(ln *base.ServerListener, logger *log.Logger) (func(), error) {
+	const op = "worker.configureForWorker"
+
 	handler, err := w.handler(HandlerProperties{ListenerConfig: ln.Config})
 	if err != nil {
 		return nil, err
 	}
 
 	cancelCtx := w.baseContext
-	server := &http.Server{
+	httpServer := &http.Server{
 		Handler:           handler,
 		ReadHeaderTimeout: 10 * time.Second,
 		ReadTimeout:       30 * time.Second,
@@ -61,80 +69,187 @@ func (w *Worker) configureForWorker(ln *base.ServerListener, logger *log.Logger)
 			return cancelCtx
 		},
 	}
-	ln.HTTPServer = server
+	ln.HTTPServer = httpServer
 
 	if ln.Config.HTTPReadHeaderTimeout > 0 {
-		server.ReadHeaderTimeout = ln.Config.HTTPReadHeaderTimeout
+		httpServer.ReadHeaderTimeout = ln.Config.HTTPReadHeaderTimeout
 	}
 	if ln.Config.HTTPReadTimeout > 0 {
-		server.ReadTimeout = ln.Config.HTTPReadTimeout
+		httpServer.ReadTimeout = ln.Config.HTTPReadTimeout
 	}
 	if ln.Config.HTTPWriteTimeout > 0 {
-		server.WriteTimeout = ln.Config.HTTPWriteTimeout
+		httpServer.WriteTimeout = ln.Config.HTTPWriteTimeout
 	}
 	if ln.Config.HTTPIdleTimeout > 0 {
-		server.IdleTimeout = ln.Config.HTTPIdleTimeout
+		httpServer.IdleTimeout = ln.Config.HTTPIdleTimeout
 	}
 
-	l := tls.NewListener(ln.ProxyListener, &tls.Config{
-		GetConfigForClient: w.getSessionTls,
-	})
+	fetchCredsFn := func(
+		ctx context.Context,
+		_ nodee.Storage,
+		req *types.FetchNodeCredentialsRequest,
+		_ ...nodee.Option,
+	) (*types.FetchNodeCredentialsResponse, error) {
+		client := w.controllerMultihopConn.Load()
+		if client == nil {
+			return nil, temperror.New(errors.New("error fetching controller connection, client is nil"))
+		}
+		multihopClient, ok := client.(multihop.MultihopServiceClient)
+		if !ok {
+			return nil, temperror.New(errors.New("client could not be understood as a multihop service client"))
+		}
+		// log.Println("proxying fetch node credentials request")
+		return multihopClient.FetchNodeCredentials(ctx, req)
+	}
 
-	return func() { go server.Serve(l) }, nil
+	generateServerCertificatesFn := func(
+		ctx context.Context,
+		_ nodee.Storage,
+		req *types.GenerateServerCertificatesRequest,
+		_ ...nodee.Option,
+	) (*types.GenerateServerCertificatesResponse, error) {
+		client := w.controllerMultihopConn.Load()
+		if client == nil {
+			return nil, temperror.New(errors.New("error fetching controller connection, client is nil"))
+		}
+		multihopClient, ok := client.(multihop.MultihopServiceClient)
+		if !ok {
+			return nil, temperror.New(errors.New("client could not be understood as a multihop service client"))
+		}
+		// log.Println("proxying generate server cert request")
+		return multihopClient.GenerateServerCertificates(ctx, req)
+	}
+
+	interceptingListener, err := protocol.NewInterceptingListener(
+		&protocol.InterceptingListenerConfiguration{
+			Context:      w.baseContext,
+			Storage:      w.WorkerAuthStorage,
+			BaseListener: ln.ProxyListener,
+			BaseTlsConfiguration: &tls.Config{
+				GetConfigForClient: w.getSessionTls,
+			},
+			FetchCredsFunc:                 fetchCredsFn,
+			GenerateServerCertificatesFunc: generateServerCertificatesFn,
+		})
+	if err != nil {
+		return nil, fmt.Errorf("error instantiating node auth listener: %w", err)
+	}
+
+	w.workerAuthSplitListener = nodeenet.NewSplitListener(interceptingListener)
+
+	downstreamServer := grpc.NewServer(
+		grpc.MaxRecvMsgSize(math.MaxInt32),
+		grpc.MaxSendMsgSize(math.MaxInt32),
+	)
+	multihopService, err := handlers.NewMultihopServiceServer(
+		w.WorkerAuthStorage,
+		false,
+		w.controllerMultihopConn,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("%s: error creating multihop service handler: %w", op, err)
+	}
+	multihop.RegisterMultihopServiceServer(downstreamServer, multihopService)
+	statusSessionService := NewWorkerProxyServiceServer(w.controllerStatusConn, w.controllerSessionConn)
+	pbs.RegisterServerCoordinationServiceServer(downstreamServer, statusSessionService)
+	pbs.RegisterSessionServiceServer(downstreamServer, statusSessionService)
+
+	ln.GrpcServer = downstreamServer
+
+	return func() {
+		go w.workerAuthSplitListener.Start()
+		go httpServer.Serve(w.workerAuthSplitListener.OtherListener())
+		go ln.GrpcServer.Serve(
+			&eventingListener{
+				ctx:    cancelCtx,
+				baseLn: w.workerAuthSplitListener.NodeEnrollmentListener(),
+			},
+		)
+	}, nil
 }
 
 func (w *Worker) stopServersAndListeners() error {
-	var closeErrors *multierror.Error
-	err := w.stopHttpServersAndListeners()
-	if err != nil {
-		closeErrors = multierror.Append(closeErrors, err)
+	var mg multierror.Group
+	mg.Go(w.stopHttpServer)
+	mg.Go(w.stopClusterGrpcServer)
+
+	// FIXME (jeff): For some reason, unlike the controller, the grpc server
+	// really likes to hang on closing. Maybe because it's never served a
+	// connection? This is a workaround to force it until I can dig in.
+	var cancel context.CancelFunc
+	if w.workerAuthSplitListener != nil {
+		var ctx context.Context
+		ctx, cancel = context.WithTimeout(w.baseContext, 2*time.Second)
+		go func() {
+			<-ctx.Done()
+			w.workerAuthSplitListener.Stop()
+			cancel()
+		}()
 	}
 
-	err = w.stopAnyListeners()
-	if err != nil {
-		closeErrors = multierror.Append(closeErrors, err)
+	stopErrors := mg.Wait()
+
+	if w.workerAuthSplitListener != nil {
+		cancel()
+		err := w.workerAuthSplitListener.Stop()
+		if err != nil {
+			stopErrors = multierror.Append(stopErrors, err)
+		}
 	}
 
-	return closeErrors.ErrorOrNil()
+	err := w.stopAnyListeners()
+	if err != nil {
+		stopErrors = multierror.Append(stopErrors, err)
+	}
+
+	return stopErrors.ErrorOrNil()
 }
 
-func (w *Worker) stopHttpServersAndListeners() error {
-	var closeErrors *multierror.Error
-	for i := range w.listeners {
-		ln := w.listeners[i]
-		if ln.HTTPServer == nil {
-			continue
-		}
-
-		ctx, cancel := context.WithTimeout(w.baseContext, ln.Config.MaxRequestDuration)
-		ln.HTTPServer.Shutdown(ctx)
-		cancel()
-
-		err := ln.ProxyListener.Close()
-		err = listenerCloseErrorCheck(ln.Config.Type, err)
-		if err != nil {
-			closeErrors = multierror.Append(closeErrors, err)
-		}
+func (w *Worker) stopHttpServer() error {
+	if w.proxyListener == nil {
+		return nil
 	}
 
-	return closeErrors.ErrorOrNil()
+	if w.proxyListener.HTTPServer == nil {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(w.baseContext, w.proxyListener.Config.MaxRequestDuration)
+	w.proxyListener.HTTPServer.Shutdown(ctx)
+	cancel()
+
+	return nil
+}
+
+func (w *Worker) stopClusterGrpcServer() error {
+	if w.proxyListener == nil {
+		return nil
+	}
+	if w.proxyListener.GrpcServer == nil {
+		return nil
+	}
+
+	w.proxyListener.GrpcServer.GracefulStop()
+	return nil
 }
 
 // stopAnyListeners does a final once over the known
 // listeners to make sure we didn't miss any;
 // expected to run at the end of stopServersAndListeners.
 func (w *Worker) stopAnyListeners() error {
+	if w.proxyListener == nil {
+		return nil
+	}
 	var closeErrors *multierror.Error
-	for _, ln := range w.listeners {
-		if ln == nil || ln.ProxyListener == nil {
-			continue
-		}
-
-		err := ln.ProxyListener.Close()
-		err = listenerCloseErrorCheck(ln.Config.Type, err)
-		if err != nil {
-			closeErrors = multierror.Append(closeErrors, err)
-		}
+	var err error
+	if w.workerAuthSplitListener != nil {
+		err = w.workerAuthSplitListener.Stop()
+	} else if w.proxyListener.ProxyListener != nil {
+		err = w.proxyListener.ProxyListener.Close()
+	}
+	err = listenerCloseErrorCheck("proxy", err)
+	if err != nil {
+		closeErrors = multierror.Append(closeErrors, err)
 	}
 
 	return closeErrors.ErrorOrNil()
@@ -160,3 +275,36 @@ func listenerCloseErrorCheck(lnType string, err error) error {
 
 	return err
 }
+
+type eventingListener struct {
+	ctx    context.Context
+	baseLn net.Listener
+}
+
+func (e *eventingListener) Accept() (net.Conn, error) {
+	const op = "worker.(eventingListener).Accept"
+	conn, err := e.baseLn.Accept()
+	if err != nil || conn == nil {
+		return conn, err
+	}
+
+	// This is all best-effort; anything going wrong here shouldn't disrupt the
+	// connection, so on error simply stop trying to get to an event
+	tlsConn, ok := conn.(*tls.Conn)
+	if ok && len(tlsConn.ConnectionState().PeerCertificates) > 0 {
+		keyId, err := nodee.KeyIdFromPkix(tlsConn.ConnectionState().PeerCertificates[0].SubjectKeyId)
+		if err == nil {
+			event.WriteSysEvent(e.ctx, op, "worker successfully authenticated", "key_id", keyId)
+		}
+	}
+
+	return conn, err
+}
+
+func (e *eventingListener) Close() error {
+	return e.baseLn.Close()
+}
+
+func (e *eventingListener) Addr() net.Addr {
+	return e.baseLn.Addr()
+}

internal/daemon/worker/listeners_test.go

@@ -3,13 +3,10 @@ package worker
 import (
 	"context"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"os"
-	"strconv"
 	"testing"
-	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/go-secure-stdlib/listenerutil"
@@ -18,30 +15,19 @@ import (
 
 func TestStartListeners(t *testing.T) {
 	testNonTlsRejected := func(t *testing.T, resp *http.Response, err error) {
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-		defer resp.Body.Close()
-		body, err := io.ReadAll(resp.Body)
-		require.NoError(t, err)
-		require.Equal(t, "Client sent an HTTP request to an HTTPS server.\n", string(body))
+		require.Error(t, err)
 	}
 
 	tests := []struct {
-		name       string
-		listeners  []*listenerutil.ListenerConfig
-		expErr     bool
-		expErrMsg  string
-		assertions func(t *testing.T, w *Worker, addrs []string)
+		name           string
+		proxyListeners []*listenerutil.ListenerConfig
+		expErr         bool
+		expErrMsg      string
+		assertions     func(t *testing.T, w *Worker, addr string)
 	}{
 		{
-			name: "one tcp listener, one unix listener",
-			listeners: []*listenerutil.ListenerConfig{
-				{
-					Type:       "tcp",
-					Purpose:    []string{"proxy"},
-					Address:    "127.0.0.1:0",
-					TLSDisable: true,
-				},
+			name: "unix listener",
+			proxyListeners: []*listenerutil.ListenerConfig{
 				{
 					Type:       "unix",
 					Purpose:    []string{"proxy"},
@@ -50,68 +36,30 @@ func TestStartListeners(t *testing.T) {
 				},
 			},
 			expErr: false,
-			assertions: func(t *testing.T, w *Worker, addrs []string) {
-				require.Len(t, addrs, 2)
-
-				resp, err := http.Get("http://" + addrs[0] + "/v1/proxy/")
-				testNonTlsRejected(t, resp, err)
-
+			assertions: func(t *testing.T, w *Worker, addr string) {
 				cl := http.Client{
 					Transport: &http.Transport{
-						Dial: func(network, addr string) (net.Conn, error) { return net.Dial("unix", addrs[1]) },
+						Dial: func(network, addr string) (net.Conn, error) { return net.Dial("unix", addr) },
 					},
 				}
-				resp, err = cl.Get("http://anything.domain/v1/proxy/")
+				resp, err := cl.Get("http://anything.domain/v1/proxy/")
 				testNonTlsRejected(t, resp, err)
 			},
 		},
 		{
-			name: "multiple tcp listeners, multiple unix listeners",
-			listeners: []*listenerutil.ListenerConfig{
+			name: "tcp listener",
+			proxyListeners: []*listenerutil.ListenerConfig{
 				{
 					Type:       "tcp",
 					Purpose:    []string{"proxy"},
 					Address:    "127.0.0.1:0",
 					TLSDisable: true,
 				},
-				{
-					Type:       "tcp",
-					Purpose:    []string{"proxy"},
-					Address:    "127.0.0.1:0",
-					TLSDisable: true,
-				},
-				{
-					Type:       "unix",
-					Purpose:    []string{"proxy"},
-					Address:    "/tmp/boundary-worker-test-start-listeners0.sock",
-					TLSDisable: true,
-				},
-				{
-					Type:       "unix",
-					Purpose:    []string{"proxy"},
-					Address:    "/tmp/boundary-worker-test-start-listeners1.sock",
-					TLSDisable: true,
-				},
 			},
 			expErr: false,
-			assertions: func(t *testing.T, w *Worker, addrs []string) {
-				require.Len(t, addrs, 4)
-
-				resp, err := http.Get("http://" + addrs[0] + "/v1/proxy/")
+			assertions: func(t *testing.T, w *Worker, addr string) {
+				resp, err := http.Get("http://" + addr + "/v1/proxy/")
 				testNonTlsRejected(t, resp, err)
-
-				resp, err = http.Get("http://" + addrs[1] + "/v1/proxy/")
-				testNonTlsRejected(t, resp, err)
-
-				for _, proxyAddr := range []string{addrs[2], addrs[3]} {
-					cl := http.Client{
-						Transport: &http.Transport{
-							Dial: func(network, addr string) (net.Conn, error) { return net.Dial("unix", proxyAddr) },
-						},
-					}
-					resp, err = cl.Get("http://anything.domain/v1/proxy")
-					testNonTlsRejected(t, resp, err)
-				}
 			},
 		},
 	}
@@ -119,7 +67,7 @@ func TestStartListeners(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			conf := NewTestWorker(t, &TestWorkerOpts{DisableAutoStart: true}).w.conf
-			conf.RawConfig.SharedConfig.Listeners = tt.listeners
+			conf.RawConfig.SharedConfig.Listeners = tt.proxyListeners
 
 			err := conf.SetupListeners(nil, conf.RawConfig.SharedConfig, []string{"proxy"})
 			require.NoError(t, err)
@@ -136,12 +84,8 @@ func TestStartListeners(t *testing.T) {
 
 			require.NoError(t, err)
 
-			addrs := make([]string, 0)
-			for _, l := range w.listeners {
-				addrs = append(addrs, l.ProxyListener.Addr().String())
-			}
 			if tt.assertions != nil {
-				tt.assertions(t, w, addrs)
+				tt.assertions(t, w, w.proxyListener.ProxyListener.Addr().String())
 			}
 		})
 	}
@@ -158,14 +102,14 @@ func TestStopHttpServersAndListeners(t *testing.T) {
 		{
 			name: "no listeners",
 			workerFn: func(t *testing.T) *Worker {
-				return &Worker{listeners: []*base.ServerListener{}}
+				return &Worker{proxyListener: &base.ServerListener{}}
 			},
 			expErr: false,
 		},
 		{
 			name: "nil listeners slice",
 			workerFn: func(t *testing.T) *Worker {
-				return &Worker{listeners: nil}
+				return &Worker{proxyListener: nil}
 			},
 			expErr: false,
 		},
@@ -173,65 +117,42 @@ func TestStopHttpServersAndListeners(t *testing.T) {
 			name: "listeners with nil http server",
 			workerFn: func(t *testing.T) *Worker {
 				return &Worker{
-					listeners: []*base.ServerListener{
-						{HTTPServer: nil},
-						{HTTPServer: nil},
-						{HTTPServer: nil},
+					proxyListener: &base.ServerListener{
+						HTTPServer: nil,
 					},
 				}
 			},
 			expErr: false,
 		},
 		{
-			name: "multiple listeners",
+			name: "valid listener with http server",
 			workerFn: func(t *testing.T) *Worker {
 				l1, err := net.Listen("tcp", "127.0.0.1:0")
 				require.NoError(t, err)
-				l2, err := net.Listen("unix", "/tmp/boundary-worker-TestStopHttpServersAndListeners-"+strconv.FormatInt(time.Now().UnixNano(), 10))
-				require.NoError(t, err)
 
 				s1 := &http.Server{}
-				s2 := &http.Server{}
 
 				go s1.Serve(l1)
-				go s2.Serve(l2)
 
 				// Make sure they're up
 				_, err = http.Get("http://" + l1.Addr().String())
 				require.NoError(t, err)
 
-				c := http.Client{Transport: &http.Transport{
-					Dial: func(network, addr string) (net.Conn, error) {
-						return net.Dial("unix", l2.Addr().String())
-					},
-				}}
-				_, err = c.Get("http://random.domain")
-				require.NoError(t, err)
-
 				return &Worker{
 					baseContext: context.Background(),
-					listeners: []*base.ServerListener{
-						{
-							ProxyListener: l1,
-							HTTPServer:    s1,
-							Config:        &listenerutil.ListenerConfig{Type: "tcp"},
-						},
-						{
-							ProxyListener: l2,
-							HTTPServer:    s2,
-							Config:        &listenerutil.ListenerConfig{Type: "tcp"},
-						},
+					proxyListener: &base.ServerListener{
+						ProxyListener: l1,
+						HTTPServer:    s1,
+						Config:        &listenerutil.ListenerConfig{Type: "tcp"},
 					},
 				}
 			},
 			assertions: func(t *testing.T, w *Worker) {
 				// Asserts the HTTP Servers are closed.
-				require.ErrorIs(t, w.listeners[0].HTTPServer.Serve(w.listeners[0].ProxyListener), http.ErrServerClosed)
-				require.ErrorIs(t, w.listeners[1].HTTPServer.Serve(w.listeners[1].ProxyListener), http.ErrServerClosed)
+				require.ErrorIs(t, w.proxyListener.HTTPServer.Serve(w.proxyListener.ProxyListener), http.ErrServerClosed)
 
 				// Asserts the underlying listeners are closed.
-				require.ErrorIs(t, w.listeners[0].ProxyListener.Close(), net.ErrClosed)
-				require.ErrorIs(t, w.listeners[1].ProxyListener.Close(), net.ErrClosed)
+				require.ErrorIs(t, w.proxyListener.ProxyListener.Close(), net.ErrClosed)
 			},
 			expErr: false,
 		},
@@ -240,7 +161,7 @@ func TestStopHttpServersAndListeners(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			w := tt.workerFn(t)
-			err := w.stopHttpServersAndListeners()
+			err := w.stopServersAndListeners()
 			if tt.expErr {
 				require.EqualError(t, err, tt.expErrStr)
 				return
@@ -265,66 +186,14 @@ func TestStopAnyListeners(t *testing.T) {
 		{
 			name: "nil listeners",
 			workerFn: func(t *testing.T) *Worker {
-				return &Worker{listeners: nil}
+				return &Worker{proxyListener: nil}
 			},
 			expErr: false,
 		},
 		{
 			name: "no listeners",
 			workerFn: func(t *testing.T) *Worker {
-				return &Worker{listeners: []*base.ServerListener{}}
-			},
-			expErr: false,
-		},
-		{
-			name: "listeners slice with nil server listeners",
-			workerFn: func(t *testing.T) *Worker {
-				return &Worker{listeners: []*base.ServerListener{nil, nil, nil}}
-			},
-			expErr: false,
-		},
-		{
-			name: "non-empty but nil listeners",
-			workerFn: func(t *testing.T) *Worker {
-				return &Worker{listeners: []*base.ServerListener{
-					{ProxyListener: nil}, {ProxyListener: nil}, {ProxyListener: nil},
-				}}
-			},
-			expErr: false,
-		},
-		{
-			name: "multiple listeners, including a closed one",
-			workerFn: func(t *testing.T) *Worker {
-				l1, err := net.Listen("tcp", "127.0.0.1:0")
-				require.NoError(t, err)
-
-				l2, err := net.Listen("unix", "/tmp/boundary-worker-TestStopAnyListeners-"+strconv.FormatInt(time.Now().UnixNano(), 10))
-				require.NoError(t, err)
-
-				l3, err := net.Listen("tcp", "127.0.0.1:0")
-				require.NoError(t, err)
-				require.NoError(t, l3.Close())
-
-				return &Worker{listeners: []*base.ServerListener{
-					{
-						Config:        &listenerutil.ListenerConfig{Type: "tcp"},
-						ProxyListener: l1,
-					},
-					{
-						Config:        &listenerutil.ListenerConfig{Type: "tcp"},
-						ProxyListener: l2,
-					},
-					{
-						Config:        &listenerutil.ListenerConfig{Type: "tcp"},
-						ProxyListener: l3,
-					},
-				}}
-			},
-			assertions: func(t *testing.T, w *Worker) {
-				for i := range w.listeners {
-					ln := w.listeners[i]
-					require.ErrorIs(t, ln.ProxyListener.Close(), net.ErrClosed)
-				}
+				return &Worker{proxyListener: &base.ServerListener{}}
 			},
 			expErr: false,
 		},

internal/daemon/worker/status.go

@@ -3,6 +3,7 @@ package worker
 import (
 	"context"
 	"errors"
+	"fmt"
 	"math/rand"
 	"time"
 
@@ -11,13 +12,14 @@ import (
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/boundary/internal/servers"
-	"github.com/hashicorp/boundary/internal/types/resource"
+	"github.com/hashicorp/go-secure-stdlib/strutil"
 	"google.golang.org/grpc/resolver"
 )
 
 type LastStatusInformation struct {
 	*pbs.StatusResponse
-	StatusTime time.Time
+	StatusTime              time.Time
+	LastCalculatedUpstreams []string
 }
 
 func (w *Worker) startStatusTicking(cancelCtx context.Context) {
@@ -90,6 +92,15 @@ func (w *Worker) WaitForNextSuccessfulStatusUpdate() error {
 
 func (w *Worker) sendWorkerStatus(cancelCtx context.Context) {
 	const op = "worker.(Worker).sendWorkerStatus"
+
+	// If we've never managed to successfully authenticate then we won't have
+	// any session information anyways and this will produce a ton of noise in
+	// observability, so suppress it
+	if !w.everAuthenticated.Load() {
+		event.WriteSysEvent(cancelCtx, op, "worker is not authenticated to an upstream, not sending status")
+		return
+	}
+
 	// First send info as-is. We'll perform cleanup duties after we
 	// get cancel/job change info back.
 	var activeJobs []*pbs.JobStatus
@@ -127,22 +138,30 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context) {
 
 	// Send status information
 	client := w.controllerStatusConn.Load().(pbs.ServerCoordinationServiceClient)
-	var tags map[string]*servers.TagValues
+	var tags []*servers.TagPair
 	// If we're not going to request a tag update, no reason to have these
 	// marshaled on every status call.
 	if w.updateTags.Load() {
-		tags = w.tags.Load().(map[string]*servers.TagValues)
+		tags = w.tags.Load().([]*servers.TagPair)
 	}
 	statusCtx, statusCancel := context.WithTimeout(cancelCtx, common.StatusTimeout)
 	defer statusCancel()
+
+	keyId := w.WorkerAuthCurrentKeyId.Load()
+
+	if w.conf.RawConfig.Worker.Name == "" && keyId == "" {
+		event.WriteError(statusCtx, op, errors.New("worker name and keyId are both empty; one is needed to identify a worker"),
+			event.WithInfoMsg("error making status request to controller"))
+	}
+
 	result, err := client.Status(statusCtx, &pbs.StatusRequest{
 		Jobs: activeJobs,
-		Worker: &servers.Server{
-			PrivateId:   w.conf.RawConfig.Worker.Name,
-			Type:        resource.Worker.String(),
+		WorkerStatus: &servers.ServerWorkerStatus{
+			Name:        w.conf.RawConfig.Worker.Name,
 			Description: w.conf.RawConfig.Worker.Description,
 			Address:     w.conf.RawConfig.Worker.PublicAddr,
 			Tags:        tags,
+			KeyId:       keyId,
 		},
 		UpdateTags: w.updateTags.Load(),
 	})
@@ -182,19 +201,27 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context) {
 		}
 	} else {
 		w.updateTags.Store(false)
-		addrs := make([]resolver.Address, 0, len(result.Controllers))
-		strAddrs := make([]string, 0, len(result.Controllers))
-		for _, v := range result.Controllers {
-			addrs = append(addrs, resolver.Address{Addr: v.Address})
-			strAddrs = append(strAddrs, v.Address)
+		// This may be nil if we are in a multiple hop scenario
+		var addrs []resolver.Address
+		var newUpstreams []string
+		if len(result.CalculatedUpstreams) > 0 {
+			addrs = make([]resolver.Address, 0, len(result.CalculatedUpstreams))
+			for _, v := range result.CalculatedUpstreams {
+				addrs = append(addrs, resolver.Address{Addr: v.Address})
+				newUpstreams = append(newUpstreams, v.Address)
+			}
+			lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
+			// Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses
+			if lastStatus != nil && !strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, newUpstreams) {
+				upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastStatus.LastCalculatedUpstreams, newUpstreams)
+				event.WriteSysEvent(cancelCtx, op, upstreamsMessage)
+				w.Resolver().UpdateState(resolver.State{Addresses: addrs})
+			} else if lastStatus == nil {
+				w.Resolver().UpdateState(resolver.State{Addresses: addrs})
+				event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first status set to: %s", newUpstreams))
+			}
 		}
-		switch len(strAddrs) {
-		case 0:
-			event.WriteError(statusCtx, op, errors.New("got no controller addresses from controller; possibly prior to first status save, not persisting"))
-		default:
-			w.Resolver().UpdateState(resolver.State{Addresses: addrs})
-		}
-		w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now()})
+		w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now(), LastCalculatedUpstreams: newUpstreams})
 
 		for _, request := range result.GetJobsRequests() {
 			switch request.GetRequestType() {

internal/daemon/worker/testing.go

@@ -4,17 +4,18 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/go-hclog"
 	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
-	"github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -31,7 +32,7 @@ type TestWorker struct {
 	name   string
 }
 
-// Worker returns the underlying controller
+// Worker returns the underlying worker
 func (tw *TestWorker) Worker() *Worker {
 	return tw.w
 }
@@ -52,10 +53,10 @@ func (tw *TestWorker) Name() string {
 	return tw.name
 }
 
-func (tw *TestWorker) ControllerAddrs() []string {
+func (tw *TestWorker) UpstreamAddrs() []string {
 	var addrs []string
 	lastStatus := tw.w.LastStatusSuccess()
-	for _, v := range lastStatus.GetControllers() {
+	for _, v := range lastStatus.GetCalculatedUpstreams() {
 		addrs = append(addrs, v.Address)
 	}
 
@@ -164,8 +165,8 @@ type TestWorkerOpts struct {
 	// Config; if not provided a dev one will be created
 	Config *config.Config
 
-	// Sets initial controller addresses
-	InitialControllers []string
+	// Sets initial upstream addresses
+	InitialUpstreams []string
 
 	// If true, the worker will not be started
 	DisableAutoStart bool
@@ -173,6 +174,12 @@ type TestWorkerOpts struct {
 	// The worker auth KMS to use, or one will be created
 	WorkerAuthKms wrapping.Wrapper
 
+	// The worker credential storage KMS to use, or one will be created
+	WorkerAuthStorageKms wrapping.Wrapper
+
+	// The location of the worker's auth storage
+	WorkerAuthStoragePath string
+
 	// The name to use for the worker, otherwise one will be randomly
 	// generated, unless provided in a non-nil Config
 	Name string
@@ -191,6 +198,9 @@ type TestWorkerOpts struct {
 	// Overrides worker's nonceFn, for cases where we want to have control
 	// over the nonce we send to the Controller
 	NonceFn randFn
+
+	// If set, override the normal auth rotation period
+	AuthRotationPeriod time.Duration
 }
 
 func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
@@ -223,8 +233,8 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 		opts.Config.Worker.Name = opts.Name
 	}
 
-	if len(opts.InitialControllers) > 0 {
-		opts.Config.Worker.Controllers = opts.InitialControllers
+	if len(opts.InitialUpstreams) > 0 {
+		opts.Config.Worker.InitialUpstreams = opts.InitialUpstreams
 	}
 
 	// Start a logger
@@ -260,6 +270,8 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 	switch {
 	case opts.WorkerAuthKms != nil:
 		tw.b.WorkerAuthKms = opts.WorkerAuthKms
+	case opts.WorkerAuthStorageKms != nil:
+		tw.b.WorkerAuthStorageKms = opts.WorkerAuthStorageKms
 	default:
 		if err := tw.b.SetupKMSes(tw.b.Context, nil, opts.Config); err != nil {
 			t.Fatal(err)
@@ -288,6 +300,8 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 		t.Fatal(err)
 	}
 
+	tw.w.TestOverrideAuthRotationPeriod = opts.AuthRotationPeriod
+
 	if opts.NonceFn != nil {
 		tw.w.nonceFn = opts.NonceFn
 	}
@@ -309,17 +323,19 @@ func (tw *TestWorker) AddClusterWorkerMember(t testing.TB, opts *TestWorkerOpts)
 	}
 	nextOpts := &TestWorkerOpts{
 		WorkerAuthKms:             tw.w.conf.WorkerAuthKms,
+		WorkerAuthStorageKms:      tw.w.conf.WorkerAuthStorageKms,
 		Name:                      opts.Name,
-		InitialControllers:        tw.ControllerAddrs(),
+		InitialUpstreams:          tw.UpstreamAddrs(),
 		Logger:                    tw.w.conf.Logger,
 		StatusGracePeriodDuration: opts.StatusGracePeriodDuration,
 	}
 	if nextOpts.Name == "" {
 		var err error
-		nextOpts.Name, err = base62.Random(5)
+		nextOpts.Name, err = db.NewPublicId("w")
 		if err != nil {
 			t.Fatal(err)
 		}
+		nextOpts.Name = strings.ToLower(nextOpts.Name)
 		event.WriteSysEvent(context.TODO(), op, "worker name generated", "name", nextOpts.Name)
 	}
 	return NewTestWorker(t, nextOpts)

internal/daemon/worker/testing_test.go

@@ -6,7 +6,9 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
 	"github.com/stretchr/testify/require"
 )
 
@@ -60,3 +62,28 @@ func TestTestWorkerLookupSessionMissing(t *testing.T) {
 	require.False(ok)
 	require.Equal(TestSessionInfo{}, actual)
 }
+
+func TestTestWorker_WorkerAuthStorageKms(t *testing.T) {
+	tests := []struct {
+		name    string
+		wrapper wrapping.Wrapper
+	}{
+		{
+			name:    "Nil Wrapper",
+			wrapper: nil,
+		},
+		{
+			name:    "Valid Wrapper",
+			wrapper: db.TestWrapper(t),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require := require.New(t)
+			tw := NewTestWorker(t, &TestWorkerOpts{
+				WorkerAuthStorageKms: tt.wrapper,
+			})
+			require.Equal(tt.wrapper, tw.Config().WorkerAuthStorageKms)
+		})
+	}
+}

internal/daemon/worker/worker.go

@@ -14,6 +14,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/mr-tron/base58"
+
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
@@ -21,13 +25,18 @@ import (
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
-	"github.com/hashicorp/boundary/internal/servers"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/hashicorp/go-secure-stdlib/mlock"
+	"github.com/hashicorp/nodeenrollment/multihop"
+	nodeenet "github.com/hashicorp/nodeenrollment/net"
+	nodeefile "github.com/hashicorp/nodeenrollment/storage/file"
+	"github.com/hashicorp/nodeenrollment/types"
 	ua "go.uber.org/atomic"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/protobuf/proto"
 )
 
 type randFn func(length int) (string, error)
@@ -42,7 +51,10 @@ type Worker struct {
 
 	tickerWg sync.WaitGroup
 
+	grpcClientConn *atomic.Value
+
 	controllerStatusConn *atomic.Value
+	everAuthenticated    *ua.Bool
 	lastStatusSuccess    *atomic.Value
 	workerStartTime      time.Time
 
@@ -51,7 +63,9 @@ type Worker struct {
 	controllerSessionConn *atomic.Value
 	sessionInfoMap        *sync.Map
 
-	listeners []*base.ServerListener
+	controllerMultihopConn *atomic.Value
+
+	proxyListener *base.ServerListener
 
 	// Used to generate a random nonce for Controller connections
 	nonceFn randFn
@@ -62,11 +76,18 @@ type Worker struct {
 	// This stores whether or not to send updated tags on the next status
 	// request. It can be set via startup in New below, or (eventually) via
 	// SIGHUP.
-	updateTags ua.Bool
+	updateTags *ua.Bool
 
-	// Test-specific options
+	// The storage for node enrollment
+	WorkerAuthStorage             *nodeefile.Storage
+	WorkerAuthCurrentKeyId        *ua.String
+	WorkerAuthRegistrationRequest string
+	workerAuthSplitListener       *nodeenet.SplitListener
+
+	// Test-specific options (and possibly hidden dev-mode flags)
 	TestOverrideX509VerifyDnsName  string
 	TestOverrideX509VerifyCertPool *x509.CertPool
+	TestOverrideAuthRotationPeriod time.Duration
 }
 
 func New(conf *Config) (*Worker, error) {
@@ -75,16 +96,21 @@ func New(conf *Config) (*Worker, error) {
 	metric.InitializeClusterClientCollectors(conf.PrometheusRegisterer)
 
 	w := &Worker{
-		conf:                  conf,
-		logger:                conf.Logger.Named("worker"),
-		started:               ua.NewBool(false),
-		controllerStatusConn:  new(atomic.Value),
-		lastStatusSuccess:     new(atomic.Value),
-		controllerResolver:    new(atomic.Value),
-		controllerSessionConn: new(atomic.Value),
-		sessionInfoMap:        new(sync.Map),
-		tags:                  new(atomic.Value),
-		nonceFn:               base62.Random,
+		conf:                   conf,
+		logger:                 conf.Logger.Named("worker"),
+		started:                ua.NewBool(false),
+		grpcClientConn:         new(atomic.Value),
+		controllerStatusConn:   new(atomic.Value),
+		everAuthenticated:      ua.NewBool(false),
+		lastStatusSuccess:      new(atomic.Value),
+		controllerResolver:     new(atomic.Value),
+		controllerSessionConn:  new(atomic.Value),
+		sessionInfoMap:         new(sync.Map),
+		controllerMultihopConn: new(atomic.Value),
+		tags:                   new(atomic.Value),
+		updateTags:             ua.NewBool(false),
+		nonceFn:                base62.Random,
+		WorkerAuthCurrentKeyId: new(ua.String),
 	}
 
 	w.lastStatusSuccess.Store((*LastStatusInformation)(nil))
@@ -121,6 +147,7 @@ func New(conf *Config) (*Worker, error) {
 		}
 	}
 
+	var listenerCount int
 	for i := range conf.Listeners {
 		l := conf.Listeners[i]
 		if l == nil || l.Config == nil || l.Config.Purpose == nil {
@@ -131,11 +158,20 @@ func New(conf *Config) (*Worker, error) {
 		}
 		switch l.Config.Purpose[0] {
 		case "proxy":
-			w.listeners = append(w.listeners, l)
+			if w.proxyListener == nil {
+				w.proxyListener = l
+			}
+			listenerCount++
 		}
 	}
-	if len(w.listeners) == 0 {
-		return nil, fmt.Errorf("no proxy listeners found")
+	if listenerCount != 1 {
+		return nil, fmt.Errorf("exactly one proxy listener is required")
+	}
+
+	w.WorkerAuthStorage, err = nodeefile.New(w.baseContext,
+		nodeefile.WithBaseDirectory(w.conf.RawConfig.Worker.AuthStoragePath))
+	if err != nil {
+		return nil, err
 	}
 
 	return w, nil
@@ -143,29 +179,144 @@ func New(conf *Config) (*Worker, error) {
 
 func (w *Worker) Start() error {
 	const op = "worker.(Worker).Start"
+
+	w.baseContext, w.baseCancel = context.WithCancel(context.Background())
+
 	if w.started.Load() {
 		event.WriteSysEvent(w.baseContext, op, "already started, skipping")
 		return nil
 	}
 
-	w.baseContext, w.baseCancel = context.WithCancel(context.Background())
-
 	scheme := strconv.FormatInt(time.Now().UnixNano(), 36)
 	controllerResolver := manual.NewBuilderWithScheme(scheme)
 	w.controllerResolver.Store(controllerResolver)
 
+	if w.conf.WorkerAuthKms == nil {
+		// In this section, we look for existing worker credentials. The two
+		// variables below store whether to create new credentials and whether
+		// to create a fetch request so it can be displayed in the worker
+		// startup info. These may be different because if initial creds have
+		// been generated on the worker side but not yet authorized/fetched from
+		// the controller, we don't want to invalidate that request on restart
+		// by generating a new set of credentials. However it's safe to output a
+		// new fetch request so we do in fact do that.
+		var createNodeAuthCreds bool
+		var createFetchRequest bool
+		nodeCreds, err := types.LoadNodeCredentials(w.baseContext, w.WorkerAuthStorage, nodeenrollment.CurrentId, nodeenrollment.WithWrapper(w.conf.WorkerAuthStorageKms))
+		switch {
+		case err == nil:
+			// It's unclear why this would ever happen -- it shouldn't -- so
+			// this is simply safety against panics if something goes
+			// catastrophically wrong
+			if nodeCreds == nil {
+				event.WriteSysEvent(w.baseContext, op, "no error loading worker auth creds but nil creds, creating new creds for registration")
+				createNodeAuthCreds = true
+				createFetchRequest = true
+				break
+			}
+
+			// Check that we have valid creds, or that we have generated creds but
+			// simply are still waiting on authentication (in which case we don't
+			// want to invalidate what we've already sent)
+			var validCreds bool
+			switch len(nodeCreds.CertificateBundles) {
+			case 0:
+				// Still waiting on initial creds, so don't invalidate the request
+				// by creating new credentials. However, we will generate and
+				// display a new valid request in case the first was lost.
+				createFetchRequest = true
+
+			default:
+				now := time.Now()
+				for _, bundle := range nodeCreds.CertificateBundles {
+					if bundle.CertificateNotBefore.AsTime().Before(now) && bundle.CertificateNotAfter.AsTime().After(now) {
+						// If we have a certificate in its validity period,
+						// everything is fine
+						validCreds = true
+						break
+					}
+				}
+
+				// Certificates are both expired, so create new credentials and
+				// output a request based on those
+				createNodeAuthCreds = !validCreds
+				createFetchRequest = !validCreds
+			}
+
+		case errors.Is(err, nodeenrollment.ErrNotFound):
+			// Nothing was found on disk, so create
+			createNodeAuthCreds = true
+			createFetchRequest = true
+
+		default:
+			// Some other type of error happened, bail out
+			return fmt.Errorf("error loading worker auth creds: %w", err)
+		}
+
+		// NOTE: this block _must_ be before the `if createFetchRequest` block
+		// or the fetch request may have no credentials to work with
+		if createNodeAuthCreds {
+			nodeCreds, err = types.NewNodeCredentials(
+				w.baseContext,
+				w.WorkerAuthStorage,
+				nodeenrollment.WithRandomReader(w.conf.SecureRandomReader),
+				nodeenrollment.WithWrapper(w.conf.WorkerAuthStorageKms),
+			)
+			if err != nil {
+				return fmt.Errorf("error generating new worker auth creds: %w", err)
+			}
+		}
+
+		if createFetchRequest {
+			if nodeCreds == nil {
+				return fmt.Errorf("need to create fetch request but worker auth creds are nil: %w", err)
+			}
+			req, err := nodeCreds.CreateFetchNodeCredentialsRequest(w.baseContext, nodeenrollment.WithRandomReader(w.conf.SecureRandomReader))
+			if err != nil {
+				return fmt.Errorf("error creating worker auth fetch credentials request: %w", err)
+			}
+			reqBytes, err := proto.Marshal(req)
+			if err != nil {
+				return fmt.Errorf("error marshaling worker auth fetch credentials request: %w", err)
+			}
+			w.WorkerAuthRegistrationRequest = base58.FastBase58Encoding(reqBytes)
+			if err != nil {
+				return fmt.Errorf("error encoding worker auth registration request: %w", err)
+			}
+			currentKeyId, err := nodeenrollment.KeyIdFromPkix(nodeCreds.CertificatePublicKeyPkix)
+			if err != nil {
+				return fmt.Errorf("error deriving worker auth key id: %w", err)
+			}
+			w.WorkerAuthCurrentKeyId.Store(currentKeyId)
+		}
+		// Regardless, we want to load the currentKeyId
+		currentKeyId, err := nodeenrollment.KeyIdFromPkix(nodeCreds.CertificatePublicKeyPkix)
+		if err != nil {
+			return fmt.Errorf("error deriving worker auth key id: %w", err)
+		}
+		w.WorkerAuthCurrentKeyId.Store(currentKeyId)
+	}
+
 	if err := w.startListeners(); err != nil {
 		return fmt.Errorf("error starting worker listeners: %w", err)
 	}
-	if err := w.startControllerConnections(); err != nil {
+	if err := w.StartControllerConnections(); err != nil {
 		return fmt.Errorf("error making controller connections: %w", err)
 	}
 
-	w.tickerWg.Add(1)
+	// Rather than deal with some of the potential error conditions for Add on
+	// the waitgroup vs. Done (in case a function exits immediately), we will
+	// always start rotation and simply exit early if we're using KMS, and
+	// always add 2 here.
+	w.tickerWg.Add(2)
 	go func() {
 		defer w.tickerWg.Done()
 		w.startStatusTicking(w.baseContext)
 	}()
+	go func() {
+		defer w.tickerWg.Done()
+		w.startAuthRotationTicking(w.baseContext)
+	}()
 
 	w.workerStartTime = time.Now()
 	w.started.Store(true)
@@ -241,19 +392,40 @@ func (w *Worker) Resolver() *manual.Resolver {
 
 func (w *Worker) ParseAndStoreTags(incoming map[string][]string) {
 	if len(incoming) == 0 {
-		w.tags.Store(map[string]*servers.TagValues{})
+		w.tags.Store([]*servers.TagPair{})
 		return
 	}
-	tags := make(map[string]*servers.TagValues, len(incoming))
-	for k, v := range incoming {
-		tags[k] = &servers.TagValues{
-			Values: append(make([]string, 0, len(v)), v...),
+	tags := []*servers.TagPair{}
+	for k, vals := range incoming {
+		for _, v := range vals {
+			tags = append(tags, &servers.TagPair{
+				Key:   k,
+				Value: v,
+			})
 		}
 	}
 	w.tags.Store(tags)
 	w.updateTags.Store(true)
 }
 
+// GrpcClientConn returns the underlying client connection
+func (w *Worker) GrpcClientConn() (*grpc.ClientConn, error) {
+	rawConn := w.grpcClientConn.Load()
+	if rawConn == nil {
+		return nil, errors.New("unable to load grpc client conn")
+	}
+	cc, ok := rawConn.(*grpc.ClientConn)
+	if !ok {
+		return nil, fmt.Errorf("invalid grpc client connection %T", rawConn)
+	}
+	if cc == nil {
+		return nil, fmt.Errorf("client connection is nil")
+	}
+
+	return cc, nil
+}
+
+// ControllerSessionConn returns the underlying session service client
 func (w *Worker) ControllerSessionConn() (pbs.SessionServiceClient, error) {
 	rawConn := w.controllerSessionConn.Load()
 	if rawConn == nil {
@@ -270,6 +442,40 @@ func (w *Worker) ControllerSessionConn() (pbs.SessionServiceClient, error) {
 	return sessClient, nil
 }
 
+// ControllerServerCoordinationServiceConn returns the underlying server coordination service client
+func (w *Worker) ControllerServerCoordinationServiceConn() (pbs.ServerCoordinationServiceClient, error) {
+	rawConn := w.controllerStatusConn.Load()
+	if rawConn == nil {
+		return nil, errors.New("unable to load controller service coordination service connection")
+	}
+	statusClient, ok := rawConn.(pbs.ServerCoordinationServiceClient)
+	if !ok {
+		return nil, fmt.Errorf("invalid service coordination service client %T", rawConn)
+	}
+	if statusClient == nil {
+		return nil, fmt.Errorf("service coordination service client is nil")
+	}
+
+	return statusClient, nil
+}
+
+// ControllerMultihopConn returns the underlying multihop service client
+func (w *Worker) ControllerMultihopConn() (multihop.MultihopServiceClient, error) {
+	rawConn := w.controllerMultihopConn.Load()
+	if rawConn == nil {
+		return nil, errors.New("unable to load controller multihop service connection")
+	}
+	multihopClient, ok := rawConn.(multihop.MultihopServiceClient)
+	if !ok {
+		return nil, fmt.Errorf("invalid multihop client %T", rawConn)
+	}
+	if multihopClient == nil {
+		return nil, fmt.Errorf("session multihop is nil")
+	}
+
+	return multihopClient, nil
+}
+
 func (w *Worker) getSessionTls(hello *tls.ClientHelloInfo) (*tls.Config, error) {
 	const op = "worker.(Worker).getSessionTls"
 	ctx := w.baseContext
@@ -300,8 +506,9 @@ func (w *Worker) getSessionTls(hello *tls.ClientHelloInfo) (*tls.Config, error)
 	defer cancel()
 
 	resp, err := conn.LookupSession(timeoutContext, &pbs.LookupSessionRequest{
-		ServerId:  w.conf.RawConfig.Worker.Name,
-		SessionId: sessionId,
+		ServerId:    w.conf.RawConfig.Worker.Name,
+		SessionId:   sessionId,
+		WorkerKeyId: w.WorkerAuthCurrentKeyId.Load(),
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error validating session: %w", err)

internal/daemon/worker/worker_proxy_service.go

@@ -0,0 +1,67 @@
+package worker
+
+import (
+	"context"
+	"sync/atomic"
+
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+)
+
+type workerProxyServiceServer struct {
+	pbs.UnimplementedServerCoordinationServiceServer
+	pbs.UnimplementedSessionServiceServer
+
+	scsClient *atomic.Value
+	ssClient  *atomic.Value
+}
+
+func NewWorkerProxyServiceServer(
+	scsClient *atomic.Value,
+	ssClient *atomic.Value,
+) *workerProxyServiceServer {
+	return &workerProxyServiceServer{
+		scsClient: scsClient,
+		ssClient:  ssClient,
+	}
+}
+
+var (
+	_ pbs.ServerCoordinationServiceServer = &workerProxyServiceServer{}
+	_ pbs.SessionServiceServer            = &workerProxyServiceServer{}
+)
+
+func (ws *workerProxyServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
+	resp, err := ws.scsClient.Load().(pbs.ServerCoordinationServiceClient).Status(ctx, req)
+
+	if resp != nil {
+		// We don't currently support distributing new addreses to workers
+		// multiple hops away so ensure they're stripped out
+		resp.CalculatedUpstreams = nil
+	}
+
+	return resp, err
+}
+
+func (ws *workerProxyServiceServer) LookupSession(ctx context.Context, req *pbs.LookupSessionRequest) (*pbs.LookupSessionResponse, error) {
+	return ws.ssClient.Load().(pbs.SessionServiceClient).LookupSession(ctx, req)
+}
+
+func (ws *workerProxyServiceServer) CancelSession(ctx context.Context, req *pbs.CancelSessionRequest) (*pbs.CancelSessionResponse, error) {
+	return ws.ssClient.Load().(pbs.SessionServiceClient).CancelSession(ctx, req)
+}
+
+func (ws *workerProxyServiceServer) ActivateSession(ctx context.Context, req *pbs.ActivateSessionRequest) (*pbs.ActivateSessionResponse, error) {
+	return ws.ssClient.Load().(pbs.SessionServiceClient).ActivateSession(ctx, req)
+}
+
+func (ws *workerProxyServiceServer) AuthorizeConnection(ctx context.Context, req *pbs.AuthorizeConnectionRequest) (*pbs.AuthorizeConnectionResponse, error) {
+	return ws.ssClient.Load().(pbs.SessionServiceClient).AuthorizeConnection(ctx, req)
+}
+
+func (ws *workerProxyServiceServer) ConnectConnection(ctx context.Context, req *pbs.ConnectConnectionRequest) (*pbs.ConnectConnectionResponse, error) {
+	return ws.ssClient.Load().(pbs.SessionServiceClient).ConnectConnection(ctx, req)
+}
+
+func (ws *workerProxyServiceServer) CloseConnection(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) {
+	return ws.ssClient.Load().(pbs.SessionServiceClient).CloseConnection(ctx, req)
+}

internal/daemon/worker/worker_test.go

@@ -1,14 +1,27 @@
 package worker
 
 import (
+	"context"
+	"crypto/rand"
+	"os"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
+	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/configutil/v2"
 	"github.com/hashicorp/go-secure-stdlib/listenerutil"
+	"github.com/hashicorp/nodeenrollment"
+	"github.com/hashicorp/nodeenrollment/registration"
+	"github.com/hashicorp/nodeenrollment/rotation"
+	nodeefile "github.com/hashicorp/nodeenrollment/storage/file"
+	"github.com/hashicorp/nodeenrollment/types"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func TestWorkerNew(t *testing.T) {
@@ -23,13 +36,13 @@ func TestWorkerNew(t *testing.T) {
 			name:      "nil listeners",
 			in:        &Config{Server: &base.Server{Listeners: nil}},
 			expErr:    true,
-			expErrMsg: "no proxy listeners found",
+			expErrMsg: "exactly one proxy listener is required",
 		},
 		{
 			name:      "zero listeners",
 			in:        &Config{Server: &base.Server{Listeners: []*base.ServerListener{}}},
 			expErr:    true,
-			expErrMsg: "no proxy listeners found",
+			expErrMsg: "exactly one proxy listener is required",
 		},
 		{
 			name: "populated with nil values",
@@ -43,7 +56,7 @@ func TestWorkerNew(t *testing.T) {
 				},
 			},
 			expErr:    true,
-			expErrMsg: "no proxy listeners found",
+			expErrMsg: "exactly one proxy listener is required",
 		},
 		{
 			name: "multiple purposes",
@@ -60,7 +73,7 @@ func TestWorkerNew(t *testing.T) {
 			expErrMsg: `found listener with multiple purposes "api,proxy"`,
 		},
 		{
-			name: "valid listeners",
+			name: "too many proxy listeners",
 			in: &Config{
 				Server: &base.Server{
 					Listeners: []*base.ServerListener{
@@ -71,10 +84,21 @@ func TestWorkerNew(t *testing.T) {
 					},
 				},
 			},
-			expErr: false,
-			assertions: func(t *testing.T, w *Worker) {
-				require.Len(t, w.listeners, 2)
+			expErr:    true,
+			expErrMsg: "exactly one proxy listener is required",
+		},
+		{
+			name: "valid listeners",
+			in: &Config{
+				Server: &base.Server{
+					Listeners: []*base.ServerListener{
+						{Config: &listenerutil.ListenerConfig{Purpose: []string{"api"}}},
+						{Config: &listenerutil.ListenerConfig{Purpose: []string{"proxy"}}},
+						{Config: &listenerutil.ListenerConfig{Purpose: []string{"cluster"}}},
+					},
+				},
 			},
+			expErr: false,
 		},
 		{
 			name: "worker nonce func is set",
@@ -112,3 +136,139 @@ func TestWorkerNew(t *testing.T) {
 		})
 	}
 }
+
+func TestSetupWorkerAuthStorage(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	ts := db.TestWrapper(t)
+	keyId, err := ts.KeyId(ctx)
+	require.NoError(t, err)
+
+	// First, just test the key ID is populated
+	tmpDir, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpDir)) })
+	tw := NewTestWorker(t, &TestWorkerOpts{
+		WorkerAuthStorageKms:  ts,
+		WorkerAuthStoragePath: tmpDir,
+		DisableAutoStart:      true,
+	})
+	t.Cleanup(tw.Shutdown)
+	err = tw.Worker().Start()
+	require.NoError(t, err)
+
+	wKeyId, err := tw.Config().WorkerAuthStorageKms.KeyId(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, keyId, wKeyId)
+
+	// Create a fresh persistent dir for the following tests
+	tmpDir, err = os.MkdirTemp("", "")
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, os.RemoveAll(tmpDir)) })
+
+	// Get an initial set of authorized node credentials
+	initStorage, err := nodeefile.New(ctx)
+	require.NoError(t, err)
+	t.Cleanup(initStorage.Cleanup)
+	_, err = rotation.RotateRootCertificates(ctx, initStorage)
+	require.NoError(t, err)
+	initNodeCreds, err := types.NewNodeCredentials(ctx, initStorage)
+	require.NoError(t, err)
+	req, err := initNodeCreds.CreateFetchNodeCredentialsRequest(ctx)
+	require.NoError(t, err)
+	_, err = registration.AuthorizeNode(ctx, initStorage, req)
+	require.NoError(t, err)
+	fetchResp, err := registration.FetchNodeCredentials(ctx, initStorage, req)
+	require.NoError(t, err)
+	initNodeCreds, err = initNodeCreds.HandleFetchNodeCredentialsResponse(ctx, initStorage, fetchResp)
+	require.NoError(t, err)
+	initKeyId, err := nodeenrollment.KeyIdFromPkix(initNodeCreds.CertificatePublicKeyPkix)
+	require.NoError(t, err)
+
+	nonce := make([]byte, nodeenrollment.NonceSize)
+	_, err = rand.Reader.Read(nonce)
+	require.NoError(t, err)
+
+	// What's going on here: in each test we are simulating a startup of a
+	// worker that has storage in various states. The input is a function to
+	// modify the current state of node credentials by using the worker's
+	// storage, but this happens before Start so we haven't done checking yet;
+	// the assertions check what the final result is.
+	tests := []struct {
+		name                   string
+		in                     func(t *testing.T, w *Worker)
+		expKeyId               string // If set, the existing key ID to expect
+		expRegistrationRequest bool   // Whether we should have seen a registration request generated
+		expError               string // Some other error
+	}{
+		{
+			name: "no creds",
+			in: func(t *testing.T, w *Worker) {
+				// Do nothing; in this case it will have already been cleared
+			},
+			expRegistrationRequest: true,
+		},
+		{
+			name: "valid creds",
+			in: func(t *testing.T, w *Worker) {
+				// Store the authorized creds
+				require.NoError(t, initNodeCreds.Store(ctx, w.WorkerAuthStorage))
+			},
+			expKeyId: initKeyId,
+		},
+		{
+			name: "existing but not validated",
+			in: func(t *testing.T, w *Worker) {
+				creds := proto.Clone(initNodeCreds).(*types.NodeCredentials)
+				creds.CertificateBundles = nil
+				creds.RegistrationNonce = nonce
+				require.NoError(t, creds.Store(ctx, w.WorkerAuthStorage))
+			},
+			expKeyId:               initKeyId,
+			expRegistrationRequest: true,
+		},
+		{
+			name: "existing and outside cert times", // Note that cert from next CA will already not be valid
+			in: func(t *testing.T, w *Worker) {
+				creds := proto.Clone(initNodeCreds).(*types.NodeCredentials)
+				creds.CertificateBundles[0].CertificateNotBefore = timestamppb.New(time.Time{})
+				creds.CertificateBundles[0].CertificateNotAfter = timestamppb.New(time.Time{})
+				require.NoError(t, creds.Store(ctx, w.WorkerAuthStorage))
+			},
+			expRegistrationRequest: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tw := NewTestWorker(t, &TestWorkerOpts{
+				WorkerAuthStoragePath: tmpDir,
+				DisableAutoStart:      true,
+			})
+			t.Cleanup(tw.Shutdown)
+
+			// Always clear out storage that was there before, ignore errors
+			_ = tw.Worker().WorkerAuthStorage.Remove(ctx, &types.NodeCredentials{Id: string(nodeenrollment.CurrentId)})
+
+			// Run node credentials modification
+			tt.in(t, tw.Worker())
+
+			// Start up to run logic
+			require.NoError(t, tw.Worker().Start())
+
+			// Validate existing key was loaded or new key was created and loaded
+			if tt.expKeyId != "" {
+				assert.Equal(t, tt.expKeyId, tw.Worker().WorkerAuthCurrentKeyId.Load())
+			} else {
+				assert.NotEmpty(t, tw.Worker().WorkerAuthCurrentKeyId.Load())
+			}
+			if tt.expRegistrationRequest {
+				assert.NotEmpty(t, tw.Worker().WorkerAuthRegistrationRequest)
+			} else {
+				assert.Empty(t, tw.Worker().WorkerAuthRegistrationRequest)
+			}
+		})
+	}
+}