Worker status refactor (#5417)

* oss: feat(proto): statistics rpc added to server coordination service

* oss: feat(session): add update statistics status service

* oss: feat(handlers): add Statistics method to worker service server

* oss: feat(worker): add statistics ticking

* oss: internal/proto: define the new RoutingInfo RPC

* oss: internal/server: repository changes for RoutingInfo RPC

* oss: internal/daemon: add RoutingInfo handler to worker service

The new RoutingInfo RPC is used to get information from the worker
that is used to make routing decisions on the controller. It also
handles the initial request from the worker.

* oss: feat(db): server_worker_session_info table

* oss: feat(proto): session info rpc for server coordination service

* oss: feat(server): worker session info request data type

* oss: feat(worker): session info ticker

* oss: feat(handler): server coordination handler for session info

* oss: feat(session): utilize session info request time for session cleanup

* oss: proto: fix some gotags comments

Some of these fields were missing comments, while others had
incorrectly formatted comments.

* oss: worker: send connection status to server

This is required for the correct tracking of orphaned connections

* oss: internal/daemon/worker: add function to wait for Statistics RPC

* oss: internal/daemon/worker: add routing info RPC

The new RoutingInfo RPC replaces part of the Status RPC,
namely the part which is used by the controller and worker
to perform decisions around the establishment of new sessions
and new worker connections.

* oss: internal/daemon/worker: remove unnecessary error check

* oss: internal/daemon/worker: lock access to config and upstreams

The new confLock protects access to the conf value and to the
addressReceivers slice.

* oss: worker: replace Status RPC with RoutingInfo

* oss: worker: deprecate the Status RPC and related types

The Status RPC and related messages have been replaced
by the RoutingInfo/SessionInfo/Statistics RPCs. The controller
will continue to support the Status RPC until two new minor
versions have been released.

* oss: internal/event: add default deny for new worker RPCs

* oss: internal/daemon/worker: allow RPC interval to be set

The new SessionInfo/RoutingInfo/Statistics RPCs run at
long intervals, which make some of our cluster tests take
a long time to complete. This allows their interval to be
configured in tests.

* oss: all: use TestWorkerRPCInterval in slow tests

This should speed up these tests significantly.

* oss: internal/tests/cluster: reset sys eventer in test

---------

Co-authored-by: Damian Debkowski <damian.debkowski@hashicorp.com>

.golangci.yml

@@ -69,3 +69,9 @@ issues:
     - linters:
         - staticcheck
       text: "SA1019: j.GetId is deprecated:"
+    - linters:
+        - staticcheck
+      text: "SA1019: pbs.StatusRequest is deprecated:"
+    - linters:
+        - staticcheck
+      text: "SA1019: pbs.StatusResponse is deprecated:"

internal/clientcache/internal/cache/repository_sessions_test.go

@@ -604,6 +604,7 @@ func TestRepository_QuerySessionsLimiting(t *testing.T) {
 }
 
 func TestDefaultSessionRetrievalFunc(t *testing.T) {
+	// This prevents us from running this test in parallel
 	oldDur := globals.RefreshReadLookbackDuration
 	globals.RefreshReadLookbackDuration = 0
 	t.Cleanup(func() {
@@ -614,28 +615,21 @@ func TestDefaultSessionRetrievalFunc(t *testing.T) {
 	tc.Client().SetToken(tc.Token().Token)
 	tarClient := targets.NewClient(tc.Client())
 	_ = worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		Name:             "test",
-		InitialUpstreams: tc.ClusterAddrs(),
-		WorkerAuthKms:    tc.Config().WorkerAuthKms,
+		Name:              "test",
+		InitialUpstreams:  tc.ClusterAddrs(),
+		WorkerAuthKms:     tc.Config().WorkerAuthKms,
+		WorkerRPCInterval: time.Second,
 	})
 
 	tar1, err := tarClient.Create(tc.Context(), "tcp", "p_1234567890", targets.WithName("tar1"), targets.WithTcpTargetDefaultPort(1), targets.WithAddress("address"))
 	require.NoError(t, err)
 	require.NotNil(t, tar1)
 
-	require.NoError(t, tc.WaitForNextWorkerStatusUpdate("test"))
-	ctx, cancel := context.WithTimeout(tc.Context(), 10*time.Second)
-	defer cancel()
-	for {
-		_, err = tarClient.AuthorizeSession(ctx, tar1.Item.Id)
-		if err == nil {
-			break
-		}
-		if err != nil && ctx.Err() != nil {
-			require.FailNow(t, "timed out waiting to authorize session without an error. Last error: %s", err.Error())
-			break
-		}
-	}
+	require.NoError(t, tc.WaitForNextWorkerRoutingInfoUpdate("test"))
+	require.Eventually(t, func() bool {
+		_, err = tarClient.AuthorizeSession(tc.Context(), tar1.Item.Id)
+		return err == nil
+	}, 30*time.Second, time.Second, "timed out waiting to authorize session without an error.")
 
 	got, refTok, err := defaultSessionFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "", nil)
 	assert.NoError(t, err)

internal/cmd/config/config.go

@@ -268,15 +268,18 @@ type Controller struct {
 	GracefulShutdownWait         any           `hcl:"graceful_shutdown_wait_duration"`
 	GracefulShutdownWaitDuration time.Duration `hcl:"-"`
 
-	// WorkerStatusGracePeriod represents the period of time (as a duration)
+	// WorkerRPCGracePeriod represents the period of time (as a duration)
 	// that the controller will wait before deciding a worker is disconnected
-	// and marking connections from it as canceling
+	// and marking connections from it as canceling. It is also used to evaluate
+	// whether a worker is available for routing based on the time since its last routing info report.
+	// For backwards compatibility this is still called worker_status_grace_period,
+	// though it is now used with both the SessionInfo and RoutingInfo RPCs.
 	//
 	// TODO: This isn't documented (on purpose) because the right place for this
 	// is central configuration so you can't drift across controllers, but we
 	// don't have that yet.
-	WorkerStatusGracePeriod         any           `hcl:"worker_status_grace_period"`
-	WorkerStatusGracePeriodDuration time.Duration `hcl:"-"`
+	WorkerRPCGracePeriod         any           `hcl:"worker_status_grace_period"`
+	WorkerRPCGracePeriodDuration time.Duration `hcl:"-"`
 
 	// LivenessTimeToStale represents the period of time (as a duration) after
 	// which it will consider other controllers to be no longer accessible,
@@ -351,13 +354,15 @@ type Worker struct {
 	Tags    map[string][]string `hcl:"-"`
 	TagsRaw any                 `hcl:"tags"`
 
-	// StatusCallTimeout represents the period of time (as a duration) that
-	// the worker will allow a status RPC call to attempt to finish before
-	// canceling it to try again.
+	// ControllerRPCCallTimeout represents the period of time (as a duration) that
+	// the worker will allow the SessionInfo, RoutingInfo and Statistics RPC calls to
+	// attempt to finish before canceling them to try again.
+	// For backwards compatibility this is still called status_call_timeout,
+	// though it is now used to control the SessionInfo, Statistics and RoutingInfo RPCs.
 	//
 	// TODO: This is currently not documented and considered internal.
-	StatusCallTimeout         any           `hcl:"status_call_timeout"`
-	StatusCallTimeoutDuration time.Duration `hcl:"-"`
+	ControllerRPCCallTimeout         any           `hcl:"status_call_timeout"`
+	ControllerRPCCallTimeoutDuration time.Duration `hcl:"-"`
 
 	// GetDownstreamWorkersTimeout represents the period of time (as a duration) timeout
 	// for GetDownstreamWorkers call in DownstreamWorkerTicker
@@ -366,14 +371,17 @@ type Worker struct {
 	GetDownstreamWorkersTimeout         any           `hcl:"get_downstream_workers_timeout"`
 	GetDownstreamWorkersTimeoutDuration time.Duration `hcl:"-"`
 
-	// SuccessfulStatusGracePeriod represents the period of time (as a duration)
-	// that the worker will wait before disconnecting connections if it cannot
-	// successfully complete a status report to a controller. This cannot be
-	// less than StatusCallTimeout.
+	// SuccessfulControllerRPCGracePeriod represents the period of time (as a duration)
+	// that the worker will wait before closing connections if it cannot
+	// successfully complete a session info report to a controller. It is also used
+	// to evaluate whether the upstreams need to be recalculated in case of repeated
+	// routing info report failures. This cannot be less than ControllerRPCCallTimeout.
+	// For backwards compatibility this is still called successful_status_grace_period,
+	// though it is now used to control the SessionInfo and RoutingInfo RPCs.
 	//
 	// TODO: This is currently not documented and considered internal.
-	SuccessfulStatusGracePeriod         any           `hcl:"successful_status_grace_period"`
-	SuccessfulStatusGracePeriodDuration time.Duration `hcl:"-"`
+	SuccessfulControllerRPCGracePeriod         any           `hcl:"successful_status_grace_period"`
+	SuccessfulControllerRPCGracePeriodDuration time.Duration `hcl:"-"`
 
 	// AuthStoragePath represents the location a worker stores its node credentials, if set
 	AuthStoragePath string `hcl:"auth_storage_path"`
@@ -400,6 +408,11 @@ type Worker struct {
 	// pre-0.13 method of using KMSes to authenticate. This is currently only
 	// supported to throw an error if used telling people they need to upgrade.
 	UseDeprecatedKmsAuthMethod bool `hcl:"use_deprecated_kms_auth_method"`
+
+	// TestWorkerRPCInterval represents the base period of time that
+	// the worker will wait between invoking the controller RPCs.
+	// This is not exposed to users and only used in tests.
+	TestWorkerRPCInterval time.Duration `hcl:"-"`
 }
 
 type Database struct {
@@ -721,19 +734,21 @@ func Parse(d string) (*Config, error) {
 			result.Controller.Scheduler.MonitorIntervalDuration = t
 		}
 
-		workerStatusGracePeriod := result.Controller.WorkerStatusGracePeriod
-		if util.IsNil(workerStatusGracePeriod) {
-			workerStatusGracePeriod = os.Getenv("BOUNDARY_CONTROLLER_WORKER_STATUS_GRACE_PERIOD")
+		workerRPCGracePeriod := result.Controller.WorkerRPCGracePeriod
+		if util.IsNil(workerRPCGracePeriod) {
+			// For backwards compatibility this is still called BOUNDARY_CONTROLLER_WORKER_STATUS_GRACE_PERIOD,
+			// though it is now used to control the SessionInfo and RoutingInfo RPCs.
+			workerRPCGracePeriod = os.Getenv("BOUNDARY_CONTROLLER_WORKER_STATUS_GRACE_PERIOD")
 		}
-		if workerStatusGracePeriod != nil {
-			t, err := parseutil.ParseDurationSecond(workerStatusGracePeriod)
+		if workerRPCGracePeriod != nil {
+			t, err := parseutil.ParseDurationSecond(workerRPCGracePeriod)
 			if err != nil {
 				return result, err
 			}
-			result.Controller.WorkerStatusGracePeriodDuration = t
+			result.Controller.WorkerRPCGracePeriodDuration = t
 		}
-		if result.Controller.WorkerStatusGracePeriodDuration < 0 {
-			return nil, errors.New("Controller worker status grace period value is negative")
+		if result.Controller.WorkerRPCGracePeriodDuration < 0 {
+			return nil, errors.New("Controller worker RPC grace period value is negative")
 		}
 
 		livenessTimeToStale := result.Controller.LivenessTimeToStale
@@ -888,8 +903,10 @@ func Parse(d string) (*Config, error) {
 			return nil, fmt.Errorf("Error parsing worker activation token: %w", err)
 		}
 
-		statusCallTimeoutDuration := result.Worker.StatusCallTimeout
+		statusCallTimeoutDuration := result.Worker.ControllerRPCCallTimeout
 		if util.IsNil(statusCallTimeoutDuration) {
+			// For backwards compatibility this is still called BOUNDARY_WORKER_STATUS_CALL_TIMEOUT,
+			// though it is now used to control the SessionInfo, Statistics and RoutingInfo RPCs.
 			statusCallTimeoutDuration = os.Getenv("BOUNDARY_WORKER_STATUS_CALL_TIMEOUT")
 		}
 		if statusCallTimeoutDuration != nil {
@@ -897,10 +914,10 @@ func Parse(d string) (*Config, error) {
 			if err != nil {
 				return result, err
 			}
-			result.Worker.StatusCallTimeoutDuration = t
+			result.Worker.ControllerRPCCallTimeoutDuration = t
 		}
-		if result.Worker.StatusCallTimeoutDuration < 0 {
-			return nil, errors.New("Status call timeout value is negative")
+		if result.Worker.ControllerRPCCallTimeoutDuration < 0 {
+			return nil, errors.New("Controller RPC timeout value is negative")
 		}
 
 		getDownstreamWorkersTimeoutDuration := result.Worker.GetDownstreamWorkersTimeout
@@ -918,19 +935,21 @@ func Parse(d string) (*Config, error) {
 			return nil, errors.New("get downstream workers timeout must be greater than 0")
 		}
 
-		successfulStatusGracePeriod := result.Worker.SuccessfulStatusGracePeriod
-		if util.IsNil(successfulStatusGracePeriod) {
-			successfulStatusGracePeriod = os.Getenv("BOUNDARY_WORKER_SUCCESSFUL_STATUS_GRACE_PERIOD")
+		successfulControllerRPCGracePeriod := result.Worker.SuccessfulControllerRPCGracePeriod
+		if util.IsNil(successfulControllerRPCGracePeriod) {
+			// For backwards compatibility this is still called BOUNDARY_WORKER_SUCCESSFUL_STATUS_GRACE_PERIOD,
+			// though it is now used to control the SessionInfo and RoutingInfo RPCs.
+			successfulControllerRPCGracePeriod = os.Getenv("BOUNDARY_WORKER_SUCCESSFUL_STATUS_GRACE_PERIOD")
 		}
-		if successfulStatusGracePeriod != nil {
-			t, err := parseutil.ParseDurationSecond(successfulStatusGracePeriod)
+		if successfulControllerRPCGracePeriod != nil {
+			t, err := parseutil.ParseDurationSecond(successfulControllerRPCGracePeriod)
 			if err != nil {
 				return result, err
 			}
-			result.Worker.SuccessfulStatusGracePeriodDuration = t
+			result.Worker.SuccessfulControllerRPCGracePeriodDuration = t
 		}
-		if result.Worker.SuccessfulStatusGracePeriodDuration < 0 {
-			return nil, errors.New("Successful status grace period value is negative")
+		if result.Worker.SuccessfulControllerRPCGracePeriodDuration < 0 {
+			return nil, errors.New("Successful controller RPC grace period value is negative")
 		}
 
 		if !util.IsNil(result.Worker.RecordingStorageMinimumAvailableCapacity) {
@@ -949,14 +968,14 @@ func Parse(d string) (*Config, error) {
 		}
 
 		switch {
-		case result.Worker.StatusCallTimeoutDuration == 0 && result.Worker.SuccessfulStatusGracePeriodDuration == 0:
+		case result.Worker.ControllerRPCCallTimeoutDuration == 0 && result.Worker.SuccessfulControllerRPCGracePeriodDuration == 0:
 			// Nothing
-		case result.Worker.StatusCallTimeoutDuration != 0 && result.Worker.SuccessfulStatusGracePeriodDuration != 0:
-			if result.Worker.StatusCallTimeoutDuration > result.Worker.SuccessfulStatusGracePeriodDuration {
-				return nil, fmt.Errorf("Worker setting for status call timeout duration must be less than or equal to successful status grace period duration")
+		case result.Worker.ControllerRPCCallTimeoutDuration != 0 && result.Worker.SuccessfulControllerRPCGracePeriodDuration != 0:
+			if result.Worker.ControllerRPCCallTimeoutDuration > result.Worker.SuccessfulControllerRPCGracePeriodDuration {
+				return nil, fmt.Errorf("Worker setting for controller rpc timeout duration must be less than or equal to successful controller rpc grace period duration")
 			}
 		default:
-			return nil, fmt.Errorf("Worker settings for status call timeout duration and successful status grace period duration must either both be set or both be empty")
+			return nil, fmt.Errorf("Worker settings for controller rpc call timeout duration and successful controller rpc grace period duration must either both be set or both be empty")
 		}
 
 		if result.Worker.TagsRaw != nil {

internal/cmd/config/config_load_test.go

@@ -378,17 +378,17 @@ func TestLoad(t *testing.T) {
 					ClusterName:                  "",
 				},
 				Worker: &config.Worker{
-					Name:                                "worker0",
-					Description:                         "A worker",
-					PublicAddr:                          "",
-					InitialUpstreams:                    []string{"boundary:9201"},
-					InitialUpstreamsRaw:                 []any{"boundary:9201"},
-					Tags:                                nil,
-					TagsRaw:                             nil,
-					SuccessfulStatusGracePeriodDuration: 0,
-					StatusCallTimeoutDuration:           0,
-					AuthStoragePath:                     "",
-					ControllerGeneratedActivationToken:  "",
+					Name:                "worker0",
+					Description:         "A worker",
+					PublicAddr:          "",
+					InitialUpstreams:    []string{"boundary:9201"},
+					InitialUpstreamsRaw: []any{"boundary:9201"},
+					Tags:                nil,
+					TagsRaw:             nil,
+					SuccessfulControllerRPCGracePeriodDuration: 0,
+					ControllerRPCCallTimeoutDuration:           0,
+					AuthStoragePath:                            "",
+					ControllerGeneratedActivationToken:         "",
 				},
 				Controller: &config.Controller{
 					Name:        "controller0",
@@ -409,18 +409,18 @@ func TestLoad(t *testing.T) {
 						JobRunInterval:  nil,
 						MonitorInterval: nil,
 					},
-					AuthTokenTimeToLive:             nil,
-					AuthTokenTimeToLiveDuration:     0,
-					AuthTokenTimeToStale:            nil,
-					AuthTokenTimeToStaleDuration:    0,
-					GracefulShutdownWait:            nil,
-					GracefulShutdownWaitDuration:    0,
-					WorkerStatusGracePeriodDuration: 0,
-					LivenessTimeToStaleDuration:     0,
-					ApiRateLimits:                   make(ratelimit.Configs, 0),
-					ApiRateLimiterMaxQuotas:         ratelimit.DefaultLimiterMaxQuotas(),
-					MaxPageSizeRaw:                  nil,
-					MaxPageSize:                     0,
+					AuthTokenTimeToLive:          nil,
+					AuthTokenTimeToLiveDuration:  0,
+					AuthTokenTimeToStale:         nil,
+					AuthTokenTimeToStaleDuration: 0,
+					GracefulShutdownWait:         nil,
+					GracefulShutdownWaitDuration: 0,
+					WorkerRPCGracePeriodDuration: 0,
+					LivenessTimeToStaleDuration:  0,
+					ApiRateLimits:                make(ratelimit.Configs, 0),
+					ApiRateLimiterMaxQuotas:      ratelimit.DefaultLimiterMaxQuotas(),
+					MaxPageSizeRaw:               nil,
+					MaxPageSize:                  0,
 				},
 				DevController:           false,
 				DevUiPassthroughDir:     "",
@@ -804,17 +804,17 @@ func TestLoad(t *testing.T) {
 					ClusterName:                  "",
 				},
 				Worker: &config.Worker{
-					Name:                                "worker0",
-					Description:                         "A worker",
-					PublicAddr:                          "",
-					InitialUpstreams:                    []string{"boundary:9201"},
-					InitialUpstreamsRaw:                 []any{"boundary:9201"},
-					Tags:                                nil,
-					TagsRaw:                             nil,
-					SuccessfulStatusGracePeriodDuration: 0,
-					StatusCallTimeoutDuration:           0,
-					AuthStoragePath:                     "",
-					ControllerGeneratedActivationToken:  "",
+					Name:                "worker0",
+					Description:         "A worker",
+					PublicAddr:          "",
+					InitialUpstreams:    []string{"boundary:9201"},
+					InitialUpstreamsRaw: []any{"boundary:9201"},
+					Tags:                nil,
+					TagsRaw:             nil,
+					SuccessfulControllerRPCGracePeriodDuration: 0,
+					ControllerRPCCallTimeoutDuration:           0,
+					AuthStoragePath:                            "",
+					ControllerGeneratedActivationToken:         "",
 				},
 				Controller: &config.Controller{
 					Name:        "controller0",
@@ -835,18 +835,18 @@ func TestLoad(t *testing.T) {
 						JobRunInterval:  nil,
 						MonitorInterval: nil,
 					},
-					AuthTokenTimeToLive:             nil,
-					AuthTokenTimeToLiveDuration:     0,
-					AuthTokenTimeToStale:            nil,
-					AuthTokenTimeToStaleDuration:    0,
-					GracefulShutdownWait:            nil,
-					GracefulShutdownWaitDuration:    0,
-					WorkerStatusGracePeriodDuration: 0,
-					LivenessTimeToStaleDuration:     0,
-					ApiRateLimits:                   make(ratelimit.Configs, 0),
-					ApiRateLimiterMaxQuotas:         ratelimit.DefaultLimiterMaxQuotas(),
-					MaxPageSizeRaw:                  nil,
-					MaxPageSize:                     0,
+					AuthTokenTimeToLive:          nil,
+					AuthTokenTimeToLiveDuration:  0,
+					AuthTokenTimeToStale:         nil,
+					AuthTokenTimeToStaleDuration: 0,
+					GracefulShutdownWait:         nil,
+					GracefulShutdownWaitDuration: 0,
+					WorkerRPCGracePeriodDuration: 0,
+					LivenessTimeToStaleDuration:  0,
+					ApiRateLimits:                make(ratelimit.Configs, 0),
+					ApiRateLimiterMaxQuotas:      ratelimit.DefaultLimiterMaxQuotas(),
+					MaxPageSizeRaw:               nil,
+					MaxPageSize:                  0,
 				},
 				DevController:           false,
 				DevUiPassthroughDir:     "",
@@ -1225,17 +1225,17 @@ func TestLoad(t *testing.T) {
 					ClusterName:                  "",
 				},
 				Worker: &config.Worker{
-					Name:                                "worker0",
-					Description:                         "A worker",
-					PublicAddr:                          "",
-					InitialUpstreams:                    []string{"boundary:9201"},
-					InitialUpstreamsRaw:                 []any{"boundary:9201"},
-					Tags:                                nil,
-					TagsRaw:                             nil,
-					SuccessfulStatusGracePeriodDuration: 0,
-					StatusCallTimeoutDuration:           0,
-					AuthStoragePath:                     "",
-					ControllerGeneratedActivationToken:  "",
+					Name:                "worker0",
+					Description:         "A worker",
+					PublicAddr:          "",
+					InitialUpstreams:    []string{"boundary:9201"},
+					InitialUpstreamsRaw: []any{"boundary:9201"},
+					Tags:                nil,
+					TagsRaw:             nil,
+					SuccessfulControllerRPCGracePeriodDuration: 0,
+					ControllerRPCCallTimeoutDuration:           0,
+					AuthStoragePath:                            "",
+					ControllerGeneratedActivationToken:         "",
 				},
 				Controller: &config.Controller{
 					Name:        "controller0",
@@ -1256,14 +1256,14 @@ func TestLoad(t *testing.T) {
 						JobRunInterval:  nil,
 						MonitorInterval: nil,
 					},
-					AuthTokenTimeToLive:             nil,
-					AuthTokenTimeToLiveDuration:     0,
-					AuthTokenTimeToStale:            nil,
-					AuthTokenTimeToStaleDuration:    0,
-					GracefulShutdownWait:            nil,
-					GracefulShutdownWaitDuration:    0,
-					WorkerStatusGracePeriodDuration: 0,
-					LivenessTimeToStaleDuration:     0,
+					AuthTokenTimeToLive:          nil,
+					AuthTokenTimeToLiveDuration:  0,
+					AuthTokenTimeToStale:         nil,
+					AuthTokenTimeToStaleDuration: 0,
+					GracefulShutdownWait:         nil,
+					GracefulShutdownWaitDuration: 0,
+					WorkerRPCGracePeriodDuration: 0,
+					LivenessTimeToStaleDuration:  0,
 					ApiRateLimits: ratelimit.Configs{
 						{
 							Resources: []string{"*"},
@@ -1665,17 +1665,17 @@ func TestLoad(t *testing.T) {
 					ClusterName:                  "",
 				},
 				Worker: &config.Worker{
-					Name:                                "worker0",
-					Description:                         "A worker",
-					PublicAddr:                          "",
-					InitialUpstreams:                    []string{"boundary:9201"},
-					InitialUpstreamsRaw:                 []any{"boundary:9201"},
-					Tags:                                nil,
-					TagsRaw:                             nil,
-					SuccessfulStatusGracePeriodDuration: 0,
-					StatusCallTimeoutDuration:           0,
-					AuthStoragePath:                     "",
-					ControllerGeneratedActivationToken:  "",
+					Name:                "worker0",
+					Description:         "A worker",
+					PublicAddr:          "",
+					InitialUpstreams:    []string{"boundary:9201"},
+					InitialUpstreamsRaw: []any{"boundary:9201"},
+					Tags:                nil,
+					TagsRaw:             nil,
+					SuccessfulControllerRPCGracePeriodDuration: 0,
+					ControllerRPCCallTimeoutDuration:           0,
+					AuthStoragePath:                            "",
+					ControllerGeneratedActivationToken:         "",
 				},
 				Controller: &config.Controller{
 					Name:        "controller0",
@@ -1696,18 +1696,18 @@ func TestLoad(t *testing.T) {
 						JobRunInterval:  nil,
 						MonitorInterval: nil,
 					},
-					AuthTokenTimeToLive:             nil,
-					AuthTokenTimeToLiveDuration:     0,
-					AuthTokenTimeToStale:            nil,
-					AuthTokenTimeToStaleDuration:    0,
-					GracefulShutdownWait:            nil,
-					GracefulShutdownWaitDuration:    0,
-					WorkerStatusGracePeriodDuration: 0,
-					LivenessTimeToStaleDuration:     0,
-					ApiRateLimits:                   make(ratelimit.Configs, 0),
-					ApiRateLimiterMaxQuotas:         ratelimit.DefaultLimiterMaxQuotas(),
-					MaxPageSizeRaw:                  nil,
-					MaxPageSize:                     0,
+					AuthTokenTimeToLive:          nil,
+					AuthTokenTimeToLiveDuration:  0,
+					AuthTokenTimeToStale:         nil,
+					AuthTokenTimeToStaleDuration: 0,
+					GracefulShutdownWait:         nil,
+					GracefulShutdownWaitDuration: 0,
+					WorkerRPCGracePeriodDuration: 0,
+					LivenessTimeToStaleDuration:  0,
+					ApiRateLimits:                make(ratelimit.Configs, 0),
+					ApiRateLimiterMaxQuotas:      ratelimit.DefaultLimiterMaxQuotas(),
+					MaxPageSizeRaw:               nil,
+					MaxPageSize:                  0,
 				},
 				DevController:           false,
 				DevUiPassthroughDir:     "",
@@ -1789,18 +1789,18 @@ func TestLoad(t *testing.T) {
 						JobRunInterval:  nil,
 						MonitorInterval: nil,
 					},
-					AuthTokenTimeToLive:             nil,
-					AuthTokenTimeToLiveDuration:     0,
-					AuthTokenTimeToStale:            nil,
-					AuthTokenTimeToStaleDuration:    0,
-					GracefulShutdownWait:            nil,
-					GracefulShutdownWaitDuration:    0,
-					WorkerStatusGracePeriodDuration: 0,
-					LivenessTimeToStaleDuration:     0,
-					ApiRateLimits:                   make(ratelimit.Configs, 0),
-					ApiRateLimiterMaxQuotas:         ratelimit.DefaultLimiterMaxQuotas(),
-					MaxPageSizeRaw:                  nil,
-					MaxPageSize:                     0,
+					AuthTokenTimeToLive:          nil,
+					AuthTokenTimeToLiveDuration:  0,
+					AuthTokenTimeToStale:         nil,
+					AuthTokenTimeToStaleDuration: 0,
+					GracefulShutdownWait:         nil,
+					GracefulShutdownWaitDuration: 0,
+					WorkerRPCGracePeriodDuration: 0,
+					LivenessTimeToStaleDuration:  0,
+					ApiRateLimits:                make(ratelimit.Configs, 0),
+					ApiRateLimiterMaxQuotas:      ratelimit.DefaultLimiterMaxQuotas(),
+					MaxPageSizeRaw:               nil,
+					MaxPageSize:                  0,
 				},
 				DevController:           false,
 				DevUiPassthroughDir:     "",

internal/daemon/cluster/handlers/worker_service.go

@@ -5,6 +5,7 @@ package handlers
 
 import (
 	"context"
+	stderrors "errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -13,6 +14,7 @@ import (
 	"github.com/hashicorp/boundary/internal/daemon/controller/common"
 	"github.com/hashicorp/boundary/internal/daemon/controller/downstream"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers"
+	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/event"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	intglobals "github.com/hashicorp/boundary/internal/globals"
@@ -95,20 +97,148 @@ func NewWorkerServiceServer(
 	}
 }
 
+func (ws *workerServiceServer) Statistics(ctx context.Context, req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) {
+	const op = "workers.(workerServiceServer).Statistics"
+	workerId := req.GetWorkerId()
+	if workerId == "" {
+		return &pbs.StatisticsResponse{}, status.Error(codes.InvalidArgument, "worker id is empty")
+	}
+	connectionRepo, err := ws.connectionRepoFn()
+	if err != nil {
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting connection repo"))
+		return &pbs.StatisticsResponse{}, status.Errorf(codes.Internal, "Error acquiring connection repo: %v", err)
+	}
+	connectionStats := []*session.Connection{}
+	for _, s := range req.GetSessions() {
+		sessionId := s.GetSessionId()
+		if sessionId == "" {
+			return &pbs.StatisticsResponse{}, status.Error(codes.InvalidArgument, "session id is empty")
+		}
+		for _, c := range s.Connections {
+			connectionId := c.GetConnectionId()
+			if connectionId == "" {
+				return &pbs.StatisticsResponse{}, status.Error(codes.InvalidArgument, "connection id is empty")
+			}
+			switch c.Status {
+			case pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED, pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED:
+				connectionStats = append(connectionStats, &session.Connection{
+					PublicId:  c.GetConnectionId(),
+					BytesUp:   c.GetBytesUp(),
+					BytesDown: c.GetBytesDown(),
+				})
+			default:
+				// Other statuses are not included in the stats sent to the domain
+			}
+		}
+	}
+	updateBytesErr := session.UpdateConnectionBytesUpDown(ctx, connectionRepo, connectionStats)
+	_, closeOrphanedErr := session.CloseOrphanedConnections(ctx, connectionRepo, workerId, connectionStats)
+	sessionErrs := stderrors.Join(updateBytesErr, closeOrphanedErr)
+	if sessionErrs != nil {
+		return &pbs.StatisticsResponse{}, errors.Wrap(ctx, sessionErrs, op)
+	}
+	return &pbs.StatisticsResponse{}, nil
+}
+
+func (ws *workerServiceServer) SessionInfo(ctx context.Context, req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) {
+	const op = "workers.(workerServiceServer).SessionInfo"
+	workerId := req.GetWorkerId()
+	if workerId == "" {
+		return &pbs.SessionInfoResponse{}, status.Error(codes.InvalidArgument, "worker id is empty")
+	}
+	sessionTypeCache := map[string]pbs.SessionType{}
+	var recordedSessionIds []string
+	var ingressedSessionIds []string
+	for _, s := range req.GetSessions() {
+		sessionId := s.GetSessionId()
+		sessionStatus := s.GetSessionStatus()
+		sessionType := s.GetSessionType()
+		switch sessionType {
+		case pbs.SessionType_SESSION_TYPE_UNSPECIFIED:
+			return &pbs.SessionInfoResponse{}, status.Errorf(codes.InvalidArgument, "unspecified session type")
+		case pbs.SessionType_SESSION_TYPE_INGRESSED:
+			if sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING || sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED {
+				// No need to see about canceling anything
+				continue
+			}
+			sessionTypeCache[sessionId] = sessionType
+			ingressedSessionIds = append(ingressedSessionIds, sessionId)
+		case pbs.SessionType_SESSION_TYPE_RECORDED:
+			if sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING || sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED {
+				// No need to see about canceling anything
+				continue
+			}
+			sessionTypeCache[sessionId] = sessionType
+			recordedSessionIds = append(recordedSessionIds, sessionId)
+		default:
+			return &pbs.SessionInfoResponse{}, status.Errorf(codes.InvalidArgument, "unknown session type: %q", s.GetSessionType().String())
+		}
+	}
+
+	sessRepo, err := ws.sessionRepoFn()
+	if err != nil {
+		return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
+	}
+
+	result := &pbs.SessionInfoResponse{}
+	nonActiveSessions, err := sessRepo.CheckIfNotActive(ctx, append(ingressedSessionIds, recordedSessionIds...))
+	if err != nil {
+		return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal,
+			"Error checking if sessions are no longer active: %v", err)
+	}
+	for _, s := range nonActiveSessions {
+		switch sessionTypeCache[s.SessionId] {
+		case pbs.SessionType_SESSION_TYPE_INGRESSED:
+			var connChanges []*pbs.Connection
+			for _, conn := range s.Connections {
+				connChanges = append(connChanges, &pbs.Connection{
+					ConnectionId: conn.GetPublicId(),
+					Status:       session.StatusClosed.ProtoVal(),
+				})
+			}
+			result.NonActiveSessions = append(result.NonActiveSessions, &pbs.Session{
+				SessionId:     s.SessionId,
+				SessionStatus: s.Status.ProtoVal(),
+				SessionType:   pbs.SessionType_SESSION_TYPE_INGRESSED,
+				Connections:   connChanges,
+			})
+		case pbs.SessionType_SESSION_TYPE_RECORDED:
+			result.NonActiveSessions = append(result.NonActiveSessions, &pbs.Session{
+				SessionId:     s.SessionId,
+				SessionStatus: s.Status.ProtoVal(),
+				SessionType:   pbs.SessionType_SESSION_TYPE_RECORDED,
+			})
+		default:
+			return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal,
+				"unknown session type found for non active session: %q", s.SessionId)
+		}
+	}
+
+	serverRepo, err := ws.serversRepoFn()
+	if err != nil {
+		return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to upsert session info status time: %v", err)
+	}
+	err = serverRepo.UpsertSessionInfo(ctx, workerId)
+	if err != nil {
+		return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, "Error updating the latest status time for the server session info: %v", err)
+	}
+	return result, nil
+}
+
+// Status is the deprecated method for worker status updates.
+// This is safe to remove after the release of Boundary v0.20.0.
 func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
 	const op = "workers.(workerServiceServer).Status"
-	// TODO: on the worker, if we get errors back from this repeatedly, do we
-	// terminate all sessions since we can't know if they were canceled?
 
 	wStat := req.GetWorkerStatus()
 	if wStat == nil {
-		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Worker sent nil status.")
+		return nil, status.Error(codes.InvalidArgument, "Worker sent nil status.")
 	}
 	switch {
 	case wStat.GetName() == "" && wStat.GetKeyId() == "":
-		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Name and keyId are not set in the request; one is required.")
+		return nil, status.Error(codes.InvalidArgument, "Name and keyId are not set in the request; one is required.")
 	case wStat.GetAddress() == "":
-		return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Address is not set but is required.")
+		return nil, status.Error(codes.InvalidArgument, "Address is not set but is required.")
 	}
 	// This Store call is currently only for testing purposes
 	ws.updateTimes.Store(wStat.GetName(), time.Now())
@@ -116,7 +246,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 	serverRepo, err := ws.serversRepoFn()
 	if err != nil {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting server repo"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to store worker status: %v", err)
+		return nil, status.Errorf(codes.Internal, "Error acquiring repo to store worker status: %v", err)
 	}
 
 	// Convert API tags to storage tags
@@ -163,7 +293,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 	workerId, err := serverRepo.UpsertWorkerStatus(ctx, wConf, opts...)
 	if err != nil {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("error storing worker status"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error storing worker status: %v", err)
+		return nil, status.Errorf(codes.Internal, "Error storing worker status: %v", err)
 	}
 
 	// update storage states
@@ -174,7 +304,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 	controllers, err := serverRepo.ListControllers(ctx, server.WithLiveness(time.Duration(ws.livenessTimeToStale.Load())))
 	if err != nil {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting current controllers"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting current controllers: %v", err)
+		return nil, status.Errorf(codes.Internal, "Error getting current controllers: %v", err)
 	}
 
 	responseControllers := []*pbs.UpstreamServer{}
@@ -189,7 +319,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 	workerAuthRepo, err := ws.workerAuthRepoFn()
 	if err != nil {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting worker auth repo"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to lookup worker auth info: %v", err)
+		return nil, status.Errorf(codes.Internal, "Error acquiring repo to lookup worker auth info: %v", err)
 	}
 
 	authorizedDownstreams := &pbs.AuthorizedDownstreamWorkerList{}
@@ -197,7 +327,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 		knownConnectedWorkers, err := serverRepo.VerifyKnownWorkers(ctx, req.GetConnectedWorkerPublicIds())
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("error getting known connected worker ids"))
-			return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting known connected worker ids: %v", err)
+			return nil, status.Errorf(codes.Internal, "Error getting known connected worker ids: %v", err)
 		}
 		authorizedDownstreams.WorkerPublicIds = knownConnectedWorkers
 	}
@@ -206,25 +336,14 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 		authorizedKeyIds, err := workerAuthRepo.FilterToAuthorizedWorkerKeyIds(ctx, req.GetConnectedUnmappedWorkerKeyIdentifiers())
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("error getting authorized unmapped worker key ids"))
-			return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting authorized worker key ids: %v", err)
+			return nil, status.Errorf(codes.Internal, "Error getting authorized worker key ids: %v", err)
 		}
 		authorizedDownstreams.UnmappedWorkerKeyIdentifiers = authorizedKeyIds
 	}
 
-	authorizedWorkerList := &pbs.AuthorizedWorkerList{}
-	if len(req.GetConnectedWorkerKeyIdentifiers()) > 0 {
-		authorizedKeyIds, err := workerAuthRepo.FilterToAuthorizedWorkerKeyIds(ctx, req.GetConnectedWorkerKeyIdentifiers())
-		if err != nil {
-			event.WriteError(ctx, op, err, event.WithInfoMsg("error getting authorized worker key ids"))
-			return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting authorized worker key ids: %v", err)
-		}
-		authorizedWorkerList.WorkerKeyIdentifiers = authorizedKeyIds
-	}
-
 	ret := &pbs.StatusResponse{
 		CalculatedUpstreams:         responseControllers,
 		WorkerId:                    workerId,
-		AuthorizedWorkers:           authorizedWorkerList,
 		AuthorizedDownstreamWorkers: authorizedDownstreams,
 	}
 
@@ -281,12 +400,12 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 	sessRepo, err := ws.sessionRepoFn()
 	if err != nil {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting sessions repo"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
+		return nil, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
 	}
 	connectionRepo, err := ws.connectionRepoFn()
 	if err != nil {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("error getting connection repo"))
-		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
+		return nil, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err)
 	}
 
 	notActive, err := session.WorkerStatusReport(ctx, sessRepo, connectionRepo, workerId, stateReport)
@@ -348,6 +467,11 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
 		})
 	}
 
+	err = serverRepo.UpsertSessionInfo(ctx, workerId)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error updating the latest status time for the server session info: %v", err)
+	}
+
 	return ret, nil
 }
 
@@ -360,8 +484,8 @@ func (ws *workerServiceServer) ListHcpbWorkers(ctx context.Context, req *pbs.Lis
 		return nil, status.Errorf(codes.Internal, "Error getting servers repo: %v", err)
 	}
 
-	// We use the livenessTimeToStale here instead of WorkerStatusGracePeriod
-	// since WorkerStatusGracePeriod is more for deciding which workers should
+	// We use the livenessTimeToStale here instead of WorkerRPCGracePeriod
+	// since WorkerRPCGracePeriod is more for deciding which workers should
 	// be used for session proxying, but here we care about providing the BYOW
 	// workers with a list of which upstreams to connect to as their upstreams.
 	managed, err := serversRepo.ListHcpbManagedWorkers(ctx, time.Duration(ws.livenessTimeToStale.Load()))
@@ -385,6 +509,102 @@ func (ws *workerServiceServer) ListHcpbWorkers(ctx context.Context, req *pbs.Lis
 	return resp, nil
 }
 
+func (ws *workerServiceServer) RoutingInfo(ctx context.Context, req *pbs.RoutingInfoRequest) (*pbs.RoutingInfoResponse, error) {
+	const op = "workers.(workerServiceServer).RoutingInfo"
+	wStat := req.GetWorkerStatus()
+	if wStat == nil {
+		return nil, status.Error(codes.InvalidArgument, "worker status is required")
+	}
+	switch {
+	case wStat.GetPublicId() == "" && wStat.GetKeyId() == "" && wStat.GetName() == "":
+		return nil, status.Error(codes.InvalidArgument, "public id, key id and name are not set in the request; one is required")
+	case wStat.GetAddress() == "":
+		return nil, status.Error(codes.InvalidArgument, "address is not set but is required")
+	case wStat.GetReleaseVersion() == "":
+		return nil, status.Error(codes.InvalidArgument, "release version is not set but is required")
+	case wStat.GetOperationalState() == "":
+		return nil, status.Error(codes.InvalidArgument, "operational state is not set but is required")
+	case wStat.GetLocalStorageState() == "":
+		return nil, status.Error(codes.InvalidArgument, "local storage state is not set but is required")
+	}
+	// This Store call is currently only for testing purposes
+	ws.updateTimes.Store(wStat.GetName(), time.Now())
+
+	serverRepo, err := ws.serversRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting server repo: %v", err)
+	}
+	workerAuthRepo, err := ws.workerAuthRepoFn()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error getting worker auth repo: %v", err)
+	}
+
+	// Convert API tags to storage tags
+	workerTags := make([]*server.Tag, 0, len(wStat.GetTags()))
+	for _, v := range wStat.GetTags() {
+		workerTags = append(workerTags, &server.Tag{
+			Key:   v.GetKey(),
+			Value: v.GetValue(),
+		})
+	}
+
+	wConf := server.NewWorker(
+		scope.Global.String(),
+		server.WithName(wStat.GetName()),
+		server.WithDescription(wStat.GetDescription()),
+		server.WithAddress(wStat.GetAddress()),
+		server.WithWorkerTags(workerTags...),
+		server.WithReleaseVersion(wStat.ReleaseVersion),
+		server.WithOperationalState(wStat.OperationalState),
+		server.WithLocalStorageState(wStat.LocalStorageState),
+	)
+	opts := []server.Option{server.WithUpdateTags(req.GetUpdateTags())}
+	if wStat.GetPublicId() != "" {
+		opts = append(opts, server.WithPublicId(wStat.GetPublicId()))
+	}
+	if wStat.GetKeyId() != "" {
+		opts = append(opts, server.WithKeyId(wStat.GetKeyId()))
+	}
+	workerId, err := serverRepo.UpsertWorkerStatus(ctx, wConf, opts...)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error storing worker status: %v", err)
+	}
+
+	if sbcStates := wStat.GetStorageBucketCredentialStates(); sbcStates != nil {
+		updateWorkerStorageBucketCredentialStatesFn(ctx, serverRepo, workerId, sbcStates)
+	}
+
+	controllers, err := serverRepo.ListControllers(ctx, server.WithLiveness(time.Duration(ws.livenessTimeToStale.Load())))
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error getting current controllers: %v", err)
+	}
+	var upstreamAddresses []string
+	for _, c := range controllers {
+		upstreamAddresses = append(upstreamAddresses, c.Address)
+	}
+
+	authorizedDownstreams, err := server.VerifyKnownAndUnmappedWorkers(
+		ctx,
+		serverRepo,
+		workerAuthRepo,
+		req.GetConnectedWorkerPublicIds(),
+		req.GetConnectedUnmappedWorkerKeyIdentifiers(),
+	)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Error getting known and authorized workers: %v", err)
+	}
+
+	ret := &pbs.RoutingInfoResponse{
+		WorkerId:                    workerId,
+		CalculatedUpstreamAddresses: upstreamAddresses,
+		AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{
+			WorkerPublicIds:              authorizedDownstreams.WorkerPublicIds,
+			UnmappedWorkerKeyIdentifiers: authorizedDownstreams.UnmappedWorkerKeyIds,
+		},
+	}
+	return ret, nil
+}
+
 // Single-hop filter lookup. We have either an egress filter or worker filter to use, if any
 // Used to verify that the worker serving this session to a client matches this filter
 func egressFilterSelector(sessionInfo *session.Session) string {

internal/daemon/cluster/handlers/worker_service_status_test.go

@@ -148,7 +148,6 @@ func TestStatus(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -228,7 +227,6 @@ func TestStatus(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 				JobsRequests: []*pbs.JobChangeRequest{
 					{
@@ -336,7 +334,6 @@ func TestStatus(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 				JobsRequests: []*pbs.JobChangeRequest{
 					{
@@ -414,7 +411,6 @@ func TestStatus(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -449,7 +445,7 @@ func TestStatus(t *testing.T) {
 			got, err := s.Status(ctx, tc.req)
 			if tc.wantErr {
 				require.Error(err)
-				assert.Equal(got, &pbs.StatusResponse{})
+				assert.Empty(got)
 				assert.Equal(tc.wantErrMsg, err.Error())
 				return
 			}
@@ -468,7 +464,6 @@ func TestStatus(t *testing.T) {
 						pbs.SessionJobInfo{},
 						pbs.MonitorSessionJobInfo{},
 						pbs.Connection{},
-						pbs.AuthorizedWorkerList{},
 						pbs.AuthorizedDownstreamWorkerList{},
 					),
 					cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"),
@@ -628,7 +623,6 @@ func TestStatusSessionClosed(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -662,7 +656,6 @@ func TestStatusSessionClosed(t *testing.T) {
 						pbs.Job_SessionInfo{},
 						pbs.SessionJobInfo{},
 						pbs.Connection{},
-						pbs.AuthorizedWorkerList{},
 						pbs.AuthorizedDownstreamWorkerList{},
 					),
 					cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"),
@@ -797,7 +790,6 @@ func TestStatusDeadConnection(t *testing.T) {
 			},
 		},
 		WorkerId:                    worker1.PublicId,
-		AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 		AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 	}
 
@@ -816,7 +808,6 @@ func TestStatusDeadConnection(t *testing.T) {
 				pbs.Job_SessionInfo{},
 				pbs.SessionJobInfo{},
 				pbs.Connection{},
-				pbs.AuthorizedWorkerList{},
 				pbs.AuthorizedDownstreamWorkerList{},
 			),
 			cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"),
@@ -952,7 +943,6 @@ func TestStatusWorkerWithKeyId(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -992,7 +982,6 @@ func TestStatusWorkerWithKeyId(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -1023,7 +1012,6 @@ func TestStatusWorkerWithKeyId(t *testing.T) {
 						pbs.Job_SessionInfo{},
 						pbs.SessionJobInfo{},
 						pbs.Connection{},
-						pbs.AuthorizedWorkerList{},
 						pbs.AuthorizedDownstreamWorkerList{},
 					),
 					cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"),
@@ -1094,7 +1082,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					Name:     worker1.GetName(),
 					Address:  worker1.GetAddress(),
 				},
-				ConnectedWorkerKeyIdentifiers: []string{},
 			},
 			want: &pbs.StatusResponse{
 				CalculatedUpstreams: []*pbs.UpstreamServer{
@@ -1104,7 +1091,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -1126,7 +1112,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					},
 				},
 				WorkerId:                    worker1.PublicId,
-				AuthorizedWorkers:           &pbs.AuthorizedWorkerList{},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
@@ -1139,7 +1124,7 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					Name:     worker1.GetName(),
 					Address:  worker1.GetAddress(),
 				},
-				ConnectedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"},
+				ConnectedUnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"},
 			},
 			want: &pbs.StatusResponse{
 				CalculatedUpstreams: []*pbs.UpstreamServer{
@@ -1149,10 +1134,9 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					},
 				},
 				WorkerId: worker1.PublicId,
-				AuthorizedWorkers: &pbs.AuthorizedWorkerList{
-					WorkerKeyIdentifiers: []string{w1KeyId, w2KeyId},
+				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{
+					UnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId},
 				},
-				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{},
 			},
 		},
 		{
@@ -1164,8 +1148,7 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					Name:     worker1.GetName(),
 					Address:  worker1.GetAddress(),
 				},
-				ConnectedWorkerKeyIdentifiers:         []string{w1KeyId, w2KeyId, "unknown"},
-				ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "unknown"},
+				ConnectedUnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"},
 				ConnectedWorkerPublicIds:              []string{w1.GetPublicId(), "unknown"},
 			},
 			want: &pbs.StatusResponse{
@@ -1176,11 +1159,8 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					},
 				},
 				WorkerId: worker1.PublicId,
-				AuthorizedWorkers: &pbs.AuthorizedWorkerList{
-					WorkerKeyIdentifiers: []string{w1KeyId, w2KeyId},
-				},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{
-					UnmappedWorkerKeyIdentifiers: []string{w2KeyId},
+					UnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId},
 					WorkerPublicIds:              []string{w1.GetPublicId()},
 				},
 			},
@@ -1194,8 +1174,7 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					Name:     worker1.GetName(),
 					Address:  worker1.GetAddress(),
 				},
-				ConnectedWorkerKeyIdentifiers:         []string{w1KeyId, w2KeyId, "unknown"},
-				ConnectedUnmappedWorkerKeyIdentifiers: []string{"unknown"},
+				ConnectedUnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"},
 				ConnectedWorkerPublicIds:              []string{w1.GetPublicId(), w2.GetPublicId(), "unknown"},
 			},
 			want: &pbs.StatusResponse{
@@ -1206,11 +1185,9 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 					},
 				},
 				WorkerId: worker1.PublicId,
-				AuthorizedWorkers: &pbs.AuthorizedWorkerList{
-					WorkerKeyIdentifiers: []string{w1KeyId, w2KeyId},
-				},
 				AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{
-					WorkerPublicIds: []string{w1.GetPublicId(), w2.GetPublicId()},
+					UnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId},
+					WorkerPublicIds:              []string{w1.GetPublicId(), w2.GetPublicId()},
 				},
 			},
 		},
@@ -1227,10 +1204,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 				assert.Equal(tc.wantErrMsg, err.Error())
 				return
 			}
-			gotAuthorizedWorkers := got.GetAuthorizedWorkers()
-			sort.Strings(gotAuthorizedWorkers.GetWorkerKeyIdentifiers())
-			wantAuthorizedWorkers := tc.want.GetAuthorizedWorkers()
-			sort.Strings(wantAuthorizedWorkers.GetWorkerKeyIdentifiers())
 			sort.Strings(got.GetAuthorizedDownstreamWorkers().GetWorkerPublicIds())
 			sort.Strings(tc.want.GetAuthorizedDownstreamWorkers().GetWorkerPublicIds())
 			sort.Strings(got.GetAuthorizedDownstreamWorkers().GetUnmappedWorkerKeyIdentifiers())
@@ -1248,7 +1221,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) {
 						pbs.Job_SessionInfo{},
 						pbs.SessionJobInfo{},
 						pbs.Connection{},
-						pbs.AuthorizedWorkerList{},
 						pbs.AuthorizedDownstreamWorkerList{},
 					),
 					cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"),

internal/daemon/cluster/handlers/worker_service_test.go

@@ -6,6 +6,7 @@ package handlers
 import (
 	"context"
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
@@ -16,12 +17,14 @@ import (
 	"github.com/hashicorp/boundary/internal/authtoken"
 	credstatic "github.com/hashicorp/boundary/internal/credential/static"
 	"github.com/hashicorp/boundary/internal/db"
+	pb "github.com/hashicorp/boundary/internal/gen/controller/servers"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	intglobals "github.com/hashicorp/boundary/internal/globals"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/internal/server/store"
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/boundary/internal/target"
 	"github.com/hashicorp/boundary/internal/target/tcp"
@@ -31,6 +34,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/ssh/testdata"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/testing/protocmp"
 )
@@ -629,3 +634,800 @@ func TestHcpbWorkers(t *testing.T) {
 	}
 	assert.ElementsMatch(expValues, gotValues)
 }
+
+func TestStatistics(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kms := kms.TestKms(t, conn, wrapper)
+	serversRepoFn := func() (*server.Repository, error) {
+		return server.NewRepository(ctx, rw, rw, kms)
+	}
+	workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) {
+		return server.NewRepositoryStorage(ctx, rw, rw, kms)
+	}
+	sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) {
+		return session.NewRepository(ctx, rw, rw, kms, opts...)
+	}
+	connectionRepoFn := func() (*session.ConnectionRepository, error) {
+		return session.NewConnectionRepository(ctx, rw, rw, kms)
+	}
+	connectionRepoErrFn := func() (*session.ConnectionRepository, error) {
+		return nil, fmt.Errorf("test error")
+	}
+	fce := &fakeControllerExtension{
+		reader: rw,
+		writer: rw,
+	}
+	cases := []struct {
+		name               string
+		workerService      *workerServiceServer
+		wantErrMsg         string
+		expectedStatusCode codes.Code
+		req                *pbs.StatisticsRequest
+	}{
+		{
+			name:               "empty worker id",
+			workerService:      NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce),
+			req:                &pbs.StatisticsRequest{},
+			wantErrMsg:         "rpc error: code = InvalidArgument desc = worker id is empty",
+			expectedStatusCode: codes.InvalidArgument,
+		},
+		{
+			name:          "missing session connection repo",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoErrFn, nil, new(sync.Map), kms, new(atomic.Int64), fce),
+			req: &pbs.StatisticsRequest{
+				WorkerId: "w_1234567890",
+				Sessions: []*pbs.SessionStatistics{
+					{},
+				},
+			},
+			wantErrMsg:         "rpc error: code = Internal desc = Error acquiring connection repo: test error",
+			expectedStatusCode: codes.Internal,
+		},
+		{
+			name:          "nil sessions",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce),
+			req: &pbs.StatisticsRequest{
+				WorkerId: "w_1234567890",
+			},
+		},
+		{
+			name:          "empty sessions",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce),
+			req: &pbs.StatisticsRequest{
+				WorkerId: "w_1234567890",
+				Sessions: []*pbs.SessionStatistics{},
+			},
+		},
+		{
+			name:          "empty session id",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce),
+			req: &pbs.StatisticsRequest{
+				WorkerId: "w_1234567890",
+				Sessions: []*pbs.SessionStatistics{
+					{},
+				},
+			},
+			wantErrMsg:         "rpc error: code = InvalidArgument desc = session id is empty",
+			expectedStatusCode: codes.InvalidArgument,
+		},
+		{
+			name:          "empty connection id",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce),
+			req: &pbs.StatisticsRequest{
+				WorkerId: "w_1234567890",
+				Sessions: []*pbs.SessionStatistics{
+					{
+						SessionId: "s_1234567890",
+						Connections: []*pbs.Connection{
+							{},
+						},
+					},
+				},
+			},
+			wantErrMsg:         "rpc error: code = InvalidArgument desc = connection id is empty",
+			expectedStatusCode: codes.InvalidArgument,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			got, err := tc.workerService.Statistics(context.Background(), tc.req)
+			require.NotNil(got)
+			if tc.wantErrMsg != "" {
+				require.Error(err)
+				assert.ErrorContains(err, tc.wantErrMsg)
+				actualStatus, ok := status.FromError(err)
+				require.True(ok)
+				assert.Equal(tc.expectedStatusCode, actualStatus.Code())
+				return
+			}
+			require.NoError(err)
+		})
+	}
+
+	// update the bytes up and bytes down for an active session and close an orphaned session connection
+	t.Run("happy path", func(t *testing.T) {
+		assert, require := assert.New(t), require.New(t)
+		connectionRepoFn := func() (*session.ConnectionRepository, error) {
+			return session.NewConnectionRepository(ctx, rw, rw, kms, session.WithWorkerStateDelay(0))
+		}
+		sessRepo, err := sessionRepoFn()
+		require.NoError(err)
+		connRepo, err := connectionRepoFn()
+		require.NoError(err)
+
+		w := server.TestKmsWorker(t, conn, wrapper)
+		org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+		authToken := authtoken.TestAuthToken(t, conn, kms, org.GetPublicId())
+		userId := authToken.GetIamUserId()
+		hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+		hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+		host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+		static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+		tar := tcp.TestTarget(
+			context.Background(),
+			t, conn, prj.GetPublicId(), "test",
+			target.WithHostSources([]string{hostSet.GetPublicId()}),
+			target.WithSessionConnectionLimit(-1),
+		)
+		s1 := session.TestSession(t, conn, wrapper, session.ComposedOf{
+			UserId:          userId,
+			HostId:          host.GetPublicId(),
+			TargetId:        tar.GetPublicId(),
+			HostSetId:       hostSet.GetPublicId(),
+			AuthTokenId:     authToken.GetPublicId(),
+			ProjectId:       prj.GetPublicId(),
+			Endpoint:        "tcp://127.0.0.1:22",
+			ConnectionLimit: 10,
+		})
+		s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, session.TestTofu(t))
+		require.NoError(err)
+		c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId())
+		require.NoError(err)
+
+		s2 := session.TestSession(t, conn, wrapper, session.ComposedOf{
+			UserId:          userId,
+			HostId:          host.GetPublicId(),
+			TargetId:        tar.GetPublicId(),
+			HostSetId:       hostSet.GetPublicId(),
+			AuthTokenId:     authToken.GetPublicId(),
+			ProjectId:       prj.GetPublicId(),
+			Endpoint:        "tcp://127.0.0.1:22",
+			ConnectionLimit: 10,
+		})
+		s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, session.TestTofu(t))
+		require.NoError(err)
+		c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
+		require.NoError(err)
+
+		var expectedBytesUp int64 = 1024
+		var expectedBytesDown int64 = 2048
+		workerService := NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce)
+		got, err := workerService.Statistics(context.Background(), &pbs.StatisticsRequest{
+			WorkerId: w.GetPublicId(),
+			Sessions: []*pbs.SessionStatistics{
+				{
+					SessionId: s1.PublicId,
+					Connections: []*pbs.Connection{
+						{
+							ConnectionId: c1.PublicId,
+							BytesUp:      expectedBytesUp,
+							BytesDown:    expectedBytesDown,
+							Status:       pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED,
+						},
+					},
+				},
+			},
+		})
+		require.NoError(err)
+		assert.NotNil(got)
+
+		conn, err := connRepo.LookupConnection(context.Background(), c1.PublicId)
+		require.NoError(err)
+		assert.Equal(expectedBytesUp, conn.BytesUp)
+		assert.Equal(expectedBytesDown, conn.BytesDown)
+		assert.Equal(session.StatusAuthorized, session.ConnectionStatusFromString(conn.Status))
+
+		conn, err = connRepo.LookupConnection(context.Background(), c2.PublicId)
+		require.NoError(err)
+		assert.Empty(conn.BytesUp)
+		assert.Empty(conn.BytesDown)
+		assert.Equal(session.StatusClosed, session.ConnectionStatusFromString(conn.Status))
+	})
+}
+
+func TestSessionInfo(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	testKms := kms.TestKms(t, conn, wrapper)
+	serversRepoFn := func() (*server.Repository, error) {
+		return server.NewRepository(ctx, rw, rw, testKms)
+	}
+	workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) {
+		return server.NewRepositoryStorage(ctx, rw, rw, testKms)
+	}
+	sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) {
+		return session.NewRepository(ctx, rw, rw, testKms, opts...)
+	}
+	connectionRepoFn := func() (*session.ConnectionRepository, error) {
+		return session.NewConnectionRepository(ctx, rw, rw, testKms)
+	}
+	serversErrRepoFn := func() (*server.Repository, error) {
+		return nil, errors.New("unknown error")
+	}
+	sessionErrRepoFn := func(opt ...session.Option) (*session.Repository, error) {
+		return nil, errors.New("unknown error")
+	}
+	fce := &fakeControllerExtension{
+		reader: rw,
+		writer: rw,
+	}
+	iamRepo := iam.TestRepo(t, conn, wrapper)
+	iam.TestScopes(t, iamRepo)
+	workerId := server.TestPkiWorker(t, conn, wrapper).PublicId
+	cases := []struct {
+		name               string
+		workerService      *workerServiceServer
+		wantErrMsg         string
+		expectedStatusCode codes.Code
+		req                *pbs.SessionInfoRequest
+		expectedResponse   *pbs.SessionInfoResponse
+	}{
+		{
+			name:               "empty worker id",
+			workerService:      NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req:                &pbs.SessionInfoRequest{},
+			wantErrMsg:         "rpc error: code = InvalidArgument desc = worker id is empty",
+			expectedStatusCode: codes.InvalidArgument,
+		},
+		{
+			name:          "invalid unspecified session type",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req: &pbs.SessionInfoRequest{
+				WorkerId: "w_1234567890",
+				Sessions: []*pbs.Session{
+					{
+						SessionId:     "s_A",
+						SessionType:   pbs.SessionType_SESSION_TYPE_UNSPECIFIED,
+						SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
+					},
+				},
+			},
+			wantErrMsg:         "rpc error: code = InvalidArgument desc = unspecified session type",
+			expectedStatusCode: codes.InvalidArgument,
+		},
+		{
+			name:          "invalid unknown session type",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req: &pbs.SessionInfoRequest{
+				WorkerId: "w_1234567890",
+				Sessions: []*pbs.Session{
+					{
+						SessionId:     "s_A",
+						SessionType:   pbs.SessionType(-1),
+						SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
+					},
+				},
+			},
+			wantErrMsg:         "rpc error: code = InvalidArgument desc = unknown session type",
+			expectedStatusCode: codes.InvalidArgument,
+		},
+		{
+			name:          "session repository error",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionErrRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req: &pbs.SessionInfoRequest{
+				WorkerId: "w_1234567890",
+			},
+			wantErrMsg:         "rpc error: code = Internal desc = Error acquiring repo to query session status: unknown error",
+			expectedStatusCode: codes.Internal,
+		},
+		{
+			name:          "server repository error",
+			workerService: NewWorkerServiceServer(serversErrRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req: &pbs.SessionInfoRequest{
+				WorkerId: "w_1234567890",
+			},
+			wantErrMsg:         "rpc error: code = Internal desc = Error acquiring repo to upsert session info status time: unknown error",
+			expectedStatusCode: codes.Internal,
+		},
+		{
+			name:          "ignore canceled and terminated sessions",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req: &pbs.SessionInfoRequest{
+				WorkerId: workerId,
+				Sessions: []*pbs.Session{
+					{
+						SessionId:     "s_A",
+						SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
+						SessionType:   pbs.SessionType_SESSION_TYPE_INGRESSED,
+					},
+					{
+						SessionId:     "s_B",
+						SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
+						SessionType:   pbs.SessionType_SESSION_TYPE_INGRESSED,
+					},
+					{
+						SessionId:     "s_C",
+						SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
+						SessionType:   pbs.SessionType_SESSION_TYPE_RECORDED,
+					},
+					{
+						SessionId:     "s_D",
+						SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
+						SessionType:   pbs.SessionType_SESSION_TYPE_RECORDED,
+					},
+				},
+			},
+			expectedResponse: &pbs.SessionInfoResponse{},
+		},
+		{
+			name:          "empty sessions",
+			workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce),
+			req: &pbs.SessionInfoRequest{
+				WorkerId: workerId,
+			},
+			expectedResponse: &pbs.SessionInfoResponse{},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			got, err := tc.workerService.SessionInfo(context.Background(), tc.req)
+			require.NotNil(got)
+			if tc.wantErrMsg != "" {
+				require.Error(err)
+				assert.ErrorContains(err, tc.wantErrMsg)
+				actualStatus, ok := status.FromError(err)
+				require.True(ok)
+				assert.Equal(tc.expectedStatusCode, actualStatus.Code())
+				return
+			}
+			require.NoError(err)
+			require.NotNil(got)
+			assert.Len(got.NonActiveSessions, len(tc.expectedResponse.NonActiveSessions))
+		})
+	}
+
+	t.Run("session canceled with active connections", func(t *testing.T) {
+		t.Parallel()
+		assert, require := assert.New(t), require.New(t)
+		sessionTypes := []pbs.SessionType{
+			pbs.SessionType_SESSION_TYPE_INGRESSED,
+			pbs.SessionType_SESSION_TYPE_RECORDED,
+		}
+		for _, sessionType := range sessionTypes {
+			t.Run(sessionType.String(), func(t *testing.T) {
+				t.Parallel()
+				ctx := context.Background()
+				conn, _ := db.TestSetup(t, "postgres")
+				rw := db.New(conn)
+				wrapper := db.TestWrapper(t)
+				testKms := kms.TestKms(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+
+				serverRepo, err := server.NewRepository(ctx, rw, rw, testKms)
+				require.NoError(err)
+				c := &store.Controller{
+					PrivateId: "test_controller1",
+					Address:   "127.0.0.1",
+				}
+				_, err = serverRepo.UpsertController(ctx, c)
+				require.NoError(err)
+
+				serversRepoFn := func() (*server.Repository, error) {
+					return serverRepo, nil
+				}
+				workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) {
+					return server.NewRepositoryStorage(ctx, rw, rw, testKms)
+				}
+				sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) {
+					return session.NewRepository(ctx, rw, rw, testKms, opts...)
+				}
+				connRepoFn := func() (*session.ConnectionRepository, error) {
+					return session.NewConnectionRepository(ctx, rw, rw, testKms)
+				}
+				fce := &fakeControllerExtension{
+					reader: rw,
+					writer: rw,
+				}
+
+				repo, err := sessionRepoFn()
+				require.NoError(err)
+				connRepo, err := connRepoFn()
+				require.NoError(err)
+
+				at := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				uId := at.GetIamUserId()
+				hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
+				h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
+				tar := tcp.TestTarget(
+					ctx,
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hs.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+
+				worker1 := server.TestKmsWorker(t, conn, wrapper)
+				sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
+					UserId:          uId,
+					HostId:          h.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hs.GetPublicId(),
+					AuthTokenId:     at.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				tofu := session.TestTofu(t)
+				sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, tofu)
+				require.NoError(err)
+				sess2 := session.TestSession(t, conn, wrapper, session.ComposedOf{
+					UserId:          uId,
+					HostId:          h.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hs.GetPublicId(),
+					AuthTokenId:     at.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				tofu2 := session.TestTofu(t)
+				sess2, _, err = repo.ActivateSession(ctx, sess2.PublicId, sess2.Version, tofu2)
+				require.NoError(err)
+
+				s := NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce)
+				require.NotNil(t, s)
+
+				_, err = connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PublicId)
+				require.NoError(err)
+
+				_, err = connRepo.AuthorizeConnection(ctx, sess2.PublicId, worker1.PublicId)
+				require.NoError(err)
+
+				_, err = repo.CancelSession(ctx, sess2.PublicId, sess.Version)
+				require.NoError(err)
+
+				got, err := s.SessionInfo(ctx, &pbs.SessionInfoRequest{
+					WorkerId: worker1.GetPublicId(),
+					Sessions: []*pbs.Session{
+						{
+							SessionId:     sess2.PublicId,
+							SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
+							SessionType:   sessionType,
+						},
+					},
+				})
+				require.NoError(err)
+				require.NotNil(got)
+				require.Len(got.NonActiveSessions, 1)
+				actualNonActiveSession := got.NonActiveSessions[0]
+				require.NotNil(actualNonActiveSession)
+				assert.Equal(sess2.PublicId, actualNonActiveSession.SessionId)
+				assert.Equal(pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING, actualNonActiveSession.SessionStatus)
+				assert.Equal(sessionType, actualNonActiveSession.SessionType)
+				assert.Empty(actualNonActiveSession.Connections)
+			})
+		}
+	})
+}
+
+func TestRoutingInfo(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kmsCache := kms.TestKms(t, conn, wrapper)
+	require.NoError(t, kmsCache.CreateKeys(context.Background(), scope.Global.String(), kms.WithRandomReader(rand.Reader)))
+
+	serversRepoFn := func() (*server.Repository, error) {
+		return server.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) {
+		return server.NewRepositoryStorage(ctx, rw, rw, kmsCache)
+	}
+	sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) {
+		return session.NewRepository(ctx, rw, rw, kmsCache, opts...)
+	}
+	connectionRepoFn := func() (*session.ConnectionRepository, error) {
+		return session.NewConnectionRepository(ctx, rw, rw, kmsCache)
+	}
+	var liveDur atomic.Int64
+	liveDur.Store(int64(1 * time.Second))
+	fce := &fakeControllerExtension{
+		reader: rw,
+		writer: rw,
+	}
+	serverRepo, err := serversRepoFn()
+	require.NoError(t, err)
+
+	// Set up resources
+	var w1KeyId, w2KeyId string
+	w1 := server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w1KeyId))
+	_ = server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w2KeyId))
+	w3 := server.TestKmsWorker(t, conn, wrapper, server.WithName("testworker3"))
+
+	c := &store.Controller{
+		PrivateId: "test_controller1",
+		Address:   "1.2.3.4",
+	}
+	_, err = serverRepo.UpsertController(ctx, c)
+	require.NoError(t, err)
+
+	s := NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kmsCache, &liveDur, fce)
+	require.NotNil(t, s)
+	require.NoError(t, err)
+
+	t.Run("Missing worker status", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{}
+		_, err := s.RoutingInfo(ctx, req)
+		require.ErrorContains(t, err, "worker status is required")
+	})
+
+	t.Run("Missing key ID and public Id", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "",
+				KeyId:    "",
+				Address:  "2.3.4.5:8080",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "Boundary v0.18.0",
+				OperationalState:  server.ActiveOperationalState.String(),
+				LocalStorageState: server.AvailableLocalStorageState.String(),
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w3.PublicId, "worker-3"},
+		}
+		_, err := s.RoutingInfo(ctx, req)
+		require.ErrorContains(t, err, "public id, key id and name are not set in the request; one is required")
+	})
+
+	t.Run("Missing address", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "",
+				KeyId:    w1KeyId,
+				Address:  "",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "Boundary v0.18.0",
+				OperationalState:  server.ActiveOperationalState.String(),
+				LocalStorageState: server.AvailableLocalStorageState.String(),
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w3.PublicId, "worker-3"},
+		}
+		_, err := s.RoutingInfo(ctx, req)
+		require.ErrorContains(t, err, "address is not set but is required")
+	})
+
+	t.Run("Missing release version", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "",
+				KeyId:    w1KeyId,
+				Address:  "2.3.4.5:8080",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "",
+				OperationalState:  server.ActiveOperationalState.String(),
+				LocalStorageState: server.AvailableLocalStorageState.String(),
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w3.PublicId, "worker-3"},
+		}
+		_, err := s.RoutingInfo(ctx, req)
+		require.ErrorContains(t, err, "release version is not set but is required")
+	})
+
+	t.Run("Missing operational state", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "",
+				KeyId:    w1KeyId,
+				Address:  "2.3.4.5:8080",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "Boundary v0.18.0",
+				OperationalState:  "",
+				LocalStorageState: server.AvailableLocalStorageState.String(),
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w3.PublicId, "worker-3"},
+		}
+		_, err := s.RoutingInfo(ctx, req)
+		require.ErrorContains(t, err, "operational state is not set but is required")
+	})
+
+	t.Run("Missing local storage state", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "",
+				KeyId:    w1KeyId,
+				Address:  "2.3.4.5:8080",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "Boundary v0.18.0",
+				OperationalState:  server.ActiveOperationalState.String(),
+				LocalStorageState: "",
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w3.PublicId, "worker-3"},
+		}
+		_, err := s.RoutingInfo(ctx, req)
+		require.ErrorContains(t, err, "local storage state is not set but is required")
+	})
+
+	t.Run("Successful first and second request (PKI worker)", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "", // Not set on the first request
+				KeyId:    w1KeyId,
+				Address:  "2.3.4.5:8080",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "Boundary v0.18.0",
+				OperationalState:  server.ActiveOperationalState.String(),
+				LocalStorageState: server.AvailableLocalStorageState.String(),
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w3.PublicId, "worker-3"},
+		}
+		resp, err := s.RoutingInfo(ctx, req)
+		require.NoError(t, err)
+		assert.NotNil(t, resp)
+
+		assert.Equal(t, resp.WorkerId, w1.PublicId)
+		assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w3.PublicId})
+
+		w1, err := serverRepo.LookupWorker(ctx, resp.WorkerId)
+		require.NoError(t, err)
+
+		assert.Equal(t, w1.Address, "2.3.4.5:8080")
+		expTags := []*server.Tag{
+			{Key: "tag1", Value: "value1"},
+			{Key: "tag2", Value: "value2"},
+		}
+		assert.Empty(
+			t,
+			cmp.Diff(
+				w1.ConfigTags,
+				server.Tags(expTags),
+			),
+		)
+		assert.Equal(t, w1.ReleaseVersion, "Boundary v0.18.0")
+		assert.EqualValues(t, w1.OperationalState, server.ActiveOperationalState)
+		assert.EqualValues(t, w1.LocalStorageState, server.AvailableLocalStorageState)
+
+		// Now send the subsequent routing info request
+		req.WorkerStatus.PublicId = w1.PublicId // Only set on subsequent requests
+		resp, err = s.RoutingInfo(ctx, req)
+		require.NoError(t, err)
+		assert.NotNil(t, resp)
+
+		assert.Equal(t, resp.WorkerId, w1.PublicId)
+		assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w3.PublicId})
+
+		w1, err = serverRepo.LookupWorker(ctx, resp.WorkerId)
+		require.NoError(t, err)
+
+		assert.Equal(t, w1.Address, "2.3.4.5:8080")
+		assert.Empty(
+			t,
+			cmp.Diff(
+				w1.ConfigTags,
+				server.Tags(expTags),
+			),
+		)
+		assert.Equal(t, w1.ReleaseVersion, "Boundary v0.18.0")
+		assert.EqualValues(t, w1.OperationalState, server.ActiveOperationalState)
+		assert.EqualValues(t, w1.LocalStorageState, server.AvailableLocalStorageState)
+	})
+
+	t.Run("Successful first and second request (KMS worker)", func(t *testing.T) {
+		req := &pbs.RoutingInfoRequest{
+			WorkerStatus: &pb.ServerWorkerStatus{
+				PublicId: "", // Not set on the first request
+				Name:     "testworker3",
+				Address:  "2.3.4.5:8080",
+				Tags: []*pb.TagPair{
+					{Key: "tag1", Value: "value1"},
+					{Key: "tag2", Value: "value2"},
+				},
+				ReleaseVersion:    "Boundary v0.18.0",
+				OperationalState:  server.ActiveOperationalState.String(),
+				LocalStorageState: server.AvailableLocalStorageState.String(),
+			},
+			UpdateTags:                            true,
+			ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"},
+			ConnectedWorkerPublicIds:              []string{w1.PublicId, "worker-4"},
+		}
+		resp, err := s.RoutingInfo(ctx, req)
+		require.NoError(t, err)
+		assert.NotNil(t, resp)
+
+		assert.Equal(t, resp.WorkerId, w3.PublicId)
+		assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w1.PublicId})
+
+		w3, err := serverRepo.LookupWorker(ctx, resp.WorkerId)
+		require.NoError(t, err)
+
+		assert.Equal(t, w3.Address, "2.3.4.5:8080")
+		expTags := []*server.Tag{
+			{Key: "tag1", Value: "value1"},
+			{Key: "tag2", Value: "value2"},
+		}
+		assert.Empty(
+			t,
+			cmp.Diff(
+				w3.ConfigTags,
+				server.Tags(expTags),
+			),
+		)
+		assert.Equal(t, w3.ReleaseVersion, "Boundary v0.18.0")
+		assert.EqualValues(t, w3.OperationalState, server.ActiveOperationalState)
+		assert.EqualValues(t, w3.LocalStorageState, server.AvailableLocalStorageState)
+
+		// Now send the subsequent routing info request
+		req.WorkerStatus.PublicId = w3.PublicId // Only set on subsequent requests
+		resp, err = s.RoutingInfo(ctx, req)
+		require.NoError(t, err)
+		assert.NotNil(t, resp)
+
+		assert.Equal(t, resp.WorkerId, w3.PublicId)
+		assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId})
+		assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w1.PublicId})
+
+		w1, err = serverRepo.LookupWorker(ctx, resp.WorkerId)
+		require.NoError(t, err)
+
+		assert.Equal(t, w3.Address, "2.3.4.5:8080")
+		assert.Empty(
+			t,
+			cmp.Diff(
+				w3.ConfigTags,
+				server.Tags(expTags),
+			),
+		)
+		assert.Equal(t, w3.ReleaseVersion, "Boundary v0.18.0")
+		assert.EqualValues(t, w3.OperationalState, server.ActiveOperationalState)
+		assert.EqualValues(t, w3.LocalStorageState, server.AvailableLocalStorageState)
+	})
+}

internal/daemon/controller/controller.go

@@ -117,11 +117,11 @@ type Controller struct {
 	clusterListener *base.ServerListener
 
 	// Used for testing and tracking worker health
-	workerStatusUpdateTimes *sync.Map
+	workerRoutingInfoUpdateTimes *sync.Map
 
 	// Timing variables. These are atomics for SIGHUP support, and are int64
 	// because they are casted to time.Duration.
-	workerStatusGracePeriod     *atomic.Int64
+	workerRPCGracePeriod        *atomic.Int64
 	livenessTimeToStale         *atomic.Int64
 	getDownstreamWorkersTimeout *atomic.Int64
 
@@ -177,19 +177,19 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 	metric.InitializeApiCollectors(conf.PrometheusRegisterer)
 	ratelimit.InitializeMetrics(conf.PrometheusRegisterer)
 	c := &Controller{
-		conf:                        conf,
-		logger:                      conf.Logger.Named("controller"),
-		started:                     ua.NewBool(false),
-		tickerWg:                    new(sync.WaitGroup),
-		schedulerWg:                 new(sync.WaitGroup),
-		workerAuthCache:             new(sync.Map),
-		workerStatusUpdateTimes:     new(sync.Map),
-		enabledPlugins:              conf.Server.EnabledPlugins,
-		apiListeners:                make([]*base.ServerListener, 0),
-		downstreamConnManager:       cluster.NewDownstreamManager(),
-		workerStatusGracePeriod:     new(atomic.Int64),
-		livenessTimeToStale:         new(atomic.Int64),
-		getDownstreamWorkersTimeout: new(atomic.Int64),
+		conf:                         conf,
+		logger:                       conf.Logger.Named("controller"),
+		started:                      ua.NewBool(false),
+		tickerWg:                     new(sync.WaitGroup),
+		schedulerWg:                  new(sync.WaitGroup),
+		workerAuthCache:              new(sync.Map),
+		workerRoutingInfoUpdateTimes: new(sync.Map),
+		enabledPlugins:               conf.Server.EnabledPlugins,
+		apiListeners:                 make([]*base.ServerListener, 0),
+		downstreamConnManager:        cluster.NewDownstreamManager(),
+		workerRPCGracePeriod:         new(atomic.Int64),
+		livenessTimeToStale:          new(atomic.Int64),
+		getDownstreamWorkersTimeout:  new(atomic.Int64),
 	}
 
 	c.started.Store(false)
@@ -223,11 +223,11 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 		}
 	}
 
-	switch conf.RawConfig.Controller.WorkerStatusGracePeriodDuration {
+	switch conf.RawConfig.Controller.WorkerRPCGracePeriodDuration {
 	case 0:
-		c.workerStatusGracePeriod.Store(int64(server.DefaultLiveness))
+		c.workerRPCGracePeriod.Store(int64(server.DefaultLiveness))
 	default:
-		c.workerStatusGracePeriod.Store(int64(conf.RawConfig.Controller.WorkerStatusGracePeriodDuration))
+		c.workerRPCGracePeriod.Store(int64(conf.RawConfig.Controller.WorkerRPCGracePeriodDuration))
 	}
 	switch conf.RawConfig.Controller.LivenessTimeToStaleDuration {
 	case 0:
@@ -623,7 +623,7 @@ func (c *Controller) registerJobs() error {
 	if err := pluginhost.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.conf.HostPlugins); err != nil {
 		return err
 	}
-	if err := session.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.workerStatusGracePeriod); err != nil {
+	if err := session.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.workerRPCGracePeriod); err != nil {
 		return err
 	}
 	var serverJobOpts []serversjob.Option
@@ -633,7 +633,7 @@ func (c *Controller) registerJobs() error {
 			serversjob.WithRotationFrequency(c.conf.TestOverrideWorkerAuthCaCertificateLifetime/2),
 		)
 	}
-	if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.ControllerExtension, c.workerStatusGracePeriod, serverJobOpts...); err != nil {
+	if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.ControllerExtension, c.workerRPCGracePeriod, serverJobOpts...); err != nil {
 		return err
 	}
 	if err := kmsjob.RegisterJobs(c.baseContext, c.scheduler, c.kms); err != nil {
@@ -675,12 +675,12 @@ func (c *Controller) Shutdown() error {
 	return nil
 }
 
-// WorkerStatusUpdateTimes returns the map, which specifically is held in _this_
+// WorkerRoutingInfoUpdateTimes returns the map, which specifically is held in _this_
 // controller, not the DB. It's used in tests to verify that a given controller
 // is receiving updates from an expected set of workers, to test out balancing
 // and auto reconnection.
-func (c *Controller) WorkerStatusUpdateTimes() *sync.Map {
-	return c.workerStatusUpdateTimes
+func (c *Controller) WorkerRoutingInfoUpdateTimes() *sync.Map {
+	return c.workerRoutingInfoUpdateTimes
 }
 
 // ReloadTimings reloads timing related parameters
@@ -694,11 +694,11 @@ func (c *Controller) ReloadTimings(newConfig *config.Config) error {
 		return errors.New(c.baseContext, errors.InvalidParameter, op, "nil config.Controller")
 	}
 
-	switch newConfig.Controller.WorkerStatusGracePeriodDuration {
+	switch newConfig.Controller.WorkerRPCGracePeriodDuration {
 	case 0:
-		c.workerStatusGracePeriod.Store(int64(server.DefaultLiveness))
+		c.workerRPCGracePeriod.Store(int64(server.DefaultLiveness))
 	default:
-		c.workerStatusGracePeriod.Store(int64(newConfig.Controller.WorkerStatusGracePeriodDuration))
+		c.workerRPCGracePeriod.Store(int64(newConfig.Controller.WorkerRPCGracePeriodDuration))
 	}
 	switch newConfig.Controller.LivenessTimeToStaleDuration {
 	case 0:

internal/daemon/controller/handler.go

@@ -243,7 +243,7 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 			c.baseContext,
 			c.IamRepoFn,
 			c.ServersRepoFn,
-			c.workerStatusGracePeriod,
+			c.workerRPCGracePeriod,
 			c.kms,
 			c.conf.RawConfig.Controller.MaxPageSize,
 			c.ControllerExtension)
@@ -266,7 +266,7 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 			c.StaticCredentialRepoFn,
 			c.TargetAliasRepoFn,
 			c.downstreamWorkers,
-			c.workerStatusGracePeriod,
+			c.workerRPCGracePeriod,
 			c.conf.RawConfig.Controller.MaxPageSize,
 			c.ControllerExtension,
 		)

internal/daemon/controller/handlers/session_recordings/session_recording_service.go

@@ -44,7 +44,7 @@ func init() {
 var NewServiceFn = func(ctx context.Context,
 	iamRepoFn common.IamRepoFactory,
 	serverRepoFn common.ServersRepoFactory,
-	workerStatusGracePeriod *atomic.Int64,
+	workerRPCGracePeriod *atomic.Int64,
 	kms *kms.Kms,
 	maxPageSize uint,
 	controllerExt intglobals.ControllerExtension,

internal/daemon/controller/handlers/targets/target_service.go

@@ -121,20 +121,20 @@ func IngressWorkerFilterUnsupported(string) error {
 type Service struct {
 	pbs.UnsafeTargetServiceServer
 
-	repoFn                  target.RepositoryFactory
-	aliasRepoFn             common.TargetAliasRepoFactory
-	iamRepoFn               common.IamRepoFactory
-	serversRepoFn           common.ServersRepoFactory
-	sessionRepoFn           session.RepositoryFactory
-	pluginHostRepoFn        common.PluginHostRepoFactory
-	staticHostRepoFn        common.StaticRepoFactory
-	vaultCredRepoFn         common.VaultCredentialRepoFactory
-	staticCredRepoFn        common.StaticCredentialRepoFactory
-	downstreams             downstream.Graph
-	kmsCache                *kms.Kms
-	workerStatusGracePeriod *atomic.Int64
-	maxPageSize             uint
-	controllerExt           intglobals.ControllerExtension
+	repoFn               target.RepositoryFactory
+	aliasRepoFn          common.TargetAliasRepoFactory
+	iamRepoFn            common.IamRepoFactory
+	serversRepoFn        common.ServersRepoFactory
+	sessionRepoFn        session.RepositoryFactory
+	pluginHostRepoFn     common.PluginHostRepoFactory
+	staticHostRepoFn     common.StaticRepoFactory
+	vaultCredRepoFn      common.VaultCredentialRepoFactory
+	staticCredRepoFn     common.StaticCredentialRepoFactory
+	downstreams          downstream.Graph
+	kmsCache             *kms.Kms
+	workerRPCGracePeriod *atomic.Int64
+	maxPageSize          uint
+	controllerExt        intglobals.ControllerExtension
 }
 
 var _ pbs.TargetServiceServer = (*Service)(nil)
@@ -153,7 +153,7 @@ func NewService(
 	staticCredRepoFn common.StaticCredentialRepoFactory,
 	aliasRepoFn common.TargetAliasRepoFactory,
 	downstreams downstream.Graph,
-	workerStatusGracePeriod *atomic.Int64,
+	workerRPCGracePeriod *atomic.Int64,
 	maxPageSize uint,
 	controllerExt intglobals.ControllerExtension,
 ) (Service, error) {
@@ -184,20 +184,20 @@ func NewService(
 		maxPageSize = uint(globals.DefaultMaxPageSize)
 	}
 	return Service{
-		repoFn:                  repoFn,
-		iamRepoFn:               iamRepoFn,
-		serversRepoFn:           serversRepoFn,
-		sessionRepoFn:           sessionRepoFn,
-		pluginHostRepoFn:        pluginHostRepoFn,
-		staticHostRepoFn:        staticHostRepoFn,
-		vaultCredRepoFn:         vaultCredRepoFn,
-		staticCredRepoFn:        staticCredRepoFn,
-		aliasRepoFn:             aliasRepoFn,
-		downstreams:             downstreams,
-		kmsCache:                kmsCache,
-		workerStatusGracePeriod: workerStatusGracePeriod,
-		maxPageSize:             maxPageSize,
-		controllerExt:           controllerExt,
+		repoFn:               repoFn,
+		iamRepoFn:            iamRepoFn,
+		serversRepoFn:        serversRepoFn,
+		sessionRepoFn:        sessionRepoFn,
+		pluginHostRepoFn:     pluginHostRepoFn,
+		staticHostRepoFn:     staticHostRepoFn,
+		vaultCredRepoFn:      vaultCredRepoFn,
+		staticCredRepoFn:     staticCredRepoFn,
+		aliasRepoFn:          aliasRepoFn,
+		downstreams:          downstreams,
+		kmsCache:             kmsCache,
+		workerRPCGracePeriod: workerRPCGracePeriod,
+		maxPageSize:          maxPageSize,
+		controllerExt:        controllerExt,
 	}, nil
 }
 
@@ -942,7 +942,7 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession
 	// Get workers and filter down to ones that can service this request
 	selectedWorkers, protoWorkerId, err := serversRepo.SelectSessionWorkers(
 		ctx,
-		time.Duration(s.workerStatusGracePeriod.Load()),
+		time.Duration(s.workerRPCGracePeriod.Load()),
 		t,
 		h,
 		s.controllerExt,

internal/daemon/controller/handlers/targets/tcp/target_service_test.go

@@ -93,8 +93,8 @@ var expectedDeprecatedWorkerFilterError = "Use egress_worker_filter instead"
 func testService(t *testing.T, ctx context.Context, conn *db.DB, kms *kms.Kms, wrapper wrapping.Wrapper) (targets.Service, error) {
 	rw := db.New(conn)
 	sche := scheduler.TestScheduler(t, conn, wrapper)
-	statusGracePeriod := new(atomic.Int64)
-	statusGracePeriod.Store(int64(server.DefaultLiveness))
+	workerRPCGracePeriod := new(atomic.Int64)
+	workerRPCGracePeriod.Store(int64(server.DefaultLiveness))
 	repoFn := func(o ...target.Option) (*target.Repository, error) {
 		return target.NewRepository(ctx, rw, rw, kms, o...)
 	}
@@ -122,7 +122,7 @@ func testService(t *testing.T, ctx context.Context, conn *db.DB, kms *kms.Kms, w
 	targetAliasRepoFn := func() (*talias.Repository, error) {
 		return talias.NewRepository(ctx, rw, rw, kms)
 	}
-	return targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil)
+	return targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil)
 }
 
 func TestGet(t *testing.T) {
@@ -3571,9 +3571,9 @@ func TestAuthorizeSession(t *testing.T) {
 		},
 	}
 
-	statusGracePeriod := new(atomic.Int64)
-	statusGracePeriod.Store(int64(server.DefaultLiveness))
-	s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil)
+	workerRPCGracePeriod := new(atomic.Int64)
+	workerRPCGracePeriod.Store(int64(server.DefaultLiveness))
+	s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil)
 	require.NoError(t, err)
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -3776,9 +3776,9 @@ func TestAuthorizeSessionTypedCredentials(t *testing.T) {
 	_ = iam.TestUserRole(t, conn, r.GetPublicId(), at.GetIamUserId())
 	_ = iam.TestRoleGrant(t, conn, r.GetPublicId(), "ids=*;type=*;actions=*")
 
-	statusGracePeriod := new(atomic.Int64)
-	statusGracePeriod.Store(int64(server.DefaultLiveness))
-	s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil)
+	workerRPCGracePeriod := new(atomic.Int64)
+	workerRPCGracePeriod.Store(int64(server.DefaultLiveness))
+	s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil)
 	require.NoError(t, err)
 
 	hc := static.TestCatalogs(t, conn, proj.GetPublicId(), 1)[0]
@@ -4380,9 +4380,9 @@ func TestAuthorizeSession_Errors(t *testing.T) {
 	}
 	org, proj := iam.TestScopes(t, iamRepo)
 
-	statusGracePeriod := new(atomic.Int64)
-	statusGracePeriod.Store(int64(server.DefaultLiveness))
-	s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil)
+	workerRPCGracePeriod := new(atomic.Int64)
+	workerRPCGracePeriod.Store(int64(server.DefaultLiveness))
+	s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil)
 	require.NoError(t, err)
 
 	// Authorized user gets full permissions

internal/daemon/controller/rpc_registration.go

@@ -47,7 +47,7 @@ func registerControllerServerCoordinationService(ctx context.Context, c *Control
 		c.SessionRepoFn,
 		c.ConnectionRepoFn,
 		c.downstreamWorkers,
-		c.workerStatusUpdateTimes,
+		c.workerRoutingInfoUpdateTimes,
 		c.kms,
 		c.livenessTimeToStale,
 		c.ControllerExtension,
@@ -74,7 +74,7 @@ func registerControllerSessionService(ctx context.Context, c *Controller, server
 		c.SessionRepoFn,
 		c.ConnectionRepoFn,
 		c.downstreamWorkers,
-		c.workerStatusUpdateTimes,
+		c.workerRoutingInfoUpdateTimes,
 		c.kms,
 		c.livenessTimeToStale,
 		c.ControllerExtension,

internal/daemon/controller/testing.go

@@ -473,8 +473,9 @@ type TestControllerOpts struct {
 	PublicClusterAddr string
 
 	// The amount of time to wait before marking connections as canceling when a
-	// worker has not reported in
-	WorkerStatusGracePeriodDuration time.Duration
+	// worker has not reported in, and whether a worker is considered to be
+	// routable based on the last time it received a routing info report.
+	WorkerRPCGracePeriod time.Duration
 
 	// The period of time after which it will consider other controllers to be
 	// no longer accessible, based on time since their last status update in the
@@ -691,8 +692,8 @@ func TestControllerConfig(t testing.TB, ctx context.Context, tc *TestController,
 		t.Fatal(err)
 	}
 
-	if opts.WorkerStatusGracePeriodDuration != 0 {
-		opts.Config.Controller.WorkerStatusGracePeriodDuration = opts.WorkerStatusGracePeriodDuration
+	if opts.WorkerRPCGracePeriod != 0 {
+		opts.Config.Controller.WorkerRPCGracePeriodDuration = opts.WorkerRPCGracePeriod
 	}
 	if opts.LivenessTimeToStaleDuration != 0 {
 		opts.Config.Controller.LivenessTimeToStaleDuration = opts.LivenessTimeToStaleDuration
@@ -862,7 +863,7 @@ func (tc *TestController) AddClusterControllerMember(t testing.TB, opts *TestCon
 		DisableAuthMethodCreation:       true,
 		DisableAutoStart:                opts.DisableAutoStart,
 		PublicClusterAddr:               opts.PublicClusterAddr,
-		WorkerStatusGracePeriodDuration: opts.WorkerStatusGracePeriodDuration,
+		WorkerRPCGracePeriod:            opts.WorkerRPCGracePeriod,
 		LivenessTimeToStaleDuration:     opts.LivenessTimeToStaleDuration,
 		WorkerAuthCaCertificateLifetime: tc.c.conf.TestOverrideWorkerAuthCaCertificateLifetime,
 		WorkerAuthDebuggingEnabled:      tc.c.conf.WorkerAuthDebuggingEnabled,
@@ -881,53 +882,45 @@ func (tc *TestController) AddClusterControllerMember(t testing.TB, opts *TestCon
 	return NewTestController(t, nextOpts)
 }
 
-// WaitForNextWorkerStatusUpdate waits for the next status check from a worker to
-// come in. If it does not come in within the default status grace
+// WaitForNextWorkerRoutingInfoUpdate waits for the next routing info RPC from a worker to
+// come in. If it does not come in within the default worker RPC grace
 // period, this function returns an error.
-func (tc *TestController) WaitForNextWorkerStatusUpdate(workerStatusName string) error {
-	const op = "controller.(TestController).WaitForNextWorkerStatusUpdate"
-	ctx := context.TODO()
-	event.WriteSysEvent(ctx, op, "waiting for next status report from worker", "worker", workerStatusName)
-	waitStatusStart := time.Now()
-	ctx, cancel := context.WithTimeout(tc.ctx, time.Duration(tc.c.workerStatusGracePeriod.Load()))
+func (tc *TestController) WaitForNextWorkerRoutingInfoUpdate(workerName string) error {
+	const op = "controller.(TestController).WaitForNextWorkerRoutingInfoUpdate"
+	waitStart := time.Now()
+	ctx, cancel := context.WithTimeout(tc.ctx, time.Duration(tc.c.workerRPCGracePeriod.Load()))
 	defer cancel()
+	event.WriteSysEvent(ctx, op, "waiting for next routing info from worker", "worker", workerName)
 	var err error
 	for {
-		if err = func() error {
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-
-			case <-time.After(time.Second):
-				// pass
-			}
-
-			return nil
-		}(); err != nil {
+		select {
+		case <-ctx.Done():
 			break
+		case <-time.After(time.Second):
+			// pass
 		}
 
-		var waitStatusCurrent time.Time
-		tc.Controller().WorkerStatusUpdateTimes().Range(func(k, v any) bool {
+		var waitCurrent time.Time
+		tc.Controller().WorkerRoutingInfoUpdateTimes().Range(func(k, v any) bool {
 			if k == nil || v == nil {
 				err = fmt.Errorf("nil key or value on entry: key=%#v value=%#v", k, v)
 				return false
 			}
 
-			workerStatusUpdateId, ok := k.(string)
+			workerUpdateId, ok := k.(string)
 			if !ok {
 				err = fmt.Errorf("unexpected type %T for key: key=%#v value=%#v", k, k, v)
 				return false
 			}
 
-			workerStatusUpdateTime, ok := v.(time.Time)
+			workerUpdateTime, ok := v.(time.Time)
 			if !ok {
 				err = fmt.Errorf("unexpected type %T for value: key=%#v value=%#v", k, k, v)
 				return false
 			}
 
-			if workerStatusUpdateId == workerStatusName {
-				waitStatusCurrent = workerStatusUpdateTime
+			if workerUpdateId == workerName {
+				waitCurrent = workerUpdateTime
 				return false
 			}
 
@@ -938,16 +931,16 @@ func (tc *TestController) WaitForNextWorkerStatusUpdate(workerStatusName string)
 			break
 		}
 
-		if waitStatusCurrent.After(waitStatusStart) {
+		if waitCurrent.After(waitStart) {
 			break
 		}
 	}
 
 	if err != nil {
-		event.WriteError(ctx, op, err, event.WithInfoMsg("error waiting for next status report from worker", "worker", workerStatusName))
+		event.WriteError(ctx, op, err, event.WithInfoMsg("error waiting for next routing info RPC from worker", "worker", workerName))
 		return err
 	}
-	event.WriteSysEvent(ctx, op, "waiting for next status report from worker received successfully", "worker", workerStatusName)
+	event.WriteSysEvent(ctx, op, "next routing info RPC from worker received successfully", "worker", workerName)
 	return nil
 }
 

internal/daemon/worker/common/common.go

@@ -11,11 +11,27 @@ import (
 
 // In the future we could make this configurable
 const (
-	// StatusInterval is the base duration used in the calculation of the random backoff
-	// during the worker status report
-	StatusInterval = 2 * time.Second
+	// SessionInfoInterval is the base duration used in the calculation of the random backoff
+	// during the worker session information report
+	SessionInfoInterval = 2 * time.Second
 
-	// DefaultStatusTimeout is the timeout duration on status calls to the controller from
+	// RoutingInfoInterval is the interval at which the worker will send routing information
+	// to the controller
+	RoutingInfoInterval = 10 * time.Second
+
+	// StatisticsInterval is the base duration used in the calculation of the random backoff
+	// during the worker statistics report
+	StatisticsInterval = 15 * time.Second
+
+	// DefaultSessionInfoTimeout is the timeout duration on SessionInfo calls to the controller from
 	// the worker
-	DefaultStatusTimeout = server.DefaultLiveness / 3
+	DefaultSessionInfoTimeout = server.DefaultLiveness / 3
+
+	// DefaultRoutingInfoTimeout is the timeout duration on routing info calls to the controller from
+	// the worker
+	DefaultRoutingInfoTimeout = server.DefaultLiveness / 2
+
+	// DefaultStatisticsTimeout is the timeout duration on Statistics calls to the controller from
+	// the worker
+	DefaultStatisticsTimeout = server.DefaultLiveness
 )

internal/daemon/worker/controller_connection.go

@@ -42,6 +42,8 @@ var HandleHcpbClusterId func(s string) string
 // StartControllerConnections starts up the resolver and initiates controller
 // connection client creation
 func (w *Worker) StartControllerConnections() error {
+	w.confAddressReceiversLock.Lock()
+	defer w.confAddressReceiversLock.Unlock()
 	const op = "worker.(Worker).StartControllerConnections"
 	initialAddrs := make([]string, 0, len(w.conf.RawConfig.Worker.InitialUpstreams))
 	for _, addr := range w.conf.RawConfig.Worker.InitialUpstreams {
@@ -261,10 +263,10 @@ func (w *Worker) workerConnectionInfo(addr string) (*structpb.Struct, error) {
 		PublicAddr:       w.conf.RawConfig.Worker.PublicAddr,
 		OperationalState: w.operationalState.Load().(server.OperationalState).String(),
 	}
-	if w.LastStatusSuccess() != nil && len(w.LastStatusSuccess().GetWorkerId()) > 0 {
+	if w.LastRoutingInfoSuccess() != nil && len(w.LastRoutingInfoSuccess().GetWorkerId()) > 0 {
 		// even though we wont have the worker the first time we dial, any
 		// redial attempts should result in the worker id being populated
-		wci.WorkerId = w.LastStatusSuccess().GetWorkerId()
+		wci.WorkerId = w.LastRoutingInfoSuccess().GetWorkerId()
 	}
 	st, err := wci.AsConnectionStateStruct()
 	if err != nil {

internal/daemon/worker/handler.go

@@ -198,14 +198,14 @@ func (w *Worker) handleProxy(listenerCfg *listenerutil.ListenerConfig, sessionMa
 			}
 		}
 
-		if w.LastStatusSuccess() == nil || w.LastStatusSuccess().WorkerId == "" {
+		if w.LastRoutingInfoSuccess() == nil || w.LastRoutingInfoSuccess().WorkerId == "" {
 			event.WriteError(ctx, op, stderrors.New("worker id is empty"))
 			if err = conn.Close(websocket.StatusInternalError, "worker id is empty"); err != nil {
 				event.WriteError(ctx, op, err, event.WithInfoMsg("error closing client connection"))
 			}
 			return
 		}
-		workerId := w.LastStatusSuccess().WorkerId
+		workerId := w.LastRoutingInfoSuccess().WorkerId
 
 		var acResp *pbs.AuthorizeConnectionResponse
 		var connsLeft int32

internal/daemon/worker/routing_info.go

@@ -0,0 +1,328 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package worker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"slices"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/event"
+	pb "github.com/hashicorp/boundary/internal/gen/controller/servers"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"github.com/hashicorp/boundary/version"
+	"github.com/hashicorp/go-secure-stdlib/parseutil"
+	"github.com/hashicorp/go-secure-stdlib/strutil"
+	"google.golang.org/grpc/connectivity"
+)
+
+var firstRoutingInfoCheckPostHooks = []func(context.Context, *Worker) error{
+	// Start session route info and statistics ticking after first routing info success
+	func(ctx context.Context, w *Worker) error {
+		w.tickerWg.Add(2)
+		go func() {
+			defer w.tickerWg.Done()
+			w.startSessionInfoTicking(w.baseContext)
+		}()
+		go func() {
+			defer w.tickerWg.Done()
+			w.startStatisticsTicking(w.baseContext)
+		}()
+		return nil
+	},
+}
+
+var downstreamWorkersFactory func(ctx context.Context, workerId string, ver string) (graph, error)
+
+var checkHCPBUpstreams func(w *Worker) bool
+
+// LastRoutingInfo represents the last successful routing info sent to the controller.
+type LastRoutingInfo struct {
+	*pbs.RoutingInfoResponse
+	RoutingInfoTime         time.Time
+	LastCalculatedUpstreams []string
+}
+
+// WaitForNextSuccessfulRoutingInfoUpdate waits for the next successful routing info. It's
+// used by testing in place of a more opaque and
+// possibly unnecessarily long sleep for things like initial controller
+// check-in, etc.
+//
+// The timeout is aligned with the worker's routing info grace period. A nil error
+// means the routing info was sent successfully.
+func (w *Worker) WaitForNextSuccessfulRoutingInfoUpdate() error {
+	const op = "worker.(Worker).WaitForNextSuccessfulRoutingInfoUpdate"
+	waitStart := time.Now()
+	ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(w.successfulRoutingInfoGracePeriod.Load()))
+	defer cancel()
+	event.WriteSysEvent(ctx, op, "waiting for next routing info report to controller")
+	for {
+		select {
+		case <-time.After(time.Second):
+			// pass
+
+		case <-ctx.Done():
+			event.WriteError(ctx, op, ctx.Err(), event.WithInfoMsg("error waiting for next routing info report to controller"))
+			return ctx.Err()
+		}
+
+		if w.lastSuccessfulRoutingInfoTime().After(waitStart) {
+			break
+		}
+	}
+
+	event.WriteSysEvent(ctx, op, "next worker routing info update sent successfully")
+	return nil
+}
+
+func (w *Worker) startRoutingInfoTicking(cancelCtx context.Context) {
+	const op = "worker.(Worker).startRoutingInfoTicking"
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-cancelCtx.Done():
+			event.WriteSysEvent(w.baseContext, op, "RoutingInfo ticking shutting down")
+			return
+
+		case <-timer.C:
+			// If we've never managed to successfully authenticate then we won't have
+			// any session information anyways and this will produce a ton of noise in
+			// observability, so skip calling the function and retry in a short duration
+			if w.everAuthenticated.Load() == authenticationStatusNeverAuthenticated {
+				timer.Reset(10 * time.Millisecond)
+				continue
+			}
+
+			w.sendWorkerRoutingInfo(cancelCtx)
+			// Desynchronize calls to controllers from workers, so we aren't always
+			// getting RoutingInfo updates at the exact same intervals, to ease the load on the DB.
+			timer.Reset(w.routingInfoInterval + getRandomInterval(r))
+		}
+	}
+}
+
+// LastRoutingInfoSuccess reports the last time we sent a successful
+// RoutingInfo request.
+func (w *Worker) LastRoutingInfoSuccess() *LastRoutingInfo {
+	s, ok := w.lastRoutingInfoSuccess.Load().(*LastRoutingInfo)
+	if !ok {
+		return nil
+	}
+	return s
+}
+
+func (w *Worker) sendWorkerRoutingInfo(cancelCtx context.Context) {
+	const op = "worker.(Worker).sendWorkerRoutingInfo"
+	// Lock access to w.conf and w.addressReceivers
+	w.confAddressReceiversLock.Lock()
+	defer w.confAddressReceiversLock.Unlock()
+
+	clientCon := w.GrpcClientConn.Load()
+	// Send RoutingInfo information
+	client := pbs.NewServerCoordinationServiceClient(clientCon)
+	var tags []*pb.TagPair
+	// If we're not going to request a tag update, no reason to have these
+	// marshaled on every RoutingInfo call.
+	if w.updateTags.Load() {
+		tags = w.tags.Load().([]*pb.TagPair)
+	}
+	ctx, cancel := context.WithTimeout(cancelCtx, time.Duration(w.routingInfoCallTimeoutDuration.Load()))
+	defer cancel()
+
+	keyId := w.WorkerAuthCurrentKeyId.Load()
+	switch {
+	case w.conf.RawConfig.Worker.Name == "" && keyId == "":
+		event.WriteError(cancelCtx, op, errors.New("worker name and keyId are both empty; at least one is needed to identify a worker"),
+			event.WithInfoMsg("error making RoutingInfo request to controller"))
+	}
+
+	var storageBucketCredentialStates map[string]*plugin.StorageBucketCredentialState
+	if w.RecordingStorage != nil {
+		storageBucketCredentialStates = w.RecordingStorage.GetStorageBucketCredentialStates()
+		// If the local storage state is unknown, and we have recording storage set, get the state from the recording storage
+		// and set it on the worker. This is done once to ensure that the worker has the correct state for the first RoutingInfo
+		// call.
+		if w.localStorageState.Load() == server.UnknownLocalStorageState {
+			w.localStorageState.Store(w.RecordingStorage.GetLocalStorageState(cancelCtx))
+		}
+	}
+	versionInfo := version.Get()
+	connectionState := w.downstreamConnManager.Connected()
+	result, err := client.RoutingInfo(ctx, &pbs.RoutingInfoRequest{
+		WorkerStatus: &pb.ServerWorkerStatus{
+			Name:                          w.conf.RawConfig.Worker.Name,
+			Description:                   w.conf.RawConfig.Worker.Description,
+			Address:                       w.conf.RawConfig.Worker.PublicAddr,
+			Tags:                          tags,
+			KeyId:                         keyId,
+			ReleaseVersion:                versionInfo.FullVersionNumber(false),
+			OperationalState:              w.operationalState.Load().(server.OperationalState).String(),
+			LocalStorageState:             w.localStorageState.Load().(server.LocalStorageState).String(),
+			StorageBucketCredentialStates: storageBucketCredentialStates,
+		},
+		ConnectedUnmappedWorkerKeyIdentifiers: connectionState.UnmappedKeyIds(),
+		ConnectedWorkerPublicIds:              connectionState.WorkerIds(),
+		UpdateTags:                            w.updateTags.Load(),
+	})
+	if err != nil {
+		event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error making routing info request to controller", "controller_address", clientCon.Target()))
+
+		// In the case that the control plane has gone down and come up with different IPs,
+		// append initial upstreams/ cluster addr to the resolver to try
+		if pastGrace, _, _ := w.isPastGrace(); pastGrace && w.GrpcClientConn.Load().GetState() == connectivity.TransientFailure {
+			lastRoutingInfo := w.lastRoutingInfoSuccess.Load().(*LastRoutingInfo)
+			if lastRoutingInfo != nil && lastRoutingInfo.LastCalculatedUpstreams != nil {
+				addrs := lastRoutingInfo.LastCalculatedUpstreams
+
+				if len(w.conf.RawConfig.Worker.InitialUpstreams) > 0 {
+					addrs = append(addrs, w.conf.RawConfig.Worker.InitialUpstreams...)
+				} else if HandleHcpbClusterId != nil && len(w.conf.RawConfig.HcpbClusterId) > 0 {
+					clusterId, err := parseutil.ParsePath(w.conf.RawConfig.HcpbClusterId)
+					if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
+						event.WriteError(cancelCtx, op, err, event.WithInfoMsg("failed to parse HCP Boundary cluster ID"))
+					} else {
+						clusterAddress := HandleHcpbClusterId(clusterId)
+						addrs = append(addrs, clusterAddress)
+					}
+				}
+
+				slices.Sort(addrs)
+				addrs = slices.Compact(addrs)
+				if slices.Equal(lastRoutingInfo.LastCalculatedUpstreams, addrs) {
+					// Nothing to update
+					return
+				}
+
+				w.updateAddresses(cancelCtx, addrs)
+				lastRoutingInfo.LastCalculatedUpstreams = addrs
+				w.lastRoutingInfoSuccess.Store(lastRoutingInfo)
+			}
+		}
+		return
+	}
+
+	w.updateTags.Store(false)
+
+	if authorized := result.GetAuthorizedDownstreamWorkers(); authorized != nil {
+		connectionState.DisconnectMissingWorkers(authorized.GetWorkerPublicIds())
+		connectionState.DisconnectMissingUnmappedKeyIds(authorized.GetUnmappedWorkerKeyIdentifiers())
+	}
+	var addrs []string
+	// This may be empty if we are in a multiple hop scenario
+	if len(result.GetCalculatedUpstreamAddresses()) > 0 {
+		addrs = result.GetCalculatedUpstreamAddresses()
+	} else if checkHCPBUpstreams != nil && checkHCPBUpstreams(w) {
+		// This is a worker that is one hop away from managed workers, so attempt to get that list
+		hcpbWorkersCtx, hcpbWorkersCancel := context.WithTimeout(cancelCtx, time.Duration(w.routingInfoCallTimeoutDuration.Load()))
+		defer hcpbWorkersCancel()
+		workersResp, err := client.ListHcpbWorkers(hcpbWorkersCtx, &pbs.ListHcpbWorkersRequest{})
+		if err != nil {
+			event.WriteError(hcpbWorkersCtx, op, err, event.WithInfoMsg("error fetching managed worker information"))
+		} else {
+			addrs = make([]string, 0, len(workersResp.Workers))
+			for _, v := range workersResp.Workers {
+				addrs = append(addrs, v.Address)
+			}
+		}
+	}
+
+	w.updateAddresses(cancelCtx, addrs)
+
+	w.lastRoutingInfoSuccess.Store(&LastRoutingInfo{RoutingInfoResponse: result, RoutingInfoTime: time.Now(), LastCalculatedUpstreams: addrs})
+
+	// If we have post hooks for after the first RoutingInfo check, run them now
+	if w.everAuthenticated.CompareAndSwap(authenticationStatusFirstAuthentication, authenticationStatusFirstRoutingInfoRpcSuccessful) {
+		if downstreamWorkersFactory != nil {
+			downstreamWorkers, err := downstreamWorkersFactory(cancelCtx, w.LastRoutingInfoSuccess().WorkerId, versionInfo.FullVersionNumber(false))
+			if err != nil {
+				event.WriteError(cancelCtx, op, err)
+				w.conf.ServerSideShutdownCh <- struct{}{}
+				return
+			}
+			w.downstreamWorkers.Store(&graphContainer{graph: downstreamWorkers})
+		}
+		for _, fn := range firstRoutingInfoCheckPostHooks {
+			if err := fn(cancelCtx, w); err != nil {
+				// If we can't verify status we can't be expected to behave
+				// properly so error and trigger shutdown
+				event.WriteError(cancelCtx, op, fmt.Errorf("error running first routing info check post hook: %w", err))
+				// We don't use a non-blocking select here to ensure that it
+				// happens; we should catch blocks in tests but we want to
+				// ensure the signal is being listened to
+				w.conf.ServerSideShutdownCh <- struct{}{}
+				return
+			}
+		}
+	}
+}
+
+func (w *Worker) lastSuccessfulRoutingInfoTime() time.Time {
+	lastRoutingInfo := w.LastRoutingInfoSuccess()
+	if lastRoutingInfo == nil {
+		return w.workerStartTime
+	}
+
+	return lastRoutingInfo.RoutingInfoTime
+}
+
+// Update address receivers and dialing listeners with new addrs
+func (w *Worker) updateAddresses(cancelCtx context.Context, addrs []string) {
+	const op = "worker.(Worker).updateAddresses"
+
+	if len(addrs) > 0 {
+		lastRoutingInfo := w.lastRoutingInfoSuccess.Load().(*LastRoutingInfo)
+		// Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses
+		if lastRoutingInfo != nil && !strutil.EquivalentSlices(lastRoutingInfo.LastCalculatedUpstreams, addrs) {
+			upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastRoutingInfo.LastCalculatedUpstreams, addrs)
+			event.WriteSysEvent(cancelCtx, op, upstreamsMessage)
+			for _, as := range w.addressReceivers {
+				as.SetAddresses(addrs)
+			}
+		} else if lastRoutingInfo == nil {
+			for _, as := range w.addressReceivers {
+				as.SetAddresses(addrs)
+			}
+			event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first RoutingInfo set to: %s", addrs))
+		}
+	}
+
+	// regardless of whether or not it's a new address, we need to set
+	// them for secondary connections
+	for _, as := range w.addressReceivers {
+		switch {
+		case as.Type() == secondaryConnectionReceiverType:
+			tmpAddrs := make([]string, len(addrs))
+			copy(tmpAddrs, addrs)
+			if len(tmpAddrs) == 0 {
+				tmpAddrs = append(tmpAddrs, w.conf.RawConfig.Worker.InitialUpstreams...)
+			}
+			as.SetAddresses(tmpAddrs)
+		}
+	}
+}
+
+func (w *Worker) isPastGrace() (bool, time.Time, time.Duration) {
+	t := w.lastSuccessfulRoutingInfoTime()
+	u := time.Duration(w.successfulRoutingInfoGracePeriod.Load())
+	v := time.Since(t)
+	return v > u, t, u
+}
+
+// getRandomInterval returns a random duration between -0.5 and 0.5 seconds (exclusive).
+func getRandomInterval(r *rand.Rand) time.Duration {
+	// 0 to 0.5 adjustment to the base
+	f := r.Float64() / 2
+	// Half a chance to be faster, not slower
+	if r.Float32() > 0.5 {
+		f = -1 * f
+	}
+	return time.Duration(f * float64(time.Second))
+}

internal/daemon/worker/routing_info_test.go

@@ -11,18 +11,19 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
+	"github.com/hashicorp/boundary/internal/daemon/worker/common"
 	"github.com/hashicorp/boundary/internal/event"
 	"github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/require"
 )
 
-func TestWorkerWaitForNextSuccessfulStatusUpdate(t *testing.T) {
+func TestWorkerWaitForNextSuccessfulRoutingInfoUpdate(t *testing.T) {
 	testConfig := event.DefaultEventerConfig()
 	testLock := &sync.Mutex{}
 	testLogger := hclog.New(&hclog.LoggerOptions{
 		Mutex: testLock,
 	})
-	err := event.InitSysEventer(testLogger, testLock, "TestWorkerWaitForNextSuccessfulStatusUpdate", event.WithEventerConfig(testConfig))
+	err := event.InitSysEventer(testLogger, testLock, "TestWorkerWaitForNextSuccessfulRoutingInfoUpdate", event.WithEventerConfig(testConfig))
 	require.NoError(t, err)
 	t.Cleanup(func() { event.TestResetSystEventer(t) })
 	for _, name := range []string{"ok", "timeout"} {
@@ -31,30 +32,30 @@ func TestWorkerWaitForNextSuccessfulStatusUpdate(t *testing.T) {
 
 			// As-needed initialization of a mock worker
 			w := &Worker{
-				logger:                      hclog.New(nil),
-				lastStatusSuccess:           new(atomic.Value),
-				baseContext:                 context.Background(),
-				successfulStatusGracePeriod: new(atomic.Int64),
+				logger:                           hclog.New(nil),
+				lastRoutingInfoSuccess:           new(atomic.Value),
+				baseContext:                      context.Background(),
+				successfulRoutingInfoGracePeriod: new(atomic.Int64),
 				conf: &Config{
 					Server: &base.Server{},
 				},
 			}
 
 			// This is present in New()
-			w.lastStatusSuccess.Store((*LastStatusInformation)(nil))
-			w.successfulStatusGracePeriod.Store(int64(time.Second * 2))
+			w.lastRoutingInfoSuccess.Store((*LastRoutingInfo)(nil))
+			w.successfulRoutingInfoGracePeriod.Store(int64(common.DefaultRoutingInfoTimeout))
 
 			var wg sync.WaitGroup
 			var err error
 			wg.Add(1)
 			go func() {
-				err = w.WaitForNextSuccessfulStatusUpdate()
+				err = w.WaitForNextSuccessfulRoutingInfoUpdate()
 				wg.Done()
 			}()
 
 			if name == "ok" {
 				time.Sleep(time.Millisecond * 100)
-				w.lastStatusSuccess.Store(&LastStatusInformation{StatusTime: time.Now()})
+				w.lastRoutingInfoSuccess.Store(&LastRoutingInfo{RoutingInfoTime: time.Now()})
 			}
 
 			wg.Wait()

internal/daemon/worker/session/manager.go

@@ -50,7 +50,7 @@ type Manager interface {
 
 	// RequestCloseConnections sends connection close requests to the
 	// controller, and sets close times within the worker. It should be called
-	// during the worker status loop and on connection exit on the proxy.
+	// during the worker session info loop and on connection exit on the proxy.
 	//
 	// The boolean indicates whether the function was successful, e.g. had any
 	// errors. Individual events will be sent for the errors if there are any.

internal/daemon/worker/session/session.go

@@ -480,7 +480,7 @@ func closeConnection(ctx context.Context, sessClient pbs.SessionServiceClient, r
 
 // closeConnections is a helper worker function that sends connection close
 // requests to the controller, and sets close times within the worker. It is
-// called during the worker status loop and on connection exit on the proxy.
+// called during the worker session info loop and on connection exit on the proxy.
 //
 // The boolean indicates whether the function was successful, e.g. had any
 // errors. Individual events will be sent for the errors if there are any.
@@ -499,9 +499,9 @@ func closeConnections(ctx context.Context, sessClient pbs.SessionServiceClient,
 	var sessionCloseInfo map[string][]*pbs.CloseConnectionResponseData
 	var err error
 
-	// TODO: This, along with the status call to the controller, probably needs a
+	// TODO: This, along with the session info call to the controller, probably needs a
 	// bit of formalization in terms of how we handle timeouts. For now, this
-	// just ensures consistency with the same status call in that it times out
+	// just ensures consistency with the same session info call in that it times out
 	// within an adequate period of time.
 	closeConnCtx, closeConnCancel := context.WithTimeout(ctx, time.Duration(CloseCallTimeout.Load()))
 	defer closeConnCancel()

internal/daemon/worker/session/testing.go

@@ -0,0 +1,59 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package session
+
+import (
+	"sync"
+	"testing"
+
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	"github.com/stretchr/testify/require"
+)
+
+// TestManager wraps a session manager. This wrapper allows direct manipulation of the sessions map for testing.
+type TestManager struct {
+	*manager
+}
+
+// NewTestManager returns a wrapped session manager that allows direct manipulation of the sessions map for testing
+func NewTestManager(t *testing.T, client pbs.SessionServiceClient) *TestManager {
+	sm, err := NewManager(client)
+	require.NoError(t, err)
+	return &TestManager{
+		manager: sm,
+	}
+}
+
+// StoreSession will mock an active local session with no connections in the session manager
+func (m *TestManager) StoreSession(sessionId string) {
+	m.manager.sessionMap.Store(sessionId, &sess{
+		lock:        sync.RWMutex{},
+		sessionId:   sessionId,
+		connInfoMap: make(map[string]*ConnInfo),
+		status:      pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
+	})
+}
+
+// StoreConnection will mock a connection for the given session. The session is expected to already exist
+// in the session manager.
+func (m *TestManager) StoreConnection(t *testing.T, sessionId, connectionId string) {
+	s, ok := m.manager.sessionMap.Load(sessionId)
+	require.True(t, ok)
+	localSession := s.(*sess)
+	info := &ConnInfo{
+		Id:     connectionId,
+		Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED,
+		BytesUp: func() int64 {
+			return 0
+		},
+		BytesDown: func() int64 {
+			return 0
+		},
+	}
+	info.connCtxCancelFunc = func() {
+		info.Status = pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED
+	}
+	localSession.connInfoMap[connectionId] = info
+	m.manager.sessionMap.Store(sessionId, localSession)
+}

internal/daemon/worker/session_info.go

@@ -0,0 +1,229 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package worker
+
+import (
+	"context"
+	stderr "errors"
+	"math/rand"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/event"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+)
+
+// lastSessionInfo holds the last successful session info RPC time.
+type lastSessionInfo struct {
+	LastSuccessfulRequestTime time.Time
+}
+
+func (w *Worker) startSessionInfoTicking(cancelCtx context.Context) {
+	const op = "worker.(Worker).startSessionInfoTicking"
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-cancelCtx.Done():
+			event.WriteSysEvent(w.baseContext, op, "session infor ticking shutting down")
+			return
+		case <-timer.C:
+			err := w.sendSessionInfo(cancelCtx)
+			if err != nil {
+				event.WriteError(w.baseContext, op, err)
+			}
+			// Add a bit of jitter to the wait, so we aren't always getting,
+			// session information updates at the exact same intervals, to
+			// ease the load on the DB.
+			timer.Reset(w.sessionInfoInterval + getRandomInterval(r))
+		}
+	}
+}
+
+func (w *Worker) sendSessionInfo(cancelCtx context.Context) error {
+	const op = "worker.(Worker).sendSessionInfo"
+	// skip when the workerId is not available
+	if w.LastRoutingInfoSuccess() == nil {
+		return errors.New(cancelCtx, errors.Internal, op, "missing latest routing info")
+	}
+	workerId := w.LastRoutingInfoSuccess().GetWorkerId()
+	if workerId == "" {
+		return errors.New(cancelCtx, errors.Internal, op, "worker id is empty")
+	}
+
+	var recordedSessionIds []string
+	if w.recorderManager != nil {
+		recSessIds, err := w.recorderManager.SessionsManaged(cancelCtx)
+		if err != nil {
+			event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error getting session ids from recorderManager"))
+		}
+		if len(recSessIds) != 0 {
+			recordedSessionIds = append(recordedSessionIds, recSessIds...)
+		}
+	}
+
+	var activeSessions []*pbs.Session
+	for _, sessionId := range recordedSessionIds {
+		activeSessions = append(activeSessions, &pbs.Session{
+			SessionId:     sessionId,
+			SessionType:   pbs.SessionType_SESSION_TYPE_RECORDED,
+			SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
+		})
+	}
+	w.sessionManager.ForEachLocalSession(func(s session.Session) bool {
+		activeSessions = append(activeSessions, &pbs.Session{
+			SessionId:     s.GetId(),
+			SessionType:   pbs.SessionType_SESSION_TYPE_INGRESSED,
+			SessionStatus: s.GetStatus(),
+		})
+		return true
+	})
+	// skip when there are no sessions to report
+	if len(activeSessions) == 0 {
+		return nil
+	}
+	clientCon := w.GrpcClientConn.Load()
+	client := pbs.NewServerCoordinationServiceClient(clientCon)
+	sessionInfoCtx, statusCancel := context.WithTimeout(cancelCtx, time.Duration(w.sessionInfoCallTimeoutDuration.Load()))
+	defer statusCancel()
+	result, err := client.SessionInfo(sessionInfoCtx, &pbs.SessionInfoRequest{
+		WorkerId: workerId,
+		Sessions: activeSessions,
+	})
+	if err != nil {
+		if isPastGrace, lastSessionInfoTime, gracePeriod := w.isPastSessionInfoGrace(); isPastGrace {
+			event.WriteError(cancelCtx, op,
+				stderr.New("session information error grace period has expired, canceling all sessions on worker"),
+				event.WithInfo("last_session_info_time", lastSessionInfoTime.String(), "grace_period", gracePeriod),
+			)
+			// Cancel connections if grace period has expired. These Connections will be closed in the
+			// database on the next successful status report, or via the Controller’s dead Worker cleanup connections job.
+			w.sessionManager.ForEachLocalSession(func(s session.Session) bool {
+				for _, connId := range s.CancelAllLocalConnections() {
+					event.WriteSysEvent(cancelCtx, op, "terminated connection due to session information grace period expiration", "session_id", s.GetId(), "connection_id", connId)
+				}
+				return true
+			})
+		}
+		return errors.Wrap(cancelCtx, err, op, errors.WithCode(errors.Internal), errors.WithMsg("error making session information request to controller: controller_address: %s", clientCon.Target()))
+	}
+
+	w.lastSessionInfoSuccess.Store(&lastSessionInfo{
+		LastSuccessfulRequestTime: time.Now(),
+	})
+
+	var nonActiveRecordedSessionIds []string
+	for _, s := range result.GetNonActiveSessions() {
+		switch s.GetSessionType() {
+		case pbs.SessionType_SESSION_TYPE_RECORDED:
+			if s.SessionStatus != pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE {
+				nonActiveRecordedSessionIds = append(nonActiveRecordedSessionIds, s.GetSessionId())
+			}
+		case pbs.SessionType_SESSION_TYPE_INGRESSED:
+			sessionId := s.GetSessionId()
+			managedSession := w.sessionManager.Get(sessionId)
+			if managedSession == nil {
+				event.WriteError(cancelCtx, op, stderr.New("session change requested but could not find local information for it"), event.WithInfo("session_id", sessionId))
+				continue
+			}
+			managedSession.ApplyLocalStatus(s.GetSessionStatus())
+			// Update connection state if there are any connections in the request.
+			for _, c := range s.GetConnections() {
+				if err := managedSession.ApplyLocalConnectionStatus(c.GetConnectionId(), c.GetStatus()); err != nil {
+					event.WriteError(cancelCtx, op, err, event.WithInfo("connection_id", c.GetConnectionId()))
+				}
+			}
+		}
+	}
+	if w.recorderManager != nil {
+		if err := w.recorderManager.ReauthorizeAllExcept(cancelCtx, nonActiveRecordedSessionIds); err != nil {
+			event.WriteError(cancelCtx, op, err)
+		}
+	}
+	// Standard cleanup: Run through current jobs. Cancel connections
+	// for any canceling session or any session that is expired.
+	w.cleanupConnections(cancelCtx, false, w.sessionManager)
+	return nil
+}
+
+func (w *Worker) isPastSessionInfoGrace() (bool, time.Time, time.Duration) {
+	t := w.workerStartTime
+	info := w.lastSessionInfoSuccess.Load().(*lastSessionInfo)
+	if info != nil {
+		t = info.LastSuccessfulRequestTime
+	}
+	u := time.Duration(w.successfulSessionInfoGracePeriod.Load())
+	v := time.Since(t)
+	return v > u, t, u
+}
+
+// cleanupConnections walks all sessions and shuts down all proxy connections.
+// After the local connections are terminated, they are requested to be marked
+// closed on the controller.
+// Additionally, sessions without connections are cleaned up from the
+// local worker's state.
+//
+// Use ignoreSessionState to ignore the state checks, this closes all
+// connections, regardless of whether or not the session is still active.
+func (w *Worker) cleanupConnections(cancelCtx context.Context, ignoreSessionState bool, sessionManager session.Manager) {
+	const op = "worker.(Worker).cleanupConnections"
+	closeInfo := make(map[string]*session.ConnectionCloseData)
+	cleanSessionIds := make([]string, 0)
+	sessionManager.ForEachLocalSession(func(s session.Session) bool {
+		switch {
+		case ignoreSessionState,
+			s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
+			s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
+			time.Until(s.GetExpiration()) < 0:
+			// Cancel connections without regard to individual connection
+			// state.
+			closedIds := s.CancelAllLocalConnections()
+			localConns := s.GetLocalConnections()
+			for _, connId := range closedIds {
+				var bytesUp, bytesDown int64
+				connInfo, ok := localConns[connId]
+				if ok {
+					bytesUp = connInfo.BytesUp()
+					bytesDown = connInfo.BytesDown()
+				}
+				closeInfo[connId] = &session.ConnectionCloseData{
+					SessionId: s.GetId(),
+					BytesUp:   bytesUp,
+					BytesDown: bytesDown,
+				}
+				event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId)
+			}
+
+			// If the session is no longer valid and all connections
+			// are marked closed, clean up the session.
+			if len(closedIds) == 0 {
+				cleanSessionIds = append(cleanSessionIds, s.GetId())
+			}
+
+		default:
+			// Cancel connections *with* regard to individual connection
+			// state (ie: only ones that the controller has requested be
+			// terminated).
+			for _, connId := range s.CancelOpenLocalConnections() {
+				closeInfo[connId] = &session.ConnectionCloseData{SessionId: s.GetId()}
+				event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId)
+			}
+		}
+
+		return true
+	})
+
+	// Note that we won't clean these from the info map until the
+	// next time we run this function
+	if len(closeInfo) > 0 {
+		// Call out to a helper to send the connection close requests to the
+		// controller, and set the close time. This functionality is shared with
+		// post-close functionality in the proxy handler.
+		_ = sessionManager.RequestCloseConnections(cancelCtx, closeInfo)
+	}
+	// Forget sessions where the session is expired/canceled and all
+	// connections are canceled and marked closed
+	sessionManager.DeleteLocalSession(cleanSessionIds)
+}

internal/daemon/worker/session_info_test.go

@@ -0,0 +1,337 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package worker
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func Test_sendSessionInfo(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name                   string
+		lastRoutingInfo        *LastRoutingInfo
+		sessionSetup           func(sm *session.TestManager)
+		expectedInternalErrMsg string
+		expectedServerErrMsg   string
+		expectedSessions       bool
+	}{
+		{
+			name:                   "nil last routing info",
+			expectedInternalErrMsg: "missing latest routing info",
+		},
+		{
+			name: "empty worker id",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{},
+			},
+			expectedInternalErrMsg: "worker id is empty",
+		},
+		{
+			name: "empty sessions",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{
+					WorkerId: "w_1234567890",
+				},
+			},
+		},
+		{
+			name: "server error",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{
+					WorkerId: "w_1234567890",
+				},
+			},
+			sessionSetup: func(sm *session.TestManager) {
+				require.NotNil(t, sm)
+				sm.StoreSession("s_1234567890")
+				sm.StoreConnection(t, "s_1234567890", "c_1234567890")
+			},
+			expectedServerErrMsg: "testing connection timeout",
+		},
+		{
+			name: "success",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{
+					WorkerId: "w_1234567890",
+				},
+			},
+			sessionSetup: func(sm *session.TestManager) {
+				require.NotNil(t, sm)
+				sm.StoreSession("s_1234567890")
+				sm.StoreConnection(t, "s_1234567890", "c_1234567890")
+			},
+			expectedSessions: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			assert, require := assert.New(t), require.New(t)
+
+			w := NewTestWorker(t, &TestWorkerOpts{
+				DisableAutoStart: true,
+				Name:             tt.name,
+			})
+			// Ensure the recorder manager is not used
+			w.Worker().recorderManager = nil
+
+			ln, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(err)
+			cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+			require.NoError(err)
+
+			mockSessionClient := pbs.NewMockSessionServiceClient()
+			mockSessionClient.CloseConnectionFn = func(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) {
+				return &pbs.CloseConnectionResponse{}, nil
+			}
+			sm := session.NewTestManager(t, mockSessionClient)
+			if tt.sessionSetup != nil {
+				tt.sessionSetup(sm)
+			}
+			w.Worker().sessionManager = sm
+
+			serverInvoked := false
+			fakeServer := &mockServerCoordinationService{
+				nextSessionInfoAssert: func(req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) {
+					serverInvoked = true
+					if tt.expectedInternalErrMsg != "" {
+						assert.Fail("server should not send session information for internal errors")
+					}
+					if tt.expectedServerErrMsg != "" {
+						return &pbs.SessionInfoResponse{}, errors.New(tt.expectedServerErrMsg)
+					}
+					if !tt.expectedSessions {
+						assert.Fail("server should not send session information when there are no managed sessions")
+					}
+					require.NotNil(req)
+
+					assert.Equal("w_1234567890", req.WorkerId)
+					require.Len(req.Sessions, 1)
+					assert.Equal("s_1234567890", req.Sessions[0].SessionId)
+
+					return &pbs.SessionInfoResponse{}, nil
+				},
+			}
+
+			srv := grpc.NewServer()
+			pbs.RegisterServerCoordinationServiceServer(srv, fakeServer)
+			w.Worker().GrpcClientConn.Store(cc)
+			w.Worker().lastRoutingInfoSuccess.Store(tt.lastRoutingInfo)
+
+			serverStarted := make(chan struct{})
+			var wg sync.WaitGroup
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				close(serverStarted)
+				assert.NoError(srv.Serve(ln))
+			}()
+
+			<-serverStarted
+			actualErr := w.Worker().sendSessionInfo(context.Background())
+
+			if tt.expectedInternalErrMsg != "" {
+				require.Error(actualErr)
+				assert.ErrorContains(actualErr, tt.expectedInternalErrMsg)
+				assert.False(serverInvoked)
+				return
+			}
+			if tt.expectedServerErrMsg != "" {
+				require.Error(actualErr)
+				assert.ErrorContains(actualErr, tt.expectedServerErrMsg)
+				assert.True(serverInvoked)
+				return
+			}
+			require.NoError(actualErr)
+			require.Equal(tt.expectedSessions, serverInvoked)
+
+			srv.GracefulStop()
+			wg.Wait()
+		})
+	}
+
+	t.Run("close non active managed sessions connections and ignore non existing sessions", func(t *testing.T) {
+		t.Parallel()
+		assert, require := assert.New(t), require.New(t)
+
+		w := NewTestWorker(t, &TestWorkerOpts{
+			DisableAutoStart: true,
+			Name:             "close non active sessions and connections",
+		})
+		// Ensure the recorder manager is not used
+		w.Worker().recorderManager = nil
+
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(err)
+		cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+		require.NoError(err)
+
+		mockSessionClient := pbs.NewMockSessionServiceClient()
+		mockSessionClient.CloseConnectionFn = func(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) {
+			return &pbs.CloseConnectionResponse{}, nil
+		}
+		sm := session.NewTestManager(t, mockSessionClient)
+		sm.StoreSession("s_1234567890")
+		sm.StoreConnection(t, "s_1234567890", "c_1234567890")
+		w.Worker().sessionManager = sm
+
+		serverInvoked := false
+		fakeServer := &mockServerCoordinationService{
+			nextSessionInfoAssert: func(req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) {
+				serverInvoked = true
+				assert.Equal("w_1234567890", req.WorkerId)
+				require.Len(req.Sessions, 1)
+				assert.Equal("s_1234567890", req.Sessions[0].SessionId)
+				return &pbs.SessionInfoResponse{
+					NonActiveSessions: []*pbs.Session{
+						{
+							SessionId:     "s_1234567890",
+							SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
+							SessionType:   pbs.SessionType_SESSION_TYPE_INGRESSED,
+							Connections: []*pbs.Connection{
+								{
+									ConnectionId: "c_1234567890",
+									Status:       pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED,
+								},
+							},
+						},
+						{
+							SessionId:     "s_dne",
+							SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
+							SessionType:   pbs.SessionType_SESSION_TYPE_INGRESSED,
+							Connections: []*pbs.Connection{
+								{
+									ConnectionId: "c_dne",
+									Status:       pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED,
+								},
+							},
+						},
+					},
+				}, nil
+			},
+		}
+
+		srv := grpc.NewServer()
+		pbs.RegisterServerCoordinationServiceServer(srv, fakeServer)
+		w.Worker().GrpcClientConn.Store(cc)
+		w.Worker().lastRoutingInfoSuccess.Store(&LastRoutingInfo{
+			RoutingInfoResponse: &pbs.RoutingInfoResponse{
+				WorkerId: "w_1234567890",
+			},
+		})
+
+		serverStarted := make(chan struct{})
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			close(serverStarted)
+			assert.NoError(srv.Serve(ln))
+		}()
+
+		<-serverStarted
+		actualErr := w.Worker().sendSessionInfo(context.Background())
+
+		require.NoError(actualErr)
+		assert.True(serverInvoked)
+		closedSession := w.Worker().sessionManager.Get("s_1234567890")
+		assert.Nil(closedSession)
+
+		srv.GracefulStop()
+		wg.Wait()
+	})
+
+	t.Run("validate closing local connections after last successful session info grace period", func(t *testing.T) {
+		t.Parallel()
+		assert, require := assert.New(t), require.New(t)
+
+		w := NewTestWorker(t, &TestWorkerOpts{
+			DisableAutoStart: true,
+			Name:             "close non active sessions and connections",
+		})
+		// Ensure the recorder manager is not used
+		w.Worker().recorderManager = nil
+
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(err)
+		cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+		require.NoError(err)
+
+		mockSessionClient := pbs.NewMockSessionServiceClient()
+		mockSessionClient.CloseConnectionFn = func(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) {
+			return &pbs.CloseConnectionResponse{}, nil
+		}
+		sm := session.NewTestManager(t, mockSessionClient)
+		sm.StoreSession("s_1234567890")
+		sm.StoreConnection(t, "s_1234567890", "c_1234567890")
+		w.Worker().sessionManager = sm
+
+		serverInvoked := false
+		fakeServer := &mockServerCoordinationService{
+			nextSessionInfoAssert: func(req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) {
+				serverInvoked = true
+				assert.Equal("w_1234567890", req.WorkerId)
+				require.Len(req.Sessions, 1)
+				assert.Equal("s_1234567890", req.Sessions[0].SessionId)
+				return &pbs.SessionInfoResponse{}, errors.New("connection timeout")
+			},
+		}
+
+		srv := grpc.NewServer()
+		pbs.RegisterServerCoordinationServiceServer(srv, fakeServer)
+		w.Worker().GrpcClientConn.Store(cc)
+		w.Worker().lastRoutingInfoSuccess.Store(&LastRoutingInfo{
+			RoutingInfoResponse: &pbs.RoutingInfoResponse{
+				WorkerId: "w_1234567890",
+			},
+		})
+		// Set the status time of the last successful session information
+		// to a value that ensures we have surpassed the grace period time.
+		gracePeriod := time.Duration(w.Worker().successfulSessionInfoGracePeriod.Load()) + (5 * time.Second)
+		w.Worker().lastSessionInfoSuccess.Store(&lastSessionInfo{
+			LastSuccessfulRequestTime: time.Now().Add(-gracePeriod),
+		})
+
+		serverStarted := make(chan struct{})
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			close(serverStarted)
+			assert.NoError(srv.Serve(ln))
+		}()
+
+		<-serverStarted
+		actualErr := w.Worker().sendSessionInfo(context.Background())
+
+		require.Error(actualErr)
+		assert.ErrorContains(actualErr, "connection timeout")
+		assert.True(serverInvoked)
+		localSession := w.Worker().sessionManager.Get("s_1234567890")
+		require.NotNil(localSession)
+		assert.Equal(pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, localSession.GetStatus())
+		localConnections := localSession.GetLocalConnections()
+		require.NotEmpty(localConnections)
+		c, ok := localConnections["c_1234567890"]
+		require.True(ok)
+		require.NotNil(c)
+		assert.Equal(pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED, c.Status)
+
+		srv.GracefulStop()
+		wg.Wait()
+	})
+}

internal/daemon/worker/statistics.go

@@ -0,0 +1,92 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package worker
+
+import (
+	"context"
+	"math/rand"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/event"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+)
+
+// lastStatistics holds the last successful statistics RPC time.
+type lastStatistics struct {
+	LastSuccessfulRequestTime time.Time
+}
+
+func (w *Worker) startStatisticsTicking(cancelCtx context.Context) {
+	const op = "worker.(Worker).startStatisticsTicking"
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-cancelCtx.Done():
+			event.WriteSysEvent(w.baseContext, op, "statistics ticking shutting down")
+			return
+		case <-timer.C:
+			err := w.sendStatistic(cancelCtx)
+			if err != nil {
+				event.WriteError(w.baseContext, op, err)
+			}
+			// Add a bit of jitter to the wait, so we aren't always getting,
+			// statistics updates at the exact same intervals, to ease the load on the DB.
+			timer.Reset(w.statisticsInterval + getRandomInterval(r))
+		}
+	}
+}
+
+func (w *Worker) sendStatistic(cancelCtx context.Context) error {
+	const op = "worker.(Worker).sendStatistic"
+	// skip when the workerId is not available
+	if w.LastRoutingInfoSuccess() == nil {
+		return errors.New(cancelCtx, errors.Internal, op, "missing latest status")
+	}
+	workerId := w.LastRoutingInfoSuccess().GetWorkerId()
+	if workerId == "" {
+		return errors.New(cancelCtx, errors.Internal, op, "worker id is empty")
+	}
+	sessions := []*pbs.SessionStatistics{}
+	w.sessionManager.ForEachLocalSession(func(s session.Session) bool {
+		localConnections := s.GetLocalConnections()
+		connections := make([]*pbs.Connection, 0, len(localConnections))
+		for connectionId, conn := range localConnections {
+			connections = append(connections, &pbs.Connection{
+				ConnectionId: connectionId,
+				BytesUp:      conn.BytesUp(),
+				BytesDown:    conn.BytesDown(),
+				Status:       conn.Status,
+			})
+		}
+		sessions = append(sessions, &pbs.SessionStatistics{
+			SessionId:   s.GetId(),
+			Connections: connections,
+		})
+		return true
+	})
+	// skip when there are no sessions to report
+	if len(sessions) == 0 {
+		return nil
+	}
+	clientCon := w.GrpcClientConn.Load()
+	client := pbs.NewServerCoordinationServiceClient(clientCon)
+	statisticsCtx, statusCancel := context.WithTimeout(cancelCtx, time.Duration(w.statisticsCallTimeoutDuration.Load()))
+	defer statusCancel()
+	_, err := client.Statistics(statisticsCtx, &pbs.StatisticsRequest{
+		WorkerId: workerId,
+		Sessions: sessions,
+	})
+	if err != nil {
+		return errors.Wrap(cancelCtx, err, op, errors.WithCode(errors.Internal), errors.WithMsg("error making statistics request to controller: controller_address: %s", clientCon.Target()))
+	}
+
+	w.lastStatisticsSuccess.Store(&lastStatistics{
+		LastSuccessfulRequestTime: time.Now(),
+	})
+
+	return nil
+}

internal/daemon/worker/statistics_test.go

@@ -0,0 +1,158 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package worker
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func Test_sendStatistic(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name                   string
+		lastRoutingInfo        *LastRoutingInfo
+		sessionSetup           func(sm *session.TestManager)
+		expectedInternalErrMsg string
+		expectedServerErrMsg   string
+		expectedSessions       bool
+	}{
+		{
+			name:                   "nil last status",
+			expectedInternalErrMsg: "missing latest status",
+		},
+		{
+			name: "empty worker id",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{},
+			},
+			expectedInternalErrMsg: "worker id is empty",
+		},
+		{
+			name: "empty sessions",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{
+					WorkerId: "w_1234567890",
+				},
+			},
+		},
+		{
+			name: "server error",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{
+					WorkerId: "w_1234567890",
+				},
+			},
+			sessionSetup: func(sm *session.TestManager) {
+				require.NotNil(t, sm)
+				sm.StoreSession("s_1234567890")
+				sm.StoreConnection(t, "s_1234567890", "c_1234567890")
+			},
+			expectedServerErrMsg: "testing connection timeout",
+		},
+		{
+			name: "success",
+			lastRoutingInfo: &LastRoutingInfo{
+				RoutingInfoResponse: &pbs.RoutingInfoResponse{
+					WorkerId: "w_1234567890",
+				},
+			},
+			sessionSetup: func(sm *session.TestManager) {
+				require.NotNil(t, sm)
+				sm.StoreSession("s_1234567890")
+				sm.StoreConnection(t, "s_1234567890", "c_1234567890")
+			},
+			expectedSessions: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			assert, require := assert.New(t), require.New(t)
+
+			w := NewTestWorker(t, &TestWorkerOpts{
+				DisableAutoStart: true,
+				Name:             tt.name,
+			})
+
+			ln, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(err)
+			cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+			require.NoError(err)
+
+			sm := session.NewTestManager(t, pbs.NewMockSessionServiceClient())
+			if tt.sessionSetup != nil {
+				tt.sessionSetup(sm)
+			}
+			w.Worker().sessionManager = sm
+
+			serverInvoked := false
+			fakeServer := &mockServerCoordinationService{
+				nextStatisticAssert: func(req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) {
+					serverInvoked = true
+					if tt.expectedInternalErrMsg != "" {
+						assert.Fail("server should not send statistics information for internal errors")
+					}
+					if tt.expectedServerErrMsg != "" {
+						return &pbs.StatisticsResponse{}, errors.New(tt.expectedServerErrMsg)
+					}
+					if !tt.expectedSessions {
+						assert.Fail("server should not send statistics information when there are no managed sessions")
+					}
+					require.NotNil(req)
+					assert.Equal("w_1234567890", req.WorkerId)
+					require.Len(req.Sessions, 1)
+					assert.Equal("s_1234567890", req.Sessions[0].SessionId)
+					require.Len(req.Sessions[0].Connections, 1)
+					assert.Equal("c_1234567890", req.Sessions[0].Connections[0].ConnectionId)
+					return &pbs.StatisticsResponse{}, nil
+				},
+			}
+			srv := grpc.NewServer()
+			pbs.RegisterServerCoordinationServiceServer(srv, fakeServer)
+			w.Worker().GrpcClientConn.Store(cc)
+			w.Worker().lastRoutingInfoSuccess.Store(tt.lastRoutingInfo)
+
+			serverStarted := make(chan struct{})
+			var wg sync.WaitGroup
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				close(serverStarted)
+				assert.NoError(srv.Serve(ln))
+			}()
+
+			<-serverStarted
+			actualErr := w.Worker().sendStatistic(context.Background())
+
+			if tt.expectedInternalErrMsg != "" {
+				require.Error(actualErr)
+				assert.ErrorContains(actualErr, tt.expectedInternalErrMsg)
+				assert.False(serverInvoked)
+				return
+			}
+			if tt.expectedServerErrMsg != "" {
+				require.Error(actualErr)
+				assert.ErrorContains(actualErr, tt.expectedServerErrMsg)
+				assert.True(serverInvoked)
+				return
+			}
+			require.NoError(actualErr)
+			require.Equal(tt.expectedSessions, serverInvoked)
+
+			srv.GracefulStop()
+			wg.Wait()
+		})
+	}
+}

internal/daemon/worker/status.go

@@ -1,519 +0,0 @@
-// Copyright (c) HashiCorp, Inc.
-// SPDX-License-Identifier: BUSL-1.1
-
-package worker
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"math/rand"
-	"time"
-
-	"github.com/hashicorp/boundary/internal/daemon/worker/common"
-	"github.com/hashicorp/boundary/internal/daemon/worker/session"
-	"github.com/hashicorp/boundary/internal/event"
-	pb "github.com/hashicorp/boundary/internal/gen/controller/servers"
-	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
-	"github.com/hashicorp/boundary/internal/server"
-	"github.com/hashicorp/boundary/sdk/pbs/plugin"
-	"github.com/hashicorp/boundary/version"
-	"github.com/hashicorp/go-secure-stdlib/parseutil"
-	"github.com/hashicorp/go-secure-stdlib/strutil"
-	"google.golang.org/grpc/connectivity"
-)
-
-var firstStatusCheckPostHooks []func(context.Context, *Worker) error
-
-var downstreamWorkersFactory func(ctx context.Context, workerId string, ver string) (graph, error)
-
-var checkHCPBUpstreams func(w *Worker) bool
-
-type LastStatusInformation struct {
-	*pbs.StatusResponse
-	StatusTime              time.Time
-	LastCalculatedUpstreams []string
-}
-
-func (w *Worker) startStatusTicking(cancelCtx context.Context, sessionManager session.Manager, addrReceivers *[]addressReceiver, recorderManager recorderManager) {
-	const op = "worker.(Worker).startStatusTicking"
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	// This function exists to desynchronize calls to controllers from
-	// workers, so we aren't always getting status updates at the exact same
-	// intervals, to ease the load on the DB.
-	getRandomInterval := func() time.Duration {
-		// 0 to 0.5 adjustment to the base
-		f := r.Float64() / 2
-		// Half a chance to be faster, not slower
-		if r.Float32() > 0.5 {
-			f = -1 * f
-		}
-		return common.StatusInterval + time.Duration(f*float64(time.Second))
-	}
-
-	timer := time.NewTimer(0)
-	for {
-		select {
-		case <-cancelCtx.Done():
-			event.WriteSysEvent(w.baseContext, op, "status ticking shutting down")
-			return
-
-		case <-timer.C:
-			// If we've never managed to successfully authenticate then we won't have
-			// any session information anyways and this will produce a ton of noise in
-			// observability, so skip calling the function and retry in a short duration
-			if w.everAuthenticated.Load() == authenticationStatusNeverAuthenticated {
-				timer.Reset(10 * time.Millisecond)
-				continue
-			}
-
-			w.sendWorkerStatus(cancelCtx, sessionManager, addrReceivers, recorderManager)
-			timer.Reset(getRandomInterval())
-		}
-	}
-}
-
-// LastStatusSuccess reports the last time we sent a successful
-// status request.
-func (w *Worker) LastStatusSuccess() *LastStatusInformation {
-	s, ok := w.lastStatusSuccess.Load().(*LastStatusInformation)
-	if !ok {
-		return nil
-	}
-	return s
-}
-
-// WaitForNextSuccessfulStatusUpdate waits for the next successful status. It's
-// used by testing (and in the future, shutdown) in place of a more opaque and
-// possibly unnecessarily long sleep for things like initial controller
-// check-in, etc.
-//
-// The timeout is aligned with the worker's status grace period. A nil error
-// means the status was sent successfully.
-func (w *Worker) WaitForNextSuccessfulStatusUpdate() error {
-	const op = "worker.(Worker).WaitForNextSuccessfulStatusUpdate"
-	waitStatusStart := time.Now()
-	ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(w.successfulStatusGracePeriod.Load()))
-	defer cancel()
-	event.WriteSysEvent(ctx, op, "waiting for next status report to controller")
-	for {
-		select {
-		case <-time.After(time.Second):
-			// pass
-
-		case <-ctx.Done():
-			event.WriteError(ctx, op, ctx.Err(), event.WithInfoMsg("error waiting for next status report to controller"))
-			return ctx.Err()
-		}
-
-		if w.lastSuccessfulStatusTime().After(waitStatusStart) {
-			break
-		}
-	}
-
-	event.WriteSysEvent(ctx, op, "next worker status update sent successfully")
-	return nil
-}
-
-func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager session.Manager, addressReceivers *[]addressReceiver, recorderManager recorderManager) {
-	const op = "worker.(Worker).sendWorkerStatus"
-	w.statusLock.Lock()
-	defer w.statusLock.Unlock()
-
-	// Collect the different session ids that are being monitored by this worker
-	var monitoredSessionIds []string
-
-	if recorderManager != nil {
-		recSessIds, err := recorderManager.SessionsManaged(cancelCtx)
-		if err != nil {
-			event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error getting session ids from recorderManager"))
-		} else {
-			monitoredSessionIds = append(monitoredSessionIds, recSessIds...)
-		}
-	}
-
-	// First send info as-is. We'll perform cleanup duties after we
-	// get cancel/job change info back.
-	var activeJobs []*pbs.JobStatus
-
-	for _, sid := range monitoredSessionIds {
-		activeJobs = append(activeJobs, &pbs.JobStatus{
-			Job: &pbs.Job{
-				Type: pbs.JOBTYPE_JOBTYPE_MONITOR_SESSION,
-				JobInfo: &pbs.Job_MonitorSessionInfo{
-					MonitorSessionInfo: &pbs.MonitorSessionJobInfo{
-						SessionId: sid,
-						Status:    pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE,
-					},
-				},
-			},
-		})
-	}
-
-	// Range over known sessions and collect info
-	sessionManager.ForEachLocalSession(func(s session.Session) bool {
-		var jobInfo pbs.SessionJobInfo
-		status := s.GetStatus()
-		sessionId := s.GetId()
-		localConnections := s.GetLocalConnections()
-		connections := make([]*pbs.Connection, 0, len(localConnections))
-		for k, v := range localConnections {
-			connections = append(connections, &pbs.Connection{
-				ConnectionId: k,
-				Status:       v.Status,
-				BytesUp:      v.BytesUp(),
-				BytesDown:    v.BytesDown(),
-			})
-		}
-		jobInfo.SessionId = sessionId
-		activeJobs = append(activeJobs, &pbs.JobStatus{
-			Job: &pbs.Job{
-				Type: pbs.JOBTYPE_JOBTYPE_SESSION,
-				JobInfo: &pbs.Job_SessionInfo{
-					SessionInfo: &pbs.SessionJobInfo{
-						SessionId:   sessionId,
-						Status:      status,
-						Connections: connections,
-					},
-				},
-			},
-		})
-		return true
-	})
-
-	clientCon := w.GrpcClientConn.Load()
-	// Send status information
-	client := pbs.NewServerCoordinationServiceClient(clientCon)
-	var tags []*pb.TagPair
-	// If we're not going to request a tag update, no reason to have these
-	// marshaled on every status call.
-	if w.updateTags.Load() {
-		tags = w.tags.Load().([]*pb.TagPair)
-	}
-	statusCtx, statusCancel := context.WithTimeout(cancelCtx, time.Duration(w.statusCallTimeoutDuration.Load()))
-	defer statusCancel()
-
-	keyId := w.WorkerAuthCurrentKeyId.Load()
-	switch {
-	case w.conf.RawConfig.Worker.Name == "" && keyId == "":
-		event.WriteError(cancelCtx, op, errors.New("worker name and keyId are both empty; at least one is needed to identify a worker"),
-			event.WithInfoMsg("error making status request to controller"))
-	}
-
-	var storageBucketCredentialStates map[string]*plugin.StorageBucketCredentialState
-	if w.RecordingStorage != nil {
-		storageBucketCredentialStates = w.RecordingStorage.GetStorageBucketCredentialStates()
-		// If the local storage state is unknown, and we have recording storage set, get the state from the recording storage
-		// and set it on the worker. This is done once to ensure that the worker has the correct state for the first status
-		// call.
-		if w.localStorageState.Load() == server.UnknownLocalStorageState {
-			w.localStorageState.Store(w.RecordingStorage.GetLocalStorageState(cancelCtx))
-		}
-	}
-	versionInfo := version.Get()
-	connectionState := w.downstreamConnManager.Connected()
-	result, err := client.Status(statusCtx, &pbs.StatusRequest{
-		Jobs: activeJobs,
-		WorkerStatus: &pb.ServerWorkerStatus{
-			Name:                          w.conf.RawConfig.Worker.Name,
-			Description:                   w.conf.RawConfig.Worker.Description,
-			Address:                       w.conf.RawConfig.Worker.PublicAddr,
-			Tags:                          tags,
-			KeyId:                         keyId,
-			ReleaseVersion:                versionInfo.FullVersionNumber(false),
-			OperationalState:              w.operationalState.Load().(server.OperationalState).String(),
-			LocalStorageState:             w.localStorageState.Load().(server.LocalStorageState).String(),
-			StorageBucketCredentialStates: storageBucketCredentialStates,
-		},
-		ConnectedWorkerKeyIdentifiers:         connectionState.AllKeyIds(),
-		ConnectedUnmappedWorkerKeyIdentifiers: connectionState.UnmappedKeyIds(),
-		ConnectedWorkerPublicIds:              connectionState.WorkerIds(),
-		UpdateTags:                            w.updateTags.Load(),
-	})
-	if err != nil {
-		event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error making status request to controller", "controller_address", clientCon.Target()))
-		// Check for last successful status. Ignore nil last status, this probably
-		// means that we've never connected to a controller, and as such probably
-		// don't have any sessions to worry about anyway.
-		//
-		// If a length of time has passed since we've been able to communicate, we
-		// want to start terminating all connections as a "break glass" kind of
-		// scenario, as there will be no way we can really tell if these
-		// connections should continue to exist.
-		if isPastGrace, lastStatusTime, gracePeriod := w.isPastGrace(); isPastGrace {
-			event.WriteError(cancelCtx, op,
-				errors.New("status error grace period has expired, canceling all sessions on worker"),
-				event.WithInfo("last_status_time", lastStatusTime.String(), "grace_period", gracePeriod),
-			)
-
-			// Cancel connections if grace period has expired. These Connections will be closed in the
-			// database on the next successful status report, or via the Controller’s dead Worker cleanup connections job.
-			sessionManager.ForEachLocalSession(func(s session.Session) bool {
-				for _, connId := range s.CancelAllLocalConnections() {
-					event.WriteSysEvent(cancelCtx, op, "terminated connection due to status grace period expiration", "session_id", s.GetId(), "connection_id", connId)
-				}
-				return true
-			})
-
-			// In the case that the control plane has gone down and come up with different IPs,
-			// append initial upstreams/ cluster addr to the resolver to try
-			if w.GrpcClientConn.Load().GetState() == connectivity.TransientFailure {
-				lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
-				if lastStatus != nil && lastStatus.LastCalculatedUpstreams != nil {
-					addrs := lastStatus.LastCalculatedUpstreams
-
-					if len(w.conf.RawConfig.Worker.InitialUpstreams) > 0 {
-						addrs = append(addrs, w.conf.RawConfig.Worker.InitialUpstreams...)
-					} else if HandleHcpbClusterId != nil && len(w.conf.RawConfig.HcpbClusterId) > 0 {
-						clusterId, err := parseutil.ParsePath(w.conf.RawConfig.HcpbClusterId)
-						if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
-							event.WriteError(cancelCtx, op, err, event.WithInfoMsg("failed to parse HCP Boundary cluster ID"))
-						} else {
-							clusterAddress := HandleHcpbClusterId(clusterId)
-							addrs = append(addrs, clusterAddress)
-						}
-					}
-
-					addrs = strutil.RemoveDuplicates(addrs, false)
-					if strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) {
-						// Nothing to update
-						return
-					}
-
-					w.updateAddresses(cancelCtx, addrs, addressReceivers)
-					lastStatus.LastCalculatedUpstreams = addrs
-					w.lastStatusSuccess.Store(lastStatus)
-				}
-			}
-
-			// Exit out of status function; our work here is done and we don't need to create closeConnection requests
-			return
-		}
-
-		// Standard cleanup: Run through current jobs. Cancel connections
-		// for any canceling session or any session that is expired.
-		w.cleanupConnections(cancelCtx, false, sessionManager)
-		return
-	}
-
-	w.updateTags.Store(false)
-
-	if authorized := result.GetAuthorizedDownstreamWorkers(); authorized != nil {
-		connectionState.DisconnectMissingWorkers(authorized.GetWorkerPublicIds())
-		connectionState.DisconnectMissingUnmappedKeyIds(authorized.GetUnmappedWorkerKeyIdentifiers())
-	} else if authorized := result.GetAuthorizedWorkers(); authorized != nil {
-		connectionState.DisconnectAllMissingKeyIds(authorized.GetWorkerKeyIdentifiers())
-	}
-	var addrs []string
-	// This may be empty if we are in a multiple hop scenario
-	if len(result.CalculatedUpstreams) > 0 {
-		addrs = make([]string, 0, len(result.CalculatedUpstreams))
-		for _, v := range result.CalculatedUpstreams {
-			addrs = append(addrs, v.Address)
-		}
-	} else if checkHCPBUpstreams != nil && checkHCPBUpstreams(w) {
-		// This is a worker that is one hop away from managed workers, so attempt to get that list
-		hcpbWorkersCtx, hcpbWorkersCancel := context.WithTimeout(cancelCtx, time.Duration(w.statusCallTimeoutDuration.Load()))
-		defer hcpbWorkersCancel()
-		workersResp, err := client.ListHcpbWorkers(hcpbWorkersCtx, &pbs.ListHcpbWorkersRequest{})
-		if err != nil {
-			event.WriteError(hcpbWorkersCtx, op, err, event.WithInfoMsg("error fetching managed worker information"))
-		} else {
-			addrs = make([]string, 0, len(workersResp.Workers))
-			for _, v := range workersResp.Workers {
-				addrs = append(addrs, v.Address)
-			}
-		}
-	}
-
-	w.updateAddresses(cancelCtx, addrs, addressReceivers)
-
-	w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now(), LastCalculatedUpstreams: addrs})
-
-	var nonActiveMonitoredSessionIds []string
-
-	for _, request := range result.GetJobsRequests() {
-		switch request.GetRequestType() {
-		case pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE:
-			switch request.GetJob().GetType() {
-			case pbs.JOBTYPE_JOBTYPE_MONITOR_SESSION:
-				si := request.GetJob().GetMonitorSessionInfo()
-				if si != nil && si.Status != pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE {
-					nonActiveMonitoredSessionIds = append(nonActiveMonitoredSessionIds, si.GetSessionId())
-				}
-			case pbs.JOBTYPE_JOBTYPE_SESSION:
-				sessInfo := request.GetJob().GetSessionInfo()
-				sessionId := sessInfo.GetSessionId()
-				si := sessionManager.Get(sessionId)
-				if si == nil {
-					event.WriteError(cancelCtx, op, errors.New("session change requested but could not find local information for it"), event.WithInfo("session_id", sessionId))
-					continue
-				}
-				si.ApplyLocalStatus(sessInfo.GetStatus())
-
-				// Update connection state if there are any connections in
-				// the request.
-				for _, conn := range sessInfo.GetConnections() {
-					if err := si.ApplyLocalConnectionStatus(conn.GetConnectionId(), conn.GetStatus()); err != nil {
-						event.WriteError(cancelCtx, op, err, event.WithInfo("connection_id", conn.GetConnectionId()))
-					}
-				}
-			}
-		}
-	}
-	if recorderManager != nil {
-		if err := recorderManager.ReauthorizeAllExcept(cancelCtx, nonActiveMonitoredSessionIds); err != nil {
-			event.WriteError(cancelCtx, op, err)
-		}
-	}
-
-	// Standard cleanup: Run through current jobs. Cancel connections
-	// for any canceling session or any session that is expired.
-	w.cleanupConnections(cancelCtx, false, sessionManager)
-
-	// If we have post hooks for after the first status check, run them now
-	if w.everAuthenticated.CompareAndSwap(authenticationStatusFirstAuthentication, authenticationStatusFirstStatusRpcSuccessful) {
-		if downstreamWorkersFactory != nil {
-			downstreamWorkers, err := downstreamWorkersFactory(cancelCtx, w.LastStatusSuccess().WorkerId, versionInfo.FullVersionNumber(false))
-			if err != nil {
-				event.WriteError(cancelCtx, op, err)
-				w.conf.ServerSideShutdownCh <- struct{}{}
-				return
-			}
-			w.downstreamWorkers.Store(&graphContainer{graph: downstreamWorkers})
-		}
-		for _, fn := range firstStatusCheckPostHooks {
-			if err := fn(cancelCtx, w); err != nil {
-				// If we can't verify status we can't be expected to behave
-				// properly so error and trigger shutdown
-				event.WriteError(cancelCtx, op, fmt.Errorf("error running first status check post hook: %w", err))
-				// We don't use a non-blocking select here to ensure that it
-				// happens; we should catch blocks in tests but we want to
-				// ensure the signal is being listened to
-				w.conf.ServerSideShutdownCh <- struct{}{}
-				return
-			}
-		}
-	}
-}
-
-// Update address receivers and dialing listeners with new addrs
-func (w *Worker) updateAddresses(cancelCtx context.Context, addrs []string, addressReceivers *[]addressReceiver) {
-	const op = "worker.(Worker).updateAddrs"
-
-	if len(addrs) > 0 {
-		lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
-		// Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses
-		if lastStatus != nil && !strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) {
-			upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastStatus.LastCalculatedUpstreams, addrs)
-			event.WriteSysEvent(cancelCtx, op, upstreamsMessage)
-			for _, as := range *addressReceivers {
-				as.SetAddresses(addrs)
-			}
-		} else if lastStatus == nil {
-			for _, as := range *addressReceivers {
-				as.SetAddresses(addrs)
-			}
-			event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first status set to: %s", addrs))
-		}
-	}
-
-	// regardless of whether or not it's a new address, we need to set
-	// them for secondary connections
-	for _, as := range *addressReceivers {
-		switch {
-		case as.Type() == secondaryConnectionReceiverType:
-			tmpAddrs := make([]string, len(addrs))
-			copy(tmpAddrs, addrs)
-			if len(tmpAddrs) == 0 {
-				tmpAddrs = append(tmpAddrs, w.conf.RawConfig.Worker.InitialUpstreams...)
-			}
-			as.SetAddresses(tmpAddrs)
-		}
-	}
-}
-
-// cleanupConnections walks all sessions and shuts down all proxy connections.
-// After the local connections are terminated, they are requested to be marked
-// close on the controller.
-// Additionally, sessions without connections are cleaned up from the
-// local worker's state.
-//
-// Use ignoreSessionState to ignore the state checks, this closes all
-// connections, regardless of whether or not the session is still active.
-func (w *Worker) cleanupConnections(cancelCtx context.Context, ignoreSessionState bool, sessionManager session.Manager) {
-	const op = "worker.(Worker).cleanupConnections"
-	closeInfo := make(map[string]*session.ConnectionCloseData)
-	cleanSessionIds := make([]string, 0)
-	sessionManager.ForEachLocalSession(func(s session.Session) bool {
-		switch {
-		case ignoreSessionState,
-			s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
-			s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
-			time.Until(s.GetExpiration()) < 0:
-			// Cancel connections without regard to individual connection
-			// state.
-			closedIds := s.CancelAllLocalConnections()
-			localConns := s.GetLocalConnections()
-			for _, connId := range closedIds {
-				var bytesUp, bytesDown int64
-				connInfo, ok := localConns[connId]
-				if ok {
-					bytesUp = connInfo.BytesUp()
-					bytesDown = connInfo.BytesDown()
-				}
-				closeInfo[connId] = &session.ConnectionCloseData{
-					SessionId: s.GetId(),
-					BytesUp:   bytesUp,
-					BytesDown: bytesDown,
-				}
-				event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId)
-			}
-
-			// If the session is no longer valid and all connections
-			// are marked closed, clean up the session.
-			if len(closedIds) == 0 {
-				cleanSessionIds = append(cleanSessionIds, s.GetId())
-			}
-
-		default:
-			// Cancel connections *with* regard to individual connection
-			// state (ie: only ones that the controller has requested be
-			// terminated).
-			for _, connId := range s.CancelOpenLocalConnections() {
-				closeInfo[connId] = &session.ConnectionCloseData{SessionId: s.GetId()}
-				event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId)
-			}
-		}
-
-		return true
-	})
-
-	// Note that we won't clean these from the info map until the
-	// next time we run this function
-	if len(closeInfo) > 0 {
-		// Call out to a helper to send the connection close requests to the
-		// controller, and set the close time. This functionality is shared with
-		// post-close functionality in the proxy handler.
-		_ = sessionManager.RequestCloseConnections(cancelCtx, closeInfo)
-	}
-	// Forget sessions where the session is expired/canceled and all
-	// connections are canceled and marked closed
-	sessionManager.DeleteLocalSession(cleanSessionIds)
-}
-
-func (w *Worker) lastSuccessfulStatusTime() time.Time {
-	lastStatus := w.LastStatusSuccess()
-	if lastStatus == nil {
-		return w.workerStartTime
-	}
-
-	return lastStatus.StatusTime
-}
-
-func (w *Worker) isPastGrace() (bool, time.Time, time.Duration) {
-	t := w.lastSuccessfulStatusTime()
-	u := time.Duration(w.successfulStatusGracePeriod.Load())
-	v := time.Since(t)
-	return v > u, t, u
-}

internal/daemon/worker/testing.go

@@ -29,6 +29,8 @@ import (
 	"github.com/mr-tron/base58"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -69,16 +71,8 @@ func (tw *TestWorker) Name() string {
 }
 
 func (tw *TestWorker) UpstreamAddrs() []string {
-	var addrs []string
-	lastStatus := tw.w.LastStatusSuccess()
-	if lastStatus == nil {
-		return addrs
-	}
-	for _, v := range lastStatus.GetCalculatedUpstreams() {
-		addrs = append(addrs, v.Address)
-	}
-
-	return addrs
+	lastRoutingInfo := tw.w.LastRoutingInfoSuccess()
+	return lastRoutingInfo.GetCalculatedUpstreamAddresses()
 }
 
 func (tw *TestWorker) ProxyAddrs() []string {
@@ -204,6 +198,10 @@ type TestWorkerOpts struct {
 	// The location of the worker's recording storage
 	WorkerRecordingStoragePath string
 
+	// The interval between each respective worker RPC invocation
+	// This sets the interval for SessionInfo, RoutingInfo and Statistics.
+	WorkerRPCInterval time.Duration
+
 	// The name to use for the worker, otherwise one will be randomly
 	// generated, unless provided in a non-nil Config
 	Name string
@@ -217,7 +215,7 @@ type TestWorkerOpts struct {
 
 	// The amount of time to wait before marking connections as closed when a
 	// connection cannot be made back to the controller
-	SuccessfulStatusGracePeriodDuration time.Duration
+	SuccessfulControllerRPCGracePeriodDuration time.Duration
 
 	// Overrides worker's nonceFn, for cases where we want to have control
 	// over the nonce we send to the Controller
@@ -286,6 +284,9 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 		if opts.Name != "" {
 			opts.Config.Worker.Name = opts.Name
 		}
+		if opts.WorkerRPCInterval > 0 {
+			opts.Config.Worker.TestWorkerRPCInterval = opts.WorkerRPCInterval
+		}
 	}
 
 	if len(opts.InitialUpstreams) > 0 {
@@ -317,8 +318,8 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 	tw.b.EnabledPlugins = append(tw.b.EnabledPlugins, base.EnabledPluginLoopback)
 	tw.name = opts.Config.Worker.Name
 
-	if opts.SuccessfulStatusGracePeriodDuration != 0 {
-		opts.Config.Worker.SuccessfulStatusGracePeriodDuration = opts.SuccessfulStatusGracePeriodDuration
+	if opts.SuccessfulControllerRPCGracePeriodDuration != 0 {
+		opts.Config.Worker.SuccessfulControllerRPCGracePeriodDuration = opts.SuccessfulControllerRPCGracePeriodDuration
 	}
 
 	serverName, err := os.Hostname()
@@ -401,14 +402,14 @@ func (tw *TestWorker) AddClusterWorkerMember(t testing.TB, opts *TestWorkerOpts)
 		opts = new(TestWorkerOpts)
 	}
 	nextOpts := &TestWorkerOpts{
-		WorkerAuthKms:                       tw.w.conf.WorkerAuthKms,
-		DownstreamWorkerAuthKms:             tw.w.conf.DownstreamWorkerAuthKms,
-		WorkerAuthStorageKms:                tw.w.conf.WorkerAuthStorageKms,
-		Name:                                opts.Name,
-		InitialUpstreams:                    tw.UpstreamAddrs(),
-		Logger:                              tw.w.conf.Logger,
-		SuccessfulStatusGracePeriodDuration: opts.SuccessfulStatusGracePeriodDuration,
-		WorkerAuthDebuggingEnabled:          tw.w.conf.WorkerAuthDebuggingEnabled,
+		WorkerAuthKms:           tw.w.conf.WorkerAuthKms,
+		DownstreamWorkerAuthKms: tw.w.conf.DownstreamWorkerAuthKms,
+		WorkerAuthStorageKms:    tw.w.conf.WorkerAuthStorageKms,
+		Name:                    opts.Name,
+		InitialUpstreams:        tw.UpstreamAddrs(),
+		Logger:                  tw.w.conf.Logger,
+		SuccessfulControllerRPCGracePeriodDuration: opts.SuccessfulControllerRPCGracePeriodDuration,
+		WorkerAuthDebuggingEnabled:                 tw.w.conf.WorkerAuthDebuggingEnabled,
 	}
 	if nextOpts.Name == "" {
 		var err error
@@ -457,3 +458,98 @@ func NewAuthorizedPkiTestWorker(t *testing.T, repo *server.Repository, name stri
 	require.NoError(t, err)
 	return w, wr.GetPublicId()
 }
+
+// mockServerCoordinationService is meant to stand in for a controller when testing
+// the methods defined by the server coordination service. It allows applying assertions and specifying
+// the return value of grpc methods by overwriting service methods.
+type mockServerCoordinationService struct {
+	pbs.UnimplementedServerCoordinationServiceServer
+	nextReqAssert         func(*pbs.StatusRequest) (*pbs.StatusResponse, error)
+	nextStatisticAssert   func(*pbs.StatisticsRequest) (*pbs.StatisticsResponse, error)
+	nextSessionInfoAssert func(*pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error)
+}
+
+func (m mockServerCoordinationService) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
+	if m.nextReqAssert != nil {
+		return m.nextReqAssert(req)
+	}
+	return nil, status.Error(codes.Unavailable, "Status not implemented")
+}
+
+func (m mockServerCoordinationService) Statistics(ctx context.Context, req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) {
+	if m.nextStatisticAssert != nil {
+		return m.nextStatisticAssert(req)
+	}
+	return nil, status.Error(codes.Unavailable, "Statistics not implemented")
+}
+
+func (m mockServerCoordinationService) SessionInfo(ctx context.Context, req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) {
+	if m.nextSessionInfoAssert != nil {
+		return m.nextSessionInfoAssert(req)
+	}
+	return nil, status.Error(codes.Unavailable, "SessionInfo not implemented")
+}
+
+var _ pbs.ServerCoordinationServiceServer = (*mockServerCoordinationService)(nil)
+
+// TestWaitForNextSuccessfulSessionInfoUpdate waits for the next successful session info. It's
+// used by testing in place of a more opaque and possibly unnecessarily long sleep for
+// things like initial controller check-in, etc.
+//
+// The timeout is aligned with the worker's session info grace period.
+func (w *Worker) TestWaitForNextSuccessfulSessionInfoUpdate(t testing.TB) {
+	t.Helper()
+	const op = "worker.(Worker).WaitForNextSuccessfulSessionInfoUpdate"
+	waitStart := time.Now()
+	ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(w.successfulSessionInfoGracePeriod.Load()))
+	defer cancel()
+	t.Log("waiting for next session info report to controller")
+	for {
+		select {
+		case <-time.After(time.Second):
+			// pass
+
+		case <-ctx.Done():
+			t.Error("error waiting for next session info report to controller")
+			return
+		}
+
+		si := w.lastSessionInfoSuccess.Load().(*lastSessionInfo)
+		if si != nil && si.LastSuccessfulRequestTime.After(waitStart) {
+			break
+		}
+	}
+
+	t.Log("next worker session info update sent successfully")
+}
+
+// TestWaitForNextSuccessfulStatisticsUpdate waits for the next successful statistics. It's
+// used by testing in place of a more opaque and possibly unnecessarily long sleep for
+// things like initial controller check-in, etc.
+//
+// The timeout is aligned with twice the worker's statistics timeout duration.
+func (w *Worker) TestWaitForNextSuccessfulStatisticsUpdate(t testing.TB) {
+	t.Helper()
+	const op = "worker.(Worker).WaitForNextSuccessfulStatisticsUpdate"
+	waitStart := time.Now()
+	ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(2*w.statisticsCallTimeoutDuration.Load()))
+	defer cancel()
+	t.Log("waiting for next statistics report to controller")
+	for {
+		select {
+		case <-time.After(time.Second):
+			// pass
+
+		case <-ctx.Done():
+			t.Error("error waiting for next statistics report to controller")
+			return
+		}
+
+		si := w.lastStatisticsSuccess.Load().(*lastStatistics)
+		if si != nil && si.LastSuccessfulRequestTime.After(waitStart) {
+			break
+		}
+	}
+
+	t.Log("next worker statistics update sent successfully")
+}

internal/daemon/worker/worker.go

@@ -11,6 +11,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -40,7 +41,6 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/hashicorp/go-secure-stdlib/mlock"
 	"github.com/hashicorp/go-secure-stdlib/pluginutil/v2"
-	"github.com/hashicorp/go-secure-stdlib/strutil"
 	"github.com/hashicorp/nodeenrollment"
 	nodeenet "github.com/hashicorp/nodeenrollment/net"
 	nodeefile "github.com/hashicorp/nodeenrollment/storage/file"
@@ -85,10 +85,10 @@ type graph interface {
 	RootId() string
 }
 
-// recorderManager updates the status updates with relevant recording
+// recorderManager updates the session info updates with relevant recording
 // information
 type recorderManager interface {
-	// ReauthorizeAllExcept should be called with the result of the status update
+	// ReauthorizeAllExcept should be called with the result of the session info update
 	// to reauthorize all recorders for the relevant sessions except the ones provided
 	ReauthorizeAllExcept(ctx context.Context, closedSessions []string) error
 	// SessionsManaged gets the list of session ids managed by this recorderManager
@@ -126,11 +126,17 @@ var hostServiceServerFactory func(
 const (
 	authenticationStatusNeverAuthenticated uint32 = iota
 	authenticationStatusFirstAuthentication
-	authenticationStatusFirstStatusRpcSuccessful
+	authenticationStatusFirstRoutingInfoRpcSuccessful
 )
 
 type Worker struct {
-	conf   *Config
+	conf *Config
+	// receives address updates and contains the grpc resolver.
+	addressReceivers []addressReceiver
+	// confAddressReceiversLock is used to protect the conf field
+	// and the addressReceivers field.
+	confAddressReceiversLock sync.Mutex
+
 	logger hclog.Logger
 
 	baseContext context.Context
@@ -145,17 +151,16 @@ type Worker struct {
 	// (mostly in tests) - which isn't thread safe. This is exported for tests.
 	GrpcClientConn atomic.Pointer[grpc.ClientConn]
 
-	// receives address updates and contains the grpc resolver.
-	addressReceivers []addressReceiver
-
 	sessionManager session.Manager
 
 	recorderManager recorderManager
 
-	everAuthenticated *ua.Uint32
-	lastStatusSuccess *atomic.Value
-	workerStartTime   time.Time
-	operationalState  *atomic.Value
+	everAuthenticated      *ua.Uint32
+	lastSessionInfoSuccess *atomic.Value
+	lastRoutingInfoSuccess *atomic.Value
+	lastStatisticsSuccess  *atomic.Value
+	workerStartTime        time.Time
+	operationalState       *atomic.Value
 	// localStorageState is the current state of the local storage.
 	// The local storage state is updated based on the local storage events.
 	localStorageState *atomic.Value
@@ -172,11 +177,9 @@ type Worker struct {
 	// Used to generate a random nonce for Controller connections
 	nonceFn randFn
 
-	// We store the current set in an atomic value so that we can add
-	// reload-on-sighup behavior later
 	tags *atomic.Value
-	// This stores whether or not to send updated tags on the next status
-	// request. It can be set via startup in New below, or (eventually) via
+	// This stores whether or not to send updated tags on the next routing info
+	// request. It can be set via startup in New below, or via
 	// SIGHUP.
 	updateTags *ua.Bool
 
@@ -195,10 +198,26 @@ type Worker struct {
 
 	// Timing variables. These are atomics for SIGHUP support, and are int64
 	// because they are casted to time.Duration.
-	successfulStatusGracePeriod         *atomic.Int64
-	statusCallTimeoutDuration           *atomic.Int64
+	successfulRoutingInfoGracePeriod *atomic.Int64
+	successfulSessionInfoGracePeriod *atomic.Int64
+	// Note: Statistics does not require a grace period,
+	// because we do not perform any special action based
+	// on whether it is successful or not, unlike for SessionInfo
+	// and RoutingInfo.
+
+	statisticsCallTimeoutDuration       *atomic.Int64
+	sessionInfoCallTimeoutDuration      *atomic.Int64
+	routingInfoCallTimeoutDuration      *atomic.Int64
 	getDownstreamWorkersTimeoutDuration *atomic.Int64
 
+	// The time intervals at which the worker will invoke the controller RPCs.
+	// Defaults to common.SessionInfoInterval, common.RoutingInfoInterval
+	// and common.StatisticsInterval and is only overridden by the
+	// TestWorkerRPCInterval test config.
+	sessionInfoInterval time.Duration
+	routingInfoInterval time.Duration
+	statisticsInterval  time.Duration
+
 	// AuthRotationNextRotation is useful in tests to understand how long to
 	// sleep
 	AuthRotationNextRotation atomic.Pointer[time.Time]
@@ -208,8 +227,6 @@ type Worker struct {
 	TestOverrideX509VerifyCertPool *x509.CertPool
 	TestOverrideAuthRotationPeriod time.Duration
 
-	statusLock sync.Mutex
-
 	downstreamConnManager *cluster.DownstreamManager
 
 	HostServiceServer wpbs.HostServiceServer
@@ -230,7 +247,9 @@ func New(ctx context.Context, conf *Config) (*Worker, error) {
 		logger:                 conf.Logger.Named("worker"),
 		started:                ua.NewBool(false),
 		everAuthenticated:      ua.NewUint32(authenticationStatusNeverAuthenticated),
-		lastStatusSuccess:      new(atomic.Value),
+		lastSessionInfoSuccess: new(atomic.Value),
+		lastRoutingInfoSuccess: new(atomic.Value),
+		lastStatisticsSuccess:  new(atomic.Value),
 		controllerMultihopConn: new(atomic.Value),
 		// controllerUpstreamMsgConn:   new(atomic.Value),
 		tags:                                new(atomic.Value),
@@ -240,8 +259,11 @@ func New(ctx context.Context, conf *Config) (*Worker, error) {
 		operationalState:                    new(atomic.Value),
 		downstreamConnManager:               cluster.NewDownstreamManager(),
 		localStorageState:                   new(atomic.Value),
-		successfulStatusGracePeriod:         new(atomic.Int64),
-		statusCallTimeoutDuration:           new(atomic.Int64),
+		successfulRoutingInfoGracePeriod:    new(atomic.Int64),
+		successfulSessionInfoGracePeriod:    new(atomic.Int64),
+		statisticsCallTimeoutDuration:       new(atomic.Int64),
+		sessionInfoCallTimeoutDuration:      new(atomic.Int64),
+		routingInfoCallTimeoutDuration:      new(atomic.Int64),
 		getDownstreamWorkersTimeoutDuration: new(atomic.Int64),
 		upstreamConnectionState:             new(atomic.Value),
 		downstreamWorkers:                   new(atomic.Pointer[graphContainer]),
@@ -249,7 +271,9 @@ func New(ctx context.Context, conf *Config) (*Worker, error) {
 
 	w.operationalState.Store(server.UnknownOperationalState)
 	w.localStorageState.Store(server.UnknownLocalStorageState)
-	w.lastStatusSuccess.Store((*LastStatusInformation)(nil))
+	w.lastSessionInfoSuccess.Store((*lastSessionInfo)(nil))
+	w.lastRoutingInfoSuccess.Store((*LastRoutingInfo)(nil))
+	w.lastStatisticsSuccess.Store((*lastStatistics)(nil))
 	scheme := strconv.FormatInt(time.Now().UnixNano(), 36)
 	controllerResolver := manual.NewBuilderWithScheme(scheme)
 	w.addressReceivers = []addressReceiver{&grpcResolverReceiver{controllerResolver}}
@@ -365,17 +389,23 @@ func New(ctx context.Context, conf *Config) (*Worker, error) {
 				err)
 		}
 	}
-	switch conf.RawConfig.Worker.SuccessfulStatusGracePeriodDuration {
+	switch conf.RawConfig.Worker.SuccessfulControllerRPCGracePeriodDuration {
 	case 0:
-		w.successfulStatusGracePeriod.Store(int64(server.DefaultLiveness))
+		w.successfulRoutingInfoGracePeriod.Store(int64(server.DefaultLiveness))
+		w.successfulSessionInfoGracePeriod.Store(int64(server.DefaultLiveness))
 	default:
-		w.successfulStatusGracePeriod.Store(int64(conf.RawConfig.Worker.SuccessfulStatusGracePeriodDuration))
+		w.successfulRoutingInfoGracePeriod.Store(int64(conf.RawConfig.Worker.SuccessfulControllerRPCGracePeriodDuration))
+		w.successfulSessionInfoGracePeriod.Store(int64(conf.RawConfig.Worker.SuccessfulControllerRPCGracePeriodDuration))
 	}
-	switch conf.RawConfig.Worker.StatusCallTimeoutDuration {
+	switch conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration {
 	case 0:
-		w.statusCallTimeoutDuration.Store(int64(common.DefaultStatusTimeout))
+		w.routingInfoCallTimeoutDuration.Store(int64(common.DefaultRoutingInfoTimeout))
+		w.statisticsCallTimeoutDuration.Store(int64(common.DefaultStatisticsTimeout))
+		w.sessionInfoCallTimeoutDuration.Store(int64(common.DefaultSessionInfoTimeout))
 	default:
-		w.statusCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.StatusCallTimeoutDuration))
+		w.routingInfoCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration))
+		w.statisticsCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration))
+		w.sessionInfoCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration))
 	}
 	switch conf.RawConfig.Worker.GetDownstreamWorkersTimeoutDuration {
 	case 0:
@@ -384,7 +414,18 @@ func New(ctx context.Context, conf *Config) (*Worker, error) {
 		w.getDownstreamWorkersTimeoutDuration.Store(int64(conf.RawConfig.Worker.GetDownstreamWorkersTimeoutDuration))
 	}
 	// FIXME: This is really ugly, but works.
-	session.CloseCallTimeout.Store(w.successfulStatusGracePeriod.Load())
+	session.CloseCallTimeout.Store(w.successfulSessionInfoGracePeriod.Load())
+
+	w.sessionInfoInterval = common.SessionInfoInterval
+	w.routingInfoInterval = common.RoutingInfoInterval
+	w.statisticsInterval = common.StatisticsInterval
+	// Override the routing info interval if it is set in the config.
+	// This should only be used by tests.
+	if conf.RawConfig.Worker.TestWorkerRPCInterval > 0 {
+		w.sessionInfoInterval = conf.RawConfig.Worker.TestWorkerRPCInterval
+		w.routingInfoInterval = conf.RawConfig.Worker.TestWorkerRPCInterval
+		w.statisticsInterval = conf.RawConfig.Worker.TestWorkerRPCInterval
+	}
 
 	if reverseConnReceiverFactory != nil {
 		var err error
@@ -443,10 +484,36 @@ func (w *Worker) Reload(ctx context.Context, newConf *config.Config) {
 
 	w.parseAndStoreTags(newConf.Worker.Tags)
 
-	if !strutil.EquivalentSlices(newConf.Worker.InitialUpstreams, w.conf.RawConfig.Worker.InitialUpstreams) {
-		w.statusLock.Lock()
-		defer w.statusLock.Unlock()
+	switch newConf.Worker.SuccessfulControllerRPCGracePeriod {
+	case 0:
+		w.successfulRoutingInfoGracePeriod.Store(int64(server.DefaultLiveness))
+		w.successfulSessionInfoGracePeriod.Store(int64(server.DefaultLiveness))
+	default:
+		w.successfulRoutingInfoGracePeriod.Store(int64(newConf.Worker.SuccessfulControllerRPCGracePeriodDuration))
+		w.successfulSessionInfoGracePeriod.Store(int64(newConf.Worker.SuccessfulControllerRPCGracePeriodDuration))
+	}
+	switch newConf.Worker.ControllerRPCCallTimeoutDuration {
+	case 0:
+		w.routingInfoCallTimeoutDuration.Store(int64(common.DefaultRoutingInfoTimeout))
+		w.statisticsCallTimeoutDuration.Store(int64(common.DefaultStatisticsTimeout))
+		w.sessionInfoCallTimeoutDuration.Store(int64(common.DefaultSessionInfoTimeout))
+	default:
+		w.routingInfoCallTimeoutDuration.Store(int64(newConf.Worker.ControllerRPCCallTimeoutDuration))
+		w.statisticsCallTimeoutDuration.Store(int64(newConf.Worker.ControllerRPCCallTimeoutDuration))
+		w.sessionInfoCallTimeoutDuration.Store(int64(newConf.Worker.ControllerRPCCallTimeoutDuration))
+	}
+	switch newConf.Worker.GetDownstreamWorkersTimeoutDuration {
+	case 0:
+		w.getDownstreamWorkersTimeoutDuration.Store(int64(server.DefaultLiveness))
+	default:
+		w.getDownstreamWorkersTimeoutDuration.Store(int64(newConf.Worker.GetDownstreamWorkersTimeoutDuration))
+	}
+	// See comment about this in worker.go
+	session.CloseCallTimeout.Store(w.successfulRoutingInfoGracePeriod.Load())
 
+	w.confAddressReceiversLock.Lock()
+	defer w.confAddressReceiversLock.Unlock()
+	if !slices.Equal(newConf.Worker.InitialUpstreams, w.conf.RawConfig.Worker.InitialUpstreams) {
 		upstreamsMessage := fmt.Sprintf(
 			"Initial Upstreams has changed; old upstreams were: %s, new upstreams are: %s",
 			w.conf.RawConfig.Worker.InitialUpstreams,
@@ -461,27 +528,6 @@ func (w *Worker) Reload(ctx context.Context, newConf *config.Config) {
 			ar.InitialAddresses(w.conf.RawConfig.Worker.InitialUpstreams)
 		}
 	}
-
-	switch newConf.Worker.SuccessfulStatusGracePeriodDuration {
-	case 0:
-		w.successfulStatusGracePeriod.Store(int64(server.DefaultLiveness))
-	default:
-		w.successfulStatusGracePeriod.Store(int64(newConf.Worker.SuccessfulStatusGracePeriodDuration))
-	}
-	switch newConf.Worker.StatusCallTimeoutDuration {
-	case 0:
-		w.statusCallTimeoutDuration.Store(int64(common.DefaultStatusTimeout))
-	default:
-		w.statusCallTimeoutDuration.Store(int64(newConf.Worker.StatusCallTimeoutDuration))
-	}
-	switch newConf.Worker.GetDownstreamWorkersTimeoutDuration {
-	case 0:
-		w.getDownstreamWorkersTimeoutDuration.Store(int64(server.DefaultLiveness))
-	default:
-		w.getDownstreamWorkersTimeoutDuration.Store(int64(newConf.Worker.GetDownstreamWorkersTimeoutDuration))
-	}
-	// See comment about this in worker.go
-	session.CloseCallTimeout.Store(w.successfulStatusGracePeriod.Load())
 }
 
 func (w *Worker) Start() error {
@@ -612,9 +658,6 @@ func (w *Worker) Start() error {
 			return fmt.Errorf("error marshaling worker auth fetch credentials request: %w", err)
 		}
 		w.WorkerAuthRegistrationRequest = base58.FastBase58Encoding(reqBytes)
-		if err != nil {
-			return fmt.Errorf("error encoding worker auth registration request: %w", err)
-		}
 		currentKeyId, err := nodeenrollment.KeyIdFromPkix(nodeCreds.CertificatePublicKeyPkix)
 		if err != nil {
 			return fmt.Errorf("error deriving worker auth key id: %w", err)
@@ -659,7 +702,7 @@ func (w *Worker) Start() error {
 	w.tickerWg.Add(2)
 	go func() {
 		defer w.tickerWg.Done()
-		w.startStatusTicking(w.baseContext, w.sessionManager, &w.addressReceivers, w.recorderManager)
+		w.startRoutingInfoTicking(w.baseContext)
 	}()
 	go func() {
 		defer w.tickerWg.Done()
@@ -669,7 +712,7 @@ func (w *Worker) Start() error {
 	if w.downstreamReceiver != nil {
 		w.tickerWg.Add(2)
 		servNameFn := func() string {
-			if s := w.LastStatusSuccess(); s != nil {
+			if s := w.LastRoutingInfoSuccess(); s != nil {
 				return s.WorkerId
 			}
 			return "unknown worker id"
@@ -706,24 +749,30 @@ func (w *Worker) GracefulShutdown() error {
 	event.WriteSysEvent(w.baseContext, op, "worker entering graceful shutdown")
 	w.operationalState.Store(server.ShutdownOperationalState)
 
-	// As long as some status has been sent in the past, wait for 2 status
-	// updates to be sent since we've updated our operational state.
-	lastStatusTime := w.lastSuccessfulStatusTime()
-	if lastStatusTime != w.workerStartTime {
-		for i := 0; i < 2; i++ {
-			for {
-				if lastStatusTime != w.lastSuccessfulStatusTime() {
-					lastStatusTime = w.lastSuccessfulStatusTime()
-					break
-				}
-				time.Sleep(time.Millisecond * 250)
+	// As long as some routing info has been sent in the past, wait for 1 routing info
+	// update to be sent since we've updated our operational state.
+	lastRoutingInfoTime := w.lastSuccessfulRoutingInfoTime()
+	if !lastRoutingInfoTime.Equal(w.workerStartTime) {
+	WaitForRoutingInfo:
+		for !lastRoutingInfoTime.Before(w.lastSuccessfulRoutingInfoTime()) {
+			select {
+			case <-w.baseContext.Done():
+				event.WriteSysEvent(w.baseContext, op, "context done waiting for routing info to be sent")
+				break WaitForRoutingInfo
+			case <-time.After(time.Millisecond * 250):
 			}
 		}
 	}
 
 	// Wait for running proxy connections to drain
+WaitForConnectionDrain:
 	for proxy.ProxyState.CurrentProxiedConnections() > 0 {
-		time.Sleep(time.Millisecond * 250)
+		select {
+		case <-w.baseContext.Done():
+			event.WriteSysEvent(w.baseContext, op, "context done waiting for connections to be drained")
+			break WaitForConnectionDrain
+		case <-time.After(time.Millisecond * 250):
+		}
 	}
 	event.WriteSysEvent(w.baseContext, op, "worker connections have drained")
 
@@ -767,30 +816,31 @@ func (w *Worker) Shutdown() error {
 	// Shut down all connections.
 	w.cleanupConnections(w.baseContext, true, w.sessionManager)
 
-	// Wait for next status request to succeed. Don't wait too long; time it out
-	// at our default liveness value, which is also our default status grace
-	// period timeout
-	waitStatusStart := time.Now()
-	nextStatusCtx, nextStatusCancel := context.WithTimeout(w.baseContext, server.DefaultLiveness)
-	defer nextStatusCancel()
-	for {
-		if err := nextStatusCtx.Err(); err != nil {
-			event.WriteError(w.baseContext, op, err, event.WithInfoMsg("error waiting for next status report to controller"))
-			break
+	// Wait for next routing info request to succeed. Don't wait too long; time it out
+	// at our default liveness value.
+	nextRoutingInfoCtx, nextRoutingInfoCancel := context.WithTimeout(w.baseContext, server.DefaultLiveness)
+	defer nextRoutingInfoCancel()
+	lastRoutingInfoTime := w.lastSuccessfulRoutingInfoTime()
+	if !lastRoutingInfoTime.Equal(w.workerStartTime) {
+	WaitForRoutingInfo:
+		for !lastRoutingInfoTime.Before(w.lastSuccessfulRoutingInfoTime()) {
+			select {
+			case <-nextRoutingInfoCtx.Done():
+				event.WriteError(w.baseContext, op, nextRoutingInfoCtx.Err(), event.WithInfoMsg("error waiting for next routing info report to controller"))
+				break WaitForRoutingInfo
+			case <-time.After(time.Millisecond * 250):
+			}
 		}
-
-		if w.lastSuccessfulStatusTime().After(waitStatusStart) {
-			break
-		}
-
-		time.Sleep(time.Second)
 	}
 
 	// Proceed with remainder of shutdown.
 	w.baseCancel()
+	// Lock to protect w.addressReceivers
+	w.confAddressReceiversLock.Lock()
 	for _, ar := range w.addressReceivers {
 		ar.SetAddresses(nil)
 	}
+	w.confAddressReceiversLock.Unlock()
 
 	if w.storageEventListener != nil {
 		err := w.storageEventListener.Shutdown(w.baseContext)
@@ -853,10 +903,10 @@ func (w *Worker) getSessionTls(sessionManager session.Manager) func(hello *tls.C
 			return nil, fmt.Errorf("could not find session ID in SNI or ALPN protos")
 		}
 
-		lastSuccess := w.LastStatusSuccess()
+		lastSuccess := w.LastRoutingInfoSuccess()
 		if lastSuccess == nil {
-			event.WriteSysEvent(ctx, op, "no last status information found at session acceptance time")
-			return nil, fmt.Errorf("no last status information found at session acceptance time")
+			event.WriteSysEvent(ctx, op, "no last routing information found at session acceptance time")
+			return nil, fmt.Errorf("no last routing information found at session acceptance time")
 		}
 
 		timeoutContext, cancel := context.WithTimeout(w.baseContext, session.ValidateSessionTimeout)

internal/daemon/worker/worker_proxy_service.go

@@ -30,6 +30,14 @@ func NewWorkerProxyServiceServer(
 	}
 }
 
+func (ws *workerProxyServiceServer) Statistics(ctx context.Context, req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) {
+	return pbs.NewServerCoordinationServiceClient(ws.cc).Statistics(ctx, req)
+}
+
+func (ws *workerProxyServiceServer) SessionInfo(ctx context.Context, req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) {
+	return pbs.NewServerCoordinationServiceClient(ws.cc).SessionInfo(ctx, req)
+}
+
 func (ws *workerProxyServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
 	resp, err := pbs.NewServerCoordinationServiceClient(ws.cc).Status(ctx, req)
 
@@ -46,6 +54,18 @@ func (ws *workerProxyServiceServer) ListHcpbWorkers(ctx context.Context, req *pb
 	return pbs.NewServerCoordinationServiceClient(ws.cc).ListHcpbWorkers(ctx, req)
 }
 
+func (ws *workerProxyServiceServer) RoutingInfo(ctx context.Context, req *pbs.RoutingInfoRequest) (*pbs.RoutingInfoResponse, error) {
+	resp, err := pbs.NewServerCoordinationServiceClient(ws.cc).RoutingInfo(ctx, req)
+
+	if resp != nil {
+		// We don't currently support distributing new addreses to workers
+		// multiple hops away so ensure they're stripped out
+		resp.CalculatedUpstreamAddresses = nil
+	}
+
+	return resp, err
+}
+
 func (ws *workerProxyServiceServer) LookupSession(ctx context.Context, req *pbs.LookupSessionRequest) (*pbs.LookupSessionResponse, error) {
 	return pbs.NewSessionServiceClient(ws.cc).LookupSession(ctx, req)
 }

internal/daemon/worker/worker_test.go

@@ -371,7 +371,7 @@ func Test_Worker_getSessionTls(t *testing.T) {
 	conf.RawConfig = &config.Config{SharedConfig: &configutil.SharedConfig{DisableMlock: true}}
 	w, err := New(context.Background(), conf)
 	require.NoError(t, err)
-	w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: &services.StatusResponse{}, StatusTime: time.Now(), LastCalculatedUpstreams: nil})
+	w.lastRoutingInfoSuccess.Store(&LastRoutingInfo{RoutingInfoResponse: &services.RoutingInfoResponse{}, RoutingInfoTime: time.Now(), LastCalculatedUpstreams: nil})
 	w.baseContext = context.Background()
 
 	t.Run("success", func(t *testing.T) {

internal/db/schema/migrations/oss/postgres/95/01_server_worker_session_info.up.sql

@@ -0,0 +1,18 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+
+create table server_worker_session_info_request (
+  worker_id wt_public_id
+  	constraint server_worker_fkey
+    	references server_worker(public_id)
+    	on delete cascade
+    	on update cascade,
+  last_request_time timestamp with time zone not null,
+  primary key (worker_id)
+);
+comment on table server_worker_session_info_request is
+  'server_worker_session_info_request is a table where each row represents the last request time a worker has reported its session information.';
+
+commit;

internal/event/cloudevents_formatter_node.go

@@ -87,13 +87,28 @@ func defaultCloudEventsDenyFilters() ([]*filter, error) {
 	const (
 		op = "event.defaultCloudEventsDenyFilters"
 		// denyWorkStatusEvents is a default filter for worker to controller API status requests
-		denyWorkStatusEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`
+		denyWorkStatusEvents      = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`
+		denyWorkSessionInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/SessionInfo"`
+		denyWorkRoutingInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/RoutingInfo"`
+		denyWorkStatisticsEvents  = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Statistics"`
 	)
-	f, err := newFilter(denyWorkStatusEvents)
+	statusFilter, err := newFilter(denyWorkStatusEvents)
 	if err != nil {
 		return nil, fmt.Errorf("%s: unable to create deny filter for worker status events '%s': %w", op, denyWorkStatusEvents, err)
 	}
-	return []*filter{f}, nil
+	sessionInfoFilter, err := newFilter(denyWorkSessionInfoEvents)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create deny filter for worker session info events '%s': %w", op, denyWorkStatusEvents, err)
+	}
+	routingInfoFilter, err := newFilter(denyWorkRoutingInfoEvents)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create deny filter for worker routing info events '%s': %w", op, denyWorkStatusEvents, err)
+	}
+	statisticsFilter, err := newFilter(denyWorkStatisticsEvents)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create deny filter for worker statistics events '%s': %w", op, denyWorkStatusEvents, err)
+	}
+	return []*filter{statusFilter, sessionInfoFilter, routingInfoFilter, statisticsFilter}, nil
 }
 
 // Rotate supports rotating the filter's wrapper. No options are currently

internal/event/cloudevents_formatter_node_test.go

@@ -36,10 +36,15 @@ func Test_newCloudEventsFormatterFilter(t *testing.T) {
 	}{
 		{
 			// default case should have default deny for filtering ServerCoordinationService/Status for observation events
-			name:     "no-opts",
-			source:   testSource,
-			format:   cloudevents.FormatJSON,
-			wantDeny: []string{`"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`},
+			name:   "no-opts",
+			source: testSource,
+			format: cloudevents.FormatJSON,
+			wantDeny: []string{
+				`"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`,
+				`"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/SessionInfo"`,
+				`"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/RoutingInfo"`,
+				`"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Statistics"`,
+			},
 		},
 		{
 			name:   "bad-allow-filter",

internal/event/hclog_formatter_node.go

@@ -84,13 +84,28 @@ func defaultHclogEventsDenyFilters() ([]*filter, error) {
 	const (
 		op = "event.defaultHclogEventsDenyFilters"
 		// denyWorkStatusEvents is a default filter for worker to controller API status requests
-		denyWorkStatusEvents = `"/RequestInfo/Method" contains "ServerCoordinationService/Status"`
+		denyWorkStatusEvents      = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`
+		denyWorkSessionInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/SessionInfo"`
+		denyWorkRoutingInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/RoutingInfo"`
+		denyWorkStatisticsEvents  = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Statistics"`
 	)
-	f, err := newFilter(denyWorkStatusEvents)
+	statusFilter, err := newFilter(denyWorkStatusEvents)
 	if err != nil {
 		return nil, fmt.Errorf("%s: unable to create deny filter for worker status events '%s': %w", op, denyWorkStatusEvents, err)
 	}
-	return []*filter{f}, nil
+	sessionInfoFilter, err := newFilter(denyWorkSessionInfoEvents)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create deny filter for worker session info events '%s': %w", op, denyWorkStatusEvents, err)
+	}
+	routingInfoFilter, err := newFilter(denyWorkRoutingInfoEvents)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create deny filter for worker routing info events '%s': %w", op, denyWorkStatusEvents, err)
+	}
+	statisticsFilter, err := newFilter(denyWorkStatisticsEvents)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create deny filter for worker statistics events '%s': %w", op, denyWorkStatusEvents, err)
+	}
+	return []*filter{statusFilter, sessionInfoFilter, routingInfoFilter, statisticsFilter}, nil
 }
 
 // Rotate supports rotating the filter's wrapper. No options are currently

internal/event/hclog_formatter_node_test.go

@@ -388,7 +388,7 @@ func Test_newHclogFormatterFilter(t *testing.T) {
 			for _, f := range got.allow {
 				assert.Contains(tt.wantAllow, f.raw)
 			}
-			assert.Len(got.deny, len(tt.wantDeny)+1) // +1 since there's always a default deny
+			assert.Len(got.deny, len(tt.wantDeny)+4) // +4 since there's always a default deny
 			defs, err := defaultHclogEventsDenyFilters()
 			require.NoError(err)
 			for _, f := range defs {

internal/gen/controller/servers/servers.pb.go

@@ -86,17 +86,17 @@ type ServerWorkerStatus struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// Id of the worker.
+	// Id of the worker, used after the first request.
 	PublicId string `protobuf:"bytes,10,opt,name=public_id,json=publicId,proto3" json:"public_id,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"`
-	// Name of the resource (optional)
+	// Name of the worker, used to identify workers in the KMS registration flow.
 	Name string `protobuf:"bytes,20,opt,name=name,proto3" json:"name,omitempty" class:"public"` // @gotags: `class:"public"`
-	// Description of the resource (optional)
+	// Description of the worker (optional). Only used by KMS workers.
 	Description string `protobuf:"bytes,25,opt,name=description,proto3" json:"description,omitempty" class:"public"` // @gotags: `class:"public"`
-	// Address for the worker. This must be set.
+	// Address for the worker.
 	Address string `protobuf:"bytes,30,opt,name=address,proto3" json:"address,omitempty" class:"public"` // @gotags: `class:"public"`
 	// Tags for workers
 	Tags []*TagPair `protobuf:"bytes,40,rep,name=tags,proto3" json:"tags,omitempty"`
-	// The key id for this worker, if applicable (optional)
+	// The key id of the worker, used to identify workers in the PKI registration flow.
 	KeyId string `protobuf:"bytes,50,opt,name=key_id,json=keyId,proto3" json:"key_id,omitempty" class:"public"` // @gotags: `class:"public"`
 	// The version of Boundary the worker binary is running
 	ReleaseVersion string `protobuf:"bytes,60,opt,name=release_version,proto3" json:"release_version,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"`

internal/gen/controller/servers/services/server_coordination_service_grpc.pb.go

@@ -24,18 +24,36 @@ const _ = grpc.SupportPackageIsVersion7
 const (
 	ServerCoordinationService_Status_FullMethodName          = "/controller.servers.services.v1.ServerCoordinationService/Status"
 	ServerCoordinationService_ListHcpbWorkers_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/ListHcpbWorkers"
+	ServerCoordinationService_Statistics_FullMethodName      = "/controller.servers.services.v1.ServerCoordinationService/Statistics"
+	ServerCoordinationService_RoutingInfo_FullMethodName     = "/controller.servers.services.v1.ServerCoordinationService/RoutingInfo"
+	ServerCoordinationService_SessionInfo_FullMethodName     = "/controller.servers.services.v1.ServerCoordinationService/SessionInfo"
 )
 
 // ServerCoordinationServiceClient is the client API for ServerCoordinationService service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type ServerCoordinationServiceClient interface {
+	// Deprecated: Do not use.
 	// Status gets worker status requests which include the ongoing jobs the worker is handling and
 	// returns the status response which includes the changes the controller would like to make to
 	// jobs as well as provide a list of the controllers in the system.
+	// This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0.
 	Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error)
 	// Returns the addresses of HCP Boundary workers, if any
 	ListHcpbWorkers(ctx context.Context, in *ListHcpbWorkersRequest, opts ...grpc.CallOption) (*ListHcpbWorkersResponse, error)
+	// Statistics is used by the worker to report non-essential statistics about its sessions and connections.
+	Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
+	// RoutingInfo is used by the worker to inform the controller of information
+	// required by the controller to make session routing decisions and any startup information.
+	// The controller may inform the worker of any downstream workers that should be disconnected.
+	// If the worker fails to successfully report its routing info to the controller,
+	// it will try again later.
+	RoutingInfo(ctx context.Context, in *RoutingInfoRequest, opts ...grpc.CallOption) (*RoutingInfoResponse, error)
+	// SessionInfo is used by the worker to inform the controller of all the sessions
+	// it is managing. The controller may inform the worker if any sessions need to be changed.
+	// If the worker repeatedly fails to successfully report its session info to the controller,
+	// it will tear down any running sessions.
+	SessionInfo(ctx context.Context, in *SessionInfoRequest, opts ...grpc.CallOption) (*SessionInfoResponse, error)
 }
 
 type serverCoordinationServiceClient struct {
@@ -46,6 +64,7 @@ func NewServerCoordinationServiceClient(cc grpc.ClientConnInterface) ServerCoord
 	return &serverCoordinationServiceClient{cc}
 }
 
+// Deprecated: Do not use.
 func (c *serverCoordinationServiceClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) {
 	out := new(StatusResponse)
 	err := c.cc.Invoke(ctx, ServerCoordinationService_Status_FullMethodName, in, out, opts...)
@@ -64,16 +83,58 @@ func (c *serverCoordinationServiceClient) ListHcpbWorkers(ctx context.Context, i
 	return out, nil
 }
 
+func (c *serverCoordinationServiceClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) {
+	out := new(StatisticsResponse)
+	err := c.cc.Invoke(ctx, ServerCoordinationService_Statistics_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *serverCoordinationServiceClient) RoutingInfo(ctx context.Context, in *RoutingInfoRequest, opts ...grpc.CallOption) (*RoutingInfoResponse, error) {
+	out := new(RoutingInfoResponse)
+	err := c.cc.Invoke(ctx, ServerCoordinationService_RoutingInfo_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *serverCoordinationServiceClient) SessionInfo(ctx context.Context, in *SessionInfoRequest, opts ...grpc.CallOption) (*SessionInfoResponse, error) {
+	out := new(SessionInfoResponse)
+	err := c.cc.Invoke(ctx, ServerCoordinationService_SessionInfo_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ServerCoordinationServiceServer is the server API for ServerCoordinationService service.
 // All implementations must embed UnimplementedServerCoordinationServiceServer
 // for forward compatibility
 type ServerCoordinationServiceServer interface {
+	// Deprecated: Do not use.
 	// Status gets worker status requests which include the ongoing jobs the worker is handling and
 	// returns the status response which includes the changes the controller would like to make to
 	// jobs as well as provide a list of the controllers in the system.
+	// This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0.
 	Status(context.Context, *StatusRequest) (*StatusResponse, error)
 	// Returns the addresses of HCP Boundary workers, if any
 	ListHcpbWorkers(context.Context, *ListHcpbWorkersRequest) (*ListHcpbWorkersResponse, error)
+	// Statistics is used by the worker to report non-essential statistics about its sessions and connections.
+	Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
+	// RoutingInfo is used by the worker to inform the controller of information
+	// required by the controller to make session routing decisions and any startup information.
+	// The controller may inform the worker of any downstream workers that should be disconnected.
+	// If the worker fails to successfully report its routing info to the controller,
+	// it will try again later.
+	RoutingInfo(context.Context, *RoutingInfoRequest) (*RoutingInfoResponse, error)
+	// SessionInfo is used by the worker to inform the controller of all the sessions
+	// it is managing. The controller may inform the worker if any sessions need to be changed.
+	// If the worker repeatedly fails to successfully report its session info to the controller,
+	// it will tear down any running sessions.
+	SessionInfo(context.Context, *SessionInfoRequest) (*SessionInfoResponse, error)
 	mustEmbedUnimplementedServerCoordinationServiceServer()
 }
 
@@ -87,6 +148,15 @@ func (UnimplementedServerCoordinationServiceServer) Status(context.Context, *Sta
 func (UnimplementedServerCoordinationServiceServer) ListHcpbWorkers(context.Context, *ListHcpbWorkersRequest) (*ListHcpbWorkersResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ListHcpbWorkers not implemented")
 }
+func (UnimplementedServerCoordinationServiceServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented")
+}
+func (UnimplementedServerCoordinationServiceServer) RoutingInfo(context.Context, *RoutingInfoRequest) (*RoutingInfoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method RoutingInfo not implemented")
+}
+func (UnimplementedServerCoordinationServiceServer) SessionInfo(context.Context, *SessionInfoRequest) (*SessionInfoResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SessionInfo not implemented")
+}
 func (UnimplementedServerCoordinationServiceServer) mustEmbedUnimplementedServerCoordinationServiceServer() {
 }
 
@@ -137,6 +207,60 @@ func _ServerCoordinationService_ListHcpbWorkers_Handler(srv interface{}, ctx con
 	return interceptor(ctx, in, info, handler)
 }
 
+func _ServerCoordinationService_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(StatisticsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ServerCoordinationServiceServer).Statistics(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: ServerCoordinationService_Statistics_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ServerCoordinationServiceServer).Statistics(ctx, req.(*StatisticsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ServerCoordinationService_RoutingInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RoutingInfoRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ServerCoordinationServiceServer).RoutingInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: ServerCoordinationService_RoutingInfo_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ServerCoordinationServiceServer).RoutingInfo(ctx, req.(*RoutingInfoRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _ServerCoordinationService_SessionInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SessionInfoRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ServerCoordinationServiceServer).SessionInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: ServerCoordinationService_SessionInfo_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ServerCoordinationServiceServer).SessionInfo(ctx, req.(*SessionInfoRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // ServerCoordinationService_ServiceDesc is the grpc.ServiceDesc for ServerCoordinationService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -152,6 +276,18 @@ var ServerCoordinationService_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "ListHcpbWorkers",
 			Handler:    _ServerCoordinationService_ListHcpbWorkers_Handler,
 		},
+		{
+			MethodName: "Statistics",
+			Handler:    _ServerCoordinationService_Statistics_Handler,
+		},
+		{
+			MethodName: "RoutingInfo",
+			Handler:    _ServerCoordinationService_RoutingInfo_Handler,
+		},
+		{
+			MethodName: "SessionInfo",
+			Handler:    _ServerCoordinationService_SessionInfo_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "controller/servers/services/v1/server_coordination_service.proto",

internal/proto/controller/servers/services/v1/server_coordination_service.proto

@@ -13,10 +13,29 @@ service ServerCoordinationService {
   // Status gets worker status requests which include the ongoing jobs the worker is handling and
   // returns the status response which includes the changes the controller would like to make to
   // jobs as well as provide a list of the controllers in the system.
-  rpc Status(StatusRequest) returns (StatusResponse) {}
+  // This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0.
+  rpc Status(StatusRequest) returns (StatusResponse) {
+    option deprecated = true;
+  }
 
   // Returns the addresses of HCP Boundary workers, if any
   rpc ListHcpbWorkers(ListHcpbWorkersRequest) returns (ListHcpbWorkersResponse) {}
+
+  // Statistics is used by the worker to report non-essential statistics about its sessions and connections.
+  rpc Statistics(StatisticsRequest) returns (StatisticsResponse) {}
+
+  // RoutingInfo is used by the worker to inform the controller of information
+  // required by the controller to make session routing decisions and any startup information.
+  // The controller may inform the worker of any downstream workers that should be disconnected.
+  // If the worker fails to successfully report its routing info to the controller,
+  // it will try again later.
+  rpc RoutingInfo(RoutingInfoRequest) returns (RoutingInfoResponse) {}
+
+  // SessionInfo is used by the worker to inform the controller of all the sessions
+  // it is managing. The controller may inform the worker if any sessions need to be changed.
+  // If the worker repeatedly fails to successfully report its session info to the controller,
+  // it will tear down any running sessions.
+  rpc SessionInfo(SessionInfoRequest) returns (SessionInfoResponse) {}
 }
 
 enum CONNECTIONSTATUS {
@@ -41,31 +60,41 @@ enum SESSIONSTATUS {
   SESSIONSTATUS_TERMINATED = 4;
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message SessionJobInfo {
+  option deprecated = true;
   string session_id = 1; // @gotags: `class:"public" eventstream:"observation"`
   SESSIONSTATUS status = 2;
   repeated Connection connections = 3;
   SessionProcessingError processing_error = 4; // @gotags: `class:"public"`
 }
 
+// This enum is deprecated and is safe to remove after the release of Boundary v0.20.0.
 enum SessionProcessingError {
+  option deprecated = true;
   SESSION_PROCESSING_ERROR_UNSPECIFIED = 0;
   SESSION_PROCESSING_ERROR_UNRECOGNIZED = 1;
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message MonitorSessionJobInfo {
+  option deprecated = true;
   string session_id = 1; // @gotags: `class:"public" eventstream:"observation"`
   SESSIONSTATUS status = 2; // @gotags: `class:"public"`
   SessionProcessingError processing_error = 3; // @gotags: `class:"public"`
 }
 
+// This enum is deprecated and is safe to remove after the release of Boundary v0.20.0.
 enum JOBTYPE {
+  option deprecated = true;
   JOBTYPE_UNSPECIFIED = 0;
   JOBTYPE_SESSION = 1;
   JOBTYPE_MONITOR_SESSION = 2;
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message Job {
+  option deprecated = true;
   JOBTYPE type = 1;
   oneof job_info {
     // This value is specified when type is JOBTYPE_SESSION.
@@ -74,13 +103,17 @@ message Job {
   }
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message JobStatus {
+  option deprecated = true;
   Job job = 1;
 }
 
 // UpstreamServer returns the controller address in the StatusResponse but may be used in the future to return
 // worker address.
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message UpstreamServer {
+  option deprecated = true;
   enum TYPE {
     TYPE_UNSPECIFIED = 0;
     TYPE_CONTROLLER = 1;
@@ -94,9 +127,11 @@ message UpstreamServer {
   string address = 20; // @gotags: `class:"public"`
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message StatusRequest {
-  reserved 10;
-  reserved "worker";
+  option deprecated = true;
+  reserved 10, 50;
+  reserved "worker", "connected_worker_key_identifiers";
 
   // Jobs which this worker wants to report the status.
   repeated JobStatus jobs = 20;
@@ -113,12 +148,6 @@ message StatusRequest {
   // get access to the key and spoof the connection, you're already compromised.
   servers.v1.ServerWorkerStatus worker_status = 40;
 
-  // The worker key identifiers presented by all downstreams connected to this
-  // worker.
-  // Deprecated. Should be removed in 0.15.0 at which time
-  // connected_unmapped_worker_key_identifiers should be used exclusively.
-  repeated string connected_worker_key_identifiers = 50 [deprecated = true];
-
   // The worker key identifiers for downstream workers for which there
   // is not a known worker id mapping for them yet.
   repeated string connected_unmapped_worker_key_identifiers = 51;
@@ -130,36 +159,36 @@ message StatusRequest {
   repeated string connected_worker_public_ids = 55;
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 enum CHANGETYPE {
+  option deprecated = true;
   CHANGETYPE_UNSPECIFIED = 0;
   // Indicates to the worker to update its knowledge of the state, which may
   // cause action to be taken.
   CHANGETYPE_UPDATE_STATE = 1;
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message JobChangeRequest {
+  option deprecated = true;
   Job job = 1;
   CHANGETYPE request_type = 2;
 }
 
-message AuthorizedWorkerList {
-  option deprecated = true;
-  // the key id of authorized workers
-  repeated string worker_key_identifiers = 1 [deprecated = true];
-}
-
 message AuthorizedDownstreamWorkerList {
   // the key id of authorized workers which do not have a worker id known yet
   // to the requester.
-  repeated string unmapped_worker_key_identifiers = 1;
+  repeated string unmapped_worker_key_identifiers = 1; // @gotags: class:"public" eventstream:"observation"
   // the public id of workers that were reported as being connected and which
   // are known by the controller.
-  repeated string worker_public_ids = 2;
+  repeated string worker_public_ids = 2; // @gotags: class:"public" eventstream:"observation"
 }
 
+// This message is deprecated and is safe to remove after the release of Boundary v0.20.0.
 message StatusResponse {
-  reserved 10;
-  reserved "controllers";
+  option deprecated = true;
+  reserved 10, 50;
+  reserved "controllers", "authorized_workers";
 
   // List of jobs and the expected state changes.  For example, this will
   // include jobs witch change type of canceled for jobs which are active on a
@@ -175,12 +204,6 @@ message StatusResponse {
   // controller does not need to do a database lookup for the id using the name field.
   string worker_id = 40; // @gotags: `class:"public" eventstream:"observation"`
 
-  // Of the worker key identifiers provided in the request, these are the ones
-  // which are authorized to remain connected.
-  // This is deprecated.  Use authorized_downstream_workers instead. This
-  // should be removed in version 0.15.0.
-  AuthorizedWorkerList authorized_workers = 50 [deprecated = true];
-
   // Of the downstream workers in the request, these are the ones
   // which are authorized to remain connected.
   AuthorizedDownstreamWorkerList authorized_downstream_workers = 51;
@@ -202,3 +225,99 @@ message ListHcpbWorkersRequest {}
 message ListHcpbWorkersResponse {
   repeated WorkerInfo workers = 1;
 }
+
+// SessionStatistics contains statistics about a session and its connections.
+message SessionStatistics {
+  // Session ID holds the ID of the session which the statistics belong to.
+  string session_id = 1; // @gotags: class:"public" eventstream:"observation"
+
+  // Connections contains statistics about all the existing connections for the given session.
+  repeated Connection connections = 2;
+}
+
+// StatisticsRequest contains a worker statistics request. This request
+// reports on information that is not important either to maintain existing
+// sessions or create new ones.
+message StatisticsRequest {
+  // The ID of the worker making the request.
+  string worker_id = 10; // @gotags: class:"public" eventstream:"observation"
+
+  // The statistics of the sessions managed by this worker
+  repeated SessionStatistics sessions = 20;
+}
+
+// StatisticsResponse contains a controller statistics response. Empty for now.
+message StatisticsResponse {}
+
+// RoutingIinfoRequest is the request message for the RoutingInfo RPC.
+message RoutingInfoRequest {
+  // The worker status of the worker making the request.
+  servers.v1.ServerWorkerStatus worker_status = 1;
+
+  // Whether the tags should be updated or not.
+  bool update_tags = 2; // @gotags: class:"public" eventstream:"observation"
+
+  // The worker key identifiers for downstream workers for which there
+  // is not a known worker id.
+  repeated string connected_unmapped_worker_key_identifiers = 3; // @gotags: class:"public" eventstream:"observation"
+
+  // The worker public ids of all downstreams connected to this worker.
+  repeated string connected_worker_public_ids = 4; // @gotags: class:"public" eventstream:"observation"
+}
+
+// RoutingInfoResponse is the response message for the RoutingInfo RPC.
+message RoutingInfoResponse {
+  // The controller-assigned ID of the worker which made the request.
+  string worker_id = 1; // @gotags: class:"public" eventstream:"observation"
+
+  // The calculated upstream addresses for the controller cluster.
+  repeated string calculated_upstream_addresses = 2; // @gotags: class:"public" eventstream:"observation"
+
+  // Of the downstream workers in the request, these are the ones
+  // which are authorized to remain connected.
+  AuthorizedDownstreamWorkerList authorized_downstream_workers = 3;
+}
+
+// Session type represents how the worker is processing a session.
+enum SessionType {
+  SESSION_TYPE_UNSPECIFIED = 0;
+  // A session where the worker is the ingress worker.
+  SESSION_TYPE_INGRESSED = 1;
+  // A session where the worker is doing the recording of the session.
+  SESSION_TYPE_RECORDED = 2;
+}
+
+// Session contains information about a worker managed session
+message Session {
+  // The ID of the session.
+  string session_id = 1; // @gotags: class:"public" eventstream:"observation"
+
+  // The status of the session.
+  SESSIONSTATUS session_status = 2; // @gotags: class:"public" eventstream:"observation"
+
+  // Represents how the worker is processing the session.
+  SessionType session_type = 3; // @gotags: class:"public" eventstream:"observation"
+
+  // Connections contains information about the connections managed by the session
+  repeated Connection connections = 4;
+}
+
+// SessionInfoRequest contains a worker session information request. This request
+// reports on currently managed sessions on the worker. This includes all ingressed
+// and recorded sessions.
+message SessionInfoRequest {
+  // The ID of the worker making the request.
+  string worker_id = 1; // @gotags: class:"public" eventstream:"observation"
+
+  // The sessions which this worker wants to report the status of.
+  // Includes both ingressed sessions and recorded sessions.
+  repeated Session sessions = 2;
+}
+
+// SessionInfoResponse contains a controller session information response. This
+// response indicates what, if any, changes should be made to the sessions.
+// This also includes any sessions the worker is monitoring but not managing directly.
+message SessionInfoResponse {
+  // List of sessions that are no longer active and whose status should be updated.
+  repeated Session non_active_sessions = 1;
+}

internal/proto/controller/servers/v1/servers.proto

@@ -17,22 +17,22 @@ message TagPair {
 
 // ServerWorkerStatus is the new message used in place of Server to relay status request info.
 message ServerWorkerStatus {
-  // Id of the worker.
+  // Id of the worker, used after the first request.
   string public_id = 10; // @gotags: `class:"public" eventstream:"observation"`
 
-  // Name of the resource (optional)
+  // Name of the worker, used to identify workers in the KMS registration flow.
   string name = 20; // @gotags: `class:"public"`
 
-  // Description of the resource (optional)
+  // Description of the worker (optional). Only used by KMS workers.
   string description = 25; // @gotags: `class:"public"`
 
-  // Address for the worker. This must be set.
+  // Address for the worker.
   string address = 30; // @gotags: `class:"public"`
 
   // Tags for workers
   repeated TagPair tags = 40;
 
-  // The key id for this worker, if applicable (optional)
+  // The key id of the worker, used to identify workers in the PKI registration flow.
   string key_id = 50; // @gotags: `class:"public"`
 
   // The version of Boundary the worker binary is running

internal/proto/controller/storage/servers/store/v1/worker.proto

@@ -159,3 +159,14 @@ message WorkerStorageBucketCredentialState {
   // @inject_tag: `gorm:"not_null"`
   timestamp.v1.Timestamp checked_at = 60;
 }
+
+// WorkerSessionInfoRequest contains information about a worker and its SessionInfo requests.
+message WorkerSessionInfoRequest {
+  // worker_id is the public id of the worker.
+  // @inject_tag: `gorm:"primary_key"`
+  string worker_id = 10;
+
+  // The last_request_time is the last time a worker has reported its session information.
+  // @inject_tag: `gorm:"default:null"`
+  timestamp.v1.Timestamp last_request_time = 20;
+}

internal/server/job/jobs.go

@@ -23,7 +23,7 @@ func RegisterJobs(
 	w db.Writer,
 	kms *kms.Kms,
 	controllerExt globals.ControllerExtension,
-	workerStatusGracePeriod *atomic.Int64,
+	workerRPCGracePeriod *atomic.Int64,
 	opt ...Option,
 ) error {
 	const op = "server.(Jobs).RegisterJobs"
@@ -40,8 +40,8 @@ func RegisterJobs(
 	if kms == nil {
 		return errors.New(ctx, errors.InvalidParameter, op, "missing kms")
 	}
-	if isNil(workerStatusGracePeriod) {
-		return errors.New(ctx, errors.InvalidParameter, op, "missing worker status grace period")
+	if isNil(workerRPCGracePeriod) {
+		return errors.New(ctx, errors.InvalidParameter, op, "missing worker RPC grace period")
 	}
 
 	rotateRootsJob, err := newRotateRootsJob(ctx, r, w, kms, opt...)
@@ -52,7 +52,7 @@ func RegisterJobs(
 		return errors.Wrap(ctx, err, op)
 	}
 
-	usbJob, err := NewUpsertWorkerStorageBucketJobFn(ctx, r, w, kms, controllerExt, workerStatusGracePeriod, scheduler)
+	usbJob, err := NewUpsertWorkerStorageBucketJobFn(ctx, r, w, kms, controllerExt, workerRPCGracePeriod, scheduler)
 	if err != nil {
 		return errors.Wrap(ctx, err, op, errors.WithMsg("error creating upsert worker storage bucket job"))
 	}

internal/server/repository_worker.go

@@ -284,8 +284,8 @@ func (r *Repository) UpsertWorkerStatus(ctx context.Context, worker *Worker, opt
 		return "", errors.New(ctx, errors.InvalidParameter, op, "scope id is empty")
 	case worker.PublicId != "":
 		return "", errors.New(ctx, errors.InvalidParameter, op, "worker id is not empty")
-	case worker.GetName() == "" && opts.withKeyId == "":
-		return "", errors.New(ctx, errors.InvalidParameter, op, "worker keyId and reported name are both empty; one is required")
+	case worker.GetName() == "" && opts.withKeyId == "" && opts.withPublicId == "":
+		return "", errors.New(ctx, errors.InvalidParameter, op, "worker keyId, name and public are all empty; one is required")
 	case worker.OperationalState == "":
 		return "", errors.New(ctx, errors.InvalidParameter, op, "worker operational state is empty")
 	case worker.LocalStorageState == "":
@@ -898,7 +898,7 @@ func (r *Repository) DeleteWorkerTags(ctx context.Context, workerId string, work
 // Additionally, it returns the data for the the protocol-aware worker that can
 // handle recording operations, if enabled.
 func (r *Repository) SelectSessionWorkers(ctx context.Context,
-	workerStatusGracePeriod time.Duration,
+	workerRPCGracePeriod time.Duration,
 	t target.Target,
 	host string,
 	ce globals.ControllerExtension,
@@ -907,12 +907,12 @@ func (r *Repository) SelectSessionWorkers(ctx context.Context,
 ) ([]WorkerAddress, string, error) {
 	const op = "server.(Repository).SelectSessionWorkers"
 
-	if workerStatusGracePeriod <= 0 {
-		workerStatusGracePeriod = DefaultLiveness
+	if workerRPCGracePeriod <= 0 {
+		workerRPCGracePeriod = DefaultLiveness
 	}
-	workerStatusGracePeriod = workerStatusGracePeriod.Truncate(time.Second)
+	workerRPCGracePeriod = workerRPCGracePeriod.Truncate(time.Second)
 
-	query := fmt.Sprintf(listSelectSessionWorkers, uint32(workerStatusGracePeriod.Seconds()))
+	query := fmt.Sprintf(listSelectSessionWorkers, uint32(workerRPCGracePeriod.Seconds()))
 	var livingWorkers []*Worker
 	_, err := r.writer.DoTx(
 		ctx,

internal/server/repository_worker_session_info.go

@@ -0,0 +1,37 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package server
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/errors"
+)
+
+// UpsertSessionInfo will update the session info latest request time to the current time for the given worker.
+func (r *Repository) UpsertSessionInfo(ctx context.Context, workerId string, _ ...Option) error {
+	const op = "server.(Repository).UpsertSessionInfo"
+	if workerId == "" {
+		return errors.New(ctx, errors.InvalidParameter, op, "missing worker id")
+	}
+	sessionInfo := NewWorkerSessionInfoRequest(workerId)
+	_, err := r.writer.DoTx(
+		ctx,
+		db.StdRetryCnt,
+		db.ExpBackoff{},
+		func(_ db.Reader, w db.Writer) error {
+			upsert := db.WithOnConflict(&db.OnConflict{
+				Target: db.Columns{"worker_id"},
+				Action: db.SetColumns([]string{"last_request_time"}),
+			})
+			return w.Create(ctx, sessionInfo, upsert)
+		},
+	)
+	if err != nil {
+		return errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("failed for %s", workerId)))
+	}
+	return nil
+}

internal/server/repository_worker_session_info_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package server
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/iam"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpsertSessionInfo(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	testKms := kms.TestKms(t, conn, wrapper)
+	testRepo, err := NewRepository(ctx, rw, rw, testKms)
+	require.NoError(t, err)
+	iamRepo := iam.TestRepo(t, conn, wrapper)
+	iam.TestScopes(t, iamRepo)
+
+	tests := []struct {
+		name           string
+		workerId       string
+		expectedErrMsg string
+	}{
+		{
+			name:           "empty worker id",
+			expectedErrMsg: "missing worker id",
+		},
+		{
+			name:           "worker does not exist",
+			workerId:       "w_dne",
+			expectedErrMsg: "server.(Repository).UpsertSessionInfo: failed for w_dne: db.DoTx: db.Create: wt_public_id_check constraint failed: check constraint violated: integrity violation: error #1000",
+		},
+		{
+			name: "success",
+			workerId: func() string {
+				return TestPkiWorker(t, conn, wrapper).PublicId
+			}(),
+		},
+		{
+			name: "upsert on conflict",
+			workerId: func() string {
+				workerId := TestPkiWorker(t, conn, wrapper).PublicId
+				require.NoError(t, testRepo.UpsertSessionInfo(context.Background(), workerId))
+				return workerId
+			}(),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			err := testRepo.UpsertSessionInfo(context.Background(), tt.workerId)
+			if tt.expectedErrMsg != "" {
+				require.Error(err)
+				assert.ErrorContains(err, tt.expectedErrMsg)
+				return
+			}
+			require.NoError(err)
+		})
+	}
+}

internal/server/repository_worker_test.go

@@ -1661,7 +1661,7 @@ func TestSelectSessionWorkers(t *testing.T) {
 		require.ErrorContains(t, err, "No workers are available to handle this session.")
 	})
 
-	t.Run("invalidWorkerStatusGracePeriod", func(t *testing.T) {
+	t.Run("invalidWorkerRPCGracePeriod", func(t *testing.T) {
 		w1 := server.TestKmsWorker(t, conn, wrapper, server.WithWorkerTags(&server.Tag{Key: "type", Value: "worker1"}))
 		require.NotNil(t, w1)
 		t.Cleanup(func() {
@@ -1688,7 +1688,7 @@ func TestSelectSessionWorkers(t *testing.T) {
 		require.Equal(t, w1.GetAddress(), was[0].Address)
 	})
 
-	t.Run("validWorkerStatusGracePeriod", func(t *testing.T) {
+	t.Run("validWorkerRPCGracePeriod", func(t *testing.T) {
 		w1 := server.TestKmsWorker(t, conn, wrapper, server.WithWorkerTags(&server.Tag{Key: "type", Value: "prod"}))
 		require.NotNil(t, w1)
 		t.Cleanup(func() {

internal/server/service_verified_workers.go

@@ -0,0 +1,52 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package server
+
+import (
+	"context"
+
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/util"
+)
+
+// AuthorizedWorkers contain authorized worker
+// public ids and unmapped worker key ids
+type AuthorizedWorkers struct {
+	WorkerPublicIds      []string
+	UnmappedWorkerKeyIds []string
+}
+
+// VerifyKnownAndUnmappedWorkers verifies the connected worker ids and unmapped worker key ids are known to the controller
+func VerifyKnownAndUnmappedWorkers(
+	ctx context.Context,
+	repo *Repository,
+	workerAuthRepo *WorkerAuthRepositoryStorage,
+	connectedWorkerIds,
+	unmappedWorkerKeyIds []string,
+) (*AuthorizedWorkers, error) {
+	const op = "server.VerifyKnownAndUnmappedWorkers"
+	switch {
+	case util.IsNil(repo):
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "repository is required")
+	case util.IsNil(workerAuthRepo):
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "worker auth repository is required")
+	}
+	var authorizedDownstreams AuthorizedWorkers
+	if len(connectedWorkerIds) > 0 {
+		knownConnectedWorkers, err := repo.VerifyKnownWorkers(ctx, connectedWorkerIds)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error getting known connected worker ids"))
+		}
+		authorizedDownstreams.WorkerPublicIds = knownConnectedWorkers
+	}
+
+	if len(unmappedWorkerKeyIds) > 0 {
+		authorizedKeyIds, err := workerAuthRepo.FilterToAuthorizedWorkerKeyIds(ctx, unmappedWorkerKeyIds)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error getting authorized unmapped worker key ids"))
+		}
+		authorizedDownstreams.UnmappedWorkerKeyIds = authorizedKeyIds
+	}
+	return &authorizedDownstreams, nil
+}

internal/server/service_verified_workers_ext_test.go

@@ -0,0 +1,61 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package server_test
+
+import (
+	"context"
+	"crypto/rand"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/internal/types/scope"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVerifyUnknownAndUnmappedWorkers(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kmsCache := kms.TestKms(t, conn, wrapper)
+	require.NoError(t, kmsCache.CreateKeys(context.Background(), scope.Global.String(), kms.WithRandomReader(rand.Reader)))
+
+	serverRepo, err := server.NewRepository(ctx, rw, rw, kmsCache)
+	require.NoError(t, err)
+	workerAuthRepo, err := server.NewRepositoryStorage(ctx, rw, rw, kmsCache)
+	require.NoError(t, err)
+
+	// Set up resources
+	var w1KeyId, w2KeyId string
+	w1 := server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w1KeyId))
+	_ = server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w2KeyId))
+	w3 := server.TestKmsWorker(t, conn, wrapper)
+
+	t.Run("With 1 PKI, 1 KMS and 1 unmapped worker", func(t *testing.T) {
+		t.Parallel()
+		authzWorkers, err := server.VerifyKnownAndUnmappedWorkers(ctx, serverRepo, workerAuthRepo, []string{w1.PublicId, w3.PublicId, "invalid-worker-id"}, []string{w2KeyId, "invalid-worker-key-id"})
+		require.NoError(t, err)
+		require.Empty(t, cmp.Diff(authzWorkers.WorkerPublicIds, []string{w1.PublicId, w3.PublicId}))
+		require.Empty(t, cmp.Diff(authzWorkers.UnmappedWorkerKeyIds, []string{w2KeyId}))
+	})
+
+	t.Run("missing repository", func(t *testing.T) {
+		t.Parallel()
+		authzWorkers, err := server.VerifyKnownAndUnmappedWorkers(ctx, nil, workerAuthRepo, []string{w1.PublicId, w3.PublicId}, []string{w2KeyId})
+		require.ErrorContains(t, err, "repository is required")
+		require.Nil(t, authzWorkers)
+	})
+
+	t.Run("missing worker auth repository", func(t *testing.T) {
+		t.Parallel()
+		authzWorkers, err := server.VerifyKnownAndUnmappedWorkers(ctx, serverRepo, nil, []string{w1.PublicId, w3.PublicId}, []string{w2KeyId})
+		require.ErrorContains(t, err, "worker auth repository is required")
+		require.Nil(t, authzWorkers)
+	})
+}

internal/server/store/worker.pb.go

@@ -528,6 +528,66 @@ func (x *WorkerStorageBucketCredentialState) GetCheckedAt() *timestamp.Timestamp
 	return nil
 }
 
+// WorkerSessionInfoRequest contains information about a worker and its SessionInfo requests.
+type WorkerSessionInfoRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// worker_id is the public id of the worker.
+	// @inject_tag: `gorm:"primary_key"`
+	WorkerId string `protobuf:"bytes,10,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty" gorm:"primary_key"`
+	// The last_request_time is the last time a worker has reported its session information.
+	// @inject_tag: `gorm:"default:null"`
+	LastRequestTime *timestamp.Timestamp `protobuf:"bytes,20,opt,name=last_request_time,json=lastRequestTime,proto3" json:"last_request_time,omitempty" gorm:"default:null"`
+}
+
+func (x *WorkerSessionInfoRequest) Reset() {
+	*x = WorkerSessionInfoRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_controller_storage_servers_store_v1_worker_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *WorkerSessionInfoRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WorkerSessionInfoRequest) ProtoMessage() {}
+
+func (x *WorkerSessionInfoRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_controller_storage_servers_store_v1_worker_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WorkerSessionInfoRequest.ProtoReflect.Descriptor instead.
+func (*WorkerSessionInfoRequest) Descriptor() ([]byte, []int) {
+	return file_controller_storage_servers_store_v1_worker_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *WorkerSessionInfoRequest) GetWorkerId() string {
+	if x != nil {
+		return x.WorkerId
+	}
+	return ""
+}
+
+func (x *WorkerSessionInfoRequest) GetLastRequestTime() *timestamp.Timestamp {
+	if x != nil {
+		return x.LastRequestTime
+	}
+	return nil
+}
+
 var File_controller_storage_servers_store_v1_worker_proto protoreflect.FileDescriptor
 
 var file_controller_storage_servers_store_v1_worker_proto_rawDesc = []byte{
@@ -617,11 +677,20 @@ var file_controller_storage_servers_store_v1_worker_proto_rawDesc = []byte{
 	0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61,
 	0x67, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x76, 0x31, 0x2e,
 	0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x68, 0x65, 0x63, 0x6b,
-	0x65, 0x64, 0x41, 0x74, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
-	0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75,
-	0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x73,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3b, 0x73, 0x74, 0x6f, 0x72,
-	0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x65, 0x64, 0x41, 0x74, 0x22, 0x8f, 0x01, 0x0a, 0x18, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53,
+	0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0a,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x12, 0x56,
+	0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74,
+	0x69, 0x6d, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74,
+	0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x74,
+	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65,
+	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62,
+	0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
+	0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3b, 0x73, 0x74,
+	0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -636,25 +705,27 @@ func file_controller_storage_servers_store_v1_worker_proto_rawDescGZIP() []byte
 	return file_controller_storage_servers_store_v1_worker_proto_rawDescData
 }
 
-var file_controller_storage_servers_store_v1_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
+var file_controller_storage_servers_store_v1_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
 var file_controller_storage_servers_store_v1_worker_proto_goTypes = []any{
 	(*Worker)(nil),    // 0: controller.storage.servers.store.v1.Worker
 	(*WorkerTag)(nil), // 1: controller.storage.servers.store.v1.WorkerTag
 	(*ApiTag)(nil),    // 2: controller.storage.servers.store.v1.ApiTag
 	(*ConfigTag)(nil), // 3: controller.storage.servers.store.v1.ConfigTag
 	(*WorkerStorageBucketCredentialState)(nil), // 4: controller.storage.servers.store.v1.WorkerStorageBucketCredentialState
-	(*timestamp.Timestamp)(nil),                // 5: controller.storage.timestamp.v1.Timestamp
+	(*WorkerSessionInfoRequest)(nil),           // 5: controller.storage.servers.store.v1.WorkerSessionInfoRequest
+	(*timestamp.Timestamp)(nil),                // 6: controller.storage.timestamp.v1.Timestamp
 }
 var file_controller_storage_servers_store_v1_worker_proto_depIdxs = []int32{
-	5, // 0: controller.storage.servers.store.v1.Worker.create_time:type_name -> controller.storage.timestamp.v1.Timestamp
-	5, // 1: controller.storage.servers.store.v1.Worker.update_time:type_name -> controller.storage.timestamp.v1.Timestamp
-	5, // 2: controller.storage.servers.store.v1.Worker.last_status_time:type_name -> controller.storage.timestamp.v1.Timestamp
-	5, // 3: controller.storage.servers.store.v1.WorkerStorageBucketCredentialState.checked_at:type_name -> controller.storage.timestamp.v1.Timestamp
-	4, // [4:4] is the sub-list for method output_type
-	4, // [4:4] is the sub-list for method input_type
-	4, // [4:4] is the sub-list for extension type_name
-	4, // [4:4] is the sub-list for extension extendee
-	0, // [0:4] is the sub-list for field type_name
+	6, // 0: controller.storage.servers.store.v1.Worker.create_time:type_name -> controller.storage.timestamp.v1.Timestamp
+	6, // 1: controller.storage.servers.store.v1.Worker.update_time:type_name -> controller.storage.timestamp.v1.Timestamp
+	6, // 2: controller.storage.servers.store.v1.Worker.last_status_time:type_name -> controller.storage.timestamp.v1.Timestamp
+	6, // 3: controller.storage.servers.store.v1.WorkerStorageBucketCredentialState.checked_at:type_name -> controller.storage.timestamp.v1.Timestamp
+	6, // 4: controller.storage.servers.store.v1.WorkerSessionInfoRequest.last_request_time:type_name -> controller.storage.timestamp.v1.Timestamp
+	5, // [5:5] is the sub-list for method output_type
+	5, // [5:5] is the sub-list for method input_type
+	5, // [5:5] is the sub-list for extension type_name
+	5, // [5:5] is the sub-list for extension extendee
+	0, // [0:5] is the sub-list for field type_name
 }
 
 func init() { file_controller_storage_servers_store_v1_worker_proto_init() }
@@ -723,6 +794,18 @@ func file_controller_storage_servers_store_v1_worker_proto_init() {
 				return nil
 			}
 		}
+		file_controller_storage_servers_store_v1_worker_proto_msgTypes[5].Exporter = func(v any, i int) any {
+			switch v := v.(*WorkerSessionInfoRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -730,7 +813,7 @@ func file_controller_storage_servers_store_v1_worker_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_controller_storage_servers_store_v1_worker_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   5,
+			NumMessages:   6,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

internal/server/worker_session_info_request.go

@@ -0,0 +1,81 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package server
+
+import (
+	"context"
+
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/db/timestamp"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/oplog"
+	"github.com/hashicorp/boundary/internal/server/store"
+	"google.golang.org/protobuf/proto"
+)
+
+// Ensure WorkerSessionInfoRequest implements interfaces
+var (
+	_ db.VetForWriter         = (*WorkerSessionInfoRequest)(nil)
+	_ oplog.ReplayableMessage = (*WorkerSessionInfoRequest)(nil)
+)
+
+// WorkerSessionInfoRequest contains the last time a worker daemon sent a status update for session information.
+type WorkerSessionInfoRequest struct {
+	*store.WorkerSessionInfoRequest
+	tableName string `gorm:"-"`
+}
+
+// NewWorkerSessionInfoRequest create a new instance of a worker session information request.
+// The LastStatusTime field is set to the current time.
+func NewWorkerSessionInfoRequest(workerId string) *WorkerSessionInfoRequest {
+	return &WorkerSessionInfoRequest{
+		WorkerSessionInfoRequest: &store.WorkerSessionInfoRequest{
+			WorkerId:        workerId,
+			LastRequestTime: timestamp.Now(),
+		},
+	}
+}
+
+// WorkerSessionInfoRequest creates an empty in-memory instance of a worker session information request.
+func AllocWorkerSessionInfoRequest() *WorkerSessionInfoRequest {
+	return &WorkerSessionInfoRequest{
+		WorkerSessionInfoRequest: &store.WorkerSessionInfoRequest{},
+	}
+}
+
+func (w *WorkerSessionInfoRequest) Clone() *WorkerSessionInfoRequest {
+	cp := proto.Clone(w.WorkerSessionInfoRequest)
+	return &WorkerSessionInfoRequest{
+		WorkerSessionInfoRequest: cp.(*store.WorkerSessionInfoRequest),
+	}
+}
+
+// VetForWrite implements db.VetForWrite() interface and validates the worker storage bucket credential state
+func (w *WorkerSessionInfoRequest) VetForWrite(ctx context.Context, _ db.Reader, opType db.OpType, _ ...db.Option) error {
+	const op = "server.(WorkerSessionInfoRequest).VetForWrite"
+	switch {
+	case w == nil:
+		return errors.New(ctx, errors.InvalidParameter, op, "missing worker session info request")
+	case w.WorkerId == "":
+		return errors.New(ctx, errors.InvalidParameter, op, "missing worker id")
+	case w.LastRequestTime == nil:
+		return errors.New(ctx, errors.InvalidParameter, op, "missing last request timestamp")
+	}
+	return nil
+}
+
+// TableName returns the tablename to override the default gorm table name
+func (w *WorkerSessionInfoRequest) TableName() string {
+	if w.tableName != "" {
+		return w.tableName
+	}
+	return "server_worker_session_info_request"
+}
+
+// SetTableName sets the tablename and satisfies the ReplayableMessage
+// interface. If the caller attempts to set the name to "" the name will be
+// reset to the default name.
+func (w *WorkerSessionInfoRequest) SetTableName(n string) {
+	w.tableName = n
+}

internal/session/job_session_cleanup.go

@@ -37,7 +37,7 @@ type sessionConnectionCleanupJob struct {
 	// The amount of time to give disconnected workers before marking their
 	// connections as closed. This should be larger than the liveness setting
 	// for the worker.
-	workerStatusGracePeriod *atomic.Int64
+	workerRPCGracePeriod *atomic.Int64
 
 	// The total number of connections closed in the last run.
 	totalClosed *atomic.Int64
@@ -47,22 +47,22 @@ type sessionConnectionCleanupJob struct {
 func newSessionConnectionCleanupJob(
 	ctx context.Context,
 	writer db.Writer,
-	workerStatusGracePeriod *atomic.Int64,
+	workerRPCGracePeriod *atomic.Int64,
 ) (*sessionConnectionCleanupJob, error) {
 	const op = "session.newNewSessionConnectionCleanupJob"
 	switch {
 	case writer == nil:
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing db writer")
-	case workerStatusGracePeriod == nil:
+	case workerRPCGracePeriod == nil:
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grace period")
-	case workerStatusGracePeriod.Load() == 0:
+	case workerRPCGracePeriod.Load() == 0:
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "grace period is zero")
 	}
 
 	return &sessionConnectionCleanupJob{
-		writer:                  writer,
-		workerStatusGracePeriod: workerStatusGracePeriod,
-		totalClosed:             new(atomic.Int64),
+		writer:               writer,
+		workerRPCGracePeriod: workerRPCGracePeriod,
+		totalClosed:          new(atomic.Int64),
 	}, nil
 }
 
@@ -100,7 +100,7 @@ func (j *sessionConnectionCleanupJob) Run(ctx context.Context, _ time.Duration)
 	j.totalClosed.Store(0)
 
 	// Run the atomic dead worker cleanup job.
-	gracePeriod := time.Duration(j.workerStatusGracePeriod.Load())
+	gracePeriod := time.Duration(j.workerRPCGracePeriod.Load())
 	results, err := j.closeConnectionsForDeadWorkers(ctx, gracePeriod)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)

internal/session/job_session_cleanup_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/scheduler"
 	"github.com/hashicorp/boundary/internal/server"
-	"github.com/hashicorp/boundary/internal/types/scope"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -54,6 +53,19 @@ func TestSessionConnectionCleanupJob(t *testing.T) {
 	worker1 := server.TestKmsWorker(t, conn, wrapper)
 	worker2 := server.TestKmsWorker(t, conn, wrapper)
 
+	updateServer := func(t *testing.T, w *server.Worker) *server.Worker {
+		t.Helper()
+		pubId := w.GetPublicId()
+		w.PublicId = ""
+		wkr, err := server.TestUpsertAndReturnWorker(ctx, t, w, serversRepo, server.WithPublicId(pubId))
+		require.NoError(err)
+		err = serversRepo.UpsertSessionInfo(ctx, pubId)
+		require.NoError(err)
+		return wkr
+	}
+	worker1 = updateServer(t, worker1)
+	worker2 = updateServer(t, worker2)
+
 	// Create a few sessions on each, activate, and authorize a connection
 	var connIds []string
 	connIdsByWorker := make(map[string][]string)
@@ -95,7 +107,7 @@ func TestSessionConnectionCleanupJob(t *testing.T) {
 
 	// Create the job.
 	job, err := newSessionConnectionCleanupJob(ctx, rw, gracePeriod)
-	job.workerStatusGracePeriod = gracePeriod // by-pass factory assert so we dont have to wait so long
+	job.workerRPCGracePeriod = gracePeriod // by-pass factory assert so we dont have to wait so long
 	require.NoError(err)
 
 	// sleep the status grace period.
@@ -103,11 +115,7 @@ func TestSessionConnectionCleanupJob(t *testing.T) {
 
 	// Push an upsert to the first worker so that its status has been
 	// updated.
-	_, err = serversRepo.UpsertWorkerStatus(ctx, server.NewWorker(scope.Global.String(),
-		server.WithName(worker1.GetName()),
-		server.WithAddress(worker1.GetAddress())),
-		server.WithPublicId(worker1.GetPublicId()))
-	require.NoError(err)
+	worker1 = updateServer(t, worker1)
 
 	// Run the job.
 	require.NoError(job.Run(ctx, 0))
@@ -280,6 +288,8 @@ func TestCloseConnectionsForDeadWorkers(t *testing.T) {
 		w.PublicId = ""
 		wkr, err := server.TestUpsertAndReturnWorker(ctx, t, w, serversRepo, server.WithPublicId(pubId))
 		require.NoError(err)
+		err = serversRepo.UpsertSessionInfo(ctx, pubId)
+		require.NoError(err)
 		return wkr
 	}
 
@@ -372,13 +382,11 @@ func TestCloseConnectionsForDeadWorkers(t *testing.T) {
 		require.NoError(err)
 		// Assert that we have one result with the appropriate ID and
 		// number of connections closed. Due to how things are
-		require.Equal([]closeConnectionsForDeadWorkersResult{
-			{
-				WorkerId:                worker1.PublicId,
-				LastUpdateTime:          timestampPbAsUTC(t, worker1.GetLastStatusTime().AsTime()),
-				NumberConnectionsClosed: 12, // 18 per server, with 6 closed already
-			},
-		}, result)
+		require.Len(result, 1)
+		require.Equal(worker1.PublicId, result[0].WorkerId)
+		require.Equal(12, result[0].NumberConnectionsClosed)
+		require.WithinDuration(timestampPbAsUTC(t, worker1.GetLastStatusTime().AsTime()), result[0].LastUpdateTime, time.Second)
+
 		// Expect all connections closed on worker1
 		requireConnectionStatus(t, worker1ConnIds, true)
 		// Expect appropriate split connection state on worker2
@@ -396,18 +404,19 @@ func TestCloseConnectionsForDeadWorkers(t *testing.T) {
 		result, err := job.closeConnectionsForDeadWorkers(ctx, gracePeriod)
 		require.NoError(err)
 		// Assert that we have one result with the appropriate ID and number of connections closed.
-		require.ElementsMatch([]closeConnectionsForDeadWorkersResult{
-			{
-				WorkerId:                worker2.PublicId,
-				LastUpdateTime:          timestampPbAsUTC(t, worker2.GetLastStatusTime().AsTime()),
-				NumberConnectionsClosed: 12, // 18 per server, with 6 closed already
-			},
-			{
-				WorkerId:                worker3.PublicId,
-				LastUpdateTime:          timestampPbAsUTC(t, worker3.GetLastStatusTime().AsTime()),
-				NumberConnectionsClosed: 12, // 18 per server, with 6 closed already
-			},
-		}, result)
+		expectedWorkers := map[string]*server.Worker{
+			worker2.PublicId: worker2,
+			worker3.PublicId: worker3,
+		}
+		require.Len(result, len(expectedWorkers))
+		for _, r := range result {
+			expectedWorker, ok := expectedWorkers[r.WorkerId]
+			require.True(ok)
+			require.Equal(expectedWorker.PublicId, r.WorkerId)
+			require.Equal(12, r.NumberConnectionsClosed)
+			require.WithinDuration(timestampPbAsUTC(t, expectedWorker.GetLastStatusTime().AsTime()), r.LastUpdateTime, time.Second)
+		}
+
 		// Expect all connections closed on worker1
 		requireConnectionStatus(t, worker1ConnIds, true)
 		// Expect all connections closed on worker2

internal/session/jobs.go

@@ -18,14 +18,14 @@ import (
 const deleteTerminatedThreshold = time.Hour
 
 // RegisterJobs registers session related jobs with the provided scheduler.
-func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writer, r db.Reader, k *kms.Kms, workerStatusGracePeriod *atomic.Int64) error {
+func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writer, r db.Reader, k *kms.Kms, workerRPCGracePeriod *atomic.Int64) error {
 	const op = "session.RegisterJobs"
 
-	if workerStatusGracePeriod == nil {
+	if workerRPCGracePeriod == nil {
 		return errors.New(ctx, errors.InvalidParameter, op, "nil grace period")
 	}
 
-	sessionConnectionCleanupJob, err := newSessionConnectionCleanupJob(ctx, w, workerStatusGracePeriod)
+	sessionConnectionCleanupJob, err := newSessionConnectionCleanupJob(ctx, w, workerRPCGracePeriod)
 	if err != nil {
 		return fmt.Errorf("error creating session cleanup job: %w", err)
 	}

internal/session/query.go

@@ -297,11 +297,11 @@ where
    with
    dead_workers (worker_id, last_update_time) as (
          select
-			w.public_id as worker_id,
-			w.last_status_time as last_update_time
-           from server_worker w
+			w.worker_id as worker_id,
+			w.last_request_time as last_update_time
+           from server_worker_session_info_request w
           where
-			w.last_status_time < wt_sub_seconds_from_now(@grace_period_seconds)
+			w.last_request_time < wt_sub_seconds_from_now(@grace_period_seconds)
    ),
    closed_connections (connection_id, worker_id) as (
          update session_connection

internal/session/repository_connection.go

@@ -108,7 +108,7 @@ func (r *ConnectionRepository) updateBytesUpBytesDown(ctx context.Context, conns
 					// we shouldn't update bytes up and down if the connection
 					// has already been closed. This also guards against
 					// potential data races where a connection closure request
-					// and a worker status update happen close to each other in
+					// and a worker statistics update happen close to each other in
 					// terms of timing.
 					db.WithWhere("closed_reason is null"),
 				)

internal/session/service_statistics.go

@@ -0,0 +1,47 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package session
+
+import (
+	"context"
+
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/event"
+)
+
+// CloseOrphanedConnections closes the orphaned connections for a given worker.
+func CloseOrphanedConnections(ctx context.Context, repo *ConnectionRepository, workerId string, connections []*Connection) ([]string, error) {
+	const op = "session.CloseOrphanedConnections"
+	if repo == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing connection repository")
+	}
+	if workerId == "" {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing worker id")
+	}
+	connectionIds := []string{}
+	for _, c := range connections {
+		connectionIds = append(connectionIds, c.GetPublicId())
+	}
+	closed, err := repo.closeOrphanedConnections(ctx, workerId, connectionIds)
+	if len(closed) > 0 {
+		event.WriteSysEvent(ctx, op, "marked unclaimed connections as closed", "worker_id", workerId, "count", len(closed))
+	}
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("failed to close orphaned connections for worker %q", workerId))
+	}
+	return closed, nil
+}
+
+// UpdateConnectionBytesUpDown processes the connection statistics for a given worker.
+func UpdateConnectionBytesUpDown(ctx context.Context, repo *ConnectionRepository, connections []*Connection) error {
+	const op = "session.UpdateConnectionBytesUpDown"
+	if repo == nil {
+		return errors.New(ctx, errors.InvalidParameter, op, "missing connection repository")
+	}
+	err := repo.updateBytesUpBytesDown(ctx, connections...)
+	if err != nil {
+		return errors.Wrap(ctx, err, op, errors.WithMsg("failed to update bytes up and down for worker connection statistics"))
+	}
+	return nil
+}

internal/session/service_statistics_test.go

@@ -0,0 +1,534 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package session
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/authtoken"
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/host/static"
+	"github.com/hashicorp/boundary/internal/iam"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/internal/server/store"
+	"github.com/hashicorp/boundary/internal/target"
+	"github.com/hashicorp/boundary/internal/target/tcp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCloseOrphanedConnections(t *testing.T) {
+	t.Parallel()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	testKms := kms.TestKms(t, conn, wrapper)
+	connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms)
+	require.NoError(t, err)
+	sessRepo, err := NewRepository(context.Background(), rw, rw, testKms)
+	require.NoError(t, err)
+
+	serverRepo, _ := server.NewRepository(context.Background(), rw, rw, testKms)
+	_, err = serverRepo.UpsertController(context.Background(), &store.Controller{
+		PrivateId: "test_controller1",
+		Address:   "127.0.0.1",
+	})
+	require.NoError(t, err)
+
+	cases := []struct {
+		name       string
+		connRepo   *ConnectionRepository
+		setup      func() (workerId string, conns []*Connection, expectedClosedConnections []string)
+		wantErrMsg string
+	}{
+		{
+			name: "missing connection repository",
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				return
+			},
+			wantErrMsg: "session.CloseOrphanedConnections: missing connection repository: parameter violation: error #100",
+		},
+		{
+			name:     "empty worker id",
+			connRepo: &ConnectionRepository{},
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				return
+			},
+			wantErrMsg: "session.CloseOrphanedConnections: missing worker id: parameter violation: error #100",
+		},
+		{
+			name:     "nil sessions",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				workerId = w.GetPublicId()
+				return
+			},
+		},
+		{
+			name:     "empty connections",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				workerId = w.GetPublicId()
+				conns = make([]*Connection, 0)
+				return
+			},
+		},
+		{
+			name:     "ignore unknown connection",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				workerId = w.GetPublicId()
+				conns = []*Connection{
+					{
+						PublicId: "c_dne",
+					},
+				}
+				return
+			},
+		},
+		{
+			name:     "session already canceled",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+				authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				userId := authToken.GetIamUserId()
+				hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+				tar := tcp.TestTarget(
+					context.Background(),
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hostSet.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+				s1 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+				require.NoError(t, err)
+				require.NoError(t, err)
+				_, err = connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
+				require.NoError(t, err)
+				_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
+				require.NoError(t, err)
+				workerId = w.GetPublicId()
+				conns = []*Connection{}
+				return
+			},
+		},
+		{
+			name:     "active session",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+				authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				userId := authToken.GetIamUserId()
+				hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+				tar := tcp.TestTarget(
+					context.Background(),
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hostSet.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+				s1 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+				require.NoError(t, err)
+				c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId())
+				require.NoError(t, err)
+				workerId = w.GetPublicId()
+				conns = []*Connection{
+					{
+						PublicId: c1.GetPublicId(),
+					},
+				}
+				return
+			},
+		},
+		{
+			name:     "closed session",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+				authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				userId := authToken.GetIamUserId()
+				hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+				tar := tcp.TestTarget(
+					context.Background(),
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hostSet.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+				s1 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+				require.NoError(t, err)
+				c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
+				require.NoError(t, err)
+				_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
+				require.NoError(t, err)
+
+				workerId = w.GetPublicId()
+				conns = []*Connection{
+					{
+						PublicId: c1.GetPublicId(),
+					},
+				}
+				return
+			},
+		},
+		{
+			name:     "multiple closed sessions",
+			connRepo: connRepo,
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+				authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				userId := authToken.GetIamUserId()
+				hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+				tar := tcp.TestTarget(
+					context.Background(),
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hostSet.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+				s1 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+				require.NoError(t, err)
+				c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
+				require.NoError(t, err)
+				_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
+				require.NoError(t, err)
+
+				s2 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t))
+				require.NoError(t, err)
+				c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
+				require.NoError(t, err)
+				_, err = sessRepo.CancelSession(context.Background(), s2.PublicId, s2.Version)
+				require.NoError(t, err)
+
+				workerId = w.GetPublicId()
+				conns = []*Connection{
+					{
+						PublicId: c1.GetPublicId(),
+					},
+					{
+						PublicId: c2.GetPublicId(),
+					},
+				}
+				return
+			},
+		},
+		{
+			name: "close orphaned connections",
+			connRepo: func() *ConnectionRepository {
+				connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms, WithWorkerStateDelay(0))
+				require.NoError(t, err)
+				return connRepo
+			}(),
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+				authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				userId := authToken.GetIamUserId()
+				hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+				tar := tcp.TestTarget(
+					context.Background(),
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hostSet.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+				s1 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+				require.NoError(t, err)
+				c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
+				require.NoError(t, err)
+				s2 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t))
+				require.NoError(t, err)
+				c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
+				require.NoError(t, err)
+				require.NotEqual(t, c1.PublicId, c2.PublicId)
+				workerId = w.GetPublicId()
+				conns = []*Connection{
+					{
+						PublicId: c2.GetPublicId(),
+					},
+				}
+				expectedClosedConnections = []string{c1.PublicId}
+				return
+			},
+		},
+		{
+			name: "multiple sessions and orphaned connections",
+			connRepo: func() *ConnectionRepository {
+				connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms, WithWorkerStateDelay(0))
+				require.NoError(t, err)
+				return connRepo
+			}(),
+			setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
+				w := server.TestKmsWorker(t, conn, wrapper)
+				org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+				authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+				userId := authToken.GetIamUserId()
+				hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+				hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+				static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+				tar := tcp.TestTarget(
+					context.Background(),
+					t, conn, prj.GetPublicId(), "test",
+					target.WithHostSources([]string{hostSet.GetPublicId()}),
+					target.WithSessionConnectionLimit(-1),
+				)
+				s1 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+				require.NoError(t, err)
+				c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
+				require.NoError(t, err)
+				_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
+				require.NoError(t, err)
+
+				s2 := TestSession(t, conn, wrapper, ComposedOf{
+					UserId:          userId,
+					HostId:          host.GetPublicId(),
+					TargetId:        tar.GetPublicId(),
+					HostSetId:       hostSet.GetPublicId(),
+					AuthTokenId:     authToken.GetPublicId(),
+					ProjectId:       prj.GetPublicId(),
+					Endpoint:        "tcp://127.0.0.1:22",
+					ConnectionLimit: 10,
+				})
+				s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t))
+				require.NoError(t, err)
+				c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
+				require.NoError(t, err)
+				c3, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
+				require.NoError(t, err)
+				_, err = sessRepo.CancelSession(context.Background(), s2.PublicId, s2.Version)
+				require.NoError(t, err)
+
+				workerId = w.GetPublicId()
+				conns = []*Connection{
+					{
+						PublicId: c1.GetPublicId(),
+					},
+					{
+						PublicId: c2.GetPublicId(),
+					},
+				}
+				expectedClosedConnections = []string{c3.PublicId}
+				return
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			require.NotNil(tc.setup)
+			workerId, stats, expectedClosedConnections := tc.setup()
+			actualClosedConnections, err := CloseOrphanedConnections(context.Background(), tc.connRepo, workerId, stats)
+			if tc.wantErrMsg != "" {
+				require.Error(err)
+				assert.ErrorContains(err, tc.wantErrMsg)
+				assert.Empty(actualClosedConnections)
+				return
+			}
+			require.NoError(err)
+			assert.Equal(expectedClosedConnections, actualClosedConnections)
+			for _, connectionId := range expectedClosedConnections {
+				conn, err := connRepo.LookupConnection(context.Background(), connectionId)
+				require.NoError(err)
+				assert.Equal(ConnectionSystemError, ClosedReason(conn.ClosedReason))
+				assert.Equal(StatusClosed, ConnectionStatusFromString(conn.Status))
+			}
+		})
+	}
+}
+
+func TestUpdateConnectionBytesUpDown(t *testing.T) {
+	t.Parallel()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	testKms := kms.TestKms(t, conn, wrapper)
+	connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms)
+	require.NoError(t, err)
+	sessRepo, err := NewRepository(context.Background(), rw, rw, testKms)
+	require.NoError(t, err)
+
+	serverRepo, _ := server.NewRepository(context.Background(), rw, rw, testKms)
+	_, err = serverRepo.UpsertController(context.Background(), &store.Controller{
+		PrivateId: "test_controller1",
+		Address:   "127.0.0.1",
+	})
+	require.NoError(t, err)
+
+	t.Run("nil connection repo", func(t *testing.T) {
+		assert, require := assert.New(t), require.New(t)
+		err := UpdateConnectionBytesUpDown(context.Background(), nil, nil)
+		require.Error(err)
+		assert.ErrorContains(err, "missing connection repository")
+	})
+
+	t.Run("bytes up and bytes down", func(t *testing.T) {
+		assert, require := assert.New(t), require.New(t)
+		w := server.TestKmsWorker(t, conn, wrapper)
+		org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+		authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
+		userId := authToken.GetIamUserId()
+		hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
+		hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
+		host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
+		static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
+		tar := tcp.TestTarget(
+			context.Background(),
+			t, conn, prj.GetPublicId(), "test",
+			target.WithHostSources([]string{hostSet.GetPublicId()}),
+			target.WithSessionConnectionLimit(-1),
+		)
+		s1 := TestSession(t, conn, wrapper, ComposedOf{
+			UserId:          userId,
+			HostId:          host.GetPublicId(),
+			TargetId:        tar.GetPublicId(),
+			HostSetId:       hostSet.GetPublicId(),
+			AuthTokenId:     authToken.GetPublicId(),
+			ProjectId:       prj.GetPublicId(),
+			Endpoint:        "tcp://127.0.0.1:22",
+			ConnectionLimit: 10,
+		})
+		s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
+		require.NoError(err)
+		c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId())
+		require.NoError(err)
+
+		// write to active session
+		var expectedBytesUp int64 = 1024
+		var expectedBytesDown int64 = 2048
+		conns := []*Connection{
+			{
+				PublicId:  c1.GetPublicId(),
+				BytesUp:   expectedBytesUp,
+				BytesDown: expectedBytesDown,
+			},
+		}
+		err = UpdateConnectionBytesUpDown(context.Background(), connRepo, conns)
+		require.NoError(err)
+		conn, err := connRepo.LookupConnection(context.Background(), c1.PublicId)
+		require.NoError(err)
+		assert.Equal(StatusAuthorized, ConnectionStatusFromString(conn.Status))
+		assert.Equal(expectedBytesUp, conn.BytesUp)
+		assert.Equal(expectedBytesDown, conn.BytesDown)
+
+		// write to canceled session
+		_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
+		require.NoError(err)
+		expectedBytesUp = 4096
+		expectedBytesDown = 8192
+		conns = []*Connection{
+			{
+				PublicId:  c1.GetPublicId(),
+				BytesUp:   expectedBytesUp,
+				BytesDown: expectedBytesDown,
+			},
+		}
+		err = UpdateConnectionBytesUpDown(context.Background(), connRepo, conns)
+		require.NoError(err)
+		conn, err = connRepo.LookupConnection(context.Background(), c1.PublicId)
+		require.NoError(err)
+		assert.Equal(expectedBytesUp, conn.BytesUp)
+		assert.Equal(expectedBytesDown, conn.BytesDown)
+	})
+}

internal/tests/api/credentials/credentials_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/internal/tests/helper"
 	"github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -462,7 +463,7 @@ func TestUpdateAfterKeyRotation(t *testing.T) {
 		WorkerAuthKms:    tc.Config().WorkerAuthKms,
 		Name:             "worker",
 	})
-	require.NoError(w.Worker().WaitForNextSuccessfulStatusUpdate())
+	helper.ExpectWorkers(t, tc, w)
 
 	// Authorize session, requires decrypting json credential
 	_, err = tgClient.AuthorizeSession(ctx, targ.Item.Id)

internal/tests/api/proxy/proxy_test.go

@@ -48,26 +48,20 @@ func TestConnectionsLeft(t *testing.T) {
 	conf, err := config.DevController()
 	require.NoError(err)
 	c1 := controller.NewTestController(t, &controller.TestControllerOpts{
-		Config:                          conf,
-		InitialResourcesSuffix:          "1234567890",
-		Logger:                          logger.Named("c1"),
-		WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
+		Config:                 conf,
+		InitialResourcesSuffix: "1234567890",
+		Logger:                 logger.Named("c1"),
+		WorkerRPCGracePeriod:   helper.DefaultControllerRPCGracePeriod,
 	})
-	defer c1.Shutdown()
 	helper.ExpectWorkers(t, c1)
 
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:                       c1.Config().WorkerAuthKms,
-		InitialUpstreams:                    c1.ClusterAddrs(),
-		Logger:                              logger.Named("w1"),
-		SuccessfulStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
-		Name:                                "w1",
+		WorkerAuthKms:    c1.Config().WorkerAuthKms,
+		InitialUpstreams: c1.ClusterAddrs(),
+		Logger:           logger.Named("w1"),
+		SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
+		Name: "w1",
 	})
-	defer w1.Shutdown()
-	err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
-	require.NoError(err)
-	err = c1.WaitForNextWorkerStatusUpdate(w1.Name())
-	require.NoError(err)
 	helper.ExpectWorkers(t, c1, w1)
 
 	// Connect target

internal/tests/api/targets/target_test.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/aliases"
@@ -27,6 +26,7 @@ import (
 	"github.com/hashicorp/boundary/internal/daemon/worker"
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/iam"
+	"github.com/hashicorp/boundary/internal/tests/helper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -862,10 +862,12 @@ func TestCreateTarget_SlashesInName(t *testing.T) {
 	tc := controller.NewTestController(t, nil)
 
 	tw, _ := worker.NewAuthorizedPkiTestWorker(t, tc.ServersRepo(), "test", tc.ClusterAddrs())
-	require.NotNil(t, tw)
+	require.NotNil(tw)
 
-	// Wait for worker to become ready
-	time.Sleep(10 * time.Second)
+	helper.ExpectWorkers(t, tc, tw)
+	// Wait for an extra routing info update for good measure
+	// TODO(https://hashicorp.atlassian.net/browse/ICU-16124): why is this necessary?
+	require.NoError(tw.Worker().WaitForNextSuccessfulRoutingInfoUpdate())
 
 	client := tc.Client()
 	token := tc.Token()

internal/tests/cluster/parallel/ipv6_listener_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/scopes"
@@ -55,10 +56,11 @@ func TestIPv6Listener(t *testing.T) {
 	require.NoError(err)
 
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		Config:           wconf,
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
-		Logger:           logger.Named("w1"),
+		Config:            wconf,
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
+		Logger:            logger.Named("w1"),
+		WorkerRPCInterval: time.Second,
 	})
 
 	wg.Add(2)

internal/tests/cluster/parallel/multi_controller_worker_test.go

@@ -50,9 +50,10 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 	wg.Wait()
 
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
-		Logger:           logger.Named("w1"),
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
+		Logger:            logger.Named("w1"),
+		WorkerRPCInterval: time.Second,
 	})
 
 	wg.Add(2)
@@ -95,9 +96,10 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 	wg.Wait()
 
 	w3 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w3"),
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  c1.ClusterAddrs(),
+		Logger:            logger.Named("w3"),
+		WorkerRPCInterval: time.Second,
 	})
 
 	wg.Add(2)
@@ -151,22 +153,28 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) {
 
 	initialUpstreams := append(c1.ClusterAddrs(), "127.0.0.9")
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:                       c1.Config().WorkerAuthKms,
-		InitialUpstreams:                    initialUpstreams,
-		Logger:                              logger.Named("w1"),
-		SuccessfulStatusGracePeriodDuration: 1 * time.Second,
+		WorkerAuthKms:    c1.Config().WorkerAuthKms,
+		InitialUpstreams: initialUpstreams,
+		Logger:           logger.Named("w1"),
+		SuccessfulControllerRPCGracePeriodDuration: 1 * time.Second,
+		WorkerRPCInterval:                          time.Second,
 	})
 
-	// Wait for worker to send status
-	helper.ExpectWorkers(t, c1, w1)
-
-	// Upstreams should be equivalent to the controller cluster addr after status updates
+	// Wait for worker to send routing info
 	require.Eventually(func() bool {
-		if w1.Worker().LastStatusSuccess() == nil {
+		if err := w1.Worker().WaitForNextSuccessfulRoutingInfoUpdate(); err != nil {
 			return false
 		}
-		return slices.Equal(c1.ClusterAddrs(), w1.Worker().LastStatusSuccess().LastCalculatedUpstreams)
-	}, 4*time.Second, 250*time.Millisecond)
+		return true
+	}, 30*time.Second, time.Second)
+
+	// Upstreams should be equivalent to the controller cluster addr after routing info updates
+	require.Eventually(func() bool {
+		if w1.Worker().LastRoutingInfoSuccess() == nil {
+			return false
+		}
+		return slices.Equal(c1.ClusterAddrs(), w1.Worker().LastRoutingInfoSuccess().LastCalculatedUpstreams)
+	}, 30*time.Second, time.Second)
 
 	// Bring down the controller
 	wg := new(sync.WaitGroup)
@@ -178,7 +186,7 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) {
 
 	// Upstreams should now match initial upstreams
 	require.Eventually(func() bool {
-		return slices.Equal(initialUpstreams, w1.Worker().LastStatusSuccess().LastCalculatedUpstreams)
-	}, 4*time.Second, 250*time.Millisecond)
+		return slices.Equal(initialUpstreams, w1.Worker().LastRoutingInfoSuccess().LastCalculatedUpstreams)
+	}, 30*time.Second, time.Second)
 	wg.Wait()
 }

internal/tests/cluster/parallel/unix_listener_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/scopes"
@@ -36,9 +37,9 @@ func TestUnixListener(t *testing.T) {
 	tempDir, err := os.MkdirTemp("", "boundary-unix-listener-test")
 	require.NoError(err)
 
-	defer func() {
+	t.Cleanup(func() {
 		require.NoError(os.RemoveAll(tempDir))
-	}()
+	})
 
 	for _, l := range conf.Listeners {
 		switch l.Purpose[0] {
@@ -81,16 +82,16 @@ func TestUnixListener(t *testing.T) {
 	require.NoError(err)
 
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		Config:           wconf,
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w1"),
+		Config:            wconf,
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  c1.ClusterAddrs(),
+		Logger:            logger.Named("w1"),
+		WorkerRPCInterval: time.Second,
 	})
 
 	helper.ExpectWorkers(t, c1, w1)
 
 	require.NoError(w1.Worker().Shutdown())
-
 	helper.ExpectWorkers(t, c1)
 
 	require.NoError(c1.Controller().Shutdown())

internal/tests/cluster/sequential/session_cleanup_test.go

@@ -14,7 +14,6 @@ import (
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/daemon/controller"
 	"github.com/hashicorp/boundary/internal/daemon/worker"
-	"github.com/hashicorp/boundary/internal/event"
 	"github.com/hashicorp/boundary/internal/server"
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/boundary/internal/tests/helper"
@@ -51,26 +50,19 @@ var timeoutBurdenCases = []timeoutBurdenType{timeoutBurdenTypeDefault, timeoutBu
 
 func controllerGracePeriod(ty timeoutBurdenType) time.Duration {
 	if ty == timeoutBurdenTypeWorker {
-		return helper.DefaultWorkerStatusGracePeriod * 10
+		return helper.DefaultControllerRPCGracePeriod * 10
 	}
 
-	return helper.DefaultWorkerStatusGracePeriod
-}
-
-func workerGracePeriod(ty timeoutBurdenType) time.Duration {
-	return helper.DefaultWorkerStatusGracePeriod
+	return helper.DefaultControllerRPCGracePeriod
 }
 
 // TestSessionCleanup is the main test for session cleanup, and
 // dispatches to the individual subtests.
 func TestSessionCleanup(t *testing.T) {
-	for _, burdenCase := range timeoutBurdenCases {
-		burdenCase := burdenCase
-		t.Run(string(burdenCase), func(t *testing.T) {
-			t.Run("single_controller", testWorkerSessionCleanupSingle(burdenCase))
-			t.Run("multi_controller", testWorkerSessionCleanupMulti(burdenCase))
-		})
-	}
+	t.Run("default/single_controller", testWorkerSessionCleanupSingle("default"))
+	t.Run("default/multi_controller", testWorkerSessionCleanupMulti("default"))
+	t.Run("worker/single_controller", testWorkerSessionCleanupSingle("worker"))
+	t.Run("worker/multi_controller", testWorkerSessionCleanupMulti("worker"))
 }
 
 func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testing.T) {
@@ -90,11 +82,13 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
 		pl, err := net.Listen("tcp", "[::1]:0")
 		require.NoError(err)
 		c1 := controller.NewTestController(t, &controller.TestControllerOpts{
-			Config:                          conf,
-			InitialResourcesSuffix:          "1234567890",
-			Logger:                          logger.Named("c1"),
-			PublicClusterAddr:               pl.Addr().String(),
-			WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase),
+			Config:                 conf,
+			InitialResourcesSuffix: "1234567890",
+			Logger:                 logger.Named("c1"),
+			PublicClusterAddr:      pl.Addr().String(),
+			WorkerRPCGracePeriod:   controllerGracePeriod(burdenCase),
+			// Run the scheduler more often to speed up cleanup of orphaned connections
+			SchedulerRunJobInterval: time.Second,
 		})
 
 		helper.ExpectWorkers(t, c1)
@@ -113,10 +107,11 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
 		require.NotEmpty(t, proxy.ListenerAddr())
 
 		w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-			WorkerAuthKms:                       c1.Config().WorkerAuthKms,
-			InitialUpstreams:                    []string{proxy.ListenerAddr()},
-			Logger:                              logger.Named("w1"),
-			SuccessfulStatusGracePeriodDuration: workerGracePeriod(burdenCase),
+			WorkerAuthKms:    c1.Config().WorkerAuthKms,
+			InitialUpstreams: []string{proxy.ListenerAddr()},
+			Logger:           logger.Named("w1"),
+			SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
+			WorkerRPCInterval:                          time.Second,
 		})
 
 		helper.ExpectWorkers(t, c1, w1)
@@ -147,11 +142,15 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
 		sConn := sess.Connect(ctx, t)
 
 		// Run initial send/receive test, make sure things are working
-		event.WriteSysEvent(ctx, op, "running initial send/recv test")
+		t.Log("running initial send/recv test")
 		sConn.TestSendRecvAll(t)
 
+		// Wait for a session info to be sent to the server, so the controller has
+		// at least one record of the connection.
+		w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
+
 		// Kill the link
-		event.WriteSysEvent(ctx, op, "pausing controller/worker link")
+		t.Log("pausing controller/worker link")
 		proxy.Pause()
 
 		// Wait for failure connection state (depends on burden case)
@@ -171,15 +170,9 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
 		sConn.TestSendRecvFail(t)
 
 		// Resume the connection, and reconnect.
-		event.WriteSysEvent(ctx, op, "resuming controller/worker link")
+		t.Log("resuming controller/worker link")
 		proxy.Resume()
-		require.Eventually(func() bool {
-			err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
-			if err != nil {
-				return false
-			}
-			return true
-		}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
+		helper.ExpectWorkers(t, c1, w1)
 
 		// Do something post-reconnect depending on burden case. Note in
 		// the default case, both worker and controller should be
@@ -195,7 +188,7 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
 		}
 
 		// Proceed with new connection test
-		event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link")
+		t.Log("connecting to new session after resuming controller/worker link")
 		sess = helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup()
 		sConn = sess.Connect(ctx, t)
 		sConn.TestSendRecvAll(t)
@@ -222,11 +215,11 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
 		pl1, err := net.Listen("tcp", "[::1]:0")
 		require.NoError(err)
 		c1 := controller.NewTestController(t, &controller.TestControllerOpts{
-			Config:                          conf1,
-			InitialResourcesSuffix:          "1234567890",
-			Logger:                          logger.Named("c1"),
-			PublicClusterAddr:               pl1.Addr().String(),
-			WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase),
+			Config:                 conf1,
+			InitialResourcesSuffix: "1234567890",
+			Logger:                 logger.Named("c1"),
+			PublicClusterAddr:      pl1.Addr().String(),
+			WorkerRPCGracePeriod:   controllerGracePeriod(burdenCase),
 		})
 
 		// ******************
@@ -235,9 +228,9 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
 		pl2, err := net.Listen("tcp", "[::1]:0")
 		require.NoError(err)
 		c2 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
-			Logger:                          logger.Named("c2"),
-			PublicClusterAddr:               pl2.Addr().String(),
-			WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase),
+			Logger:               logger.Named("c2"),
+			PublicClusterAddr:    pl2.Addr().String(),
+			WorkerRPCGracePeriod: controllerGracePeriod(burdenCase),
 		})
 
 		wg := new(sync.WaitGroup)
@@ -286,11 +279,13 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
 		// ** Worker **
 		// ************
 		w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-			WorkerAuthKms:                       c1.Config().WorkerAuthKms,
-			InitialUpstreams:                    []string{p1.ListenerAddr(), p2.ListenerAddr()},
-			Logger:                              logger.Named("w1"),
-			SuccessfulStatusGracePeriodDuration: workerGracePeriod(burdenCase),
+			WorkerAuthKms:    c1.Config().WorkerAuthKms,
+			InitialUpstreams: []string{p1.ListenerAddr(), p2.ListenerAddr()},
+			Logger:           logger.Named("w1"),
+			SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
+			WorkerRPCInterval:                          time.Second,
 		})
+
 		wg.Add(2)
 		go func() {
 			defer wg.Done()
@@ -328,39 +323,30 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
 		sConn := sess.Connect(ctx, t)
 
 		// Run initial send/receive test, make sure things are working
-		event.WriteSysEvent(ctx, op, "running initial send/recv test")
+		t.Log("running initial send/recv test")
 		sConn.TestSendRecvAll(t)
 
+		// Wait for a session info to be sent to the server, so the controller has
+		// at least one record of the connection.
+		w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
+
 		// Kill connection to first controller, and run test again, should
 		// pass, deferring to other controller. Wait for the next
 		// successful status report to ensure this.
-		event.WriteSysEvent(ctx, op, "pausing link to controller #1")
+		t.Log("pausing link to controller #1")
 		p1.Pause()
-		require.Eventually(func() bool {
-			err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
-			if err != nil {
-				return false
-			}
-			return true
-		}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
+		w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
 		sConn.TestSendRecvAll(t)
 
 		// Resume first controller, pause second. This one should work too.
-		event.WriteSysEvent(ctx, op, "pausing link to controller #2, resuming #1")
+		t.Log("pausing link to controller #2, resuming #1")
 		p1.Resume()
 		p2.Pause()
-		require.Eventually(func() bool {
-			err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
-			if err != nil {
-				return false
-			}
-			return true
-		}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
-		sConn.TestSendRecvAll(t)
+		w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
 
 		// Kill the first controller connection again. This one should fail
 		// due to lack of any connection.
-		event.WriteSysEvent(ctx, op, "pausing link to controller #1 again, both connections should be offline")
+		t.Log("pausing link to controller #1 again, both connections should be offline")
 		p1.Pause()
 
 		// Wait for failure connection state (depends on burden case)
@@ -382,16 +368,10 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
 		sConn.TestSendRecvFail(t)
 
 		// Finally resume both, try again. Should behave as per normal.
-		event.WriteSysEvent(ctx, op, "resuming connections to both controllers")
+		t.Log("resuming connections to both controllers")
 		p1.Resume()
 		p2.Resume()
-		require.Eventually(func() bool {
-			err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
-			if err != nil {
-				return false
-			}
-			return true
-		}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
+		w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
 
 		// Do something post-reconnect depending on burden case. Note in
 		// the default case, both worker and controller should be
@@ -407,7 +387,7 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
 		}
 
 		// Proceed with new connection test
-		event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link")
+		t.Log("connecting to new session after resuming controller/worker link")
 		sess = helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup()
 		sConn = sess.Connect(ctx, t)
 		sConn.TestSendRecvAll(t)

internal/tests/cluster/sequential/worker_bytesupdown_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"net"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/api/sessions"
 	"github.com/hashicorp/boundary/api/targets"
@@ -45,10 +46,10 @@ func TestWorkerBytesUpDown(t *testing.T) {
 	}
 
 	c1 := controller.NewTestController(t, &controller.TestControllerOpts{
-		Config:                          conf,
-		InitialResourcesSuffix:          "1234567890",
-		Logger:                          logger.Named("c1"),
-		WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
+		Config:                 conf,
+		InitialResourcesSuffix: "1234567890",
+		Logger:                 logger.Named("c1"),
+		WorkerRPCGracePeriod:   helper.DefaultControllerRPCGracePeriod,
 	})
 
 	helper.ExpectWorkers(t, c1)
@@ -65,11 +66,12 @@ func TestWorkerBytesUpDown(t *testing.T) {
 	t.Cleanup(func() { _ = proxy.Close() })
 
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:                       c1.Config().WorkerAuthKms,
-		InitialUpstreams:                    []string{proxy.ListenerAddr()},
-		Logger:                              logger.Named("w1"),
-		SuccessfulStatusGracePeriodDuration: helper.DefaultSuccessfulStatusGracePeriod,
-		EnableIPv6:                          true,
+		WorkerAuthKms:    c1.Config().WorkerAuthKms,
+		InitialUpstreams: []string{proxy.ListenerAddr()},
+		Logger:           logger.Named("w1"),
+		SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
+		EnableIPv6:        true,
+		WorkerRPCInterval: time.Second,
 	})
 
 	helper.ExpectWorkers(t, c1, w1)
@@ -105,8 +107,8 @@ func TestWorkerBytesUpDown(t *testing.T) {
 	conn := sess.Connect(ctx, t)
 	conn.TestSendRecvAll(t)
 
-	// Wait for next status and then check DB for bytes up and down
-	helper.ExpectWorkers(t, c1, w1)
+	// Wait for next statistics update and then check DB for bytes up and down
+	w1.Worker().TestWaitForNextSuccessfulStatisticsUpdate(t)
 
 	dbConns1, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId)
 	require.NoError(err)
@@ -122,11 +124,11 @@ func TestWorkerBytesUpDown(t *testing.T) {
 	require.NotZero(apiRes1.Item.Connections[0].BytesUp)
 	require.NotZero(apiRes1.Item.Connections[0].BytesDown)
 
-	// Send/recv some more traffic, close connection, wait for the next status
+	// Send/recv some more traffic, close connection, wait for the next statistics
 	// update and check everything again.
 	conn.TestSendRecvAll(t)
 	require.NoError(conn.Close())
-	helper.ExpectWorkers(t, c1, w1)
+	w1.Worker().TestWaitForNextSuccessfulStatisticsUpdate(t)
 
 	dbConns2, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId)
 	require.NoError(err)

internal/tests/cluster/sequential/worker_proxy_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"net"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/internal/cmd/config"
@@ -46,10 +47,10 @@ func TestWorkerSessionProxyMultipleConnections(t *testing.T) {
 	}
 
 	c1 := controller.NewTestController(t, &controller.TestControllerOpts{
-		Config:                          conf,
-		InitialResourcesSuffix:          "1234567890",
-		Logger:                          logger.Named("c1"),
-		WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
+		Config:                 conf,
+		InitialResourcesSuffix: "1234567890",
+		Logger:                 logger.Named("c1"),
+		WorkerRPCGracePeriod:   helper.DefaultControllerRPCGracePeriod,
 	})
 
 	helper.ExpectWorkers(t, c1)
@@ -66,11 +67,12 @@ func TestWorkerSessionProxyMultipleConnections(t *testing.T) {
 	require.NotEmpty(t, proxy.ListenerAddr())
 
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:                       c1.Config().WorkerAuthKms,
-		InitialUpstreams:                    []string{proxy.ListenerAddr()},
-		Logger:                              logger.Named("w1"),
-		SuccessfulStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
-		EnableIPv6:                          true,
+		WorkerAuthKms:    c1.Config().WorkerAuthKms,
+		InitialUpstreams: []string{proxy.ListenerAddr()},
+		Logger:           logger.Named("w1"),
+		SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
+		EnableIPv6:        true,
+		WorkerRPCInterval: time.Second,
 	})
 
 	helper.ExpectWorkers(t, c1, w1)

internal/tests/cluster/sequential/worker_tagging_test.go

@@ -5,6 +5,7 @@ package sequential
 
 import (
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/internal/cmd/config"
@@ -57,10 +58,11 @@ func TestWorkerTagging(t *testing.T) {
 		"foo":    {"bar"},
 	}
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		Config:           conf,
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w1"),
+		Config:            conf,
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  c1.ClusterAddrs(),
+		Logger:            logger.Named("w1"),
+		WorkerRPCInterval: time.Second,
 	})
 	w1Addr := w1.ProxyAddrs()[0]
 
@@ -73,10 +75,11 @@ func TestWorkerTagging(t *testing.T) {
 		"az":     {"one", "two", "three"},
 	}
 	w2 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		Config:           conf,
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w2"),
+		Config:            conf,
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  c1.ClusterAddrs(),
+		Logger:            logger.Named("w2"),
+		WorkerRPCInterval: time.Second,
 	})
 	w2Addr := w2.ProxyAddrs()[0]
 
@@ -89,10 +92,11 @@ func TestWorkerTagging(t *testing.T) {
 		"az":     {"one", "three"},
 	}
 	w3 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		Config:           conf,
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w3"),
+		Config:            conf,
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  c1.ClusterAddrs(),
+		Logger:            logger.Named("w3"),
+		WorkerRPCInterval: time.Second,
 	})
 	w3Addr := w3.ProxyAddrs()[0]
 

internal/tests/cluster/sequential/x509_verification_test.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"sync"
 	"testing"
+	"time"
 
 	apiproxy "github.com/hashicorp/boundary/api/proxy"
 	"github.com/hashicorp/boundary/api/targets"
@@ -43,6 +44,14 @@ func TestCustomX509Verification_Client(t *testing.T) {
 		Name:  "test",
 	})
 	req.NoError(event.InitSysEventer(logger, testLock, "use-TestCustomX509Verification", event.WithEventerConfig(&ec.EventerConfig)))
+	t.Cleanup(func() {
+		event.TestResetSystEventer(t)
+	})
+	t.Cleanup(func() {
+		all, err := io.ReadAll(ec.AllEvents)
+		require.NoError(t, err)
+		t.Log(string(all))
+	})
 
 	conf, err := config.DevController()
 	conf.Eventing = &ec.EventerConfig
@@ -57,12 +66,12 @@ func TestCustomX509Verification_Client(t *testing.T) {
 	conf.Eventing = &ec.EventerConfig
 	req.NoError(err)
 	w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-		WorkerAuthKms:    c1.Config().WorkerAuthKms,
-		InitialUpstreams: c1.ClusterAddrs(),
-		Logger:           logger.Named("w1"),
+		WorkerAuthKms:     c1.Config().WorkerAuthKms,
+		InitialUpstreams:  c1.ClusterAddrs(),
+		Logger:            logger.Named("w1"),
+		WorkerRPCInterval: time.Second,
 	})
 
-	// Give time for it to connect
 	helper.ExpectWorkers(t, c1, w1)
 
 	// Connect target
@@ -221,8 +230,9 @@ func testCustomX509Verification_Server(ec event.TestConfig, certPool *x509.CertP
 		conf.Eventing = &ec.EventerConfig
 		req.NoError(err)
 		w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
-			WorkerAuthKms:    c1.Config().WorkerAuthKms,
-			InitialUpstreams: c1.ClusterAddrs(),
+			WorkerAuthKms:     c1.Config().WorkerAuthKms,
+			InitialUpstreams:  c1.ClusterAddrs(),
+			WorkerRPCInterval: time.Second,
 		})
 		w1.Worker().TestOverrideX509VerifyCertPool = certPool
 		w1.Worker().TestOverrideX509VerifyDnsName = dnsName

internal/tests/helper/testing_helper.go

@@ -27,22 +27,14 @@ import (
 )
 
 const (
-	DefaultWorkerStatusGracePeriod           = time.Second * 15
-	DefaultSuccessfulStatusGracePeriod       = time.Second * 15
+	DefaultControllerRPCGracePeriod          = time.Second * 15
 	expectConnectionStateOnControllerTimeout = time.Minute * 2
-	expectConnectionStateOnWorkerTimeout     = DefaultWorkerStatusGracePeriod * 3
+	expectConnectionStateOnWorkerTimeout     = DefaultControllerRPCGracePeriod * 3
 
 	// This is the interval that we check states on in the worker. It
 	// needs to be particularly granular to ensure that we allow for
 	// adequate time to catch any edge cases where the connection state
 	// disappears from the worker before we can update the state.
-	//
-	// As of this writing, the (hardcoded) status request interval on
-	// the worker is 2 seconds, and a session is removed from the state
-	// when it has no connections left on the *next* pass. A one second
-	// interval would only mean two chances to check with a high
-	// possibility of skew; while it seems okay I'm still not 100%
-	// comfortable with this little resolution.
 	expectConnectionStateOnWorkerInterval = time.Millisecond * 100
 )
 
@@ -465,22 +457,27 @@ func NewTestTcpServer(t *testing.T) *TestTcpServer {
 
 // ExpectWorkers is a blocking call, where the method validates that the expected workers
 // can be found in the controllers status update. If the provided list of workers is empty,
-// this method will validate that the controller worker status is empty.
+// this method will validate that the controller worker routing info is not called.
 func ExpectWorkers(t *testing.T, c *controller.TestController, workers ...*worker.TestWorker) {
+	t.Helper()
 	// validate the controller has no reported workers
 	if len(workers) == 0 {
-		require.Eventually(t, func() bool {
+		assert.Eventually(t, func() bool {
 			workers := []string{}
-			c.Controller().WorkerStatusUpdateTimes().Range(func(workerId, lastStatusTime any) bool {
-				require.NotNil(t, workerId)
-				require.NotNil(t, lastStatusTime)
-				if time.Since(lastStatusTime.(time.Time)) < DefaultSuccessfulStatusGracePeriod {
+			c.Controller().WorkerRoutingInfoUpdateTimes().Range(func(workerId, lastRoutingInfo any) bool {
+				if assert.NotNil(t, workerId) {
+					return false
+				}
+				if assert.NotNil(t, lastRoutingInfo) {
+					return false
+				}
+				if time.Since(lastRoutingInfo.(time.Time)) < DefaultControllerRPCGracePeriod {
 					workers = append(workers, workerId.(string))
 				}
 				return true
 			})
 			return len(workers) == 0
-		}, 2*DefaultSuccessfulStatusGracePeriod, 250*time.Millisecond)
+		}, 2*DefaultControllerRPCGracePeriod, 250*time.Millisecond)
 		return
 	}
 
@@ -490,11 +487,11 @@ func ExpectWorkers(t *testing.T, c *controller.TestController, workers ...*worke
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			require.NoError(t, w.Worker().WaitForNextSuccessfulStatusUpdate())
-			require.Eventually(t, func() bool {
-				_, ok := c.Controller().WorkerStatusUpdateTimes().Load(w.Name())
+			assert.NoError(t, w.Worker().WaitForNextSuccessfulRoutingInfoUpdate())
+			assert.Eventually(t, func() bool {
+				_, ok := c.Controller().WorkerRoutingInfoUpdateTimes().Load(w.Name())
 				return ok
-			}, 30*time.Second, 100*time.Millisecond)
+			}, 30*time.Second, time.Second)
 		}()
 	}
 	wg.Wait()