From ec0a3f964bc7b58654221acb4d07cc7bdd8aa44a Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Thu, 9 Jan 2025 10:26:37 -0800 Subject: [PATCH] Worker status refactor (#5417) * oss: feat(proto): statistics rpc added to server coordination service * oss: feat(session): add update statistics status service * oss: feat(handlers): add Statistics method to worker service server * oss: feat(worker): add statistics ticking * oss: internal/proto: define the new RoutingInfo RPC * oss: internal/server: repository changes for RoutingInfo RPC * oss: internal/daemon: add RoutingInfo handler to worker service The new RoutingInfo RPC is used to get information from the worker that is used to make routing decisions on the controller. It also handles the initial request from the worker. * oss: feat(db): server_worker_session_info table * oss: feat(proto): session info rpc for server coordination service * oss: feat(server): worker session info request data type * oss: feat(worker): session info ticker * oss: feat(handler): server coordination handler for session info * oss: feat(session): utilize session info request time for session cleanup * oss: proto: fix some gotags comments Some of these fields were missing comments, while others had incorrectly formatted comments. * oss: worker: send connection status to server This is required for the correct tracking of orphaned connections * oss: internal/daemon/worker: add function to wait for Statistics RPC * oss: internal/daemon/worker: add routing info RPC The new RoutingInfo RPC replaces part of the Status RPC, namely the part which is used by the controller and worker to perform decisions around the establishment of new sessions and new worker connections. * oss: internal/daemon/worker: remove unnecessary error check * oss: internal/daemon/worker: lock access to config and upstreams The new confLock protects access to the conf value and to the addressReceivers slice. * oss: worker: replace Status RPC with RoutingInfo * oss: worker: deprecate the Status RPC and related types The Status RPC and related messages have been replaced by the RoutingInfo/SessionInfo/Statistics RPCs. The controller will continue to support the Status RPC until two new minor versions have been released. * oss: internal/event: add default deny for new worker RPCs * oss: internal/daemon/worker: allow RPC interval to be set The new SessionInfo/RoutingInfo/Statistics RPCs run at long intervals, which make some of our cluster tests take a long time to complete. This allows their interval to be configured in tests. * oss: all: use TestWorkerRPCInterval in slow tests This should speed up these tests significantly. * oss: internal/tests/cluster: reset sys eventer in test --------- Co-authored-by: Damian Debkowski --- .golangci.yml | 6 + .../cache/repository_sessions_test.go | 26 +- internal/cmd/config/config.go | 99 +- internal/cmd/config/config_load_test.go | 200 +-- .../daemon/cluster/handlers/worker_service.go | 272 +++- .../handlers/worker_service_status_test.go | 46 +- .../cluster/handlers/worker_service_test.go | 802 ++++++++++ internal/daemon/controller/controller.go | 52 +- internal/daemon/controller/handler.go | 4 +- .../session_recording_service.go | 2 +- .../handlers/targets/target_service.go | 60 +- .../targets/tcp/target_service_test.go | 24 +- .../daemon/controller/rpc_registration.go | 4 +- internal/daemon/controller/testing.go | 59 +- internal/daemon/worker/common/common.go | 26 +- .../daemon/worker/controller_connection.go | 6 +- internal/daemon/worker/handler.go | 4 +- internal/daemon/worker/routing_info.go | 328 ++++ .../{status_test.go => routing_info_test.go} | 21 +- internal/daemon/worker/session/manager.go | 2 +- internal/daemon/worker/session/session.go | 6 +- internal/daemon/worker/session/testing.go | 59 + internal/daemon/worker/session_info.go | 229 +++ internal/daemon/worker/session_info_test.go | 337 ++++ internal/daemon/worker/statistics.go | 92 ++ internal/daemon/worker/statistics_test.go | 158 ++ internal/daemon/worker/status.go | 519 ------- internal/daemon/worker/testing.go | 138 +- internal/daemon/worker/worker.go | 232 +-- .../daemon/worker/worker_proxy_service.go | 20 + internal/daemon/worker/worker_test.go | 2 +- .../95/01_server_worker_session_info.up.sql | 18 + internal/event/cloudevents_formatter_node.go | 21 +- .../event/cloudevents_formatter_node_test.go | 13 +- internal/event/hclog_formatter_node.go | 21 +- internal/event/hclog_formatter_node_test.go | 2 +- internal/gen/controller/servers/servers.pb.go | 10 +- .../server_coordination_service.pb.go | 1372 +++++++++++++---- .../server_coordination_service_grpc.pb.go | 136 ++ .../v1/server_coordination_service.proto | 169 +- .../proto/controller/servers/v1/servers.proto | 10 +- .../storage/servers/store/v1/worker.proto | 11 + internal/server/job/jobs.go | 8 +- internal/server/repository_worker.go | 14 +- .../server/repository_worker_session_info.go | 37 + .../repository_worker_session_info_test.go | 71 + internal/server/repository_worker_test.go | 4 +- internal/server/service_verified_workers.go | 52 + .../service_verified_workers_ext_test.go | 61 + internal/server/store/worker.pb.go | 117 +- .../server/worker_session_info_request.go | 81 + internal/session/job_session_cleanup.go | 16 +- internal/session/job_session_cleanup_test.go | 61 +- internal/session/jobs.go | 6 +- internal/session/query.go | 8 +- internal/session/repository_connection.go | 2 +- internal/session/service_statistics.go | 47 + internal/session/service_statistics_test.go | 534 +++++++ .../tests/api/credentials/credentials_test.go | 3 +- internal/tests/api/proxy/proxy_test.go | 24 +- internal/tests/api/targets/target_test.go | 10 +- .../cluster/parallel/ipv6_listener_test.go | 10 +- .../parallel/multi_controller_worker_test.go | 46 +- .../cluster/parallel/unix_listener_test.go | 15 +- .../sequential/session_cleanup_test.go | 128 +- .../sequential/worker_bytesupdown_test.go | 28 +- .../cluster/sequential/worker_proxy_test.go | 20 +- .../cluster/sequential/worker_tagging_test.go | 28 +- .../sequential/x509_verification_test.go | 22 +- internal/tests/helper/testing_helper.go | 39 +- 70 files changed, 5492 insertions(+), 1618 deletions(-) create mode 100644 internal/daemon/worker/routing_info.go rename internal/daemon/worker/{status_test.go => routing_info_test.go} (62%) create mode 100644 internal/daemon/worker/session/testing.go create mode 100644 internal/daemon/worker/session_info.go create mode 100644 internal/daemon/worker/session_info_test.go create mode 100644 internal/daemon/worker/statistics.go create mode 100644 internal/daemon/worker/statistics_test.go delete mode 100644 internal/daemon/worker/status.go create mode 100644 internal/db/schema/migrations/oss/postgres/95/01_server_worker_session_info.up.sql create mode 100644 internal/server/repository_worker_session_info.go create mode 100644 internal/server/repository_worker_session_info_test.go create mode 100644 internal/server/service_verified_workers.go create mode 100644 internal/server/service_verified_workers_ext_test.go create mode 100644 internal/server/worker_session_info_request.go create mode 100644 internal/session/service_statistics.go create mode 100644 internal/session/service_statistics_test.go diff --git a/.golangci.yml b/.golangci.yml index 9d12f78d4..608d91266 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -69,3 +69,9 @@ issues: - linters: - staticcheck text: "SA1019: j.GetId is deprecated:" + - linters: + - staticcheck + text: "SA1019: pbs.StatusRequest is deprecated:" + - linters: + - staticcheck + text: "SA1019: pbs.StatusResponse is deprecated:" diff --git a/internal/clientcache/internal/cache/repository_sessions_test.go b/internal/clientcache/internal/cache/repository_sessions_test.go index 5cc31dca0..466f917d8 100644 --- a/internal/clientcache/internal/cache/repository_sessions_test.go +++ b/internal/clientcache/internal/cache/repository_sessions_test.go @@ -604,6 +604,7 @@ func TestRepository_QuerySessionsLimiting(t *testing.T) { } func TestDefaultSessionRetrievalFunc(t *testing.T) { + // This prevents us from running this test in parallel oldDur := globals.RefreshReadLookbackDuration globals.RefreshReadLookbackDuration = 0 t.Cleanup(func() { @@ -614,28 +615,21 @@ func TestDefaultSessionRetrievalFunc(t *testing.T) { tc.Client().SetToken(tc.Token().Token) tarClient := targets.NewClient(tc.Client()) _ = worker.NewTestWorker(t, &worker.TestWorkerOpts{ - Name: "test", - InitialUpstreams: tc.ClusterAddrs(), - WorkerAuthKms: tc.Config().WorkerAuthKms, + Name: "test", + InitialUpstreams: tc.ClusterAddrs(), + WorkerAuthKms: tc.Config().WorkerAuthKms, + WorkerRPCInterval: time.Second, }) tar1, err := tarClient.Create(tc.Context(), "tcp", "p_1234567890", targets.WithName("tar1"), targets.WithTcpTargetDefaultPort(1), targets.WithAddress("address")) require.NoError(t, err) require.NotNil(t, tar1) - require.NoError(t, tc.WaitForNextWorkerStatusUpdate("test")) - ctx, cancel := context.WithTimeout(tc.Context(), 10*time.Second) - defer cancel() - for { - _, err = tarClient.AuthorizeSession(ctx, tar1.Item.Id) - if err == nil { - break - } - if err != nil && ctx.Err() != nil { - require.FailNow(t, "timed out waiting to authorize session without an error. Last error: %s", err.Error()) - break - } - } + require.NoError(t, tc.WaitForNextWorkerRoutingInfoUpdate("test")) + require.Eventually(t, func() bool { + _, err = tarClient.AuthorizeSession(tc.Context(), tar1.Item.Id) + return err == nil + }, 30*time.Second, time.Second, "timed out waiting to authorize session without an error.") got, refTok, err := defaultSessionFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "", nil) assert.NoError(t, err) diff --git a/internal/cmd/config/config.go b/internal/cmd/config/config.go index b66ea720b..2b6f23ed3 100644 --- a/internal/cmd/config/config.go +++ b/internal/cmd/config/config.go @@ -268,15 +268,18 @@ type Controller struct { GracefulShutdownWait any `hcl:"graceful_shutdown_wait_duration"` GracefulShutdownWaitDuration time.Duration `hcl:"-"` - // WorkerStatusGracePeriod represents the period of time (as a duration) + // WorkerRPCGracePeriod represents the period of time (as a duration) // that the controller will wait before deciding a worker is disconnected - // and marking connections from it as canceling + // and marking connections from it as canceling. It is also used to evaluate + // whether a worker is available for routing based on the time since its last routing info report. + // For backwards compatibility this is still called worker_status_grace_period, + // though it is now used with both the SessionInfo and RoutingInfo RPCs. // // TODO: This isn't documented (on purpose) because the right place for this // is central configuration so you can't drift across controllers, but we // don't have that yet. - WorkerStatusGracePeriod any `hcl:"worker_status_grace_period"` - WorkerStatusGracePeriodDuration time.Duration `hcl:"-"` + WorkerRPCGracePeriod any `hcl:"worker_status_grace_period"` + WorkerRPCGracePeriodDuration time.Duration `hcl:"-"` // LivenessTimeToStale represents the period of time (as a duration) after // which it will consider other controllers to be no longer accessible, @@ -351,13 +354,15 @@ type Worker struct { Tags map[string][]string `hcl:"-"` TagsRaw any `hcl:"tags"` - // StatusCallTimeout represents the period of time (as a duration) that - // the worker will allow a status RPC call to attempt to finish before - // canceling it to try again. + // ControllerRPCCallTimeout represents the period of time (as a duration) that + // the worker will allow the SessionInfo, RoutingInfo and Statistics RPC calls to + // attempt to finish before canceling them to try again. + // For backwards compatibility this is still called status_call_timeout, + // though it is now used to control the SessionInfo, Statistics and RoutingInfo RPCs. // // TODO: This is currently not documented and considered internal. - StatusCallTimeout any `hcl:"status_call_timeout"` - StatusCallTimeoutDuration time.Duration `hcl:"-"` + ControllerRPCCallTimeout any `hcl:"status_call_timeout"` + ControllerRPCCallTimeoutDuration time.Duration `hcl:"-"` // GetDownstreamWorkersTimeout represents the period of time (as a duration) timeout // for GetDownstreamWorkers call in DownstreamWorkerTicker @@ -366,14 +371,17 @@ type Worker struct { GetDownstreamWorkersTimeout any `hcl:"get_downstream_workers_timeout"` GetDownstreamWorkersTimeoutDuration time.Duration `hcl:"-"` - // SuccessfulStatusGracePeriod represents the period of time (as a duration) - // that the worker will wait before disconnecting connections if it cannot - // successfully complete a status report to a controller. This cannot be - // less than StatusCallTimeout. + // SuccessfulControllerRPCGracePeriod represents the period of time (as a duration) + // that the worker will wait before closing connections if it cannot + // successfully complete a session info report to a controller. It is also used + // to evaluate whether the upstreams need to be recalculated in case of repeated + // routing info report failures. This cannot be less than ControllerRPCCallTimeout. + // For backwards compatibility this is still called successful_status_grace_period, + // though it is now used to control the SessionInfo and RoutingInfo RPCs. // // TODO: This is currently not documented and considered internal. - SuccessfulStatusGracePeriod any `hcl:"successful_status_grace_period"` - SuccessfulStatusGracePeriodDuration time.Duration `hcl:"-"` + SuccessfulControllerRPCGracePeriod any `hcl:"successful_status_grace_period"` + SuccessfulControllerRPCGracePeriodDuration time.Duration `hcl:"-"` // AuthStoragePath represents the location a worker stores its node credentials, if set AuthStoragePath string `hcl:"auth_storage_path"` @@ -400,6 +408,11 @@ type Worker struct { // pre-0.13 method of using KMSes to authenticate. This is currently only // supported to throw an error if used telling people they need to upgrade. UseDeprecatedKmsAuthMethod bool `hcl:"use_deprecated_kms_auth_method"` + + // TestWorkerRPCInterval represents the base period of time that + // the worker will wait between invoking the controller RPCs. + // This is not exposed to users and only used in tests. + TestWorkerRPCInterval time.Duration `hcl:"-"` } type Database struct { @@ -721,19 +734,21 @@ func Parse(d string) (*Config, error) { result.Controller.Scheduler.MonitorIntervalDuration = t } - workerStatusGracePeriod := result.Controller.WorkerStatusGracePeriod - if util.IsNil(workerStatusGracePeriod) { - workerStatusGracePeriod = os.Getenv("BOUNDARY_CONTROLLER_WORKER_STATUS_GRACE_PERIOD") + workerRPCGracePeriod := result.Controller.WorkerRPCGracePeriod + if util.IsNil(workerRPCGracePeriod) { + // For backwards compatibility this is still called BOUNDARY_CONTROLLER_WORKER_STATUS_GRACE_PERIOD, + // though it is now used to control the SessionInfo and RoutingInfo RPCs. + workerRPCGracePeriod = os.Getenv("BOUNDARY_CONTROLLER_WORKER_STATUS_GRACE_PERIOD") } - if workerStatusGracePeriod != nil { - t, err := parseutil.ParseDurationSecond(workerStatusGracePeriod) + if workerRPCGracePeriod != nil { + t, err := parseutil.ParseDurationSecond(workerRPCGracePeriod) if err != nil { return result, err } - result.Controller.WorkerStatusGracePeriodDuration = t + result.Controller.WorkerRPCGracePeriodDuration = t } - if result.Controller.WorkerStatusGracePeriodDuration < 0 { - return nil, errors.New("Controller worker status grace period value is negative") + if result.Controller.WorkerRPCGracePeriodDuration < 0 { + return nil, errors.New("Controller worker RPC grace period value is negative") } livenessTimeToStale := result.Controller.LivenessTimeToStale @@ -888,8 +903,10 @@ func Parse(d string) (*Config, error) { return nil, fmt.Errorf("Error parsing worker activation token: %w", err) } - statusCallTimeoutDuration := result.Worker.StatusCallTimeout + statusCallTimeoutDuration := result.Worker.ControllerRPCCallTimeout if util.IsNil(statusCallTimeoutDuration) { + // For backwards compatibility this is still called BOUNDARY_WORKER_STATUS_CALL_TIMEOUT, + // though it is now used to control the SessionInfo, Statistics and RoutingInfo RPCs. statusCallTimeoutDuration = os.Getenv("BOUNDARY_WORKER_STATUS_CALL_TIMEOUT") } if statusCallTimeoutDuration != nil { @@ -897,10 +914,10 @@ func Parse(d string) (*Config, error) { if err != nil { return result, err } - result.Worker.StatusCallTimeoutDuration = t + result.Worker.ControllerRPCCallTimeoutDuration = t } - if result.Worker.StatusCallTimeoutDuration < 0 { - return nil, errors.New("Status call timeout value is negative") + if result.Worker.ControllerRPCCallTimeoutDuration < 0 { + return nil, errors.New("Controller RPC timeout value is negative") } getDownstreamWorkersTimeoutDuration := result.Worker.GetDownstreamWorkersTimeout @@ -918,19 +935,21 @@ func Parse(d string) (*Config, error) { return nil, errors.New("get downstream workers timeout must be greater than 0") } - successfulStatusGracePeriod := result.Worker.SuccessfulStatusGracePeriod - if util.IsNil(successfulStatusGracePeriod) { - successfulStatusGracePeriod = os.Getenv("BOUNDARY_WORKER_SUCCESSFUL_STATUS_GRACE_PERIOD") + successfulControllerRPCGracePeriod := result.Worker.SuccessfulControllerRPCGracePeriod + if util.IsNil(successfulControllerRPCGracePeriod) { + // For backwards compatibility this is still called BOUNDARY_WORKER_SUCCESSFUL_STATUS_GRACE_PERIOD, + // though it is now used to control the SessionInfo and RoutingInfo RPCs. + successfulControllerRPCGracePeriod = os.Getenv("BOUNDARY_WORKER_SUCCESSFUL_STATUS_GRACE_PERIOD") } - if successfulStatusGracePeriod != nil { - t, err := parseutil.ParseDurationSecond(successfulStatusGracePeriod) + if successfulControllerRPCGracePeriod != nil { + t, err := parseutil.ParseDurationSecond(successfulControllerRPCGracePeriod) if err != nil { return result, err } - result.Worker.SuccessfulStatusGracePeriodDuration = t + result.Worker.SuccessfulControllerRPCGracePeriodDuration = t } - if result.Worker.SuccessfulStatusGracePeriodDuration < 0 { - return nil, errors.New("Successful status grace period value is negative") + if result.Worker.SuccessfulControllerRPCGracePeriodDuration < 0 { + return nil, errors.New("Successful controller RPC grace period value is negative") } if !util.IsNil(result.Worker.RecordingStorageMinimumAvailableCapacity) { @@ -949,14 +968,14 @@ func Parse(d string) (*Config, error) { } switch { - case result.Worker.StatusCallTimeoutDuration == 0 && result.Worker.SuccessfulStatusGracePeriodDuration == 0: + case result.Worker.ControllerRPCCallTimeoutDuration == 0 && result.Worker.SuccessfulControllerRPCGracePeriodDuration == 0: // Nothing - case result.Worker.StatusCallTimeoutDuration != 0 && result.Worker.SuccessfulStatusGracePeriodDuration != 0: - if result.Worker.StatusCallTimeoutDuration > result.Worker.SuccessfulStatusGracePeriodDuration { - return nil, fmt.Errorf("Worker setting for status call timeout duration must be less than or equal to successful status grace period duration") + case result.Worker.ControllerRPCCallTimeoutDuration != 0 && result.Worker.SuccessfulControllerRPCGracePeriodDuration != 0: + if result.Worker.ControllerRPCCallTimeoutDuration > result.Worker.SuccessfulControllerRPCGracePeriodDuration { + return nil, fmt.Errorf("Worker setting for controller rpc timeout duration must be less than or equal to successful controller rpc grace period duration") } default: - return nil, fmt.Errorf("Worker settings for status call timeout duration and successful status grace period duration must either both be set or both be empty") + return nil, fmt.Errorf("Worker settings for controller rpc call timeout duration and successful controller rpc grace period duration must either both be set or both be empty") } if result.Worker.TagsRaw != nil { diff --git a/internal/cmd/config/config_load_test.go b/internal/cmd/config/config_load_test.go index 3a4239817..e57114726 100644 --- a/internal/cmd/config/config_load_test.go +++ b/internal/cmd/config/config_load_test.go @@ -378,17 +378,17 @@ func TestLoad(t *testing.T) { ClusterName: "", }, Worker: &config.Worker{ - Name: "worker0", - Description: "A worker", - PublicAddr: "", - InitialUpstreams: []string{"boundary:9201"}, - InitialUpstreamsRaw: []any{"boundary:9201"}, - Tags: nil, - TagsRaw: nil, - SuccessfulStatusGracePeriodDuration: 0, - StatusCallTimeoutDuration: 0, - AuthStoragePath: "", - ControllerGeneratedActivationToken: "", + Name: "worker0", + Description: "A worker", + PublicAddr: "", + InitialUpstreams: []string{"boundary:9201"}, + InitialUpstreamsRaw: []any{"boundary:9201"}, + Tags: nil, + TagsRaw: nil, + SuccessfulControllerRPCGracePeriodDuration: 0, + ControllerRPCCallTimeoutDuration: 0, + AuthStoragePath: "", + ControllerGeneratedActivationToken: "", }, Controller: &config.Controller{ Name: "controller0", @@ -409,18 +409,18 @@ func TestLoad(t *testing.T) { JobRunInterval: nil, MonitorInterval: nil, }, - AuthTokenTimeToLive: nil, - AuthTokenTimeToLiveDuration: 0, - AuthTokenTimeToStale: nil, - AuthTokenTimeToStaleDuration: 0, - GracefulShutdownWait: nil, - GracefulShutdownWaitDuration: 0, - WorkerStatusGracePeriodDuration: 0, - LivenessTimeToStaleDuration: 0, - ApiRateLimits: make(ratelimit.Configs, 0), - ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), - MaxPageSizeRaw: nil, - MaxPageSize: 0, + AuthTokenTimeToLive: nil, + AuthTokenTimeToLiveDuration: 0, + AuthTokenTimeToStale: nil, + AuthTokenTimeToStaleDuration: 0, + GracefulShutdownWait: nil, + GracefulShutdownWaitDuration: 0, + WorkerRPCGracePeriodDuration: 0, + LivenessTimeToStaleDuration: 0, + ApiRateLimits: make(ratelimit.Configs, 0), + ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), + MaxPageSizeRaw: nil, + MaxPageSize: 0, }, DevController: false, DevUiPassthroughDir: "", @@ -804,17 +804,17 @@ func TestLoad(t *testing.T) { ClusterName: "", }, Worker: &config.Worker{ - Name: "worker0", - Description: "A worker", - PublicAddr: "", - InitialUpstreams: []string{"boundary:9201"}, - InitialUpstreamsRaw: []any{"boundary:9201"}, - Tags: nil, - TagsRaw: nil, - SuccessfulStatusGracePeriodDuration: 0, - StatusCallTimeoutDuration: 0, - AuthStoragePath: "", - ControllerGeneratedActivationToken: "", + Name: "worker0", + Description: "A worker", + PublicAddr: "", + InitialUpstreams: []string{"boundary:9201"}, + InitialUpstreamsRaw: []any{"boundary:9201"}, + Tags: nil, + TagsRaw: nil, + SuccessfulControllerRPCGracePeriodDuration: 0, + ControllerRPCCallTimeoutDuration: 0, + AuthStoragePath: "", + ControllerGeneratedActivationToken: "", }, Controller: &config.Controller{ Name: "controller0", @@ -835,18 +835,18 @@ func TestLoad(t *testing.T) { JobRunInterval: nil, MonitorInterval: nil, }, - AuthTokenTimeToLive: nil, - AuthTokenTimeToLiveDuration: 0, - AuthTokenTimeToStale: nil, - AuthTokenTimeToStaleDuration: 0, - GracefulShutdownWait: nil, - GracefulShutdownWaitDuration: 0, - WorkerStatusGracePeriodDuration: 0, - LivenessTimeToStaleDuration: 0, - ApiRateLimits: make(ratelimit.Configs, 0), - ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), - MaxPageSizeRaw: nil, - MaxPageSize: 0, + AuthTokenTimeToLive: nil, + AuthTokenTimeToLiveDuration: 0, + AuthTokenTimeToStale: nil, + AuthTokenTimeToStaleDuration: 0, + GracefulShutdownWait: nil, + GracefulShutdownWaitDuration: 0, + WorkerRPCGracePeriodDuration: 0, + LivenessTimeToStaleDuration: 0, + ApiRateLimits: make(ratelimit.Configs, 0), + ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), + MaxPageSizeRaw: nil, + MaxPageSize: 0, }, DevController: false, DevUiPassthroughDir: "", @@ -1225,17 +1225,17 @@ func TestLoad(t *testing.T) { ClusterName: "", }, Worker: &config.Worker{ - Name: "worker0", - Description: "A worker", - PublicAddr: "", - InitialUpstreams: []string{"boundary:9201"}, - InitialUpstreamsRaw: []any{"boundary:9201"}, - Tags: nil, - TagsRaw: nil, - SuccessfulStatusGracePeriodDuration: 0, - StatusCallTimeoutDuration: 0, - AuthStoragePath: "", - ControllerGeneratedActivationToken: "", + Name: "worker0", + Description: "A worker", + PublicAddr: "", + InitialUpstreams: []string{"boundary:9201"}, + InitialUpstreamsRaw: []any{"boundary:9201"}, + Tags: nil, + TagsRaw: nil, + SuccessfulControllerRPCGracePeriodDuration: 0, + ControllerRPCCallTimeoutDuration: 0, + AuthStoragePath: "", + ControllerGeneratedActivationToken: "", }, Controller: &config.Controller{ Name: "controller0", @@ -1256,14 +1256,14 @@ func TestLoad(t *testing.T) { JobRunInterval: nil, MonitorInterval: nil, }, - AuthTokenTimeToLive: nil, - AuthTokenTimeToLiveDuration: 0, - AuthTokenTimeToStale: nil, - AuthTokenTimeToStaleDuration: 0, - GracefulShutdownWait: nil, - GracefulShutdownWaitDuration: 0, - WorkerStatusGracePeriodDuration: 0, - LivenessTimeToStaleDuration: 0, + AuthTokenTimeToLive: nil, + AuthTokenTimeToLiveDuration: 0, + AuthTokenTimeToStale: nil, + AuthTokenTimeToStaleDuration: 0, + GracefulShutdownWait: nil, + GracefulShutdownWaitDuration: 0, + WorkerRPCGracePeriodDuration: 0, + LivenessTimeToStaleDuration: 0, ApiRateLimits: ratelimit.Configs{ { Resources: []string{"*"}, @@ -1665,17 +1665,17 @@ func TestLoad(t *testing.T) { ClusterName: "", }, Worker: &config.Worker{ - Name: "worker0", - Description: "A worker", - PublicAddr: "", - InitialUpstreams: []string{"boundary:9201"}, - InitialUpstreamsRaw: []any{"boundary:9201"}, - Tags: nil, - TagsRaw: nil, - SuccessfulStatusGracePeriodDuration: 0, - StatusCallTimeoutDuration: 0, - AuthStoragePath: "", - ControllerGeneratedActivationToken: "", + Name: "worker0", + Description: "A worker", + PublicAddr: "", + InitialUpstreams: []string{"boundary:9201"}, + InitialUpstreamsRaw: []any{"boundary:9201"}, + Tags: nil, + TagsRaw: nil, + SuccessfulControllerRPCGracePeriodDuration: 0, + ControllerRPCCallTimeoutDuration: 0, + AuthStoragePath: "", + ControllerGeneratedActivationToken: "", }, Controller: &config.Controller{ Name: "controller0", @@ -1696,18 +1696,18 @@ func TestLoad(t *testing.T) { JobRunInterval: nil, MonitorInterval: nil, }, - AuthTokenTimeToLive: nil, - AuthTokenTimeToLiveDuration: 0, - AuthTokenTimeToStale: nil, - AuthTokenTimeToStaleDuration: 0, - GracefulShutdownWait: nil, - GracefulShutdownWaitDuration: 0, - WorkerStatusGracePeriodDuration: 0, - LivenessTimeToStaleDuration: 0, - ApiRateLimits: make(ratelimit.Configs, 0), - ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), - MaxPageSizeRaw: nil, - MaxPageSize: 0, + AuthTokenTimeToLive: nil, + AuthTokenTimeToLiveDuration: 0, + AuthTokenTimeToStale: nil, + AuthTokenTimeToStaleDuration: 0, + GracefulShutdownWait: nil, + GracefulShutdownWaitDuration: 0, + WorkerRPCGracePeriodDuration: 0, + LivenessTimeToStaleDuration: 0, + ApiRateLimits: make(ratelimit.Configs, 0), + ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), + MaxPageSizeRaw: nil, + MaxPageSize: 0, }, DevController: false, DevUiPassthroughDir: "", @@ -1789,18 +1789,18 @@ func TestLoad(t *testing.T) { JobRunInterval: nil, MonitorInterval: nil, }, - AuthTokenTimeToLive: nil, - AuthTokenTimeToLiveDuration: 0, - AuthTokenTimeToStale: nil, - AuthTokenTimeToStaleDuration: 0, - GracefulShutdownWait: nil, - GracefulShutdownWaitDuration: 0, - WorkerStatusGracePeriodDuration: 0, - LivenessTimeToStaleDuration: 0, - ApiRateLimits: make(ratelimit.Configs, 0), - ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), - MaxPageSizeRaw: nil, - MaxPageSize: 0, + AuthTokenTimeToLive: nil, + AuthTokenTimeToLiveDuration: 0, + AuthTokenTimeToStale: nil, + AuthTokenTimeToStaleDuration: 0, + GracefulShutdownWait: nil, + GracefulShutdownWaitDuration: 0, + WorkerRPCGracePeriodDuration: 0, + LivenessTimeToStaleDuration: 0, + ApiRateLimits: make(ratelimit.Configs, 0), + ApiRateLimiterMaxQuotas: ratelimit.DefaultLimiterMaxQuotas(), + MaxPageSizeRaw: nil, + MaxPageSize: 0, }, DevController: false, DevUiPassthroughDir: "", diff --git a/internal/daemon/cluster/handlers/worker_service.go b/internal/daemon/cluster/handlers/worker_service.go index 2b77e823c..fa6416222 100644 --- a/internal/daemon/cluster/handlers/worker_service.go +++ b/internal/daemon/cluster/handlers/worker_service.go @@ -5,6 +5,7 @@ package handlers import ( "context" + stderrors "errors" "fmt" "sync" "sync/atomic" @@ -13,6 +14,7 @@ import ( "github.com/hashicorp/boundary/internal/daemon/controller/common" "github.com/hashicorp/boundary/internal/daemon/controller/downstream" "github.com/hashicorp/boundary/internal/daemon/controller/handlers" + "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/internal/event" pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" intglobals "github.com/hashicorp/boundary/internal/globals" @@ -95,20 +97,148 @@ func NewWorkerServiceServer( } } +func (ws *workerServiceServer) Statistics(ctx context.Context, req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) { + const op = "workers.(workerServiceServer).Statistics" + workerId := req.GetWorkerId() + if workerId == "" { + return &pbs.StatisticsResponse{}, status.Error(codes.InvalidArgument, "worker id is empty") + } + connectionRepo, err := ws.connectionRepoFn() + if err != nil { + event.WriteError(ctx, op, err, event.WithInfoMsg("error getting connection repo")) + return &pbs.StatisticsResponse{}, status.Errorf(codes.Internal, "Error acquiring connection repo: %v", err) + } + connectionStats := []*session.Connection{} + for _, s := range req.GetSessions() { + sessionId := s.GetSessionId() + if sessionId == "" { + return &pbs.StatisticsResponse{}, status.Error(codes.InvalidArgument, "session id is empty") + } + for _, c := range s.Connections { + connectionId := c.GetConnectionId() + if connectionId == "" { + return &pbs.StatisticsResponse{}, status.Error(codes.InvalidArgument, "connection id is empty") + } + switch c.Status { + case pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED, pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED: + connectionStats = append(connectionStats, &session.Connection{ + PublicId: c.GetConnectionId(), + BytesUp: c.GetBytesUp(), + BytesDown: c.GetBytesDown(), + }) + default: + // Other statuses are not included in the stats sent to the domain + } + } + } + updateBytesErr := session.UpdateConnectionBytesUpDown(ctx, connectionRepo, connectionStats) + _, closeOrphanedErr := session.CloseOrphanedConnections(ctx, connectionRepo, workerId, connectionStats) + sessionErrs := stderrors.Join(updateBytesErr, closeOrphanedErr) + if sessionErrs != nil { + return &pbs.StatisticsResponse{}, errors.Wrap(ctx, sessionErrs, op) + } + return &pbs.StatisticsResponse{}, nil +} + +func (ws *workerServiceServer) SessionInfo(ctx context.Context, req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) { + const op = "workers.(workerServiceServer).SessionInfo" + workerId := req.GetWorkerId() + if workerId == "" { + return &pbs.SessionInfoResponse{}, status.Error(codes.InvalidArgument, "worker id is empty") + } + sessionTypeCache := map[string]pbs.SessionType{} + var recordedSessionIds []string + var ingressedSessionIds []string + for _, s := range req.GetSessions() { + sessionId := s.GetSessionId() + sessionStatus := s.GetSessionStatus() + sessionType := s.GetSessionType() + switch sessionType { + case pbs.SessionType_SESSION_TYPE_UNSPECIFIED: + return &pbs.SessionInfoResponse{}, status.Errorf(codes.InvalidArgument, "unspecified session type") + case pbs.SessionType_SESSION_TYPE_INGRESSED: + if sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING || sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED { + // No need to see about canceling anything + continue + } + sessionTypeCache[sessionId] = sessionType + ingressedSessionIds = append(ingressedSessionIds, sessionId) + case pbs.SessionType_SESSION_TYPE_RECORDED: + if sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING || sessionStatus == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED { + // No need to see about canceling anything + continue + } + sessionTypeCache[sessionId] = sessionType + recordedSessionIds = append(recordedSessionIds, sessionId) + default: + return &pbs.SessionInfoResponse{}, status.Errorf(codes.InvalidArgument, "unknown session type: %q", s.GetSessionType().String()) + } + } + + sessRepo, err := ws.sessionRepoFn() + if err != nil { + return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err) + } + + result := &pbs.SessionInfoResponse{} + nonActiveSessions, err := sessRepo.CheckIfNotActive(ctx, append(ingressedSessionIds, recordedSessionIds...)) + if err != nil { + return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, + "Error checking if sessions are no longer active: %v", err) + } + for _, s := range nonActiveSessions { + switch sessionTypeCache[s.SessionId] { + case pbs.SessionType_SESSION_TYPE_INGRESSED: + var connChanges []*pbs.Connection + for _, conn := range s.Connections { + connChanges = append(connChanges, &pbs.Connection{ + ConnectionId: conn.GetPublicId(), + Status: session.StatusClosed.ProtoVal(), + }) + } + result.NonActiveSessions = append(result.NonActiveSessions, &pbs.Session{ + SessionId: s.SessionId, + SessionStatus: s.Status.ProtoVal(), + SessionType: pbs.SessionType_SESSION_TYPE_INGRESSED, + Connections: connChanges, + }) + case pbs.SessionType_SESSION_TYPE_RECORDED: + result.NonActiveSessions = append(result.NonActiveSessions, &pbs.Session{ + SessionId: s.SessionId, + SessionStatus: s.Status.ProtoVal(), + SessionType: pbs.SessionType_SESSION_TYPE_RECORDED, + }) + default: + return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, + "unknown session type found for non active session: %q", s.SessionId) + } + } + + serverRepo, err := ws.serversRepoFn() + if err != nil { + return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to upsert session info status time: %v", err) + } + err = serverRepo.UpsertSessionInfo(ctx, workerId) + if err != nil { + return &pbs.SessionInfoResponse{}, status.Errorf(codes.Internal, "Error updating the latest status time for the server session info: %v", err) + } + return result, nil +} + +// Status is the deprecated method for worker status updates. +// This is safe to remove after the release of Boundary v0.20.0. func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) { const op = "workers.(workerServiceServer).Status" - // TODO: on the worker, if we get errors back from this repeatedly, do we - // terminate all sessions since we can't know if they were canceled? wStat := req.GetWorkerStatus() if wStat == nil { - return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Worker sent nil status.") + return nil, status.Error(codes.InvalidArgument, "Worker sent nil status.") } switch { case wStat.GetName() == "" && wStat.GetKeyId() == "": - return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Name and keyId are not set in the request; one is required.") + return nil, status.Error(codes.InvalidArgument, "Name and keyId are not set in the request; one is required.") case wStat.GetAddress() == "": - return &pbs.StatusResponse{}, status.Error(codes.InvalidArgument, "Address is not set but is required.") + return nil, status.Error(codes.InvalidArgument, "Address is not set but is required.") } // This Store call is currently only for testing purposes ws.updateTimes.Store(wStat.GetName(), time.Now()) @@ -116,7 +246,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques serverRepo, err := ws.serversRepoFn() if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting server repo")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to store worker status: %v", err) + return nil, status.Errorf(codes.Internal, "Error acquiring repo to store worker status: %v", err) } // Convert API tags to storage tags @@ -163,7 +293,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques workerId, err := serverRepo.UpsertWorkerStatus(ctx, wConf, opts...) if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error storing worker status")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error storing worker status: %v", err) + return nil, status.Errorf(codes.Internal, "Error storing worker status: %v", err) } // update storage states @@ -174,7 +304,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques controllers, err := serverRepo.ListControllers(ctx, server.WithLiveness(time.Duration(ws.livenessTimeToStale.Load()))) if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting current controllers")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting current controllers: %v", err) + return nil, status.Errorf(codes.Internal, "Error getting current controllers: %v", err) } responseControllers := []*pbs.UpstreamServer{} @@ -189,7 +319,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques workerAuthRepo, err := ws.workerAuthRepoFn() if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting worker auth repo")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to lookup worker auth info: %v", err) + return nil, status.Errorf(codes.Internal, "Error acquiring repo to lookup worker auth info: %v", err) } authorizedDownstreams := &pbs.AuthorizedDownstreamWorkerList{} @@ -197,7 +327,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques knownConnectedWorkers, err := serverRepo.VerifyKnownWorkers(ctx, req.GetConnectedWorkerPublicIds()) if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting known connected worker ids")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting known connected worker ids: %v", err) + return nil, status.Errorf(codes.Internal, "Error getting known connected worker ids: %v", err) } authorizedDownstreams.WorkerPublicIds = knownConnectedWorkers } @@ -206,25 +336,14 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques authorizedKeyIds, err := workerAuthRepo.FilterToAuthorizedWorkerKeyIds(ctx, req.GetConnectedUnmappedWorkerKeyIdentifiers()) if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting authorized unmapped worker key ids")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting authorized worker key ids: %v", err) + return nil, status.Errorf(codes.Internal, "Error getting authorized worker key ids: %v", err) } authorizedDownstreams.UnmappedWorkerKeyIdentifiers = authorizedKeyIds } - authorizedWorkerList := &pbs.AuthorizedWorkerList{} - if len(req.GetConnectedWorkerKeyIdentifiers()) > 0 { - authorizedKeyIds, err := workerAuthRepo.FilterToAuthorizedWorkerKeyIds(ctx, req.GetConnectedWorkerKeyIdentifiers()) - if err != nil { - event.WriteError(ctx, op, err, event.WithInfoMsg("error getting authorized worker key ids")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error getting authorized worker key ids: %v", err) - } - authorizedWorkerList.WorkerKeyIdentifiers = authorizedKeyIds - } - ret := &pbs.StatusResponse{ CalculatedUpstreams: responseControllers, WorkerId: workerId, - AuthorizedWorkers: authorizedWorkerList, AuthorizedDownstreamWorkers: authorizedDownstreams, } @@ -281,12 +400,12 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques sessRepo, err := ws.sessionRepoFn() if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting sessions repo")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err) + return nil, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err) } connectionRepo, err := ws.connectionRepoFn() if err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error getting connection repo")) - return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err) + return nil, status.Errorf(codes.Internal, "Error acquiring repo to query session status: %v", err) } notActive, err := session.WorkerStatusReport(ctx, sessRepo, connectionRepo, workerId, stateReport) @@ -348,6 +467,11 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques }) } + err = serverRepo.UpsertSessionInfo(ctx, workerId) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error updating the latest status time for the server session info: %v", err) + } + return ret, nil } @@ -360,8 +484,8 @@ func (ws *workerServiceServer) ListHcpbWorkers(ctx context.Context, req *pbs.Lis return nil, status.Errorf(codes.Internal, "Error getting servers repo: %v", err) } - // We use the livenessTimeToStale here instead of WorkerStatusGracePeriod - // since WorkerStatusGracePeriod is more for deciding which workers should + // We use the livenessTimeToStale here instead of WorkerRPCGracePeriod + // since WorkerRPCGracePeriod is more for deciding which workers should // be used for session proxying, but here we care about providing the BYOW // workers with a list of which upstreams to connect to as their upstreams. managed, err := serversRepo.ListHcpbManagedWorkers(ctx, time.Duration(ws.livenessTimeToStale.Load())) @@ -385,6 +509,102 @@ func (ws *workerServiceServer) ListHcpbWorkers(ctx context.Context, req *pbs.Lis return resp, nil } +func (ws *workerServiceServer) RoutingInfo(ctx context.Context, req *pbs.RoutingInfoRequest) (*pbs.RoutingInfoResponse, error) { + const op = "workers.(workerServiceServer).RoutingInfo" + wStat := req.GetWorkerStatus() + if wStat == nil { + return nil, status.Error(codes.InvalidArgument, "worker status is required") + } + switch { + case wStat.GetPublicId() == "" && wStat.GetKeyId() == "" && wStat.GetName() == "": + return nil, status.Error(codes.InvalidArgument, "public id, key id and name are not set in the request; one is required") + case wStat.GetAddress() == "": + return nil, status.Error(codes.InvalidArgument, "address is not set but is required") + case wStat.GetReleaseVersion() == "": + return nil, status.Error(codes.InvalidArgument, "release version is not set but is required") + case wStat.GetOperationalState() == "": + return nil, status.Error(codes.InvalidArgument, "operational state is not set but is required") + case wStat.GetLocalStorageState() == "": + return nil, status.Error(codes.InvalidArgument, "local storage state is not set but is required") + } + // This Store call is currently only for testing purposes + ws.updateTimes.Store(wStat.GetName(), time.Now()) + + serverRepo, err := ws.serversRepoFn() + if err != nil { + return nil, status.Errorf(codes.Internal, "error getting server repo: %v", err) + } + workerAuthRepo, err := ws.workerAuthRepoFn() + if err != nil { + return nil, status.Errorf(codes.Internal, "error getting worker auth repo: %v", err) + } + + // Convert API tags to storage tags + workerTags := make([]*server.Tag, 0, len(wStat.GetTags())) + for _, v := range wStat.GetTags() { + workerTags = append(workerTags, &server.Tag{ + Key: v.GetKey(), + Value: v.GetValue(), + }) + } + + wConf := server.NewWorker( + scope.Global.String(), + server.WithName(wStat.GetName()), + server.WithDescription(wStat.GetDescription()), + server.WithAddress(wStat.GetAddress()), + server.WithWorkerTags(workerTags...), + server.WithReleaseVersion(wStat.ReleaseVersion), + server.WithOperationalState(wStat.OperationalState), + server.WithLocalStorageState(wStat.LocalStorageState), + ) + opts := []server.Option{server.WithUpdateTags(req.GetUpdateTags())} + if wStat.GetPublicId() != "" { + opts = append(opts, server.WithPublicId(wStat.GetPublicId())) + } + if wStat.GetKeyId() != "" { + opts = append(opts, server.WithKeyId(wStat.GetKeyId())) + } + workerId, err := serverRepo.UpsertWorkerStatus(ctx, wConf, opts...) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error storing worker status: %v", err) + } + + if sbcStates := wStat.GetStorageBucketCredentialStates(); sbcStates != nil { + updateWorkerStorageBucketCredentialStatesFn(ctx, serverRepo, workerId, sbcStates) + } + + controllers, err := serverRepo.ListControllers(ctx, server.WithLiveness(time.Duration(ws.livenessTimeToStale.Load()))) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error getting current controllers: %v", err) + } + var upstreamAddresses []string + for _, c := range controllers { + upstreamAddresses = append(upstreamAddresses, c.Address) + } + + authorizedDownstreams, err := server.VerifyKnownAndUnmappedWorkers( + ctx, + serverRepo, + workerAuthRepo, + req.GetConnectedWorkerPublicIds(), + req.GetConnectedUnmappedWorkerKeyIdentifiers(), + ) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error getting known and authorized workers: %v", err) + } + + ret := &pbs.RoutingInfoResponse{ + WorkerId: workerId, + CalculatedUpstreamAddresses: upstreamAddresses, + AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{ + WorkerPublicIds: authorizedDownstreams.WorkerPublicIds, + UnmappedWorkerKeyIdentifiers: authorizedDownstreams.UnmappedWorkerKeyIds, + }, + } + return ret, nil +} + // Single-hop filter lookup. We have either an egress filter or worker filter to use, if any // Used to verify that the worker serving this session to a client matches this filter func egressFilterSelector(sessionInfo *session.Session) string { diff --git a/internal/daemon/cluster/handlers/worker_service_status_test.go b/internal/daemon/cluster/handlers/worker_service_status_test.go index 1dc27f1a7..aea455ed1 100644 --- a/internal/daemon/cluster/handlers/worker_service_status_test.go +++ b/internal/daemon/cluster/handlers/worker_service_status_test.go @@ -148,7 +148,6 @@ func TestStatus(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -228,7 +227,6 @@ func TestStatus(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, JobsRequests: []*pbs.JobChangeRequest{ { @@ -336,7 +334,6 @@ func TestStatus(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, JobsRequests: []*pbs.JobChangeRequest{ { @@ -414,7 +411,6 @@ func TestStatus(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -449,7 +445,7 @@ func TestStatus(t *testing.T) { got, err := s.Status(ctx, tc.req) if tc.wantErr { require.Error(err) - assert.Equal(got, &pbs.StatusResponse{}) + assert.Empty(got) assert.Equal(tc.wantErrMsg, err.Error()) return } @@ -468,7 +464,6 @@ func TestStatus(t *testing.T) { pbs.SessionJobInfo{}, pbs.MonitorSessionJobInfo{}, pbs.Connection{}, - pbs.AuthorizedWorkerList{}, pbs.AuthorizedDownstreamWorkerList{}, ), cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"), @@ -628,7 +623,6 @@ func TestStatusSessionClosed(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -662,7 +656,6 @@ func TestStatusSessionClosed(t *testing.T) { pbs.Job_SessionInfo{}, pbs.SessionJobInfo{}, pbs.Connection{}, - pbs.AuthorizedWorkerList{}, pbs.AuthorizedDownstreamWorkerList{}, ), cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"), @@ -797,7 +790,6 @@ func TestStatusDeadConnection(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, } @@ -816,7 +808,6 @@ func TestStatusDeadConnection(t *testing.T) { pbs.Job_SessionInfo{}, pbs.SessionJobInfo{}, pbs.Connection{}, - pbs.AuthorizedWorkerList{}, pbs.AuthorizedDownstreamWorkerList{}, ), cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"), @@ -952,7 +943,6 @@ func TestStatusWorkerWithKeyId(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -992,7 +982,6 @@ func TestStatusWorkerWithKeyId(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -1023,7 +1012,6 @@ func TestStatusWorkerWithKeyId(t *testing.T) { pbs.Job_SessionInfo{}, pbs.SessionJobInfo{}, pbs.Connection{}, - pbs.AuthorizedWorkerList{}, pbs.AuthorizedDownstreamWorkerList{}, ), cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"), @@ -1094,7 +1082,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) { Name: worker1.GetName(), Address: worker1.GetAddress(), }, - ConnectedWorkerKeyIdentifiers: []string{}, }, want: &pbs.StatusResponse{ CalculatedUpstreams: []*pbs.UpstreamServer{ @@ -1104,7 +1091,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -1126,7 +1112,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{}, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, @@ -1139,7 +1124,7 @@ func TestStatusAuthorizedWorkers(t *testing.T) { Name: worker1.GetName(), Address: worker1.GetAddress(), }, - ConnectedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"}, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"}, }, want: &pbs.StatusResponse{ CalculatedUpstreams: []*pbs.UpstreamServer{ @@ -1149,10 +1134,9 @@ func TestStatusAuthorizedWorkers(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{ - WorkerKeyIdentifiers: []string{w1KeyId, w2KeyId}, + AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{ + UnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId}, }, - AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{}, }, }, { @@ -1164,8 +1148,7 @@ func TestStatusAuthorizedWorkers(t *testing.T) { Name: worker1.GetName(), Address: worker1.GetAddress(), }, - ConnectedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"}, - ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "unknown"}, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"}, ConnectedWorkerPublicIds: []string{w1.GetPublicId(), "unknown"}, }, want: &pbs.StatusResponse{ @@ -1176,11 +1159,8 @@ func TestStatusAuthorizedWorkers(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{ - WorkerKeyIdentifiers: []string{w1KeyId, w2KeyId}, - }, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{ - UnmappedWorkerKeyIdentifiers: []string{w2KeyId}, + UnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId}, WorkerPublicIds: []string{w1.GetPublicId()}, }, }, @@ -1194,8 +1174,7 @@ func TestStatusAuthorizedWorkers(t *testing.T) { Name: worker1.GetName(), Address: worker1.GetAddress(), }, - ConnectedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"}, - ConnectedUnmappedWorkerKeyIdentifiers: []string{"unknown"}, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId, "unknown"}, ConnectedWorkerPublicIds: []string{w1.GetPublicId(), w2.GetPublicId(), "unknown"}, }, want: &pbs.StatusResponse{ @@ -1206,11 +1185,9 @@ func TestStatusAuthorizedWorkers(t *testing.T) { }, }, WorkerId: worker1.PublicId, - AuthorizedWorkers: &pbs.AuthorizedWorkerList{ - WorkerKeyIdentifiers: []string{w1KeyId, w2KeyId}, - }, AuthorizedDownstreamWorkers: &pbs.AuthorizedDownstreamWorkerList{ - WorkerPublicIds: []string{w1.GetPublicId(), w2.GetPublicId()}, + UnmappedWorkerKeyIdentifiers: []string{w1KeyId, w2KeyId}, + WorkerPublicIds: []string{w1.GetPublicId(), w2.GetPublicId()}, }, }, }, @@ -1227,10 +1204,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) { assert.Equal(tc.wantErrMsg, err.Error()) return } - gotAuthorizedWorkers := got.GetAuthorizedWorkers() - sort.Strings(gotAuthorizedWorkers.GetWorkerKeyIdentifiers()) - wantAuthorizedWorkers := tc.want.GetAuthorizedWorkers() - sort.Strings(wantAuthorizedWorkers.GetWorkerKeyIdentifiers()) sort.Strings(got.GetAuthorizedDownstreamWorkers().GetWorkerPublicIds()) sort.Strings(tc.want.GetAuthorizedDownstreamWorkers().GetWorkerPublicIds()) sort.Strings(got.GetAuthorizedDownstreamWorkers().GetUnmappedWorkerKeyIdentifiers()) @@ -1248,7 +1221,6 @@ func TestStatusAuthorizedWorkers(t *testing.T) { pbs.Job_SessionInfo{}, pbs.SessionJobInfo{}, pbs.Connection{}, - pbs.AuthorizedWorkerList{}, pbs.AuthorizedDownstreamWorkerList{}, ), cmpopts.IgnoreFields(pb.ServerWorkerStatus{}, "Tags"), diff --git a/internal/daemon/cluster/handlers/worker_service_test.go b/internal/daemon/cluster/handlers/worker_service_test.go index d4325ea99..d850cf123 100644 --- a/internal/daemon/cluster/handlers/worker_service_test.go +++ b/internal/daemon/cluster/handlers/worker_service_test.go @@ -6,6 +6,7 @@ package handlers import ( "context" "crypto/rand" + "errors" "fmt" "sync" "sync/atomic" @@ -16,12 +17,14 @@ import ( "github.com/hashicorp/boundary/internal/authtoken" credstatic "github.com/hashicorp/boundary/internal/credential/static" "github.com/hashicorp/boundary/internal/db" + pb "github.com/hashicorp/boundary/internal/gen/controller/servers" pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" intglobals "github.com/hashicorp/boundary/internal/globals" "github.com/hashicorp/boundary/internal/host/static" "github.com/hashicorp/boundary/internal/iam" "github.com/hashicorp/boundary/internal/kms" "github.com/hashicorp/boundary/internal/server" + "github.com/hashicorp/boundary/internal/server/store" "github.com/hashicorp/boundary/internal/session" "github.com/hashicorp/boundary/internal/target" "github.com/hashicorp/boundary/internal/target/tcp" @@ -31,6 +34,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/crypto/ssh/testdata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" ) @@ -629,3 +634,800 @@ func TestHcpbWorkers(t *testing.T) { } assert.ElementsMatch(expValues, gotValues) } + +func TestStatistics(t *testing.T) { + t.Parallel() + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kms := kms.TestKms(t, conn, wrapper) + serversRepoFn := func() (*server.Repository, error) { + return server.NewRepository(ctx, rw, rw, kms) + } + workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) { + return server.NewRepositoryStorage(ctx, rw, rw, kms) + } + sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) { + return session.NewRepository(ctx, rw, rw, kms, opts...) + } + connectionRepoFn := func() (*session.ConnectionRepository, error) { + return session.NewConnectionRepository(ctx, rw, rw, kms) + } + connectionRepoErrFn := func() (*session.ConnectionRepository, error) { + return nil, fmt.Errorf("test error") + } + fce := &fakeControllerExtension{ + reader: rw, + writer: rw, + } + cases := []struct { + name string + workerService *workerServiceServer + wantErrMsg string + expectedStatusCode codes.Code + req *pbs.StatisticsRequest + }{ + { + name: "empty worker id", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce), + req: &pbs.StatisticsRequest{}, + wantErrMsg: "rpc error: code = InvalidArgument desc = worker id is empty", + expectedStatusCode: codes.InvalidArgument, + }, + { + name: "missing session connection repo", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoErrFn, nil, new(sync.Map), kms, new(atomic.Int64), fce), + req: &pbs.StatisticsRequest{ + WorkerId: "w_1234567890", + Sessions: []*pbs.SessionStatistics{ + {}, + }, + }, + wantErrMsg: "rpc error: code = Internal desc = Error acquiring connection repo: test error", + expectedStatusCode: codes.Internal, + }, + { + name: "nil sessions", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce), + req: &pbs.StatisticsRequest{ + WorkerId: "w_1234567890", + }, + }, + { + name: "empty sessions", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce), + req: &pbs.StatisticsRequest{ + WorkerId: "w_1234567890", + Sessions: []*pbs.SessionStatistics{}, + }, + }, + { + name: "empty session id", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce), + req: &pbs.StatisticsRequest{ + WorkerId: "w_1234567890", + Sessions: []*pbs.SessionStatistics{ + {}, + }, + }, + wantErrMsg: "rpc error: code = InvalidArgument desc = session id is empty", + expectedStatusCode: codes.InvalidArgument, + }, + { + name: "empty connection id", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce), + req: &pbs.StatisticsRequest{ + WorkerId: "w_1234567890", + Sessions: []*pbs.SessionStatistics{ + { + SessionId: "s_1234567890", + Connections: []*pbs.Connection{ + {}, + }, + }, + }, + }, + wantErrMsg: "rpc error: code = InvalidArgument desc = connection id is empty", + expectedStatusCode: codes.InvalidArgument, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + got, err := tc.workerService.Statistics(context.Background(), tc.req) + require.NotNil(got) + if tc.wantErrMsg != "" { + require.Error(err) + assert.ErrorContains(err, tc.wantErrMsg) + actualStatus, ok := status.FromError(err) + require.True(ok) + assert.Equal(tc.expectedStatusCode, actualStatus.Code()) + return + } + require.NoError(err) + }) + } + + // update the bytes up and bytes down for an active session and close an orphaned session connection + t.Run("happy path", func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + connectionRepoFn := func() (*session.ConnectionRepository, error) { + return session.NewConnectionRepository(ctx, rw, rw, kms, session.WithWorkerStateDelay(0)) + } + sessRepo, err := sessionRepoFn() + require.NoError(err) + connRepo, err := connectionRepoFn() + require.NoError(err) + + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, kms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := session.TestSession(t, conn, wrapper, session.ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, session.TestTofu(t)) + require.NoError(err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId()) + require.NoError(err) + + s2 := session.TestSession(t, conn, wrapper, session.ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, session.TestTofu(t)) + require.NoError(err) + c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId) + require.NoError(err) + + var expectedBytesUp int64 = 1024 + var expectedBytesDown int64 = 2048 + workerService := NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kms, new(atomic.Int64), fce) + got, err := workerService.Statistics(context.Background(), &pbs.StatisticsRequest{ + WorkerId: w.GetPublicId(), + Sessions: []*pbs.SessionStatistics{ + { + SessionId: s1.PublicId, + Connections: []*pbs.Connection{ + { + ConnectionId: c1.PublicId, + BytesUp: expectedBytesUp, + BytesDown: expectedBytesDown, + Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED, + }, + }, + }, + }, + }) + require.NoError(err) + assert.NotNil(got) + + conn, err := connRepo.LookupConnection(context.Background(), c1.PublicId) + require.NoError(err) + assert.Equal(expectedBytesUp, conn.BytesUp) + assert.Equal(expectedBytesDown, conn.BytesDown) + assert.Equal(session.StatusAuthorized, session.ConnectionStatusFromString(conn.Status)) + + conn, err = connRepo.LookupConnection(context.Background(), c2.PublicId) + require.NoError(err) + assert.Empty(conn.BytesUp) + assert.Empty(conn.BytesDown) + assert.Equal(session.StatusClosed, session.ConnectionStatusFromString(conn.Status)) + }) +} + +func TestSessionInfo(t *testing.T) { + t.Parallel() + + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + testKms := kms.TestKms(t, conn, wrapper) + serversRepoFn := func() (*server.Repository, error) { + return server.NewRepository(ctx, rw, rw, testKms) + } + workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) { + return server.NewRepositoryStorage(ctx, rw, rw, testKms) + } + sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) { + return session.NewRepository(ctx, rw, rw, testKms, opts...) + } + connectionRepoFn := func() (*session.ConnectionRepository, error) { + return session.NewConnectionRepository(ctx, rw, rw, testKms) + } + serversErrRepoFn := func() (*server.Repository, error) { + return nil, errors.New("unknown error") + } + sessionErrRepoFn := func(opt ...session.Option) (*session.Repository, error) { + return nil, errors.New("unknown error") + } + fce := &fakeControllerExtension{ + reader: rw, + writer: rw, + } + iamRepo := iam.TestRepo(t, conn, wrapper) + iam.TestScopes(t, iamRepo) + workerId := server.TestPkiWorker(t, conn, wrapper).PublicId + cases := []struct { + name string + workerService *workerServiceServer + wantErrMsg string + expectedStatusCode codes.Code + req *pbs.SessionInfoRequest + expectedResponse *pbs.SessionInfoResponse + }{ + { + name: "empty worker id", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{}, + wantErrMsg: "rpc error: code = InvalidArgument desc = worker id is empty", + expectedStatusCode: codes.InvalidArgument, + }, + { + name: "invalid unspecified session type", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{ + WorkerId: "w_1234567890", + Sessions: []*pbs.Session{ + { + SessionId: "s_A", + SessionType: pbs.SessionType_SESSION_TYPE_UNSPECIFIED, + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, + }, + }, + }, + wantErrMsg: "rpc error: code = InvalidArgument desc = unspecified session type", + expectedStatusCode: codes.InvalidArgument, + }, + { + name: "invalid unknown session type", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{ + WorkerId: "w_1234567890", + Sessions: []*pbs.Session{ + { + SessionId: "s_A", + SessionType: pbs.SessionType(-1), + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, + }, + }, + }, + wantErrMsg: "rpc error: code = InvalidArgument desc = unknown session type", + expectedStatusCode: codes.InvalidArgument, + }, + { + name: "session repository error", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionErrRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{ + WorkerId: "w_1234567890", + }, + wantErrMsg: "rpc error: code = Internal desc = Error acquiring repo to query session status: unknown error", + expectedStatusCode: codes.Internal, + }, + { + name: "server repository error", + workerService: NewWorkerServiceServer(serversErrRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{ + WorkerId: "w_1234567890", + }, + wantErrMsg: "rpc error: code = Internal desc = Error acquiring repo to upsert session info status time: unknown error", + expectedStatusCode: codes.Internal, + }, + { + name: "ignore canceled and terminated sessions", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{ + WorkerId: workerId, + Sessions: []*pbs.Session{ + { + SessionId: "s_A", + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING, + SessionType: pbs.SessionType_SESSION_TYPE_INGRESSED, + }, + { + SessionId: "s_B", + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED, + SessionType: pbs.SessionType_SESSION_TYPE_INGRESSED, + }, + { + SessionId: "s_C", + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING, + SessionType: pbs.SessionType_SESSION_TYPE_RECORDED, + }, + { + SessionId: "s_D", + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED, + SessionType: pbs.SessionType_SESSION_TYPE_RECORDED, + }, + }, + }, + expectedResponse: &pbs.SessionInfoResponse{}, + }, + { + name: "empty sessions", + workerService: NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce), + req: &pbs.SessionInfoRequest{ + WorkerId: workerId, + }, + expectedResponse: &pbs.SessionInfoResponse{}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + got, err := tc.workerService.SessionInfo(context.Background(), tc.req) + require.NotNil(got) + if tc.wantErrMsg != "" { + require.Error(err) + assert.ErrorContains(err, tc.wantErrMsg) + actualStatus, ok := status.FromError(err) + require.True(ok) + assert.Equal(tc.expectedStatusCode, actualStatus.Code()) + return + } + require.NoError(err) + require.NotNil(got) + assert.Len(got.NonActiveSessions, len(tc.expectedResponse.NonActiveSessions)) + }) + } + + t.Run("session canceled with active connections", func(t *testing.T) { + t.Parallel() + assert, require := assert.New(t), require.New(t) + sessionTypes := []pbs.SessionType{ + pbs.SessionType_SESSION_TYPE_INGRESSED, + pbs.SessionType_SESSION_TYPE_RECORDED, + } + for _, sessionType := range sessionTypes { + t.Run(sessionType.String(), func(t *testing.T) { + t.Parallel() + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + testKms := kms.TestKms(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + + serverRepo, err := server.NewRepository(ctx, rw, rw, testKms) + require.NoError(err) + c := &store.Controller{ + PrivateId: "test_controller1", + Address: "127.0.0.1", + } + _, err = serverRepo.UpsertController(ctx, c) + require.NoError(err) + + serversRepoFn := func() (*server.Repository, error) { + return serverRepo, nil + } + workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) { + return server.NewRepositoryStorage(ctx, rw, rw, testKms) + } + sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) { + return session.NewRepository(ctx, rw, rw, testKms, opts...) + } + connRepoFn := func() (*session.ConnectionRepository, error) { + return session.NewConnectionRepository(ctx, rw, rw, testKms) + } + fce := &fakeControllerExtension{ + reader: rw, + writer: rw, + } + + repo, err := sessionRepoFn() + require.NoError(err) + connRepo, err := connRepoFn() + require.NoError(err) + + at := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + uId := at.GetIamUserId() + hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0] + h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h}) + tar := tcp.TestTarget( + ctx, + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hs.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + + worker1 := server.TestKmsWorker(t, conn, wrapper) + sess := session.TestSession(t, conn, wrapper, session.ComposedOf{ + UserId: uId, + HostId: h.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hs.GetPublicId(), + AuthTokenId: at.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + tofu := session.TestTofu(t) + sess, _, err = repo.ActivateSession(ctx, sess.PublicId, sess.Version, tofu) + require.NoError(err) + sess2 := session.TestSession(t, conn, wrapper, session.ComposedOf{ + UserId: uId, + HostId: h.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hs.GetPublicId(), + AuthTokenId: at.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + tofu2 := session.TestTofu(t) + sess2, _, err = repo.ActivateSession(ctx, sess2.PublicId, sess2.Version, tofu2) + require.NoError(err) + + s := NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connRepoFn, nil, new(sync.Map), testKms, new(atomic.Int64), fce) + require.NotNil(t, s) + + _, err = connRepo.AuthorizeConnection(ctx, sess.PublicId, worker1.PublicId) + require.NoError(err) + + _, err = connRepo.AuthorizeConnection(ctx, sess2.PublicId, worker1.PublicId) + require.NoError(err) + + _, err = repo.CancelSession(ctx, sess2.PublicId, sess.Version) + require.NoError(err) + + got, err := s.SessionInfo(ctx, &pbs.SessionInfoRequest{ + WorkerId: worker1.GetPublicId(), + Sessions: []*pbs.Session{ + { + SessionId: sess2.PublicId, + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, + SessionType: sessionType, + }, + }, + }) + require.NoError(err) + require.NotNil(got) + require.Len(got.NonActiveSessions, 1) + actualNonActiveSession := got.NonActiveSessions[0] + require.NotNil(actualNonActiveSession) + assert.Equal(sess2.PublicId, actualNonActiveSession.SessionId) + assert.Equal(pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING, actualNonActiveSession.SessionStatus) + assert.Equal(sessionType, actualNonActiveSession.SessionType) + assert.Empty(actualNonActiveSession.Connections) + }) + } + }) +} + +func TestRoutingInfo(t *testing.T) { + t.Parallel() + + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kmsCache := kms.TestKms(t, conn, wrapper) + require.NoError(t, kmsCache.CreateKeys(context.Background(), scope.Global.String(), kms.WithRandomReader(rand.Reader))) + + serversRepoFn := func() (*server.Repository, error) { + return server.NewRepository(ctx, rw, rw, kmsCache) + } + workerAuthRepoFn := func() (*server.WorkerAuthRepositoryStorage, error) { + return server.NewRepositoryStorage(ctx, rw, rw, kmsCache) + } + sessionRepoFn := func(opts ...session.Option) (*session.Repository, error) { + return session.NewRepository(ctx, rw, rw, kmsCache, opts...) + } + connectionRepoFn := func() (*session.ConnectionRepository, error) { + return session.NewConnectionRepository(ctx, rw, rw, kmsCache) + } + var liveDur atomic.Int64 + liveDur.Store(int64(1 * time.Second)) + fce := &fakeControllerExtension{ + reader: rw, + writer: rw, + } + serverRepo, err := serversRepoFn() + require.NoError(t, err) + + // Set up resources + var w1KeyId, w2KeyId string + w1 := server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w1KeyId)) + _ = server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w2KeyId)) + w3 := server.TestKmsWorker(t, conn, wrapper, server.WithName("testworker3")) + + c := &store.Controller{ + PrivateId: "test_controller1", + Address: "1.2.3.4", + } + _, err = serverRepo.UpsertController(ctx, c) + require.NoError(t, err) + + s := NewWorkerServiceServer(serversRepoFn, workerAuthRepoFn, sessionRepoFn, connectionRepoFn, nil, new(sync.Map), kmsCache, &liveDur, fce) + require.NotNil(t, s) + require.NoError(t, err) + + t.Run("Missing worker status", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{} + _, err := s.RoutingInfo(ctx, req) + require.ErrorContains(t, err, "worker status is required") + }) + + t.Run("Missing key ID and public Id", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", + KeyId: "", + Address: "2.3.4.5:8080", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "Boundary v0.18.0", + OperationalState: server.ActiveOperationalState.String(), + LocalStorageState: server.AvailableLocalStorageState.String(), + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w3.PublicId, "worker-3"}, + } + _, err := s.RoutingInfo(ctx, req) + require.ErrorContains(t, err, "public id, key id and name are not set in the request; one is required") + }) + + t.Run("Missing address", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", + KeyId: w1KeyId, + Address: "", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "Boundary v0.18.0", + OperationalState: server.ActiveOperationalState.String(), + LocalStorageState: server.AvailableLocalStorageState.String(), + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w3.PublicId, "worker-3"}, + } + _, err := s.RoutingInfo(ctx, req) + require.ErrorContains(t, err, "address is not set but is required") + }) + + t.Run("Missing release version", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", + KeyId: w1KeyId, + Address: "2.3.4.5:8080", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "", + OperationalState: server.ActiveOperationalState.String(), + LocalStorageState: server.AvailableLocalStorageState.String(), + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w3.PublicId, "worker-3"}, + } + _, err := s.RoutingInfo(ctx, req) + require.ErrorContains(t, err, "release version is not set but is required") + }) + + t.Run("Missing operational state", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", + KeyId: w1KeyId, + Address: "2.3.4.5:8080", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "Boundary v0.18.0", + OperationalState: "", + LocalStorageState: server.AvailableLocalStorageState.String(), + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w3.PublicId, "worker-3"}, + } + _, err := s.RoutingInfo(ctx, req) + require.ErrorContains(t, err, "operational state is not set but is required") + }) + + t.Run("Missing local storage state", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", + KeyId: w1KeyId, + Address: "2.3.4.5:8080", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "Boundary v0.18.0", + OperationalState: server.ActiveOperationalState.String(), + LocalStorageState: "", + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w3.PublicId, "worker-3"}, + } + _, err := s.RoutingInfo(ctx, req) + require.ErrorContains(t, err, "local storage state is not set but is required") + }) + + t.Run("Successful first and second request (PKI worker)", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", // Not set on the first request + KeyId: w1KeyId, + Address: "2.3.4.5:8080", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "Boundary v0.18.0", + OperationalState: server.ActiveOperationalState.String(), + LocalStorageState: server.AvailableLocalStorageState.String(), + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w3.PublicId, "worker-3"}, + } + resp, err := s.RoutingInfo(ctx, req) + require.NoError(t, err) + assert.NotNil(t, resp) + + assert.Equal(t, resp.WorkerId, w1.PublicId) + assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w3.PublicId}) + + w1, err := serverRepo.LookupWorker(ctx, resp.WorkerId) + require.NoError(t, err) + + assert.Equal(t, w1.Address, "2.3.4.5:8080") + expTags := []*server.Tag{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + } + assert.Empty( + t, + cmp.Diff( + w1.ConfigTags, + server.Tags(expTags), + ), + ) + assert.Equal(t, w1.ReleaseVersion, "Boundary v0.18.0") + assert.EqualValues(t, w1.OperationalState, server.ActiveOperationalState) + assert.EqualValues(t, w1.LocalStorageState, server.AvailableLocalStorageState) + + // Now send the subsequent routing info request + req.WorkerStatus.PublicId = w1.PublicId // Only set on subsequent requests + resp, err = s.RoutingInfo(ctx, req) + require.NoError(t, err) + assert.NotNil(t, resp) + + assert.Equal(t, resp.WorkerId, w1.PublicId) + assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w3.PublicId}) + + w1, err = serverRepo.LookupWorker(ctx, resp.WorkerId) + require.NoError(t, err) + + assert.Equal(t, w1.Address, "2.3.4.5:8080") + assert.Empty( + t, + cmp.Diff( + w1.ConfigTags, + server.Tags(expTags), + ), + ) + assert.Equal(t, w1.ReleaseVersion, "Boundary v0.18.0") + assert.EqualValues(t, w1.OperationalState, server.ActiveOperationalState) + assert.EqualValues(t, w1.LocalStorageState, server.AvailableLocalStorageState) + }) + + t.Run("Successful first and second request (KMS worker)", func(t *testing.T) { + req := &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + PublicId: "", // Not set on the first request + Name: "testworker3", + Address: "2.3.4.5:8080", + Tags: []*pb.TagPair{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + }, + ReleaseVersion: "Boundary v0.18.0", + OperationalState: server.ActiveOperationalState.String(), + LocalStorageState: server.AvailableLocalStorageState.String(), + }, + UpdateTags: true, + ConnectedUnmappedWorkerKeyIdentifiers: []string{w2KeyId, "worker-key-2"}, + ConnectedWorkerPublicIds: []string{w1.PublicId, "worker-4"}, + } + resp, err := s.RoutingInfo(ctx, req) + require.NoError(t, err) + assert.NotNil(t, resp) + + assert.Equal(t, resp.WorkerId, w3.PublicId) + assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w1.PublicId}) + + w3, err := serverRepo.LookupWorker(ctx, resp.WorkerId) + require.NoError(t, err) + + assert.Equal(t, w3.Address, "2.3.4.5:8080") + expTags := []*server.Tag{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2", Value: "value2"}, + } + assert.Empty( + t, + cmp.Diff( + w3.ConfigTags, + server.Tags(expTags), + ), + ) + assert.Equal(t, w3.ReleaseVersion, "Boundary v0.18.0") + assert.EqualValues(t, w3.OperationalState, server.ActiveOperationalState) + assert.EqualValues(t, w3.LocalStorageState, server.AvailableLocalStorageState) + + // Now send the subsequent routing info request + req.WorkerStatus.PublicId = w3.PublicId // Only set on subsequent requests + resp, err = s.RoutingInfo(ctx, req) + require.NoError(t, err) + assert.NotNil(t, resp) + + assert.Equal(t, resp.WorkerId, w3.PublicId) + assert.Equal(t, resp.CalculatedUpstreamAddresses, []string{c.Address}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.UnmappedWorkerKeyIdentifiers, []string{w2KeyId}) + assert.Equal(t, resp.AuthorizedDownstreamWorkers.WorkerPublicIds, []string{w1.PublicId}) + + w1, err = serverRepo.LookupWorker(ctx, resp.WorkerId) + require.NoError(t, err) + + assert.Equal(t, w3.Address, "2.3.4.5:8080") + assert.Empty( + t, + cmp.Diff( + w3.ConfigTags, + server.Tags(expTags), + ), + ) + assert.Equal(t, w3.ReleaseVersion, "Boundary v0.18.0") + assert.EqualValues(t, w3.OperationalState, server.ActiveOperationalState) + assert.EqualValues(t, w3.LocalStorageState, server.AvailableLocalStorageState) + }) +} diff --git a/internal/daemon/controller/controller.go b/internal/daemon/controller/controller.go index 68122885d..d7818da80 100644 --- a/internal/daemon/controller/controller.go +++ b/internal/daemon/controller/controller.go @@ -117,11 +117,11 @@ type Controller struct { clusterListener *base.ServerListener // Used for testing and tracking worker health - workerStatusUpdateTimes *sync.Map + workerRoutingInfoUpdateTimes *sync.Map // Timing variables. These are atomics for SIGHUP support, and are int64 // because they are casted to time.Duration. - workerStatusGracePeriod *atomic.Int64 + workerRPCGracePeriod *atomic.Int64 livenessTimeToStale *atomic.Int64 getDownstreamWorkersTimeout *atomic.Int64 @@ -177,19 +177,19 @@ func New(ctx context.Context, conf *Config) (*Controller, error) { metric.InitializeApiCollectors(conf.PrometheusRegisterer) ratelimit.InitializeMetrics(conf.PrometheusRegisterer) c := &Controller{ - conf: conf, - logger: conf.Logger.Named("controller"), - started: ua.NewBool(false), - tickerWg: new(sync.WaitGroup), - schedulerWg: new(sync.WaitGroup), - workerAuthCache: new(sync.Map), - workerStatusUpdateTimes: new(sync.Map), - enabledPlugins: conf.Server.EnabledPlugins, - apiListeners: make([]*base.ServerListener, 0), - downstreamConnManager: cluster.NewDownstreamManager(), - workerStatusGracePeriod: new(atomic.Int64), - livenessTimeToStale: new(atomic.Int64), - getDownstreamWorkersTimeout: new(atomic.Int64), + conf: conf, + logger: conf.Logger.Named("controller"), + started: ua.NewBool(false), + tickerWg: new(sync.WaitGroup), + schedulerWg: new(sync.WaitGroup), + workerAuthCache: new(sync.Map), + workerRoutingInfoUpdateTimes: new(sync.Map), + enabledPlugins: conf.Server.EnabledPlugins, + apiListeners: make([]*base.ServerListener, 0), + downstreamConnManager: cluster.NewDownstreamManager(), + workerRPCGracePeriod: new(atomic.Int64), + livenessTimeToStale: new(atomic.Int64), + getDownstreamWorkersTimeout: new(atomic.Int64), } c.started.Store(false) @@ -223,11 +223,11 @@ func New(ctx context.Context, conf *Config) (*Controller, error) { } } - switch conf.RawConfig.Controller.WorkerStatusGracePeriodDuration { + switch conf.RawConfig.Controller.WorkerRPCGracePeriodDuration { case 0: - c.workerStatusGracePeriod.Store(int64(server.DefaultLiveness)) + c.workerRPCGracePeriod.Store(int64(server.DefaultLiveness)) default: - c.workerStatusGracePeriod.Store(int64(conf.RawConfig.Controller.WorkerStatusGracePeriodDuration)) + c.workerRPCGracePeriod.Store(int64(conf.RawConfig.Controller.WorkerRPCGracePeriodDuration)) } switch conf.RawConfig.Controller.LivenessTimeToStaleDuration { case 0: @@ -623,7 +623,7 @@ func (c *Controller) registerJobs() error { if err := pluginhost.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.conf.HostPlugins); err != nil { return err } - if err := session.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.workerStatusGracePeriod); err != nil { + if err := session.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.workerRPCGracePeriod); err != nil { return err } var serverJobOpts []serversjob.Option @@ -633,7 +633,7 @@ func (c *Controller) registerJobs() error { serversjob.WithRotationFrequency(c.conf.TestOverrideWorkerAuthCaCertificateLifetime/2), ) } - if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.ControllerExtension, c.workerStatusGracePeriod, serverJobOpts...); err != nil { + if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.ControllerExtension, c.workerRPCGracePeriod, serverJobOpts...); err != nil { return err } if err := kmsjob.RegisterJobs(c.baseContext, c.scheduler, c.kms); err != nil { @@ -675,12 +675,12 @@ func (c *Controller) Shutdown() error { return nil } -// WorkerStatusUpdateTimes returns the map, which specifically is held in _this_ +// WorkerRoutingInfoUpdateTimes returns the map, which specifically is held in _this_ // controller, not the DB. It's used in tests to verify that a given controller // is receiving updates from an expected set of workers, to test out balancing // and auto reconnection. -func (c *Controller) WorkerStatusUpdateTimes() *sync.Map { - return c.workerStatusUpdateTimes +func (c *Controller) WorkerRoutingInfoUpdateTimes() *sync.Map { + return c.workerRoutingInfoUpdateTimes } // ReloadTimings reloads timing related parameters @@ -694,11 +694,11 @@ func (c *Controller) ReloadTimings(newConfig *config.Config) error { return errors.New(c.baseContext, errors.InvalidParameter, op, "nil config.Controller") } - switch newConfig.Controller.WorkerStatusGracePeriodDuration { + switch newConfig.Controller.WorkerRPCGracePeriodDuration { case 0: - c.workerStatusGracePeriod.Store(int64(server.DefaultLiveness)) + c.workerRPCGracePeriod.Store(int64(server.DefaultLiveness)) default: - c.workerStatusGracePeriod.Store(int64(newConfig.Controller.WorkerStatusGracePeriodDuration)) + c.workerRPCGracePeriod.Store(int64(newConfig.Controller.WorkerRPCGracePeriodDuration)) } switch newConfig.Controller.LivenessTimeToStaleDuration { case 0: diff --git a/internal/daemon/controller/handler.go b/internal/daemon/controller/handler.go index b220fc307..f5ac0a5fd 100644 --- a/internal/daemon/controller/handler.go +++ b/internal/daemon/controller/handler.go @@ -243,7 +243,7 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error { c.baseContext, c.IamRepoFn, c.ServersRepoFn, - c.workerStatusGracePeriod, + c.workerRPCGracePeriod, c.kms, c.conf.RawConfig.Controller.MaxPageSize, c.ControllerExtension) @@ -266,7 +266,7 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error { c.StaticCredentialRepoFn, c.TargetAliasRepoFn, c.downstreamWorkers, - c.workerStatusGracePeriod, + c.workerRPCGracePeriod, c.conf.RawConfig.Controller.MaxPageSize, c.ControllerExtension, ) diff --git a/internal/daemon/controller/handlers/session_recordings/session_recording_service.go b/internal/daemon/controller/handlers/session_recordings/session_recording_service.go index 137b6faa1..ef3caa8f5 100644 --- a/internal/daemon/controller/handlers/session_recordings/session_recording_service.go +++ b/internal/daemon/controller/handlers/session_recordings/session_recording_service.go @@ -44,7 +44,7 @@ func init() { var NewServiceFn = func(ctx context.Context, iamRepoFn common.IamRepoFactory, serverRepoFn common.ServersRepoFactory, - workerStatusGracePeriod *atomic.Int64, + workerRPCGracePeriod *atomic.Int64, kms *kms.Kms, maxPageSize uint, controllerExt intglobals.ControllerExtension, diff --git a/internal/daemon/controller/handlers/targets/target_service.go b/internal/daemon/controller/handlers/targets/target_service.go index 0a8d3e4f1..ea1c8128d 100644 --- a/internal/daemon/controller/handlers/targets/target_service.go +++ b/internal/daemon/controller/handlers/targets/target_service.go @@ -121,20 +121,20 @@ func IngressWorkerFilterUnsupported(string) error { type Service struct { pbs.UnsafeTargetServiceServer - repoFn target.RepositoryFactory - aliasRepoFn common.TargetAliasRepoFactory - iamRepoFn common.IamRepoFactory - serversRepoFn common.ServersRepoFactory - sessionRepoFn session.RepositoryFactory - pluginHostRepoFn common.PluginHostRepoFactory - staticHostRepoFn common.StaticRepoFactory - vaultCredRepoFn common.VaultCredentialRepoFactory - staticCredRepoFn common.StaticCredentialRepoFactory - downstreams downstream.Graph - kmsCache *kms.Kms - workerStatusGracePeriod *atomic.Int64 - maxPageSize uint - controllerExt intglobals.ControllerExtension + repoFn target.RepositoryFactory + aliasRepoFn common.TargetAliasRepoFactory + iamRepoFn common.IamRepoFactory + serversRepoFn common.ServersRepoFactory + sessionRepoFn session.RepositoryFactory + pluginHostRepoFn common.PluginHostRepoFactory + staticHostRepoFn common.StaticRepoFactory + vaultCredRepoFn common.VaultCredentialRepoFactory + staticCredRepoFn common.StaticCredentialRepoFactory + downstreams downstream.Graph + kmsCache *kms.Kms + workerRPCGracePeriod *atomic.Int64 + maxPageSize uint + controllerExt intglobals.ControllerExtension } var _ pbs.TargetServiceServer = (*Service)(nil) @@ -153,7 +153,7 @@ func NewService( staticCredRepoFn common.StaticCredentialRepoFactory, aliasRepoFn common.TargetAliasRepoFactory, downstreams downstream.Graph, - workerStatusGracePeriod *atomic.Int64, + workerRPCGracePeriod *atomic.Int64, maxPageSize uint, controllerExt intglobals.ControllerExtension, ) (Service, error) { @@ -184,20 +184,20 @@ func NewService( maxPageSize = uint(globals.DefaultMaxPageSize) } return Service{ - repoFn: repoFn, - iamRepoFn: iamRepoFn, - serversRepoFn: serversRepoFn, - sessionRepoFn: sessionRepoFn, - pluginHostRepoFn: pluginHostRepoFn, - staticHostRepoFn: staticHostRepoFn, - vaultCredRepoFn: vaultCredRepoFn, - staticCredRepoFn: staticCredRepoFn, - aliasRepoFn: aliasRepoFn, - downstreams: downstreams, - kmsCache: kmsCache, - workerStatusGracePeriod: workerStatusGracePeriod, - maxPageSize: maxPageSize, - controllerExt: controllerExt, + repoFn: repoFn, + iamRepoFn: iamRepoFn, + serversRepoFn: serversRepoFn, + sessionRepoFn: sessionRepoFn, + pluginHostRepoFn: pluginHostRepoFn, + staticHostRepoFn: staticHostRepoFn, + vaultCredRepoFn: vaultCredRepoFn, + staticCredRepoFn: staticCredRepoFn, + aliasRepoFn: aliasRepoFn, + downstreams: downstreams, + kmsCache: kmsCache, + workerRPCGracePeriod: workerRPCGracePeriod, + maxPageSize: maxPageSize, + controllerExt: controllerExt, }, nil } @@ -942,7 +942,7 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession // Get workers and filter down to ones that can service this request selectedWorkers, protoWorkerId, err := serversRepo.SelectSessionWorkers( ctx, - time.Duration(s.workerStatusGracePeriod.Load()), + time.Duration(s.workerRPCGracePeriod.Load()), t, h, s.controllerExt, diff --git a/internal/daemon/controller/handlers/targets/tcp/target_service_test.go b/internal/daemon/controller/handlers/targets/tcp/target_service_test.go index 1b2b7fae1..7079a10de 100644 --- a/internal/daemon/controller/handlers/targets/tcp/target_service_test.go +++ b/internal/daemon/controller/handlers/targets/tcp/target_service_test.go @@ -93,8 +93,8 @@ var expectedDeprecatedWorkerFilterError = "Use egress_worker_filter instead" func testService(t *testing.T, ctx context.Context, conn *db.DB, kms *kms.Kms, wrapper wrapping.Wrapper) (targets.Service, error) { rw := db.New(conn) sche := scheduler.TestScheduler(t, conn, wrapper) - statusGracePeriod := new(atomic.Int64) - statusGracePeriod.Store(int64(server.DefaultLiveness)) + workerRPCGracePeriod := new(atomic.Int64) + workerRPCGracePeriod.Store(int64(server.DefaultLiveness)) repoFn := func(o ...target.Option) (*target.Repository, error) { return target.NewRepository(ctx, rw, rw, kms, o...) } @@ -122,7 +122,7 @@ func testService(t *testing.T, ctx context.Context, conn *db.DB, kms *kms.Kms, w targetAliasRepoFn := func() (*talias.Repository, error) { return talias.NewRepository(ctx, rw, rw, kms) } - return targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil) + return targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil) } func TestGet(t *testing.T) { @@ -3571,9 +3571,9 @@ func TestAuthorizeSession(t *testing.T) { }, } - statusGracePeriod := new(atomic.Int64) - statusGracePeriod.Store(int64(server.DefaultLiveness)) - s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil) + workerRPCGracePeriod := new(atomic.Int64) + workerRPCGracePeriod.Store(int64(server.DefaultLiveness)) + s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil) require.NoError(t, err) for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -3776,9 +3776,9 @@ func TestAuthorizeSessionTypedCredentials(t *testing.T) { _ = iam.TestUserRole(t, conn, r.GetPublicId(), at.GetIamUserId()) _ = iam.TestRoleGrant(t, conn, r.GetPublicId(), "ids=*;type=*;actions=*") - statusGracePeriod := new(atomic.Int64) - statusGracePeriod.Store(int64(server.DefaultLiveness)) - s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil) + workerRPCGracePeriod := new(atomic.Int64) + workerRPCGracePeriod.Store(int64(server.DefaultLiveness)) + s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil) require.NoError(t, err) hc := static.TestCatalogs(t, conn, proj.GetPublicId(), 1)[0] @@ -4380,9 +4380,9 @@ func TestAuthorizeSession_Errors(t *testing.T) { } org, proj := iam.TestScopes(t, iamRepo) - statusGracePeriod := new(atomic.Int64) - statusGracePeriod.Store(int64(server.DefaultLiveness)) - s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, statusGracePeriod, 1000, nil) + workerRPCGracePeriod := new(atomic.Int64) + workerRPCGracePeriod.Store(int64(server.DefaultLiveness)) + s, err := targets.NewService(ctx, kms, repoFn, iamRepoFn, serversRepoFn, sessionRepoFn, pluginHostRepoFn, staticHostRepoFn, vaultCredRepoFn, staticCredRepoFn, targetAliasRepoFn, nil, workerRPCGracePeriod, 1000, nil) require.NoError(t, err) // Authorized user gets full permissions diff --git a/internal/daemon/controller/rpc_registration.go b/internal/daemon/controller/rpc_registration.go index 8e9da95ea..7ad75934f 100644 --- a/internal/daemon/controller/rpc_registration.go +++ b/internal/daemon/controller/rpc_registration.go @@ -47,7 +47,7 @@ func registerControllerServerCoordinationService(ctx context.Context, c *Control c.SessionRepoFn, c.ConnectionRepoFn, c.downstreamWorkers, - c.workerStatusUpdateTimes, + c.workerRoutingInfoUpdateTimes, c.kms, c.livenessTimeToStale, c.ControllerExtension, @@ -74,7 +74,7 @@ func registerControllerSessionService(ctx context.Context, c *Controller, server c.SessionRepoFn, c.ConnectionRepoFn, c.downstreamWorkers, - c.workerStatusUpdateTimes, + c.workerRoutingInfoUpdateTimes, c.kms, c.livenessTimeToStale, c.ControllerExtension, diff --git a/internal/daemon/controller/testing.go b/internal/daemon/controller/testing.go index 7803b5ba9..aa8e5304d 100644 --- a/internal/daemon/controller/testing.go +++ b/internal/daemon/controller/testing.go @@ -473,8 +473,9 @@ type TestControllerOpts struct { PublicClusterAddr string // The amount of time to wait before marking connections as canceling when a - // worker has not reported in - WorkerStatusGracePeriodDuration time.Duration + // worker has not reported in, and whether a worker is considered to be + // routable based on the last time it received a routing info report. + WorkerRPCGracePeriod time.Duration // The period of time after which it will consider other controllers to be // no longer accessible, based on time since their last status update in the @@ -691,8 +692,8 @@ func TestControllerConfig(t testing.TB, ctx context.Context, tc *TestController, t.Fatal(err) } - if opts.WorkerStatusGracePeriodDuration != 0 { - opts.Config.Controller.WorkerStatusGracePeriodDuration = opts.WorkerStatusGracePeriodDuration + if opts.WorkerRPCGracePeriod != 0 { + opts.Config.Controller.WorkerRPCGracePeriodDuration = opts.WorkerRPCGracePeriod } if opts.LivenessTimeToStaleDuration != 0 { opts.Config.Controller.LivenessTimeToStaleDuration = opts.LivenessTimeToStaleDuration @@ -862,7 +863,7 @@ func (tc *TestController) AddClusterControllerMember(t testing.TB, opts *TestCon DisableAuthMethodCreation: true, DisableAutoStart: opts.DisableAutoStart, PublicClusterAddr: opts.PublicClusterAddr, - WorkerStatusGracePeriodDuration: opts.WorkerStatusGracePeriodDuration, + WorkerRPCGracePeriod: opts.WorkerRPCGracePeriod, LivenessTimeToStaleDuration: opts.LivenessTimeToStaleDuration, WorkerAuthCaCertificateLifetime: tc.c.conf.TestOverrideWorkerAuthCaCertificateLifetime, WorkerAuthDebuggingEnabled: tc.c.conf.WorkerAuthDebuggingEnabled, @@ -881,53 +882,45 @@ func (tc *TestController) AddClusterControllerMember(t testing.TB, opts *TestCon return NewTestController(t, nextOpts) } -// WaitForNextWorkerStatusUpdate waits for the next status check from a worker to -// come in. If it does not come in within the default status grace +// WaitForNextWorkerRoutingInfoUpdate waits for the next routing info RPC from a worker to +// come in. If it does not come in within the default worker RPC grace // period, this function returns an error. -func (tc *TestController) WaitForNextWorkerStatusUpdate(workerStatusName string) error { - const op = "controller.(TestController).WaitForNextWorkerStatusUpdate" - ctx := context.TODO() - event.WriteSysEvent(ctx, op, "waiting for next status report from worker", "worker", workerStatusName) - waitStatusStart := time.Now() - ctx, cancel := context.WithTimeout(tc.ctx, time.Duration(tc.c.workerStatusGracePeriod.Load())) +func (tc *TestController) WaitForNextWorkerRoutingInfoUpdate(workerName string) error { + const op = "controller.(TestController).WaitForNextWorkerRoutingInfoUpdate" + waitStart := time.Now() + ctx, cancel := context.WithTimeout(tc.ctx, time.Duration(tc.c.workerRPCGracePeriod.Load())) defer cancel() + event.WriteSysEvent(ctx, op, "waiting for next routing info from worker", "worker", workerName) var err error for { - if err = func() error { - select { - case <-ctx.Done(): - return ctx.Err() - - case <-time.After(time.Second): - // pass - } - - return nil - }(); err != nil { + select { + case <-ctx.Done(): break + case <-time.After(time.Second): + // pass } - var waitStatusCurrent time.Time - tc.Controller().WorkerStatusUpdateTimes().Range(func(k, v any) bool { + var waitCurrent time.Time + tc.Controller().WorkerRoutingInfoUpdateTimes().Range(func(k, v any) bool { if k == nil || v == nil { err = fmt.Errorf("nil key or value on entry: key=%#v value=%#v", k, v) return false } - workerStatusUpdateId, ok := k.(string) + workerUpdateId, ok := k.(string) if !ok { err = fmt.Errorf("unexpected type %T for key: key=%#v value=%#v", k, k, v) return false } - workerStatusUpdateTime, ok := v.(time.Time) + workerUpdateTime, ok := v.(time.Time) if !ok { err = fmt.Errorf("unexpected type %T for value: key=%#v value=%#v", k, k, v) return false } - if workerStatusUpdateId == workerStatusName { - waitStatusCurrent = workerStatusUpdateTime + if workerUpdateId == workerName { + waitCurrent = workerUpdateTime return false } @@ -938,16 +931,16 @@ func (tc *TestController) WaitForNextWorkerStatusUpdate(workerStatusName string) break } - if waitStatusCurrent.After(waitStatusStart) { + if waitCurrent.After(waitStart) { break } } if err != nil { - event.WriteError(ctx, op, err, event.WithInfoMsg("error waiting for next status report from worker", "worker", workerStatusName)) + event.WriteError(ctx, op, err, event.WithInfoMsg("error waiting for next routing info RPC from worker", "worker", workerName)) return err } - event.WriteSysEvent(ctx, op, "waiting for next status report from worker received successfully", "worker", workerStatusName) + event.WriteSysEvent(ctx, op, "next routing info RPC from worker received successfully", "worker", workerName) return nil } diff --git a/internal/daemon/worker/common/common.go b/internal/daemon/worker/common/common.go index 0b6ca43ca..7deed1aaa 100644 --- a/internal/daemon/worker/common/common.go +++ b/internal/daemon/worker/common/common.go @@ -11,11 +11,27 @@ import ( // In the future we could make this configurable const ( - // StatusInterval is the base duration used in the calculation of the random backoff - // during the worker status report - StatusInterval = 2 * time.Second + // SessionInfoInterval is the base duration used in the calculation of the random backoff + // during the worker session information report + SessionInfoInterval = 2 * time.Second - // DefaultStatusTimeout is the timeout duration on status calls to the controller from + // RoutingInfoInterval is the interval at which the worker will send routing information + // to the controller + RoutingInfoInterval = 10 * time.Second + + // StatisticsInterval is the base duration used in the calculation of the random backoff + // during the worker statistics report + StatisticsInterval = 15 * time.Second + + // DefaultSessionInfoTimeout is the timeout duration on SessionInfo calls to the controller from // the worker - DefaultStatusTimeout = server.DefaultLiveness / 3 + DefaultSessionInfoTimeout = server.DefaultLiveness / 3 + + // DefaultRoutingInfoTimeout is the timeout duration on routing info calls to the controller from + // the worker + DefaultRoutingInfoTimeout = server.DefaultLiveness / 2 + + // DefaultStatisticsTimeout is the timeout duration on Statistics calls to the controller from + // the worker + DefaultStatisticsTimeout = server.DefaultLiveness ) diff --git a/internal/daemon/worker/controller_connection.go b/internal/daemon/worker/controller_connection.go index d766ae00c..d57175950 100644 --- a/internal/daemon/worker/controller_connection.go +++ b/internal/daemon/worker/controller_connection.go @@ -42,6 +42,8 @@ var HandleHcpbClusterId func(s string) string // StartControllerConnections starts up the resolver and initiates controller // connection client creation func (w *Worker) StartControllerConnections() error { + w.confAddressReceiversLock.Lock() + defer w.confAddressReceiversLock.Unlock() const op = "worker.(Worker).StartControllerConnections" initialAddrs := make([]string, 0, len(w.conf.RawConfig.Worker.InitialUpstreams)) for _, addr := range w.conf.RawConfig.Worker.InitialUpstreams { @@ -261,10 +263,10 @@ func (w *Worker) workerConnectionInfo(addr string) (*structpb.Struct, error) { PublicAddr: w.conf.RawConfig.Worker.PublicAddr, OperationalState: w.operationalState.Load().(server.OperationalState).String(), } - if w.LastStatusSuccess() != nil && len(w.LastStatusSuccess().GetWorkerId()) > 0 { + if w.LastRoutingInfoSuccess() != nil && len(w.LastRoutingInfoSuccess().GetWorkerId()) > 0 { // even though we wont have the worker the first time we dial, any // redial attempts should result in the worker id being populated - wci.WorkerId = w.LastStatusSuccess().GetWorkerId() + wci.WorkerId = w.LastRoutingInfoSuccess().GetWorkerId() } st, err := wci.AsConnectionStateStruct() if err != nil { diff --git a/internal/daemon/worker/handler.go b/internal/daemon/worker/handler.go index 90e743e39..9b1e099ed 100644 --- a/internal/daemon/worker/handler.go +++ b/internal/daemon/worker/handler.go @@ -198,14 +198,14 @@ func (w *Worker) handleProxy(listenerCfg *listenerutil.ListenerConfig, sessionMa } } - if w.LastStatusSuccess() == nil || w.LastStatusSuccess().WorkerId == "" { + if w.LastRoutingInfoSuccess() == nil || w.LastRoutingInfoSuccess().WorkerId == "" { event.WriteError(ctx, op, stderrors.New("worker id is empty")) if err = conn.Close(websocket.StatusInternalError, "worker id is empty"); err != nil { event.WriteError(ctx, op, err, event.WithInfoMsg("error closing client connection")) } return } - workerId := w.LastStatusSuccess().WorkerId + workerId := w.LastRoutingInfoSuccess().WorkerId var acResp *pbs.AuthorizeConnectionResponse var connsLeft int32 diff --git a/internal/daemon/worker/routing_info.go b/internal/daemon/worker/routing_info.go new file mode 100644 index 000000000..74e0469f6 --- /dev/null +++ b/internal/daemon/worker/routing_info.go @@ -0,0 +1,328 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package worker + +import ( + "context" + "errors" + "fmt" + "math/rand" + "slices" + "time" + + "github.com/hashicorp/boundary/internal/event" + pb "github.com/hashicorp/boundary/internal/gen/controller/servers" + pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" + "github.com/hashicorp/boundary/internal/server" + "github.com/hashicorp/boundary/sdk/pbs/plugin" + "github.com/hashicorp/boundary/version" + "github.com/hashicorp/go-secure-stdlib/parseutil" + "github.com/hashicorp/go-secure-stdlib/strutil" + "google.golang.org/grpc/connectivity" +) + +var firstRoutingInfoCheckPostHooks = []func(context.Context, *Worker) error{ + // Start session route info and statistics ticking after first routing info success + func(ctx context.Context, w *Worker) error { + w.tickerWg.Add(2) + go func() { + defer w.tickerWg.Done() + w.startSessionInfoTicking(w.baseContext) + }() + go func() { + defer w.tickerWg.Done() + w.startStatisticsTicking(w.baseContext) + }() + return nil + }, +} + +var downstreamWorkersFactory func(ctx context.Context, workerId string, ver string) (graph, error) + +var checkHCPBUpstreams func(w *Worker) bool + +// LastRoutingInfo represents the last successful routing info sent to the controller. +type LastRoutingInfo struct { + *pbs.RoutingInfoResponse + RoutingInfoTime time.Time + LastCalculatedUpstreams []string +} + +// WaitForNextSuccessfulRoutingInfoUpdate waits for the next successful routing info. It's +// used by testing in place of a more opaque and +// possibly unnecessarily long sleep for things like initial controller +// check-in, etc. +// +// The timeout is aligned with the worker's routing info grace period. A nil error +// means the routing info was sent successfully. +func (w *Worker) WaitForNextSuccessfulRoutingInfoUpdate() error { + const op = "worker.(Worker).WaitForNextSuccessfulRoutingInfoUpdate" + waitStart := time.Now() + ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(w.successfulRoutingInfoGracePeriod.Load())) + defer cancel() + event.WriteSysEvent(ctx, op, "waiting for next routing info report to controller") + for { + select { + case <-time.After(time.Second): + // pass + + case <-ctx.Done(): + event.WriteError(ctx, op, ctx.Err(), event.WithInfoMsg("error waiting for next routing info report to controller")) + return ctx.Err() + } + + if w.lastSuccessfulRoutingInfoTime().After(waitStart) { + break + } + } + + event.WriteSysEvent(ctx, op, "next worker routing info update sent successfully") + return nil +} + +func (w *Worker) startRoutingInfoTicking(cancelCtx context.Context) { + const op = "worker.(Worker).startRoutingInfoTicking" + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + timer := time.NewTimer(0) + for { + select { + case <-cancelCtx.Done(): + event.WriteSysEvent(w.baseContext, op, "RoutingInfo ticking shutting down") + return + + case <-timer.C: + // If we've never managed to successfully authenticate then we won't have + // any session information anyways and this will produce a ton of noise in + // observability, so skip calling the function and retry in a short duration + if w.everAuthenticated.Load() == authenticationStatusNeverAuthenticated { + timer.Reset(10 * time.Millisecond) + continue + } + + w.sendWorkerRoutingInfo(cancelCtx) + // Desynchronize calls to controllers from workers, so we aren't always + // getting RoutingInfo updates at the exact same intervals, to ease the load on the DB. + timer.Reset(w.routingInfoInterval + getRandomInterval(r)) + } + } +} + +// LastRoutingInfoSuccess reports the last time we sent a successful +// RoutingInfo request. +func (w *Worker) LastRoutingInfoSuccess() *LastRoutingInfo { + s, ok := w.lastRoutingInfoSuccess.Load().(*LastRoutingInfo) + if !ok { + return nil + } + return s +} + +func (w *Worker) sendWorkerRoutingInfo(cancelCtx context.Context) { + const op = "worker.(Worker).sendWorkerRoutingInfo" + // Lock access to w.conf and w.addressReceivers + w.confAddressReceiversLock.Lock() + defer w.confAddressReceiversLock.Unlock() + + clientCon := w.GrpcClientConn.Load() + // Send RoutingInfo information + client := pbs.NewServerCoordinationServiceClient(clientCon) + var tags []*pb.TagPair + // If we're not going to request a tag update, no reason to have these + // marshaled on every RoutingInfo call. + if w.updateTags.Load() { + tags = w.tags.Load().([]*pb.TagPair) + } + ctx, cancel := context.WithTimeout(cancelCtx, time.Duration(w.routingInfoCallTimeoutDuration.Load())) + defer cancel() + + keyId := w.WorkerAuthCurrentKeyId.Load() + switch { + case w.conf.RawConfig.Worker.Name == "" && keyId == "": + event.WriteError(cancelCtx, op, errors.New("worker name and keyId are both empty; at least one is needed to identify a worker"), + event.WithInfoMsg("error making RoutingInfo request to controller")) + } + + var storageBucketCredentialStates map[string]*plugin.StorageBucketCredentialState + if w.RecordingStorage != nil { + storageBucketCredentialStates = w.RecordingStorage.GetStorageBucketCredentialStates() + // If the local storage state is unknown, and we have recording storage set, get the state from the recording storage + // and set it on the worker. This is done once to ensure that the worker has the correct state for the first RoutingInfo + // call. + if w.localStorageState.Load() == server.UnknownLocalStorageState { + w.localStorageState.Store(w.RecordingStorage.GetLocalStorageState(cancelCtx)) + } + } + versionInfo := version.Get() + connectionState := w.downstreamConnManager.Connected() + result, err := client.RoutingInfo(ctx, &pbs.RoutingInfoRequest{ + WorkerStatus: &pb.ServerWorkerStatus{ + Name: w.conf.RawConfig.Worker.Name, + Description: w.conf.RawConfig.Worker.Description, + Address: w.conf.RawConfig.Worker.PublicAddr, + Tags: tags, + KeyId: keyId, + ReleaseVersion: versionInfo.FullVersionNumber(false), + OperationalState: w.operationalState.Load().(server.OperationalState).String(), + LocalStorageState: w.localStorageState.Load().(server.LocalStorageState).String(), + StorageBucketCredentialStates: storageBucketCredentialStates, + }, + ConnectedUnmappedWorkerKeyIdentifiers: connectionState.UnmappedKeyIds(), + ConnectedWorkerPublicIds: connectionState.WorkerIds(), + UpdateTags: w.updateTags.Load(), + }) + if err != nil { + event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error making routing info request to controller", "controller_address", clientCon.Target())) + + // In the case that the control plane has gone down and come up with different IPs, + // append initial upstreams/ cluster addr to the resolver to try + if pastGrace, _, _ := w.isPastGrace(); pastGrace && w.GrpcClientConn.Load().GetState() == connectivity.TransientFailure { + lastRoutingInfo := w.lastRoutingInfoSuccess.Load().(*LastRoutingInfo) + if lastRoutingInfo != nil && lastRoutingInfo.LastCalculatedUpstreams != nil { + addrs := lastRoutingInfo.LastCalculatedUpstreams + + if len(w.conf.RawConfig.Worker.InitialUpstreams) > 0 { + addrs = append(addrs, w.conf.RawConfig.Worker.InitialUpstreams...) + } else if HandleHcpbClusterId != nil && len(w.conf.RawConfig.HcpbClusterId) > 0 { + clusterId, err := parseutil.ParsePath(w.conf.RawConfig.HcpbClusterId) + if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { + event.WriteError(cancelCtx, op, err, event.WithInfoMsg("failed to parse HCP Boundary cluster ID")) + } else { + clusterAddress := HandleHcpbClusterId(clusterId) + addrs = append(addrs, clusterAddress) + } + } + + slices.Sort(addrs) + addrs = slices.Compact(addrs) + if slices.Equal(lastRoutingInfo.LastCalculatedUpstreams, addrs) { + // Nothing to update + return + } + + w.updateAddresses(cancelCtx, addrs) + lastRoutingInfo.LastCalculatedUpstreams = addrs + w.lastRoutingInfoSuccess.Store(lastRoutingInfo) + } + } + return + } + + w.updateTags.Store(false) + + if authorized := result.GetAuthorizedDownstreamWorkers(); authorized != nil { + connectionState.DisconnectMissingWorkers(authorized.GetWorkerPublicIds()) + connectionState.DisconnectMissingUnmappedKeyIds(authorized.GetUnmappedWorkerKeyIdentifiers()) + } + var addrs []string + // This may be empty if we are in a multiple hop scenario + if len(result.GetCalculatedUpstreamAddresses()) > 0 { + addrs = result.GetCalculatedUpstreamAddresses() + } else if checkHCPBUpstreams != nil && checkHCPBUpstreams(w) { + // This is a worker that is one hop away from managed workers, so attempt to get that list + hcpbWorkersCtx, hcpbWorkersCancel := context.WithTimeout(cancelCtx, time.Duration(w.routingInfoCallTimeoutDuration.Load())) + defer hcpbWorkersCancel() + workersResp, err := client.ListHcpbWorkers(hcpbWorkersCtx, &pbs.ListHcpbWorkersRequest{}) + if err != nil { + event.WriteError(hcpbWorkersCtx, op, err, event.WithInfoMsg("error fetching managed worker information")) + } else { + addrs = make([]string, 0, len(workersResp.Workers)) + for _, v := range workersResp.Workers { + addrs = append(addrs, v.Address) + } + } + } + + w.updateAddresses(cancelCtx, addrs) + + w.lastRoutingInfoSuccess.Store(&LastRoutingInfo{RoutingInfoResponse: result, RoutingInfoTime: time.Now(), LastCalculatedUpstreams: addrs}) + + // If we have post hooks for after the first RoutingInfo check, run them now + if w.everAuthenticated.CompareAndSwap(authenticationStatusFirstAuthentication, authenticationStatusFirstRoutingInfoRpcSuccessful) { + if downstreamWorkersFactory != nil { + downstreamWorkers, err := downstreamWorkersFactory(cancelCtx, w.LastRoutingInfoSuccess().WorkerId, versionInfo.FullVersionNumber(false)) + if err != nil { + event.WriteError(cancelCtx, op, err) + w.conf.ServerSideShutdownCh <- struct{}{} + return + } + w.downstreamWorkers.Store(&graphContainer{graph: downstreamWorkers}) + } + for _, fn := range firstRoutingInfoCheckPostHooks { + if err := fn(cancelCtx, w); err != nil { + // If we can't verify status we can't be expected to behave + // properly so error and trigger shutdown + event.WriteError(cancelCtx, op, fmt.Errorf("error running first routing info check post hook: %w", err)) + // We don't use a non-blocking select here to ensure that it + // happens; we should catch blocks in tests but we want to + // ensure the signal is being listened to + w.conf.ServerSideShutdownCh <- struct{}{} + return + } + } + } +} + +func (w *Worker) lastSuccessfulRoutingInfoTime() time.Time { + lastRoutingInfo := w.LastRoutingInfoSuccess() + if lastRoutingInfo == nil { + return w.workerStartTime + } + + return lastRoutingInfo.RoutingInfoTime +} + +// Update address receivers and dialing listeners with new addrs +func (w *Worker) updateAddresses(cancelCtx context.Context, addrs []string) { + const op = "worker.(Worker).updateAddresses" + + if len(addrs) > 0 { + lastRoutingInfo := w.lastRoutingInfoSuccess.Load().(*LastRoutingInfo) + // Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses + if lastRoutingInfo != nil && !strutil.EquivalentSlices(lastRoutingInfo.LastCalculatedUpstreams, addrs) { + upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastRoutingInfo.LastCalculatedUpstreams, addrs) + event.WriteSysEvent(cancelCtx, op, upstreamsMessage) + for _, as := range w.addressReceivers { + as.SetAddresses(addrs) + } + } else if lastRoutingInfo == nil { + for _, as := range w.addressReceivers { + as.SetAddresses(addrs) + } + event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first RoutingInfo set to: %s", addrs)) + } + } + + // regardless of whether or not it's a new address, we need to set + // them for secondary connections + for _, as := range w.addressReceivers { + switch { + case as.Type() == secondaryConnectionReceiverType: + tmpAddrs := make([]string, len(addrs)) + copy(tmpAddrs, addrs) + if len(tmpAddrs) == 0 { + tmpAddrs = append(tmpAddrs, w.conf.RawConfig.Worker.InitialUpstreams...) + } + as.SetAddresses(tmpAddrs) + } + } +} + +func (w *Worker) isPastGrace() (bool, time.Time, time.Duration) { + t := w.lastSuccessfulRoutingInfoTime() + u := time.Duration(w.successfulRoutingInfoGracePeriod.Load()) + v := time.Since(t) + return v > u, t, u +} + +// getRandomInterval returns a random duration between -0.5 and 0.5 seconds (exclusive). +func getRandomInterval(r *rand.Rand) time.Duration { + // 0 to 0.5 adjustment to the base + f := r.Float64() / 2 + // Half a chance to be faster, not slower + if r.Float32() > 0.5 { + f = -1 * f + } + return time.Duration(f * float64(time.Second)) +} diff --git a/internal/daemon/worker/status_test.go b/internal/daemon/worker/routing_info_test.go similarity index 62% rename from internal/daemon/worker/status_test.go rename to internal/daemon/worker/routing_info_test.go index 34ff14f34..dac2ddf8c 100644 --- a/internal/daemon/worker/status_test.go +++ b/internal/daemon/worker/routing_info_test.go @@ -11,18 +11,19 @@ import ( "time" "github.com/hashicorp/boundary/internal/cmd/base" + "github.com/hashicorp/boundary/internal/daemon/worker/common" "github.com/hashicorp/boundary/internal/event" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/require" ) -func TestWorkerWaitForNextSuccessfulStatusUpdate(t *testing.T) { +func TestWorkerWaitForNextSuccessfulRoutingInfoUpdate(t *testing.T) { testConfig := event.DefaultEventerConfig() testLock := &sync.Mutex{} testLogger := hclog.New(&hclog.LoggerOptions{ Mutex: testLock, }) - err := event.InitSysEventer(testLogger, testLock, "TestWorkerWaitForNextSuccessfulStatusUpdate", event.WithEventerConfig(testConfig)) + err := event.InitSysEventer(testLogger, testLock, "TestWorkerWaitForNextSuccessfulRoutingInfoUpdate", event.WithEventerConfig(testConfig)) require.NoError(t, err) t.Cleanup(func() { event.TestResetSystEventer(t) }) for _, name := range []string{"ok", "timeout"} { @@ -31,30 +32,30 @@ func TestWorkerWaitForNextSuccessfulStatusUpdate(t *testing.T) { // As-needed initialization of a mock worker w := &Worker{ - logger: hclog.New(nil), - lastStatusSuccess: new(atomic.Value), - baseContext: context.Background(), - successfulStatusGracePeriod: new(atomic.Int64), + logger: hclog.New(nil), + lastRoutingInfoSuccess: new(atomic.Value), + baseContext: context.Background(), + successfulRoutingInfoGracePeriod: new(atomic.Int64), conf: &Config{ Server: &base.Server{}, }, } // This is present in New() - w.lastStatusSuccess.Store((*LastStatusInformation)(nil)) - w.successfulStatusGracePeriod.Store(int64(time.Second * 2)) + w.lastRoutingInfoSuccess.Store((*LastRoutingInfo)(nil)) + w.successfulRoutingInfoGracePeriod.Store(int64(common.DefaultRoutingInfoTimeout)) var wg sync.WaitGroup var err error wg.Add(1) go func() { - err = w.WaitForNextSuccessfulStatusUpdate() + err = w.WaitForNextSuccessfulRoutingInfoUpdate() wg.Done() }() if name == "ok" { time.Sleep(time.Millisecond * 100) - w.lastStatusSuccess.Store(&LastStatusInformation{StatusTime: time.Now()}) + w.lastRoutingInfoSuccess.Store(&LastRoutingInfo{RoutingInfoTime: time.Now()}) } wg.Wait() diff --git a/internal/daemon/worker/session/manager.go b/internal/daemon/worker/session/manager.go index ba356af7e..6a52b4aad 100644 --- a/internal/daemon/worker/session/manager.go +++ b/internal/daemon/worker/session/manager.go @@ -50,7 +50,7 @@ type Manager interface { // RequestCloseConnections sends connection close requests to the // controller, and sets close times within the worker. It should be called - // during the worker status loop and on connection exit on the proxy. + // during the worker session info loop and on connection exit on the proxy. // // The boolean indicates whether the function was successful, e.g. had any // errors. Individual events will be sent for the errors if there are any. diff --git a/internal/daemon/worker/session/session.go b/internal/daemon/worker/session/session.go index e684ce0a5..62140670b 100644 --- a/internal/daemon/worker/session/session.go +++ b/internal/daemon/worker/session/session.go @@ -480,7 +480,7 @@ func closeConnection(ctx context.Context, sessClient pbs.SessionServiceClient, r // closeConnections is a helper worker function that sends connection close // requests to the controller, and sets close times within the worker. It is -// called during the worker status loop and on connection exit on the proxy. +// called during the worker session info loop and on connection exit on the proxy. // // The boolean indicates whether the function was successful, e.g. had any // errors. Individual events will be sent for the errors if there are any. @@ -499,9 +499,9 @@ func closeConnections(ctx context.Context, sessClient pbs.SessionServiceClient, var sessionCloseInfo map[string][]*pbs.CloseConnectionResponseData var err error - // TODO: This, along with the status call to the controller, probably needs a + // TODO: This, along with the session info call to the controller, probably needs a // bit of formalization in terms of how we handle timeouts. For now, this - // just ensures consistency with the same status call in that it times out + // just ensures consistency with the same session info call in that it times out // within an adequate period of time. closeConnCtx, closeConnCancel := context.WithTimeout(ctx, time.Duration(CloseCallTimeout.Load())) defer closeConnCancel() diff --git a/internal/daemon/worker/session/testing.go b/internal/daemon/worker/session/testing.go new file mode 100644 index 000000000..ed47c5a34 --- /dev/null +++ b/internal/daemon/worker/session/testing.go @@ -0,0 +1,59 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package session + +import ( + "sync" + "testing" + + pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" + "github.com/stretchr/testify/require" +) + +// TestManager wraps a session manager. This wrapper allows direct manipulation of the sessions map for testing. +type TestManager struct { + *manager +} + +// NewTestManager returns a wrapped session manager that allows direct manipulation of the sessions map for testing +func NewTestManager(t *testing.T, client pbs.SessionServiceClient) *TestManager { + sm, err := NewManager(client) + require.NoError(t, err) + return &TestManager{ + manager: sm, + } +} + +// StoreSession will mock an active local session with no connections in the session manager +func (m *TestManager) StoreSession(sessionId string) { + m.manager.sessionMap.Store(sessionId, &sess{ + lock: sync.RWMutex{}, + sessionId: sessionId, + connInfoMap: make(map[string]*ConnInfo), + status: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, + }) +} + +// StoreConnection will mock a connection for the given session. The session is expected to already exist +// in the session manager. +func (m *TestManager) StoreConnection(t *testing.T, sessionId, connectionId string) { + s, ok := m.manager.sessionMap.Load(sessionId) + require.True(t, ok) + localSession := s.(*sess) + info := &ConnInfo{ + Id: connectionId, + Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED, + BytesUp: func() int64 { + return 0 + }, + BytesDown: func() int64 { + return 0 + }, + } + info.connCtxCancelFunc = func() { + info.Status = pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED + } + localSession.connInfoMap[connectionId] = info + m.manager.sessionMap.Store(sessionId, localSession) +} diff --git a/internal/daemon/worker/session_info.go b/internal/daemon/worker/session_info.go new file mode 100644 index 000000000..3c6439fdd --- /dev/null +++ b/internal/daemon/worker/session_info.go @@ -0,0 +1,229 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package worker + +import ( + "context" + stderr "errors" + "math/rand" + "time" + + "github.com/hashicorp/boundary/internal/daemon/worker/session" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/event" + pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" +) + +// lastSessionInfo holds the last successful session info RPC time. +type lastSessionInfo struct { + LastSuccessfulRequestTime time.Time +} + +func (w *Worker) startSessionInfoTicking(cancelCtx context.Context) { + const op = "worker.(Worker).startSessionInfoTicking" + r := rand.New(rand.NewSource(time.Now().UnixNano())) + timer := time.NewTimer(0) + for { + select { + case <-cancelCtx.Done(): + event.WriteSysEvent(w.baseContext, op, "session infor ticking shutting down") + return + case <-timer.C: + err := w.sendSessionInfo(cancelCtx) + if err != nil { + event.WriteError(w.baseContext, op, err) + } + // Add a bit of jitter to the wait, so we aren't always getting, + // session information updates at the exact same intervals, to + // ease the load on the DB. + timer.Reset(w.sessionInfoInterval + getRandomInterval(r)) + } + } +} + +func (w *Worker) sendSessionInfo(cancelCtx context.Context) error { + const op = "worker.(Worker).sendSessionInfo" + // skip when the workerId is not available + if w.LastRoutingInfoSuccess() == nil { + return errors.New(cancelCtx, errors.Internal, op, "missing latest routing info") + } + workerId := w.LastRoutingInfoSuccess().GetWorkerId() + if workerId == "" { + return errors.New(cancelCtx, errors.Internal, op, "worker id is empty") + } + + var recordedSessionIds []string + if w.recorderManager != nil { + recSessIds, err := w.recorderManager.SessionsManaged(cancelCtx) + if err != nil { + event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error getting session ids from recorderManager")) + } + if len(recSessIds) != 0 { + recordedSessionIds = append(recordedSessionIds, recSessIds...) + } + } + + var activeSessions []*pbs.Session + for _, sessionId := range recordedSessionIds { + activeSessions = append(activeSessions, &pbs.Session{ + SessionId: sessionId, + SessionType: pbs.SessionType_SESSION_TYPE_RECORDED, + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, + }) + } + w.sessionManager.ForEachLocalSession(func(s session.Session) bool { + activeSessions = append(activeSessions, &pbs.Session{ + SessionId: s.GetId(), + SessionType: pbs.SessionType_SESSION_TYPE_INGRESSED, + SessionStatus: s.GetStatus(), + }) + return true + }) + // skip when there are no sessions to report + if len(activeSessions) == 0 { + return nil + } + clientCon := w.GrpcClientConn.Load() + client := pbs.NewServerCoordinationServiceClient(clientCon) + sessionInfoCtx, statusCancel := context.WithTimeout(cancelCtx, time.Duration(w.sessionInfoCallTimeoutDuration.Load())) + defer statusCancel() + result, err := client.SessionInfo(sessionInfoCtx, &pbs.SessionInfoRequest{ + WorkerId: workerId, + Sessions: activeSessions, + }) + if err != nil { + if isPastGrace, lastSessionInfoTime, gracePeriod := w.isPastSessionInfoGrace(); isPastGrace { + event.WriteError(cancelCtx, op, + stderr.New("session information error grace period has expired, canceling all sessions on worker"), + event.WithInfo("last_session_info_time", lastSessionInfoTime.String(), "grace_period", gracePeriod), + ) + // Cancel connections if grace period has expired. These Connections will be closed in the + // database on the next successful status report, or via the Controller’s dead Worker cleanup connections job. + w.sessionManager.ForEachLocalSession(func(s session.Session) bool { + for _, connId := range s.CancelAllLocalConnections() { + event.WriteSysEvent(cancelCtx, op, "terminated connection due to session information grace period expiration", "session_id", s.GetId(), "connection_id", connId) + } + return true + }) + } + return errors.Wrap(cancelCtx, err, op, errors.WithCode(errors.Internal), errors.WithMsg("error making session information request to controller: controller_address: %s", clientCon.Target())) + } + + w.lastSessionInfoSuccess.Store(&lastSessionInfo{ + LastSuccessfulRequestTime: time.Now(), + }) + + var nonActiveRecordedSessionIds []string + for _, s := range result.GetNonActiveSessions() { + switch s.GetSessionType() { + case pbs.SessionType_SESSION_TYPE_RECORDED: + if s.SessionStatus != pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE { + nonActiveRecordedSessionIds = append(nonActiveRecordedSessionIds, s.GetSessionId()) + } + case pbs.SessionType_SESSION_TYPE_INGRESSED: + sessionId := s.GetSessionId() + managedSession := w.sessionManager.Get(sessionId) + if managedSession == nil { + event.WriteError(cancelCtx, op, stderr.New("session change requested but could not find local information for it"), event.WithInfo("session_id", sessionId)) + continue + } + managedSession.ApplyLocalStatus(s.GetSessionStatus()) + // Update connection state if there are any connections in the request. + for _, c := range s.GetConnections() { + if err := managedSession.ApplyLocalConnectionStatus(c.GetConnectionId(), c.GetStatus()); err != nil { + event.WriteError(cancelCtx, op, err, event.WithInfo("connection_id", c.GetConnectionId())) + } + } + } + } + if w.recorderManager != nil { + if err := w.recorderManager.ReauthorizeAllExcept(cancelCtx, nonActiveRecordedSessionIds); err != nil { + event.WriteError(cancelCtx, op, err) + } + } + // Standard cleanup: Run through current jobs. Cancel connections + // for any canceling session or any session that is expired. + w.cleanupConnections(cancelCtx, false, w.sessionManager) + return nil +} + +func (w *Worker) isPastSessionInfoGrace() (bool, time.Time, time.Duration) { + t := w.workerStartTime + info := w.lastSessionInfoSuccess.Load().(*lastSessionInfo) + if info != nil { + t = info.LastSuccessfulRequestTime + } + u := time.Duration(w.successfulSessionInfoGracePeriod.Load()) + v := time.Since(t) + return v > u, t, u +} + +// cleanupConnections walks all sessions and shuts down all proxy connections. +// After the local connections are terminated, they are requested to be marked +// closed on the controller. +// Additionally, sessions without connections are cleaned up from the +// local worker's state. +// +// Use ignoreSessionState to ignore the state checks, this closes all +// connections, regardless of whether or not the session is still active. +func (w *Worker) cleanupConnections(cancelCtx context.Context, ignoreSessionState bool, sessionManager session.Manager) { + const op = "worker.(Worker).cleanupConnections" + closeInfo := make(map[string]*session.ConnectionCloseData) + cleanSessionIds := make([]string, 0) + sessionManager.ForEachLocalSession(func(s session.Session) bool { + switch { + case ignoreSessionState, + s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING, + s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED, + time.Until(s.GetExpiration()) < 0: + // Cancel connections without regard to individual connection + // state. + closedIds := s.CancelAllLocalConnections() + localConns := s.GetLocalConnections() + for _, connId := range closedIds { + var bytesUp, bytesDown int64 + connInfo, ok := localConns[connId] + if ok { + bytesUp = connInfo.BytesUp() + bytesDown = connInfo.BytesDown() + } + closeInfo[connId] = &session.ConnectionCloseData{ + SessionId: s.GetId(), + BytesUp: bytesUp, + BytesDown: bytesDown, + } + event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId) + } + + // If the session is no longer valid and all connections + // are marked closed, clean up the session. + if len(closedIds) == 0 { + cleanSessionIds = append(cleanSessionIds, s.GetId()) + } + + default: + // Cancel connections *with* regard to individual connection + // state (ie: only ones that the controller has requested be + // terminated). + for _, connId := range s.CancelOpenLocalConnections() { + closeInfo[connId] = &session.ConnectionCloseData{SessionId: s.GetId()} + event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId) + } + } + + return true + }) + + // Note that we won't clean these from the info map until the + // next time we run this function + if len(closeInfo) > 0 { + // Call out to a helper to send the connection close requests to the + // controller, and set the close time. This functionality is shared with + // post-close functionality in the proxy handler. + _ = sessionManager.RequestCloseConnections(cancelCtx, closeInfo) + } + // Forget sessions where the session is expired/canceled and all + // connections are canceled and marked closed + sessionManager.DeleteLocalSession(cleanSessionIds) +} diff --git a/internal/daemon/worker/session_info_test.go b/internal/daemon/worker/session_info_test.go new file mode 100644 index 000000000..ac647f1c9 --- /dev/null +++ b/internal/daemon/worker/session_info_test.go @@ -0,0 +1,337 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package worker + +import ( + "context" + "errors" + "net" + "sync" + "testing" + "time" + + "github.com/hashicorp/boundary/internal/daemon/worker/session" + pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func Test_sendSessionInfo(t *testing.T) { + t.Parallel() + tests := []struct { + name string + lastRoutingInfo *LastRoutingInfo + sessionSetup func(sm *session.TestManager) + expectedInternalErrMsg string + expectedServerErrMsg string + expectedSessions bool + }{ + { + name: "nil last routing info", + expectedInternalErrMsg: "missing latest routing info", + }, + { + name: "empty worker id", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{}, + }, + expectedInternalErrMsg: "worker id is empty", + }, + { + name: "empty sessions", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }, + }, + { + name: "server error", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }, + sessionSetup: func(sm *session.TestManager) { + require.NotNil(t, sm) + sm.StoreSession("s_1234567890") + sm.StoreConnection(t, "s_1234567890", "c_1234567890") + }, + expectedServerErrMsg: "testing connection timeout", + }, + { + name: "success", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }, + sessionSetup: func(sm *session.TestManager) { + require.NotNil(t, sm) + sm.StoreSession("s_1234567890") + sm.StoreConnection(t, "s_1234567890", "c_1234567890") + }, + expectedSessions: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert, require := assert.New(t), require.New(t) + + w := NewTestWorker(t, &TestWorkerOpts{ + DisableAutoStart: true, + Name: tt.name, + }) + // Ensure the recorder manager is not used + w.Worker().recorderManager = nil + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(err) + cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(err) + + mockSessionClient := pbs.NewMockSessionServiceClient() + mockSessionClient.CloseConnectionFn = func(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) { + return &pbs.CloseConnectionResponse{}, nil + } + sm := session.NewTestManager(t, mockSessionClient) + if tt.sessionSetup != nil { + tt.sessionSetup(sm) + } + w.Worker().sessionManager = sm + + serverInvoked := false + fakeServer := &mockServerCoordinationService{ + nextSessionInfoAssert: func(req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) { + serverInvoked = true + if tt.expectedInternalErrMsg != "" { + assert.Fail("server should not send session information for internal errors") + } + if tt.expectedServerErrMsg != "" { + return &pbs.SessionInfoResponse{}, errors.New(tt.expectedServerErrMsg) + } + if !tt.expectedSessions { + assert.Fail("server should not send session information when there are no managed sessions") + } + require.NotNil(req) + + assert.Equal("w_1234567890", req.WorkerId) + require.Len(req.Sessions, 1) + assert.Equal("s_1234567890", req.Sessions[0].SessionId) + + return &pbs.SessionInfoResponse{}, nil + }, + } + + srv := grpc.NewServer() + pbs.RegisterServerCoordinationServiceServer(srv, fakeServer) + w.Worker().GrpcClientConn.Store(cc) + w.Worker().lastRoutingInfoSuccess.Store(tt.lastRoutingInfo) + + serverStarted := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + close(serverStarted) + assert.NoError(srv.Serve(ln)) + }() + + <-serverStarted + actualErr := w.Worker().sendSessionInfo(context.Background()) + + if tt.expectedInternalErrMsg != "" { + require.Error(actualErr) + assert.ErrorContains(actualErr, tt.expectedInternalErrMsg) + assert.False(serverInvoked) + return + } + if tt.expectedServerErrMsg != "" { + require.Error(actualErr) + assert.ErrorContains(actualErr, tt.expectedServerErrMsg) + assert.True(serverInvoked) + return + } + require.NoError(actualErr) + require.Equal(tt.expectedSessions, serverInvoked) + + srv.GracefulStop() + wg.Wait() + }) + } + + t.Run("close non active managed sessions connections and ignore non existing sessions", func(t *testing.T) { + t.Parallel() + assert, require := assert.New(t), require.New(t) + + w := NewTestWorker(t, &TestWorkerOpts{ + DisableAutoStart: true, + Name: "close non active sessions and connections", + }) + // Ensure the recorder manager is not used + w.Worker().recorderManager = nil + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(err) + cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(err) + + mockSessionClient := pbs.NewMockSessionServiceClient() + mockSessionClient.CloseConnectionFn = func(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) { + return &pbs.CloseConnectionResponse{}, nil + } + sm := session.NewTestManager(t, mockSessionClient) + sm.StoreSession("s_1234567890") + sm.StoreConnection(t, "s_1234567890", "c_1234567890") + w.Worker().sessionManager = sm + + serverInvoked := false + fakeServer := &mockServerCoordinationService{ + nextSessionInfoAssert: func(req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) { + serverInvoked = true + assert.Equal("w_1234567890", req.WorkerId) + require.Len(req.Sessions, 1) + assert.Equal("s_1234567890", req.Sessions[0].SessionId) + return &pbs.SessionInfoResponse{ + NonActiveSessions: []*pbs.Session{ + { + SessionId: "s_1234567890", + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED, + SessionType: pbs.SessionType_SESSION_TYPE_INGRESSED, + Connections: []*pbs.Connection{ + { + ConnectionId: "c_1234567890", + Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED, + }, + }, + }, + { + SessionId: "s_dne", + SessionStatus: pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED, + SessionType: pbs.SessionType_SESSION_TYPE_INGRESSED, + Connections: []*pbs.Connection{ + { + ConnectionId: "c_dne", + Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED, + }, + }, + }, + }, + }, nil + }, + } + + srv := grpc.NewServer() + pbs.RegisterServerCoordinationServiceServer(srv, fakeServer) + w.Worker().GrpcClientConn.Store(cc) + w.Worker().lastRoutingInfoSuccess.Store(&LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }) + + serverStarted := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + close(serverStarted) + assert.NoError(srv.Serve(ln)) + }() + + <-serverStarted + actualErr := w.Worker().sendSessionInfo(context.Background()) + + require.NoError(actualErr) + assert.True(serverInvoked) + closedSession := w.Worker().sessionManager.Get("s_1234567890") + assert.Nil(closedSession) + + srv.GracefulStop() + wg.Wait() + }) + + t.Run("validate closing local connections after last successful session info grace period", func(t *testing.T) { + t.Parallel() + assert, require := assert.New(t), require.New(t) + + w := NewTestWorker(t, &TestWorkerOpts{ + DisableAutoStart: true, + Name: "close non active sessions and connections", + }) + // Ensure the recorder manager is not used + w.Worker().recorderManager = nil + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(err) + cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(err) + + mockSessionClient := pbs.NewMockSessionServiceClient() + mockSessionClient.CloseConnectionFn = func(ctx context.Context, req *pbs.CloseConnectionRequest) (*pbs.CloseConnectionResponse, error) { + return &pbs.CloseConnectionResponse{}, nil + } + sm := session.NewTestManager(t, mockSessionClient) + sm.StoreSession("s_1234567890") + sm.StoreConnection(t, "s_1234567890", "c_1234567890") + w.Worker().sessionManager = sm + + serverInvoked := false + fakeServer := &mockServerCoordinationService{ + nextSessionInfoAssert: func(req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) { + serverInvoked = true + assert.Equal("w_1234567890", req.WorkerId) + require.Len(req.Sessions, 1) + assert.Equal("s_1234567890", req.Sessions[0].SessionId) + return &pbs.SessionInfoResponse{}, errors.New("connection timeout") + }, + } + + srv := grpc.NewServer() + pbs.RegisterServerCoordinationServiceServer(srv, fakeServer) + w.Worker().GrpcClientConn.Store(cc) + w.Worker().lastRoutingInfoSuccess.Store(&LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }) + // Set the status time of the last successful session information + // to a value that ensures we have surpassed the grace period time. + gracePeriod := time.Duration(w.Worker().successfulSessionInfoGracePeriod.Load()) + (5 * time.Second) + w.Worker().lastSessionInfoSuccess.Store(&lastSessionInfo{ + LastSuccessfulRequestTime: time.Now().Add(-gracePeriod), + }) + + serverStarted := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + close(serverStarted) + assert.NoError(srv.Serve(ln)) + }() + + <-serverStarted + actualErr := w.Worker().sendSessionInfo(context.Background()) + + require.Error(actualErr) + assert.ErrorContains(actualErr, "connection timeout") + assert.True(serverInvoked) + localSession := w.Worker().sessionManager.Get("s_1234567890") + require.NotNil(localSession) + assert.Equal(pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, localSession.GetStatus()) + localConnections := localSession.GetLocalConnections() + require.NotEmpty(localConnections) + c, ok := localConnections["c_1234567890"] + require.True(ok) + require.NotNil(c) + assert.Equal(pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED, c.Status) + + srv.GracefulStop() + wg.Wait() + }) +} diff --git a/internal/daemon/worker/statistics.go b/internal/daemon/worker/statistics.go new file mode 100644 index 000000000..490487fae --- /dev/null +++ b/internal/daemon/worker/statistics.go @@ -0,0 +1,92 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package worker + +import ( + "context" + "math/rand" + "time" + + "github.com/hashicorp/boundary/internal/daemon/worker/session" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/event" + pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" +) + +// lastStatistics holds the last successful statistics RPC time. +type lastStatistics struct { + LastSuccessfulRequestTime time.Time +} + +func (w *Worker) startStatisticsTicking(cancelCtx context.Context) { + const op = "worker.(Worker).startStatisticsTicking" + r := rand.New(rand.NewSource(time.Now().UnixNano())) + timer := time.NewTimer(0) + for { + select { + case <-cancelCtx.Done(): + event.WriteSysEvent(w.baseContext, op, "statistics ticking shutting down") + return + case <-timer.C: + err := w.sendStatistic(cancelCtx) + if err != nil { + event.WriteError(w.baseContext, op, err) + } + // Add a bit of jitter to the wait, so we aren't always getting, + // statistics updates at the exact same intervals, to ease the load on the DB. + timer.Reset(w.statisticsInterval + getRandomInterval(r)) + } + } +} + +func (w *Worker) sendStatistic(cancelCtx context.Context) error { + const op = "worker.(Worker).sendStatistic" + // skip when the workerId is not available + if w.LastRoutingInfoSuccess() == nil { + return errors.New(cancelCtx, errors.Internal, op, "missing latest status") + } + workerId := w.LastRoutingInfoSuccess().GetWorkerId() + if workerId == "" { + return errors.New(cancelCtx, errors.Internal, op, "worker id is empty") + } + sessions := []*pbs.SessionStatistics{} + w.sessionManager.ForEachLocalSession(func(s session.Session) bool { + localConnections := s.GetLocalConnections() + connections := make([]*pbs.Connection, 0, len(localConnections)) + for connectionId, conn := range localConnections { + connections = append(connections, &pbs.Connection{ + ConnectionId: connectionId, + BytesUp: conn.BytesUp(), + BytesDown: conn.BytesDown(), + Status: conn.Status, + }) + } + sessions = append(sessions, &pbs.SessionStatistics{ + SessionId: s.GetId(), + Connections: connections, + }) + return true + }) + // skip when there are no sessions to report + if len(sessions) == 0 { + return nil + } + clientCon := w.GrpcClientConn.Load() + client := pbs.NewServerCoordinationServiceClient(clientCon) + statisticsCtx, statusCancel := context.WithTimeout(cancelCtx, time.Duration(w.statisticsCallTimeoutDuration.Load())) + defer statusCancel() + _, err := client.Statistics(statisticsCtx, &pbs.StatisticsRequest{ + WorkerId: workerId, + Sessions: sessions, + }) + if err != nil { + return errors.Wrap(cancelCtx, err, op, errors.WithCode(errors.Internal), errors.WithMsg("error making statistics request to controller: controller_address: %s", clientCon.Target())) + } + + w.lastStatisticsSuccess.Store(&lastStatistics{ + LastSuccessfulRequestTime: time.Now(), + }) + + return nil +} diff --git a/internal/daemon/worker/statistics_test.go b/internal/daemon/worker/statistics_test.go new file mode 100644 index 000000000..f70939f3c --- /dev/null +++ b/internal/daemon/worker/statistics_test.go @@ -0,0 +1,158 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package worker + +import ( + "context" + "errors" + "net" + "sync" + "testing" + + "github.com/hashicorp/boundary/internal/daemon/worker/session" + pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func Test_sendStatistic(t *testing.T) { + t.Parallel() + tests := []struct { + name string + lastRoutingInfo *LastRoutingInfo + sessionSetup func(sm *session.TestManager) + expectedInternalErrMsg string + expectedServerErrMsg string + expectedSessions bool + }{ + { + name: "nil last status", + expectedInternalErrMsg: "missing latest status", + }, + { + name: "empty worker id", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{}, + }, + expectedInternalErrMsg: "worker id is empty", + }, + { + name: "empty sessions", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }, + }, + { + name: "server error", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }, + sessionSetup: func(sm *session.TestManager) { + require.NotNil(t, sm) + sm.StoreSession("s_1234567890") + sm.StoreConnection(t, "s_1234567890", "c_1234567890") + }, + expectedServerErrMsg: "testing connection timeout", + }, + { + name: "success", + lastRoutingInfo: &LastRoutingInfo{ + RoutingInfoResponse: &pbs.RoutingInfoResponse{ + WorkerId: "w_1234567890", + }, + }, + sessionSetup: func(sm *session.TestManager) { + require.NotNil(t, sm) + sm.StoreSession("s_1234567890") + sm.StoreConnection(t, "s_1234567890", "c_1234567890") + }, + expectedSessions: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert, require := assert.New(t), require.New(t) + + w := NewTestWorker(t, &TestWorkerOpts{ + DisableAutoStart: true, + Name: tt.name, + }) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(err) + cc, err := grpc.Dial(ln.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(err) + + sm := session.NewTestManager(t, pbs.NewMockSessionServiceClient()) + if tt.sessionSetup != nil { + tt.sessionSetup(sm) + } + w.Worker().sessionManager = sm + + serverInvoked := false + fakeServer := &mockServerCoordinationService{ + nextStatisticAssert: func(req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) { + serverInvoked = true + if tt.expectedInternalErrMsg != "" { + assert.Fail("server should not send statistics information for internal errors") + } + if tt.expectedServerErrMsg != "" { + return &pbs.StatisticsResponse{}, errors.New(tt.expectedServerErrMsg) + } + if !tt.expectedSessions { + assert.Fail("server should not send statistics information when there are no managed sessions") + } + require.NotNil(req) + assert.Equal("w_1234567890", req.WorkerId) + require.Len(req.Sessions, 1) + assert.Equal("s_1234567890", req.Sessions[0].SessionId) + require.Len(req.Sessions[0].Connections, 1) + assert.Equal("c_1234567890", req.Sessions[0].Connections[0].ConnectionId) + return &pbs.StatisticsResponse{}, nil + }, + } + srv := grpc.NewServer() + pbs.RegisterServerCoordinationServiceServer(srv, fakeServer) + w.Worker().GrpcClientConn.Store(cc) + w.Worker().lastRoutingInfoSuccess.Store(tt.lastRoutingInfo) + + serverStarted := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + close(serverStarted) + assert.NoError(srv.Serve(ln)) + }() + + <-serverStarted + actualErr := w.Worker().sendStatistic(context.Background()) + + if tt.expectedInternalErrMsg != "" { + require.Error(actualErr) + assert.ErrorContains(actualErr, tt.expectedInternalErrMsg) + assert.False(serverInvoked) + return + } + if tt.expectedServerErrMsg != "" { + require.Error(actualErr) + assert.ErrorContains(actualErr, tt.expectedServerErrMsg) + assert.True(serverInvoked) + return + } + require.NoError(actualErr) + require.Equal(tt.expectedSessions, serverInvoked) + + srv.GracefulStop() + wg.Wait() + }) + } +} diff --git a/internal/daemon/worker/status.go b/internal/daemon/worker/status.go deleted file mode 100644 index 9cdb5a4fa..000000000 --- a/internal/daemon/worker/status.go +++ /dev/null @@ -1,519 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package worker - -import ( - "context" - "errors" - "fmt" - "math/rand" - "time" - - "github.com/hashicorp/boundary/internal/daemon/worker/common" - "github.com/hashicorp/boundary/internal/daemon/worker/session" - "github.com/hashicorp/boundary/internal/event" - pb "github.com/hashicorp/boundary/internal/gen/controller/servers" - pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services" - "github.com/hashicorp/boundary/internal/server" - "github.com/hashicorp/boundary/sdk/pbs/plugin" - "github.com/hashicorp/boundary/version" - "github.com/hashicorp/go-secure-stdlib/parseutil" - "github.com/hashicorp/go-secure-stdlib/strutil" - "google.golang.org/grpc/connectivity" -) - -var firstStatusCheckPostHooks []func(context.Context, *Worker) error - -var downstreamWorkersFactory func(ctx context.Context, workerId string, ver string) (graph, error) - -var checkHCPBUpstreams func(w *Worker) bool - -type LastStatusInformation struct { - *pbs.StatusResponse - StatusTime time.Time - LastCalculatedUpstreams []string -} - -func (w *Worker) startStatusTicking(cancelCtx context.Context, sessionManager session.Manager, addrReceivers *[]addressReceiver, recorderManager recorderManager) { - const op = "worker.(Worker).startStatusTicking" - r := rand.New(rand.NewSource(time.Now().UnixNano())) - // This function exists to desynchronize calls to controllers from - // workers, so we aren't always getting status updates at the exact same - // intervals, to ease the load on the DB. - getRandomInterval := func() time.Duration { - // 0 to 0.5 adjustment to the base - f := r.Float64() / 2 - // Half a chance to be faster, not slower - if r.Float32() > 0.5 { - f = -1 * f - } - return common.StatusInterval + time.Duration(f*float64(time.Second)) - } - - timer := time.NewTimer(0) - for { - select { - case <-cancelCtx.Done(): - event.WriteSysEvent(w.baseContext, op, "status ticking shutting down") - return - - case <-timer.C: - // If we've never managed to successfully authenticate then we won't have - // any session information anyways and this will produce a ton of noise in - // observability, so skip calling the function and retry in a short duration - if w.everAuthenticated.Load() == authenticationStatusNeverAuthenticated { - timer.Reset(10 * time.Millisecond) - continue - } - - w.sendWorkerStatus(cancelCtx, sessionManager, addrReceivers, recorderManager) - timer.Reset(getRandomInterval()) - } - } -} - -// LastStatusSuccess reports the last time we sent a successful -// status request. -func (w *Worker) LastStatusSuccess() *LastStatusInformation { - s, ok := w.lastStatusSuccess.Load().(*LastStatusInformation) - if !ok { - return nil - } - return s -} - -// WaitForNextSuccessfulStatusUpdate waits for the next successful status. It's -// used by testing (and in the future, shutdown) in place of a more opaque and -// possibly unnecessarily long sleep for things like initial controller -// check-in, etc. -// -// The timeout is aligned with the worker's status grace period. A nil error -// means the status was sent successfully. -func (w *Worker) WaitForNextSuccessfulStatusUpdate() error { - const op = "worker.(Worker).WaitForNextSuccessfulStatusUpdate" - waitStatusStart := time.Now() - ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(w.successfulStatusGracePeriod.Load())) - defer cancel() - event.WriteSysEvent(ctx, op, "waiting for next status report to controller") - for { - select { - case <-time.After(time.Second): - // pass - - case <-ctx.Done(): - event.WriteError(ctx, op, ctx.Err(), event.WithInfoMsg("error waiting for next status report to controller")) - return ctx.Err() - } - - if w.lastSuccessfulStatusTime().After(waitStatusStart) { - break - } - } - - event.WriteSysEvent(ctx, op, "next worker status update sent successfully") - return nil -} - -func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager session.Manager, addressReceivers *[]addressReceiver, recorderManager recorderManager) { - const op = "worker.(Worker).sendWorkerStatus" - w.statusLock.Lock() - defer w.statusLock.Unlock() - - // Collect the different session ids that are being monitored by this worker - var monitoredSessionIds []string - - if recorderManager != nil { - recSessIds, err := recorderManager.SessionsManaged(cancelCtx) - if err != nil { - event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error getting session ids from recorderManager")) - } else { - monitoredSessionIds = append(monitoredSessionIds, recSessIds...) - } - } - - // First send info as-is. We'll perform cleanup duties after we - // get cancel/job change info back. - var activeJobs []*pbs.JobStatus - - for _, sid := range monitoredSessionIds { - activeJobs = append(activeJobs, &pbs.JobStatus{ - Job: &pbs.Job{ - Type: pbs.JOBTYPE_JOBTYPE_MONITOR_SESSION, - JobInfo: &pbs.Job_MonitorSessionInfo{ - MonitorSessionInfo: &pbs.MonitorSessionJobInfo{ - SessionId: sid, - Status: pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE, - }, - }, - }, - }) - } - - // Range over known sessions and collect info - sessionManager.ForEachLocalSession(func(s session.Session) bool { - var jobInfo pbs.SessionJobInfo - status := s.GetStatus() - sessionId := s.GetId() - localConnections := s.GetLocalConnections() - connections := make([]*pbs.Connection, 0, len(localConnections)) - for k, v := range localConnections { - connections = append(connections, &pbs.Connection{ - ConnectionId: k, - Status: v.Status, - BytesUp: v.BytesUp(), - BytesDown: v.BytesDown(), - }) - } - jobInfo.SessionId = sessionId - activeJobs = append(activeJobs, &pbs.JobStatus{ - Job: &pbs.Job{ - Type: pbs.JOBTYPE_JOBTYPE_SESSION, - JobInfo: &pbs.Job_SessionInfo{ - SessionInfo: &pbs.SessionJobInfo{ - SessionId: sessionId, - Status: status, - Connections: connections, - }, - }, - }, - }) - return true - }) - - clientCon := w.GrpcClientConn.Load() - // Send status information - client := pbs.NewServerCoordinationServiceClient(clientCon) - var tags []*pb.TagPair - // If we're not going to request a tag update, no reason to have these - // marshaled on every status call. - if w.updateTags.Load() { - tags = w.tags.Load().([]*pb.TagPair) - } - statusCtx, statusCancel := context.WithTimeout(cancelCtx, time.Duration(w.statusCallTimeoutDuration.Load())) - defer statusCancel() - - keyId := w.WorkerAuthCurrentKeyId.Load() - switch { - case w.conf.RawConfig.Worker.Name == "" && keyId == "": - event.WriteError(cancelCtx, op, errors.New("worker name and keyId are both empty; at least one is needed to identify a worker"), - event.WithInfoMsg("error making status request to controller")) - } - - var storageBucketCredentialStates map[string]*plugin.StorageBucketCredentialState - if w.RecordingStorage != nil { - storageBucketCredentialStates = w.RecordingStorage.GetStorageBucketCredentialStates() - // If the local storage state is unknown, and we have recording storage set, get the state from the recording storage - // and set it on the worker. This is done once to ensure that the worker has the correct state for the first status - // call. - if w.localStorageState.Load() == server.UnknownLocalStorageState { - w.localStorageState.Store(w.RecordingStorage.GetLocalStorageState(cancelCtx)) - } - } - versionInfo := version.Get() - connectionState := w.downstreamConnManager.Connected() - result, err := client.Status(statusCtx, &pbs.StatusRequest{ - Jobs: activeJobs, - WorkerStatus: &pb.ServerWorkerStatus{ - Name: w.conf.RawConfig.Worker.Name, - Description: w.conf.RawConfig.Worker.Description, - Address: w.conf.RawConfig.Worker.PublicAddr, - Tags: tags, - KeyId: keyId, - ReleaseVersion: versionInfo.FullVersionNumber(false), - OperationalState: w.operationalState.Load().(server.OperationalState).String(), - LocalStorageState: w.localStorageState.Load().(server.LocalStorageState).String(), - StorageBucketCredentialStates: storageBucketCredentialStates, - }, - ConnectedWorkerKeyIdentifiers: connectionState.AllKeyIds(), - ConnectedUnmappedWorkerKeyIdentifiers: connectionState.UnmappedKeyIds(), - ConnectedWorkerPublicIds: connectionState.WorkerIds(), - UpdateTags: w.updateTags.Load(), - }) - if err != nil { - event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error making status request to controller", "controller_address", clientCon.Target())) - // Check for last successful status. Ignore nil last status, this probably - // means that we've never connected to a controller, and as such probably - // don't have any sessions to worry about anyway. - // - // If a length of time has passed since we've been able to communicate, we - // want to start terminating all connections as a "break glass" kind of - // scenario, as there will be no way we can really tell if these - // connections should continue to exist. - if isPastGrace, lastStatusTime, gracePeriod := w.isPastGrace(); isPastGrace { - event.WriteError(cancelCtx, op, - errors.New("status error grace period has expired, canceling all sessions on worker"), - event.WithInfo("last_status_time", lastStatusTime.String(), "grace_period", gracePeriod), - ) - - // Cancel connections if grace period has expired. These Connections will be closed in the - // database on the next successful status report, or via the Controller’s dead Worker cleanup connections job. - sessionManager.ForEachLocalSession(func(s session.Session) bool { - for _, connId := range s.CancelAllLocalConnections() { - event.WriteSysEvent(cancelCtx, op, "terminated connection due to status grace period expiration", "session_id", s.GetId(), "connection_id", connId) - } - return true - }) - - // In the case that the control plane has gone down and come up with different IPs, - // append initial upstreams/ cluster addr to the resolver to try - if w.GrpcClientConn.Load().GetState() == connectivity.TransientFailure { - lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation) - if lastStatus != nil && lastStatus.LastCalculatedUpstreams != nil { - addrs := lastStatus.LastCalculatedUpstreams - - if len(w.conf.RawConfig.Worker.InitialUpstreams) > 0 { - addrs = append(addrs, w.conf.RawConfig.Worker.InitialUpstreams...) - } else if HandleHcpbClusterId != nil && len(w.conf.RawConfig.HcpbClusterId) > 0 { - clusterId, err := parseutil.ParsePath(w.conf.RawConfig.HcpbClusterId) - if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { - event.WriteError(cancelCtx, op, err, event.WithInfoMsg("failed to parse HCP Boundary cluster ID")) - } else { - clusterAddress := HandleHcpbClusterId(clusterId) - addrs = append(addrs, clusterAddress) - } - } - - addrs = strutil.RemoveDuplicates(addrs, false) - if strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) { - // Nothing to update - return - } - - w.updateAddresses(cancelCtx, addrs, addressReceivers) - lastStatus.LastCalculatedUpstreams = addrs - w.lastStatusSuccess.Store(lastStatus) - } - } - - // Exit out of status function; our work here is done and we don't need to create closeConnection requests - return - } - - // Standard cleanup: Run through current jobs. Cancel connections - // for any canceling session or any session that is expired. - w.cleanupConnections(cancelCtx, false, sessionManager) - return - } - - w.updateTags.Store(false) - - if authorized := result.GetAuthorizedDownstreamWorkers(); authorized != nil { - connectionState.DisconnectMissingWorkers(authorized.GetWorkerPublicIds()) - connectionState.DisconnectMissingUnmappedKeyIds(authorized.GetUnmappedWorkerKeyIdentifiers()) - } else if authorized := result.GetAuthorizedWorkers(); authorized != nil { - connectionState.DisconnectAllMissingKeyIds(authorized.GetWorkerKeyIdentifiers()) - } - var addrs []string - // This may be empty if we are in a multiple hop scenario - if len(result.CalculatedUpstreams) > 0 { - addrs = make([]string, 0, len(result.CalculatedUpstreams)) - for _, v := range result.CalculatedUpstreams { - addrs = append(addrs, v.Address) - } - } else if checkHCPBUpstreams != nil && checkHCPBUpstreams(w) { - // This is a worker that is one hop away from managed workers, so attempt to get that list - hcpbWorkersCtx, hcpbWorkersCancel := context.WithTimeout(cancelCtx, time.Duration(w.statusCallTimeoutDuration.Load())) - defer hcpbWorkersCancel() - workersResp, err := client.ListHcpbWorkers(hcpbWorkersCtx, &pbs.ListHcpbWorkersRequest{}) - if err != nil { - event.WriteError(hcpbWorkersCtx, op, err, event.WithInfoMsg("error fetching managed worker information")) - } else { - addrs = make([]string, 0, len(workersResp.Workers)) - for _, v := range workersResp.Workers { - addrs = append(addrs, v.Address) - } - } - } - - w.updateAddresses(cancelCtx, addrs, addressReceivers) - - w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now(), LastCalculatedUpstreams: addrs}) - - var nonActiveMonitoredSessionIds []string - - for _, request := range result.GetJobsRequests() { - switch request.GetRequestType() { - case pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE: - switch request.GetJob().GetType() { - case pbs.JOBTYPE_JOBTYPE_MONITOR_SESSION: - si := request.GetJob().GetMonitorSessionInfo() - if si != nil && si.Status != pbs.SESSIONSTATUS_SESSIONSTATUS_ACTIVE { - nonActiveMonitoredSessionIds = append(nonActiveMonitoredSessionIds, si.GetSessionId()) - } - case pbs.JOBTYPE_JOBTYPE_SESSION: - sessInfo := request.GetJob().GetSessionInfo() - sessionId := sessInfo.GetSessionId() - si := sessionManager.Get(sessionId) - if si == nil { - event.WriteError(cancelCtx, op, errors.New("session change requested but could not find local information for it"), event.WithInfo("session_id", sessionId)) - continue - } - si.ApplyLocalStatus(sessInfo.GetStatus()) - - // Update connection state if there are any connections in - // the request. - for _, conn := range sessInfo.GetConnections() { - if err := si.ApplyLocalConnectionStatus(conn.GetConnectionId(), conn.GetStatus()); err != nil { - event.WriteError(cancelCtx, op, err, event.WithInfo("connection_id", conn.GetConnectionId())) - } - } - } - } - } - if recorderManager != nil { - if err := recorderManager.ReauthorizeAllExcept(cancelCtx, nonActiveMonitoredSessionIds); err != nil { - event.WriteError(cancelCtx, op, err) - } - } - - // Standard cleanup: Run through current jobs. Cancel connections - // for any canceling session or any session that is expired. - w.cleanupConnections(cancelCtx, false, sessionManager) - - // If we have post hooks for after the first status check, run them now - if w.everAuthenticated.CompareAndSwap(authenticationStatusFirstAuthentication, authenticationStatusFirstStatusRpcSuccessful) { - if downstreamWorkersFactory != nil { - downstreamWorkers, err := downstreamWorkersFactory(cancelCtx, w.LastStatusSuccess().WorkerId, versionInfo.FullVersionNumber(false)) - if err != nil { - event.WriteError(cancelCtx, op, err) - w.conf.ServerSideShutdownCh <- struct{}{} - return - } - w.downstreamWorkers.Store(&graphContainer{graph: downstreamWorkers}) - } - for _, fn := range firstStatusCheckPostHooks { - if err := fn(cancelCtx, w); err != nil { - // If we can't verify status we can't be expected to behave - // properly so error and trigger shutdown - event.WriteError(cancelCtx, op, fmt.Errorf("error running first status check post hook: %w", err)) - // We don't use a non-blocking select here to ensure that it - // happens; we should catch blocks in tests but we want to - // ensure the signal is being listened to - w.conf.ServerSideShutdownCh <- struct{}{} - return - } - } - } -} - -// Update address receivers and dialing listeners with new addrs -func (w *Worker) updateAddresses(cancelCtx context.Context, addrs []string, addressReceivers *[]addressReceiver) { - const op = "worker.(Worker).updateAddrs" - - if len(addrs) > 0 { - lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation) - // Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses - if lastStatus != nil && !strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) { - upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastStatus.LastCalculatedUpstreams, addrs) - event.WriteSysEvent(cancelCtx, op, upstreamsMessage) - for _, as := range *addressReceivers { - as.SetAddresses(addrs) - } - } else if lastStatus == nil { - for _, as := range *addressReceivers { - as.SetAddresses(addrs) - } - event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first status set to: %s", addrs)) - } - } - - // regardless of whether or not it's a new address, we need to set - // them for secondary connections - for _, as := range *addressReceivers { - switch { - case as.Type() == secondaryConnectionReceiverType: - tmpAddrs := make([]string, len(addrs)) - copy(tmpAddrs, addrs) - if len(tmpAddrs) == 0 { - tmpAddrs = append(tmpAddrs, w.conf.RawConfig.Worker.InitialUpstreams...) - } - as.SetAddresses(tmpAddrs) - } - } -} - -// cleanupConnections walks all sessions and shuts down all proxy connections. -// After the local connections are terminated, they are requested to be marked -// close on the controller. -// Additionally, sessions without connections are cleaned up from the -// local worker's state. -// -// Use ignoreSessionState to ignore the state checks, this closes all -// connections, regardless of whether or not the session is still active. -func (w *Worker) cleanupConnections(cancelCtx context.Context, ignoreSessionState bool, sessionManager session.Manager) { - const op = "worker.(Worker).cleanupConnections" - closeInfo := make(map[string]*session.ConnectionCloseData) - cleanSessionIds := make([]string, 0) - sessionManager.ForEachLocalSession(func(s session.Session) bool { - switch { - case ignoreSessionState, - s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING, - s.GetStatus() == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED, - time.Until(s.GetExpiration()) < 0: - // Cancel connections without regard to individual connection - // state. - closedIds := s.CancelAllLocalConnections() - localConns := s.GetLocalConnections() - for _, connId := range closedIds { - var bytesUp, bytesDown int64 - connInfo, ok := localConns[connId] - if ok { - bytesUp = connInfo.BytesUp() - bytesDown = connInfo.BytesDown() - } - closeInfo[connId] = &session.ConnectionCloseData{ - SessionId: s.GetId(), - BytesUp: bytesUp, - BytesDown: bytesDown, - } - event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId) - } - - // If the session is no longer valid and all connections - // are marked closed, clean up the session. - if len(closedIds) == 0 { - cleanSessionIds = append(cleanSessionIds, s.GetId()) - } - - default: - // Cancel connections *with* regard to individual connection - // state (ie: only ones that the controller has requested be - // terminated). - for _, connId := range s.CancelOpenLocalConnections() { - closeInfo[connId] = &session.ConnectionCloseData{SessionId: s.GetId()} - event.WriteSysEvent(cancelCtx, op, "terminated connection due to cancellation or expiration", "session_id", s.GetId(), "connection_id", connId) - } - } - - return true - }) - - // Note that we won't clean these from the info map until the - // next time we run this function - if len(closeInfo) > 0 { - // Call out to a helper to send the connection close requests to the - // controller, and set the close time. This functionality is shared with - // post-close functionality in the proxy handler. - _ = sessionManager.RequestCloseConnections(cancelCtx, closeInfo) - } - // Forget sessions where the session is expired/canceled and all - // connections are canceled and marked closed - sessionManager.DeleteLocalSession(cleanSessionIds) -} - -func (w *Worker) lastSuccessfulStatusTime() time.Time { - lastStatus := w.LastStatusSuccess() - if lastStatus == nil { - return w.workerStartTime - } - - return lastStatus.StatusTime -} - -func (w *Worker) isPastGrace() (bool, time.Time, time.Duration) { - t := w.lastSuccessfulStatusTime() - u := time.Duration(w.successfulStatusGracePeriod.Load()) - v := time.Since(t) - return v > u, t, u -} diff --git a/internal/daemon/worker/testing.go b/internal/daemon/worker/testing.go index a20135d9a..8a2f77676 100644 --- a/internal/daemon/worker/testing.go +++ b/internal/daemon/worker/testing.go @@ -29,6 +29,8 @@ import ( "github.com/mr-tron/base58" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) @@ -69,16 +71,8 @@ func (tw *TestWorker) Name() string { } func (tw *TestWorker) UpstreamAddrs() []string { - var addrs []string - lastStatus := tw.w.LastStatusSuccess() - if lastStatus == nil { - return addrs - } - for _, v := range lastStatus.GetCalculatedUpstreams() { - addrs = append(addrs, v.Address) - } - - return addrs + lastRoutingInfo := tw.w.LastRoutingInfoSuccess() + return lastRoutingInfo.GetCalculatedUpstreamAddresses() } func (tw *TestWorker) ProxyAddrs() []string { @@ -204,6 +198,10 @@ type TestWorkerOpts struct { // The location of the worker's recording storage WorkerRecordingStoragePath string + // The interval between each respective worker RPC invocation + // This sets the interval for SessionInfo, RoutingInfo and Statistics. + WorkerRPCInterval time.Duration + // The name to use for the worker, otherwise one will be randomly // generated, unless provided in a non-nil Config Name string @@ -217,7 +215,7 @@ type TestWorkerOpts struct { // The amount of time to wait before marking connections as closed when a // connection cannot be made back to the controller - SuccessfulStatusGracePeriodDuration time.Duration + SuccessfulControllerRPCGracePeriodDuration time.Duration // Overrides worker's nonceFn, for cases where we want to have control // over the nonce we send to the Controller @@ -286,6 +284,9 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker { if opts.Name != "" { opts.Config.Worker.Name = opts.Name } + if opts.WorkerRPCInterval > 0 { + opts.Config.Worker.TestWorkerRPCInterval = opts.WorkerRPCInterval + } } if len(opts.InitialUpstreams) > 0 { @@ -317,8 +318,8 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker { tw.b.EnabledPlugins = append(tw.b.EnabledPlugins, base.EnabledPluginLoopback) tw.name = opts.Config.Worker.Name - if opts.SuccessfulStatusGracePeriodDuration != 0 { - opts.Config.Worker.SuccessfulStatusGracePeriodDuration = opts.SuccessfulStatusGracePeriodDuration + if opts.SuccessfulControllerRPCGracePeriodDuration != 0 { + opts.Config.Worker.SuccessfulControllerRPCGracePeriodDuration = opts.SuccessfulControllerRPCGracePeriodDuration } serverName, err := os.Hostname() @@ -401,14 +402,14 @@ func (tw *TestWorker) AddClusterWorkerMember(t testing.TB, opts *TestWorkerOpts) opts = new(TestWorkerOpts) } nextOpts := &TestWorkerOpts{ - WorkerAuthKms: tw.w.conf.WorkerAuthKms, - DownstreamWorkerAuthKms: tw.w.conf.DownstreamWorkerAuthKms, - WorkerAuthStorageKms: tw.w.conf.WorkerAuthStorageKms, - Name: opts.Name, - InitialUpstreams: tw.UpstreamAddrs(), - Logger: tw.w.conf.Logger, - SuccessfulStatusGracePeriodDuration: opts.SuccessfulStatusGracePeriodDuration, - WorkerAuthDebuggingEnabled: tw.w.conf.WorkerAuthDebuggingEnabled, + WorkerAuthKms: tw.w.conf.WorkerAuthKms, + DownstreamWorkerAuthKms: tw.w.conf.DownstreamWorkerAuthKms, + WorkerAuthStorageKms: tw.w.conf.WorkerAuthStorageKms, + Name: opts.Name, + InitialUpstreams: tw.UpstreamAddrs(), + Logger: tw.w.conf.Logger, + SuccessfulControllerRPCGracePeriodDuration: opts.SuccessfulControllerRPCGracePeriodDuration, + WorkerAuthDebuggingEnabled: tw.w.conf.WorkerAuthDebuggingEnabled, } if nextOpts.Name == "" { var err error @@ -457,3 +458,98 @@ func NewAuthorizedPkiTestWorker(t *testing.T, repo *server.Repository, name stri require.NoError(t, err) return w, wr.GetPublicId() } + +// mockServerCoordinationService is meant to stand in for a controller when testing +// the methods defined by the server coordination service. It allows applying assertions and specifying +// the return value of grpc methods by overwriting service methods. +type mockServerCoordinationService struct { + pbs.UnimplementedServerCoordinationServiceServer + nextReqAssert func(*pbs.StatusRequest) (*pbs.StatusResponse, error) + nextStatisticAssert func(*pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) + nextSessionInfoAssert func(*pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) +} + +func (m mockServerCoordinationService) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) { + if m.nextReqAssert != nil { + return m.nextReqAssert(req) + } + return nil, status.Error(codes.Unavailable, "Status not implemented") +} + +func (m mockServerCoordinationService) Statistics(ctx context.Context, req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) { + if m.nextStatisticAssert != nil { + return m.nextStatisticAssert(req) + } + return nil, status.Error(codes.Unavailable, "Statistics not implemented") +} + +func (m mockServerCoordinationService) SessionInfo(ctx context.Context, req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) { + if m.nextSessionInfoAssert != nil { + return m.nextSessionInfoAssert(req) + } + return nil, status.Error(codes.Unavailable, "SessionInfo not implemented") +} + +var _ pbs.ServerCoordinationServiceServer = (*mockServerCoordinationService)(nil) + +// TestWaitForNextSuccessfulSessionInfoUpdate waits for the next successful session info. It's +// used by testing in place of a more opaque and possibly unnecessarily long sleep for +// things like initial controller check-in, etc. +// +// The timeout is aligned with the worker's session info grace period. +func (w *Worker) TestWaitForNextSuccessfulSessionInfoUpdate(t testing.TB) { + t.Helper() + const op = "worker.(Worker).WaitForNextSuccessfulSessionInfoUpdate" + waitStart := time.Now() + ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(w.successfulSessionInfoGracePeriod.Load())) + defer cancel() + t.Log("waiting for next session info report to controller") + for { + select { + case <-time.After(time.Second): + // pass + + case <-ctx.Done(): + t.Error("error waiting for next session info report to controller") + return + } + + si := w.lastSessionInfoSuccess.Load().(*lastSessionInfo) + if si != nil && si.LastSuccessfulRequestTime.After(waitStart) { + break + } + } + + t.Log("next worker session info update sent successfully") +} + +// TestWaitForNextSuccessfulStatisticsUpdate waits for the next successful statistics. It's +// used by testing in place of a more opaque and possibly unnecessarily long sleep for +// things like initial controller check-in, etc. +// +// The timeout is aligned with twice the worker's statistics timeout duration. +func (w *Worker) TestWaitForNextSuccessfulStatisticsUpdate(t testing.TB) { + t.Helper() + const op = "worker.(Worker).WaitForNextSuccessfulStatisticsUpdate" + waitStart := time.Now() + ctx, cancel := context.WithTimeout(w.baseContext, time.Duration(2*w.statisticsCallTimeoutDuration.Load())) + defer cancel() + t.Log("waiting for next statistics report to controller") + for { + select { + case <-time.After(time.Second): + // pass + + case <-ctx.Done(): + t.Error("error waiting for next statistics report to controller") + return + } + + si := w.lastStatisticsSuccess.Load().(*lastStatistics) + if si != nil && si.LastSuccessfulRequestTime.After(waitStart) { + break + } + } + + t.Log("next worker statistics update sent successfully") +} diff --git a/internal/daemon/worker/worker.go b/internal/daemon/worker/worker.go index e8c034292..cade23b67 100644 --- a/internal/daemon/worker/worker.go +++ b/internal/daemon/worker/worker.go @@ -11,6 +11,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "slices" "strconv" "strings" "sync" @@ -40,7 +41,6 @@ import ( "github.com/hashicorp/go-secure-stdlib/base62" "github.com/hashicorp/go-secure-stdlib/mlock" "github.com/hashicorp/go-secure-stdlib/pluginutil/v2" - "github.com/hashicorp/go-secure-stdlib/strutil" "github.com/hashicorp/nodeenrollment" nodeenet "github.com/hashicorp/nodeenrollment/net" nodeefile "github.com/hashicorp/nodeenrollment/storage/file" @@ -85,10 +85,10 @@ type graph interface { RootId() string } -// recorderManager updates the status updates with relevant recording +// recorderManager updates the session info updates with relevant recording // information type recorderManager interface { - // ReauthorizeAllExcept should be called with the result of the status update + // ReauthorizeAllExcept should be called with the result of the session info update // to reauthorize all recorders for the relevant sessions except the ones provided ReauthorizeAllExcept(ctx context.Context, closedSessions []string) error // SessionsManaged gets the list of session ids managed by this recorderManager @@ -126,11 +126,17 @@ var hostServiceServerFactory func( const ( authenticationStatusNeverAuthenticated uint32 = iota authenticationStatusFirstAuthentication - authenticationStatusFirstStatusRpcSuccessful + authenticationStatusFirstRoutingInfoRpcSuccessful ) type Worker struct { - conf *Config + conf *Config + // receives address updates and contains the grpc resolver. + addressReceivers []addressReceiver + // confAddressReceiversLock is used to protect the conf field + // and the addressReceivers field. + confAddressReceiversLock sync.Mutex + logger hclog.Logger baseContext context.Context @@ -145,17 +151,16 @@ type Worker struct { // (mostly in tests) - which isn't thread safe. This is exported for tests. GrpcClientConn atomic.Pointer[grpc.ClientConn] - // receives address updates and contains the grpc resolver. - addressReceivers []addressReceiver - sessionManager session.Manager recorderManager recorderManager - everAuthenticated *ua.Uint32 - lastStatusSuccess *atomic.Value - workerStartTime time.Time - operationalState *atomic.Value + everAuthenticated *ua.Uint32 + lastSessionInfoSuccess *atomic.Value + lastRoutingInfoSuccess *atomic.Value + lastStatisticsSuccess *atomic.Value + workerStartTime time.Time + operationalState *atomic.Value // localStorageState is the current state of the local storage. // The local storage state is updated based on the local storage events. localStorageState *atomic.Value @@ -172,11 +177,9 @@ type Worker struct { // Used to generate a random nonce for Controller connections nonceFn randFn - // We store the current set in an atomic value so that we can add - // reload-on-sighup behavior later tags *atomic.Value - // This stores whether or not to send updated tags on the next status - // request. It can be set via startup in New below, or (eventually) via + // This stores whether or not to send updated tags on the next routing info + // request. It can be set via startup in New below, or via // SIGHUP. updateTags *ua.Bool @@ -195,10 +198,26 @@ type Worker struct { // Timing variables. These are atomics for SIGHUP support, and are int64 // because they are casted to time.Duration. - successfulStatusGracePeriod *atomic.Int64 - statusCallTimeoutDuration *atomic.Int64 + successfulRoutingInfoGracePeriod *atomic.Int64 + successfulSessionInfoGracePeriod *atomic.Int64 + // Note: Statistics does not require a grace period, + // because we do not perform any special action based + // on whether it is successful or not, unlike for SessionInfo + // and RoutingInfo. + + statisticsCallTimeoutDuration *atomic.Int64 + sessionInfoCallTimeoutDuration *atomic.Int64 + routingInfoCallTimeoutDuration *atomic.Int64 getDownstreamWorkersTimeoutDuration *atomic.Int64 + // The time intervals at which the worker will invoke the controller RPCs. + // Defaults to common.SessionInfoInterval, common.RoutingInfoInterval + // and common.StatisticsInterval and is only overridden by the + // TestWorkerRPCInterval test config. + sessionInfoInterval time.Duration + routingInfoInterval time.Duration + statisticsInterval time.Duration + // AuthRotationNextRotation is useful in tests to understand how long to // sleep AuthRotationNextRotation atomic.Pointer[time.Time] @@ -208,8 +227,6 @@ type Worker struct { TestOverrideX509VerifyCertPool *x509.CertPool TestOverrideAuthRotationPeriod time.Duration - statusLock sync.Mutex - downstreamConnManager *cluster.DownstreamManager HostServiceServer wpbs.HostServiceServer @@ -230,7 +247,9 @@ func New(ctx context.Context, conf *Config) (*Worker, error) { logger: conf.Logger.Named("worker"), started: ua.NewBool(false), everAuthenticated: ua.NewUint32(authenticationStatusNeverAuthenticated), - lastStatusSuccess: new(atomic.Value), + lastSessionInfoSuccess: new(atomic.Value), + lastRoutingInfoSuccess: new(atomic.Value), + lastStatisticsSuccess: new(atomic.Value), controllerMultihopConn: new(atomic.Value), // controllerUpstreamMsgConn: new(atomic.Value), tags: new(atomic.Value), @@ -240,8 +259,11 @@ func New(ctx context.Context, conf *Config) (*Worker, error) { operationalState: new(atomic.Value), downstreamConnManager: cluster.NewDownstreamManager(), localStorageState: new(atomic.Value), - successfulStatusGracePeriod: new(atomic.Int64), - statusCallTimeoutDuration: new(atomic.Int64), + successfulRoutingInfoGracePeriod: new(atomic.Int64), + successfulSessionInfoGracePeriod: new(atomic.Int64), + statisticsCallTimeoutDuration: new(atomic.Int64), + sessionInfoCallTimeoutDuration: new(atomic.Int64), + routingInfoCallTimeoutDuration: new(atomic.Int64), getDownstreamWorkersTimeoutDuration: new(atomic.Int64), upstreamConnectionState: new(atomic.Value), downstreamWorkers: new(atomic.Pointer[graphContainer]), @@ -249,7 +271,9 @@ func New(ctx context.Context, conf *Config) (*Worker, error) { w.operationalState.Store(server.UnknownOperationalState) w.localStorageState.Store(server.UnknownLocalStorageState) - w.lastStatusSuccess.Store((*LastStatusInformation)(nil)) + w.lastSessionInfoSuccess.Store((*lastSessionInfo)(nil)) + w.lastRoutingInfoSuccess.Store((*LastRoutingInfo)(nil)) + w.lastStatisticsSuccess.Store((*lastStatistics)(nil)) scheme := strconv.FormatInt(time.Now().UnixNano(), 36) controllerResolver := manual.NewBuilderWithScheme(scheme) w.addressReceivers = []addressReceiver{&grpcResolverReceiver{controllerResolver}} @@ -365,17 +389,23 @@ func New(ctx context.Context, conf *Config) (*Worker, error) { err) } } - switch conf.RawConfig.Worker.SuccessfulStatusGracePeriodDuration { + switch conf.RawConfig.Worker.SuccessfulControllerRPCGracePeriodDuration { case 0: - w.successfulStatusGracePeriod.Store(int64(server.DefaultLiveness)) + w.successfulRoutingInfoGracePeriod.Store(int64(server.DefaultLiveness)) + w.successfulSessionInfoGracePeriod.Store(int64(server.DefaultLiveness)) default: - w.successfulStatusGracePeriod.Store(int64(conf.RawConfig.Worker.SuccessfulStatusGracePeriodDuration)) + w.successfulRoutingInfoGracePeriod.Store(int64(conf.RawConfig.Worker.SuccessfulControllerRPCGracePeriodDuration)) + w.successfulSessionInfoGracePeriod.Store(int64(conf.RawConfig.Worker.SuccessfulControllerRPCGracePeriodDuration)) } - switch conf.RawConfig.Worker.StatusCallTimeoutDuration { + switch conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration { case 0: - w.statusCallTimeoutDuration.Store(int64(common.DefaultStatusTimeout)) + w.routingInfoCallTimeoutDuration.Store(int64(common.DefaultRoutingInfoTimeout)) + w.statisticsCallTimeoutDuration.Store(int64(common.DefaultStatisticsTimeout)) + w.sessionInfoCallTimeoutDuration.Store(int64(common.DefaultSessionInfoTimeout)) default: - w.statusCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.StatusCallTimeoutDuration)) + w.routingInfoCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration)) + w.statisticsCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration)) + w.sessionInfoCallTimeoutDuration.Store(int64(conf.RawConfig.Worker.ControllerRPCCallTimeoutDuration)) } switch conf.RawConfig.Worker.GetDownstreamWorkersTimeoutDuration { case 0: @@ -384,7 +414,18 @@ func New(ctx context.Context, conf *Config) (*Worker, error) { w.getDownstreamWorkersTimeoutDuration.Store(int64(conf.RawConfig.Worker.GetDownstreamWorkersTimeoutDuration)) } // FIXME: This is really ugly, but works. - session.CloseCallTimeout.Store(w.successfulStatusGracePeriod.Load()) + session.CloseCallTimeout.Store(w.successfulSessionInfoGracePeriod.Load()) + + w.sessionInfoInterval = common.SessionInfoInterval + w.routingInfoInterval = common.RoutingInfoInterval + w.statisticsInterval = common.StatisticsInterval + // Override the routing info interval if it is set in the config. + // This should only be used by tests. + if conf.RawConfig.Worker.TestWorkerRPCInterval > 0 { + w.sessionInfoInterval = conf.RawConfig.Worker.TestWorkerRPCInterval + w.routingInfoInterval = conf.RawConfig.Worker.TestWorkerRPCInterval + w.statisticsInterval = conf.RawConfig.Worker.TestWorkerRPCInterval + } if reverseConnReceiverFactory != nil { var err error @@ -443,10 +484,36 @@ func (w *Worker) Reload(ctx context.Context, newConf *config.Config) { w.parseAndStoreTags(newConf.Worker.Tags) - if !strutil.EquivalentSlices(newConf.Worker.InitialUpstreams, w.conf.RawConfig.Worker.InitialUpstreams) { - w.statusLock.Lock() - defer w.statusLock.Unlock() + switch newConf.Worker.SuccessfulControllerRPCGracePeriod { + case 0: + w.successfulRoutingInfoGracePeriod.Store(int64(server.DefaultLiveness)) + w.successfulSessionInfoGracePeriod.Store(int64(server.DefaultLiveness)) + default: + w.successfulRoutingInfoGracePeriod.Store(int64(newConf.Worker.SuccessfulControllerRPCGracePeriodDuration)) + w.successfulSessionInfoGracePeriod.Store(int64(newConf.Worker.SuccessfulControllerRPCGracePeriodDuration)) + } + switch newConf.Worker.ControllerRPCCallTimeoutDuration { + case 0: + w.routingInfoCallTimeoutDuration.Store(int64(common.DefaultRoutingInfoTimeout)) + w.statisticsCallTimeoutDuration.Store(int64(common.DefaultStatisticsTimeout)) + w.sessionInfoCallTimeoutDuration.Store(int64(common.DefaultSessionInfoTimeout)) + default: + w.routingInfoCallTimeoutDuration.Store(int64(newConf.Worker.ControllerRPCCallTimeoutDuration)) + w.statisticsCallTimeoutDuration.Store(int64(newConf.Worker.ControllerRPCCallTimeoutDuration)) + w.sessionInfoCallTimeoutDuration.Store(int64(newConf.Worker.ControllerRPCCallTimeoutDuration)) + } + switch newConf.Worker.GetDownstreamWorkersTimeoutDuration { + case 0: + w.getDownstreamWorkersTimeoutDuration.Store(int64(server.DefaultLiveness)) + default: + w.getDownstreamWorkersTimeoutDuration.Store(int64(newConf.Worker.GetDownstreamWorkersTimeoutDuration)) + } + // See comment about this in worker.go + session.CloseCallTimeout.Store(w.successfulRoutingInfoGracePeriod.Load()) + w.confAddressReceiversLock.Lock() + defer w.confAddressReceiversLock.Unlock() + if !slices.Equal(newConf.Worker.InitialUpstreams, w.conf.RawConfig.Worker.InitialUpstreams) { upstreamsMessage := fmt.Sprintf( "Initial Upstreams has changed; old upstreams were: %s, new upstreams are: %s", w.conf.RawConfig.Worker.InitialUpstreams, @@ -461,27 +528,6 @@ func (w *Worker) Reload(ctx context.Context, newConf *config.Config) { ar.InitialAddresses(w.conf.RawConfig.Worker.InitialUpstreams) } } - - switch newConf.Worker.SuccessfulStatusGracePeriodDuration { - case 0: - w.successfulStatusGracePeriod.Store(int64(server.DefaultLiveness)) - default: - w.successfulStatusGracePeriod.Store(int64(newConf.Worker.SuccessfulStatusGracePeriodDuration)) - } - switch newConf.Worker.StatusCallTimeoutDuration { - case 0: - w.statusCallTimeoutDuration.Store(int64(common.DefaultStatusTimeout)) - default: - w.statusCallTimeoutDuration.Store(int64(newConf.Worker.StatusCallTimeoutDuration)) - } - switch newConf.Worker.GetDownstreamWorkersTimeoutDuration { - case 0: - w.getDownstreamWorkersTimeoutDuration.Store(int64(server.DefaultLiveness)) - default: - w.getDownstreamWorkersTimeoutDuration.Store(int64(newConf.Worker.GetDownstreamWorkersTimeoutDuration)) - } - // See comment about this in worker.go - session.CloseCallTimeout.Store(w.successfulStatusGracePeriod.Load()) } func (w *Worker) Start() error { @@ -612,9 +658,6 @@ func (w *Worker) Start() error { return fmt.Errorf("error marshaling worker auth fetch credentials request: %w", err) } w.WorkerAuthRegistrationRequest = base58.FastBase58Encoding(reqBytes) - if err != nil { - return fmt.Errorf("error encoding worker auth registration request: %w", err) - } currentKeyId, err := nodeenrollment.KeyIdFromPkix(nodeCreds.CertificatePublicKeyPkix) if err != nil { return fmt.Errorf("error deriving worker auth key id: %w", err) @@ -659,7 +702,7 @@ func (w *Worker) Start() error { w.tickerWg.Add(2) go func() { defer w.tickerWg.Done() - w.startStatusTicking(w.baseContext, w.sessionManager, &w.addressReceivers, w.recorderManager) + w.startRoutingInfoTicking(w.baseContext) }() go func() { defer w.tickerWg.Done() @@ -669,7 +712,7 @@ func (w *Worker) Start() error { if w.downstreamReceiver != nil { w.tickerWg.Add(2) servNameFn := func() string { - if s := w.LastStatusSuccess(); s != nil { + if s := w.LastRoutingInfoSuccess(); s != nil { return s.WorkerId } return "unknown worker id" @@ -706,24 +749,30 @@ func (w *Worker) GracefulShutdown() error { event.WriteSysEvent(w.baseContext, op, "worker entering graceful shutdown") w.operationalState.Store(server.ShutdownOperationalState) - // As long as some status has been sent in the past, wait for 2 status - // updates to be sent since we've updated our operational state. - lastStatusTime := w.lastSuccessfulStatusTime() - if lastStatusTime != w.workerStartTime { - for i := 0; i < 2; i++ { - for { - if lastStatusTime != w.lastSuccessfulStatusTime() { - lastStatusTime = w.lastSuccessfulStatusTime() - break - } - time.Sleep(time.Millisecond * 250) + // As long as some routing info has been sent in the past, wait for 1 routing info + // update to be sent since we've updated our operational state. + lastRoutingInfoTime := w.lastSuccessfulRoutingInfoTime() + if !lastRoutingInfoTime.Equal(w.workerStartTime) { + WaitForRoutingInfo: + for !lastRoutingInfoTime.Before(w.lastSuccessfulRoutingInfoTime()) { + select { + case <-w.baseContext.Done(): + event.WriteSysEvent(w.baseContext, op, "context done waiting for routing info to be sent") + break WaitForRoutingInfo + case <-time.After(time.Millisecond * 250): } } } // Wait for running proxy connections to drain +WaitForConnectionDrain: for proxy.ProxyState.CurrentProxiedConnections() > 0 { - time.Sleep(time.Millisecond * 250) + select { + case <-w.baseContext.Done(): + event.WriteSysEvent(w.baseContext, op, "context done waiting for connections to be drained") + break WaitForConnectionDrain + case <-time.After(time.Millisecond * 250): + } } event.WriteSysEvent(w.baseContext, op, "worker connections have drained") @@ -767,30 +816,31 @@ func (w *Worker) Shutdown() error { // Shut down all connections. w.cleanupConnections(w.baseContext, true, w.sessionManager) - // Wait for next status request to succeed. Don't wait too long; time it out - // at our default liveness value, which is also our default status grace - // period timeout - waitStatusStart := time.Now() - nextStatusCtx, nextStatusCancel := context.WithTimeout(w.baseContext, server.DefaultLiveness) - defer nextStatusCancel() - for { - if err := nextStatusCtx.Err(); err != nil { - event.WriteError(w.baseContext, op, err, event.WithInfoMsg("error waiting for next status report to controller")) - break + // Wait for next routing info request to succeed. Don't wait too long; time it out + // at our default liveness value. + nextRoutingInfoCtx, nextRoutingInfoCancel := context.WithTimeout(w.baseContext, server.DefaultLiveness) + defer nextRoutingInfoCancel() + lastRoutingInfoTime := w.lastSuccessfulRoutingInfoTime() + if !lastRoutingInfoTime.Equal(w.workerStartTime) { + WaitForRoutingInfo: + for !lastRoutingInfoTime.Before(w.lastSuccessfulRoutingInfoTime()) { + select { + case <-nextRoutingInfoCtx.Done(): + event.WriteError(w.baseContext, op, nextRoutingInfoCtx.Err(), event.WithInfoMsg("error waiting for next routing info report to controller")) + break WaitForRoutingInfo + case <-time.After(time.Millisecond * 250): + } } - - if w.lastSuccessfulStatusTime().After(waitStatusStart) { - break - } - - time.Sleep(time.Second) } // Proceed with remainder of shutdown. w.baseCancel() + // Lock to protect w.addressReceivers + w.confAddressReceiversLock.Lock() for _, ar := range w.addressReceivers { ar.SetAddresses(nil) } + w.confAddressReceiversLock.Unlock() if w.storageEventListener != nil { err := w.storageEventListener.Shutdown(w.baseContext) @@ -853,10 +903,10 @@ func (w *Worker) getSessionTls(sessionManager session.Manager) func(hello *tls.C return nil, fmt.Errorf("could not find session ID in SNI or ALPN protos") } - lastSuccess := w.LastStatusSuccess() + lastSuccess := w.LastRoutingInfoSuccess() if lastSuccess == nil { - event.WriteSysEvent(ctx, op, "no last status information found at session acceptance time") - return nil, fmt.Errorf("no last status information found at session acceptance time") + event.WriteSysEvent(ctx, op, "no last routing information found at session acceptance time") + return nil, fmt.Errorf("no last routing information found at session acceptance time") } timeoutContext, cancel := context.WithTimeout(w.baseContext, session.ValidateSessionTimeout) diff --git a/internal/daemon/worker/worker_proxy_service.go b/internal/daemon/worker/worker_proxy_service.go index 5bd351b84..d786106f3 100644 --- a/internal/daemon/worker/worker_proxy_service.go +++ b/internal/daemon/worker/worker_proxy_service.go @@ -30,6 +30,14 @@ func NewWorkerProxyServiceServer( } } +func (ws *workerProxyServiceServer) Statistics(ctx context.Context, req *pbs.StatisticsRequest) (*pbs.StatisticsResponse, error) { + return pbs.NewServerCoordinationServiceClient(ws.cc).Statistics(ctx, req) +} + +func (ws *workerProxyServiceServer) SessionInfo(ctx context.Context, req *pbs.SessionInfoRequest) (*pbs.SessionInfoResponse, error) { + return pbs.NewServerCoordinationServiceClient(ws.cc).SessionInfo(ctx, req) +} + func (ws *workerProxyServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) { resp, err := pbs.NewServerCoordinationServiceClient(ws.cc).Status(ctx, req) @@ -46,6 +54,18 @@ func (ws *workerProxyServiceServer) ListHcpbWorkers(ctx context.Context, req *pb return pbs.NewServerCoordinationServiceClient(ws.cc).ListHcpbWorkers(ctx, req) } +func (ws *workerProxyServiceServer) RoutingInfo(ctx context.Context, req *pbs.RoutingInfoRequest) (*pbs.RoutingInfoResponse, error) { + resp, err := pbs.NewServerCoordinationServiceClient(ws.cc).RoutingInfo(ctx, req) + + if resp != nil { + // We don't currently support distributing new addreses to workers + // multiple hops away so ensure they're stripped out + resp.CalculatedUpstreamAddresses = nil + } + + return resp, err +} + func (ws *workerProxyServiceServer) LookupSession(ctx context.Context, req *pbs.LookupSessionRequest) (*pbs.LookupSessionResponse, error) { return pbs.NewSessionServiceClient(ws.cc).LookupSession(ctx, req) } diff --git a/internal/daemon/worker/worker_test.go b/internal/daemon/worker/worker_test.go index 387d91aa6..5607c8f37 100644 --- a/internal/daemon/worker/worker_test.go +++ b/internal/daemon/worker/worker_test.go @@ -371,7 +371,7 @@ func Test_Worker_getSessionTls(t *testing.T) { conf.RawConfig = &config.Config{SharedConfig: &configutil.SharedConfig{DisableMlock: true}} w, err := New(context.Background(), conf) require.NoError(t, err) - w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: &services.StatusResponse{}, StatusTime: time.Now(), LastCalculatedUpstreams: nil}) + w.lastRoutingInfoSuccess.Store(&LastRoutingInfo{RoutingInfoResponse: &services.RoutingInfoResponse{}, RoutingInfoTime: time.Now(), LastCalculatedUpstreams: nil}) w.baseContext = context.Background() t.Run("success", func(t *testing.T) { diff --git a/internal/db/schema/migrations/oss/postgres/95/01_server_worker_session_info.up.sql b/internal/db/schema/migrations/oss/postgres/95/01_server_worker_session_info.up.sql new file mode 100644 index 000000000..66497ebbf --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/95/01_server_worker_session_info.up.sql @@ -0,0 +1,18 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: BUSL-1.1 + +begin; + +create table server_worker_session_info_request ( + worker_id wt_public_id + constraint server_worker_fkey + references server_worker(public_id) + on delete cascade + on update cascade, + last_request_time timestamp with time zone not null, + primary key (worker_id) +); +comment on table server_worker_session_info_request is + 'server_worker_session_info_request is a table where each row represents the last request time a worker has reported its session information.'; + +commit; \ No newline at end of file diff --git a/internal/event/cloudevents_formatter_node.go b/internal/event/cloudevents_formatter_node.go index 9a24c78f2..7dd943c9f 100644 --- a/internal/event/cloudevents_formatter_node.go +++ b/internal/event/cloudevents_formatter_node.go @@ -87,13 +87,28 @@ func defaultCloudEventsDenyFilters() ([]*filter, error) { const ( op = "event.defaultCloudEventsDenyFilters" // denyWorkStatusEvents is a default filter for worker to controller API status requests - denyWorkStatusEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"` + denyWorkStatusEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"` + denyWorkSessionInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/SessionInfo"` + denyWorkRoutingInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/RoutingInfo"` + denyWorkStatisticsEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Statistics"` ) - f, err := newFilter(denyWorkStatusEvents) + statusFilter, err := newFilter(denyWorkStatusEvents) if err != nil { return nil, fmt.Errorf("%s: unable to create deny filter for worker status events '%s': %w", op, denyWorkStatusEvents, err) } - return []*filter{f}, nil + sessionInfoFilter, err := newFilter(denyWorkSessionInfoEvents) + if err != nil { + return nil, fmt.Errorf("%s: unable to create deny filter for worker session info events '%s': %w", op, denyWorkStatusEvents, err) + } + routingInfoFilter, err := newFilter(denyWorkRoutingInfoEvents) + if err != nil { + return nil, fmt.Errorf("%s: unable to create deny filter for worker routing info events '%s': %w", op, denyWorkStatusEvents, err) + } + statisticsFilter, err := newFilter(denyWorkStatisticsEvents) + if err != nil { + return nil, fmt.Errorf("%s: unable to create deny filter for worker statistics events '%s': %w", op, denyWorkStatusEvents, err) + } + return []*filter{statusFilter, sessionInfoFilter, routingInfoFilter, statisticsFilter}, nil } // Rotate supports rotating the filter's wrapper. No options are currently diff --git a/internal/event/cloudevents_formatter_node_test.go b/internal/event/cloudevents_formatter_node_test.go index 58f2a1e15..e0184af7d 100644 --- a/internal/event/cloudevents_formatter_node_test.go +++ b/internal/event/cloudevents_formatter_node_test.go @@ -36,10 +36,15 @@ func Test_newCloudEventsFormatterFilter(t *testing.T) { }{ { // default case should have default deny for filtering ServerCoordinationService/Status for observation events - name: "no-opts", - source: testSource, - format: cloudevents.FormatJSON, - wantDeny: []string{`"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`}, + name: "no-opts", + source: testSource, + format: cloudevents.FormatJSON, + wantDeny: []string{ + `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"`, + `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/SessionInfo"`, + `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/RoutingInfo"`, + `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Statistics"`, + }, }, { name: "bad-allow-filter", diff --git a/internal/event/hclog_formatter_node.go b/internal/event/hclog_formatter_node.go index 54b5f7012..727472810 100644 --- a/internal/event/hclog_formatter_node.go +++ b/internal/event/hclog_formatter_node.go @@ -84,13 +84,28 @@ func defaultHclogEventsDenyFilters() ([]*filter, error) { const ( op = "event.defaultHclogEventsDenyFilters" // denyWorkStatusEvents is a default filter for worker to controller API status requests - denyWorkStatusEvents = `"/RequestInfo/Method" contains "ServerCoordinationService/Status"` + denyWorkStatusEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Status"` + denyWorkSessionInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/SessionInfo"` + denyWorkRoutingInfoEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/RoutingInfo"` + denyWorkStatisticsEvents = `"/type" contains "observation" and "/data/request_info/method" contains "ServerCoordinationService/Statistics"` ) - f, err := newFilter(denyWorkStatusEvents) + statusFilter, err := newFilter(denyWorkStatusEvents) if err != nil { return nil, fmt.Errorf("%s: unable to create deny filter for worker status events '%s': %w", op, denyWorkStatusEvents, err) } - return []*filter{f}, nil + sessionInfoFilter, err := newFilter(denyWorkSessionInfoEvents) + if err != nil { + return nil, fmt.Errorf("%s: unable to create deny filter for worker session info events '%s': %w", op, denyWorkStatusEvents, err) + } + routingInfoFilter, err := newFilter(denyWorkRoutingInfoEvents) + if err != nil { + return nil, fmt.Errorf("%s: unable to create deny filter for worker routing info events '%s': %w", op, denyWorkStatusEvents, err) + } + statisticsFilter, err := newFilter(denyWorkStatisticsEvents) + if err != nil { + return nil, fmt.Errorf("%s: unable to create deny filter for worker statistics events '%s': %w", op, denyWorkStatusEvents, err) + } + return []*filter{statusFilter, sessionInfoFilter, routingInfoFilter, statisticsFilter}, nil } // Rotate supports rotating the filter's wrapper. No options are currently diff --git a/internal/event/hclog_formatter_node_test.go b/internal/event/hclog_formatter_node_test.go index a4c84d569..4b1682047 100644 --- a/internal/event/hclog_formatter_node_test.go +++ b/internal/event/hclog_formatter_node_test.go @@ -388,7 +388,7 @@ func Test_newHclogFormatterFilter(t *testing.T) { for _, f := range got.allow { assert.Contains(tt.wantAllow, f.raw) } - assert.Len(got.deny, len(tt.wantDeny)+1) // +1 since there's always a default deny + assert.Len(got.deny, len(tt.wantDeny)+4) // +4 since there's always a default deny defs, err := defaultHclogEventsDenyFilters() require.NoError(err) for _, f := range defs { diff --git a/internal/gen/controller/servers/servers.pb.go b/internal/gen/controller/servers/servers.pb.go index c9c6a7f68..93cc3dda0 100644 --- a/internal/gen/controller/servers/servers.pb.go +++ b/internal/gen/controller/servers/servers.pb.go @@ -86,17 +86,17 @@ type ServerWorkerStatus struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Id of the worker. + // Id of the worker, used after the first request. PublicId string `protobuf:"bytes,10,opt,name=public_id,json=publicId,proto3" json:"public_id,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"` - // Name of the resource (optional) + // Name of the worker, used to identify workers in the KMS registration flow. Name string `protobuf:"bytes,20,opt,name=name,proto3" json:"name,omitempty" class:"public"` // @gotags: `class:"public"` - // Description of the resource (optional) + // Description of the worker (optional). Only used by KMS workers. Description string `protobuf:"bytes,25,opt,name=description,proto3" json:"description,omitempty" class:"public"` // @gotags: `class:"public"` - // Address for the worker. This must be set. + // Address for the worker. Address string `protobuf:"bytes,30,opt,name=address,proto3" json:"address,omitempty" class:"public"` // @gotags: `class:"public"` // Tags for workers Tags []*TagPair `protobuf:"bytes,40,rep,name=tags,proto3" json:"tags,omitempty"` - // The key id for this worker, if applicable (optional) + // The key id of the worker, used to identify workers in the PKI registration flow. KeyId string `protobuf:"bytes,50,opt,name=key_id,json=keyId,proto3" json:"key_id,omitempty" class:"public"` // @gotags: `class:"public"` // The version of Boundary the worker binary is running ReleaseVersion string `protobuf:"bytes,60,opt,name=release_version,proto3" json:"release_version,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"` diff --git a/internal/gen/controller/servers/services/server_coordination_service.pb.go b/internal/gen/controller/servers/services/server_coordination_service.pb.go index 55856a81b..63fb7273e 100644 --- a/internal/gen/controller/servers/services/server_coordination_service.pb.go +++ b/internal/gen/controller/servers/services/server_coordination_service.pb.go @@ -131,6 +131,9 @@ func (SESSIONSTATUS) EnumDescriptor() ([]byte, []int) { return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{1} } +// This enum is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type SessionProcessingError int32 const ( @@ -177,6 +180,9 @@ func (SessionProcessingError) EnumDescriptor() ([]byte, []int) { return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{2} } +// This enum is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type JOBTYPE int32 const ( @@ -226,6 +232,9 @@ func (JOBTYPE) EnumDescriptor() ([]byte, []int) { return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{3} } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type CHANGETYPE int32 const ( @@ -274,6 +283,58 @@ func (CHANGETYPE) EnumDescriptor() ([]byte, []int) { return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{4} } +// Session type represents how the worker is processing a session. +type SessionType int32 + +const ( + SessionType_SESSION_TYPE_UNSPECIFIED SessionType = 0 + // A session where the worker is the ingress worker. + SessionType_SESSION_TYPE_INGRESSED SessionType = 1 + // A session where the worker is doing the recording of the session. + SessionType_SESSION_TYPE_RECORDED SessionType = 2 +) + +// Enum value maps for SessionType. +var ( + SessionType_name = map[int32]string{ + 0: "SESSION_TYPE_UNSPECIFIED", + 1: "SESSION_TYPE_INGRESSED", + 2: "SESSION_TYPE_RECORDED", + } + SessionType_value = map[string]int32{ + "SESSION_TYPE_UNSPECIFIED": 0, + "SESSION_TYPE_INGRESSED": 1, + "SESSION_TYPE_RECORDED": 2, + } +) + +func (x SessionType) Enum() *SessionType { + p := new(SessionType) + *p = x + return p +} + +func (x SessionType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SessionType) Descriptor() protoreflect.EnumDescriptor { + return file_controller_servers_services_v1_server_coordination_service_proto_enumTypes[5].Descriptor() +} + +func (SessionType) Type() protoreflect.EnumType { + return &file_controller_servers_services_v1_server_coordination_service_proto_enumTypes[5] +} + +func (x SessionType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SessionType.Descriptor instead. +func (SessionType) EnumDescriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{5} +} + type UpstreamServer_TYPE int32 const ( @@ -307,11 +368,11 @@ func (x UpstreamServer_TYPE) String() string { } func (UpstreamServer_TYPE) Descriptor() protoreflect.EnumDescriptor { - return file_controller_servers_services_v1_server_coordination_service_proto_enumTypes[5].Descriptor() + return file_controller_servers_services_v1_server_coordination_service_proto_enumTypes[6].Descriptor() } func (UpstreamServer_TYPE) Type() protoreflect.EnumType { - return &file_controller_servers_services_v1_server_coordination_service_proto_enumTypes[5] + return &file_controller_servers_services_v1_server_coordination_service_proto_enumTypes[6] } func (x UpstreamServer_TYPE) Number() protoreflect.EnumNumber { @@ -394,6 +455,9 @@ func (x *Connection) GetBytesDown() int64 { return 0 } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type SessionJobInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -465,6 +529,9 @@ func (x *SessionJobInfo) GetProcessingError() SessionProcessingError { return SessionProcessingError_SESSION_PROCESSING_ERROR_UNSPECIFIED } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type MonitorSessionJobInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -528,6 +595,9 @@ func (x *MonitorSessionJobInfo) GetProcessingError() SessionProcessingError { return SessionProcessingError_SESSION_PROCESSING_ERROR_UNSPECIFIED } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type Job struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -618,6 +688,9 @@ func (*Job_SessionInfo) isJob_JobInfo() {} func (*Job_MonitorSessionInfo) isJob_JobInfo() {} +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type JobStatus struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -667,6 +740,9 @@ func (x *JobStatus) GetJob() *Job { // UpstreamServer returns the controller address in the StatusResponse but may be used in the future to return // worker address. +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type UpstreamServer struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -724,6 +800,9 @@ func (x *UpstreamServer) GetAddress() string { return "" } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type StatusRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -741,13 +820,6 @@ type StatusRequest struct { // is easier and going the other route doesn't provide much benefit -- if you // get access to the key and spoof the connection, you're already compromised. WorkerStatus *servers.ServerWorkerStatus `protobuf:"bytes,40,opt,name=worker_status,json=workerStatus,proto3" json:"worker_status,omitempty"` - // The worker key identifiers presented by all downstreams connected to this - // worker. - // Deprecated. Should be removed in 0.15.0 at which time - // connected_unmapped_worker_key_identifiers should be used exclusively. - // - // Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. - ConnectedWorkerKeyIdentifiers []string `protobuf:"bytes,50,rep,name=connected_worker_key_identifiers,json=connectedWorkerKeyIdentifiers,proto3" json:"connected_worker_key_identifiers,omitempty"` // The worker key identifiers for downstream workers for which there // is not a known worker id mapping for them yet. ConnectedUnmappedWorkerKeyIdentifiers []string `protobuf:"bytes,51,rep,name=connected_unmapped_worker_key_identifiers,json=connectedUnmappedWorkerKeyIdentifiers,proto3" json:"connected_unmapped_worker_key_identifiers,omitempty"` @@ -811,14 +883,6 @@ func (x *StatusRequest) GetWorkerStatus() *servers.ServerWorkerStatus { return nil } -// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. -func (x *StatusRequest) GetConnectedWorkerKeyIdentifiers() []string { - if x != nil { - return x.ConnectedWorkerKeyIdentifiers - } - return nil -} - func (x *StatusRequest) GetConnectedUnmappedWorkerKeyIdentifiers() []string { if x != nil { return x.ConnectedUnmappedWorkerKeyIdentifiers @@ -833,6 +897,9 @@ func (x *StatusRequest) GetConnectedWorkerPublicIds() []string { return nil } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type JobChangeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -888,58 +955,6 @@ func (x *JobChangeRequest) GetRequestType() CHANGETYPE { return CHANGETYPE_CHANGETYPE_UNSPECIFIED } -// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. -type AuthorizedWorkerList struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // the key id of authorized workers - // - // Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. - WorkerKeyIdentifiers []string `protobuf:"bytes,1,rep,name=worker_key_identifiers,json=workerKeyIdentifiers,proto3" json:"worker_key_identifiers,omitempty"` -} - -func (x *AuthorizedWorkerList) Reset() { - *x = AuthorizedWorkerList{} - if protoimpl.UnsafeEnabled { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AuthorizedWorkerList) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AuthorizedWorkerList) ProtoMessage() {} - -func (x *AuthorizedWorkerList) ProtoReflect() protoreflect.Message { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AuthorizedWorkerList.ProtoReflect.Descriptor instead. -func (*AuthorizedWorkerList) Descriptor() ([]byte, []int) { - return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{8} -} - -// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. -func (x *AuthorizedWorkerList) GetWorkerKeyIdentifiers() []string { - if x != nil { - return x.WorkerKeyIdentifiers - } - return nil -} - type AuthorizedDownstreamWorkerList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -947,16 +962,16 @@ type AuthorizedDownstreamWorkerList struct { // the key id of authorized workers which do not have a worker id known yet // to the requester. - UnmappedWorkerKeyIdentifiers []string `protobuf:"bytes,1,rep,name=unmapped_worker_key_identifiers,json=unmappedWorkerKeyIdentifiers,proto3" json:"unmapped_worker_key_identifiers,omitempty"` + UnmappedWorkerKeyIdentifiers []string `protobuf:"bytes,1,rep,name=unmapped_worker_key_identifiers,json=unmappedWorkerKeyIdentifiers,proto3" json:"unmapped_worker_key_identifiers,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" // the public id of workers that were reported as being connected and which // are known by the controller. - WorkerPublicIds []string `protobuf:"bytes,2,rep,name=worker_public_ids,json=workerPublicIds,proto3" json:"worker_public_ids,omitempty"` + WorkerPublicIds []string `protobuf:"bytes,2,rep,name=worker_public_ids,json=workerPublicIds,proto3" json:"worker_public_ids,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" } func (x *AuthorizedDownstreamWorkerList) Reset() { *x = AuthorizedDownstreamWorkerList{} if protoimpl.UnsafeEnabled { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[9] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -969,7 +984,7 @@ func (x *AuthorizedDownstreamWorkerList) String() string { func (*AuthorizedDownstreamWorkerList) ProtoMessage() {} func (x *AuthorizedDownstreamWorkerList) ProtoReflect() protoreflect.Message { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[9] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -982,7 +997,7 @@ func (x *AuthorizedDownstreamWorkerList) ProtoReflect() protoreflect.Message { // Deprecated: Use AuthorizedDownstreamWorkerList.ProtoReflect.Descriptor instead. func (*AuthorizedDownstreamWorkerList) Descriptor() ([]byte, []int) { - return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{9} + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{8} } func (x *AuthorizedDownstreamWorkerList) GetUnmappedWorkerKeyIdentifiers() []string { @@ -999,6 +1014,9 @@ func (x *AuthorizedDownstreamWorkerList) GetWorkerPublicIds() []string { return nil } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. +// +// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. type StatusResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1015,13 +1033,6 @@ type StatusResponse struct { // The ID of the worker which made the request. The worker can send this value in subsequent requests so the // controller does not need to do a database lookup for the id using the name field. WorkerId string `protobuf:"bytes,40,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"` - // Of the worker key identifiers provided in the request, these are the ones - // which are authorized to remain connected. - // This is deprecated. Use authorized_downstream_workers instead. This - // should be removed in version 0.15.0. - // - // Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. - AuthorizedWorkers *AuthorizedWorkerList `protobuf:"bytes,50,opt,name=authorized_workers,json=authorizedWorkers,proto3" json:"authorized_workers,omitempty"` // Of the downstream workers in the request, these are the ones // which are authorized to remain connected. AuthorizedDownstreamWorkers *AuthorizedDownstreamWorkerList `protobuf:"bytes,51,opt,name=authorized_downstream_workers,json=authorizedDownstreamWorkers,proto3" json:"authorized_downstream_workers,omitempty"` @@ -1030,7 +1041,7 @@ type StatusResponse struct { func (x *StatusResponse) Reset() { *x = StatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[10] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1043,7 +1054,7 @@ func (x *StatusResponse) String() string { func (*StatusResponse) ProtoMessage() {} func (x *StatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[10] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1056,7 +1067,7 @@ func (x *StatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StatusResponse.ProtoReflect.Descriptor instead. func (*StatusResponse) Descriptor() ([]byte, []int) { - return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{10} + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{9} } func (x *StatusResponse) GetJobsRequests() []*JobChangeRequest { @@ -1080,14 +1091,6 @@ func (x *StatusResponse) GetWorkerId() string { return "" } -// Deprecated: Marked as deprecated in controller/servers/services/v1/server_coordination_service.proto. -func (x *StatusResponse) GetAuthorizedWorkers() *AuthorizedWorkerList { - if x != nil { - return x.AuthorizedWorkers - } - return nil -} - func (x *StatusResponse) GetAuthorizedDownstreamWorkers() *AuthorizedDownstreamWorkerList { if x != nil { return x.AuthorizedDownstreamWorkers @@ -1110,7 +1113,7 @@ type WorkerInfo struct { func (x *WorkerInfo) Reset() { *x = WorkerInfo{} if protoimpl.UnsafeEnabled { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[11] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1123,7 +1126,7 @@ func (x *WorkerInfo) String() string { func (*WorkerInfo) ProtoMessage() {} func (x *WorkerInfo) ProtoReflect() protoreflect.Message { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[11] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1136,7 +1139,7 @@ func (x *WorkerInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerInfo.ProtoReflect.Descriptor instead. func (*WorkerInfo) Descriptor() ([]byte, []int) { - return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{11} + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{10} } func (x *WorkerInfo) GetId() string { @@ -1163,7 +1166,7 @@ type ListHcpbWorkersRequest struct { func (x *ListHcpbWorkersRequest) Reset() { *x = ListHcpbWorkersRequest{} if protoimpl.UnsafeEnabled { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[12] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1176,7 +1179,7 @@ func (x *ListHcpbWorkersRequest) String() string { func (*ListHcpbWorkersRequest) ProtoMessage() {} func (x *ListHcpbWorkersRequest) ProtoReflect() protoreflect.Message { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[12] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1189,7 +1192,7 @@ func (x *ListHcpbWorkersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListHcpbWorkersRequest.ProtoReflect.Descriptor instead. func (*ListHcpbWorkersRequest) Descriptor() ([]byte, []int) { - return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{12} + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{11} } // A response containing worker information @@ -1204,7 +1207,7 @@ type ListHcpbWorkersResponse struct { func (x *ListHcpbWorkersResponse) Reset() { *x = ListHcpbWorkersResponse{} if protoimpl.UnsafeEnabled { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[13] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1217,7 +1220,7 @@ func (x *ListHcpbWorkersResponse) String() string { func (*ListHcpbWorkersResponse) ProtoMessage() {} func (x *ListHcpbWorkersResponse) ProtoReflect() protoreflect.Message { - mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[13] + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1230,7 +1233,7 @@ func (x *ListHcpbWorkersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListHcpbWorkersResponse.ProtoReflect.Descriptor instead. func (*ListHcpbWorkersResponse) Descriptor() ([]byte, []int) { - return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{13} + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{12} } func (x *ListHcpbWorkersResponse) GetWorkers() []*WorkerInfo { @@ -1240,6 +1243,496 @@ func (x *ListHcpbWorkersResponse) GetWorkers() []*WorkerInfo { return nil } +// SessionStatistics contains statistics about a session and its connections. +type SessionStatistics struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Session ID holds the ID of the session which the statistics belong to. + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // Connections contains statistics about all the existing connections for the given session. + Connections []*Connection `protobuf:"bytes,2,rep,name=connections,proto3" json:"connections,omitempty"` +} + +func (x *SessionStatistics) Reset() { + *x = SessionStatistics{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionStatistics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionStatistics) ProtoMessage() {} + +func (x *SessionStatistics) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionStatistics.ProtoReflect.Descriptor instead. +func (*SessionStatistics) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{13} +} + +func (x *SessionStatistics) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *SessionStatistics) GetConnections() []*Connection { + if x != nil { + return x.Connections + } + return nil +} + +// StatisticsRequest contains a worker statistics request. This request +// reports on information that is not important either to maintain existing +// sessions or create new ones. +type StatisticsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The ID of the worker making the request. + WorkerId string `protobuf:"bytes,10,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // The statistics of the sessions managed by this worker + Sessions []*SessionStatistics `protobuf:"bytes,20,rep,name=sessions,proto3" json:"sessions,omitempty"` +} + +func (x *StatisticsRequest) Reset() { + *x = StatisticsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StatisticsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StatisticsRequest) ProtoMessage() {} + +func (x *StatisticsRequest) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StatisticsRequest.ProtoReflect.Descriptor instead. +func (*StatisticsRequest) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{14} +} + +func (x *StatisticsRequest) GetWorkerId() string { + if x != nil { + return x.WorkerId + } + return "" +} + +func (x *StatisticsRequest) GetSessions() []*SessionStatistics { + if x != nil { + return x.Sessions + } + return nil +} + +// StatisticsResponse contains a controller statistics response. Empty for now. +type StatisticsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *StatisticsResponse) Reset() { + *x = StatisticsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StatisticsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StatisticsResponse) ProtoMessage() {} + +func (x *StatisticsResponse) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StatisticsResponse.ProtoReflect.Descriptor instead. +func (*StatisticsResponse) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{15} +} + +// RoutingIinfoRequest is the request message for the RoutingInfo RPC. +type RoutingInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The worker status of the worker making the request. + WorkerStatus *servers.ServerWorkerStatus `protobuf:"bytes,1,opt,name=worker_status,json=workerStatus,proto3" json:"worker_status,omitempty"` + // Whether the tags should be updated or not. + UpdateTags bool `protobuf:"varint,2,opt,name=update_tags,json=updateTags,proto3" json:"update_tags,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // The worker key identifiers for downstream workers for which there + // is not a known worker id. + ConnectedUnmappedWorkerKeyIdentifiers []string `protobuf:"bytes,3,rep,name=connected_unmapped_worker_key_identifiers,json=connectedUnmappedWorkerKeyIdentifiers,proto3" json:"connected_unmapped_worker_key_identifiers,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // The worker public ids of all downstreams connected to this worker. + ConnectedWorkerPublicIds []string `protobuf:"bytes,4,rep,name=connected_worker_public_ids,json=connectedWorkerPublicIds,proto3" json:"connected_worker_public_ids,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" +} + +func (x *RoutingInfoRequest) Reset() { + *x = RoutingInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RoutingInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RoutingInfoRequest) ProtoMessage() {} + +func (x *RoutingInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RoutingInfoRequest.ProtoReflect.Descriptor instead. +func (*RoutingInfoRequest) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{16} +} + +func (x *RoutingInfoRequest) GetWorkerStatus() *servers.ServerWorkerStatus { + if x != nil { + return x.WorkerStatus + } + return nil +} + +func (x *RoutingInfoRequest) GetUpdateTags() bool { + if x != nil { + return x.UpdateTags + } + return false +} + +func (x *RoutingInfoRequest) GetConnectedUnmappedWorkerKeyIdentifiers() []string { + if x != nil { + return x.ConnectedUnmappedWorkerKeyIdentifiers + } + return nil +} + +func (x *RoutingInfoRequest) GetConnectedWorkerPublicIds() []string { + if x != nil { + return x.ConnectedWorkerPublicIds + } + return nil +} + +// RoutingInfoResponse is the response message for the RoutingInfo RPC. +type RoutingInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The controller-assigned ID of the worker which made the request. + WorkerId string `protobuf:"bytes,1,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // The calculated upstream addresses for the controller cluster. + CalculatedUpstreamAddresses []string `protobuf:"bytes,2,rep,name=calculated_upstream_addresses,json=calculatedUpstreamAddresses,proto3" json:"calculated_upstream_addresses,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // Of the downstream workers in the request, these are the ones + // which are authorized to remain connected. + AuthorizedDownstreamWorkers *AuthorizedDownstreamWorkerList `protobuf:"bytes,3,opt,name=authorized_downstream_workers,json=authorizedDownstreamWorkers,proto3" json:"authorized_downstream_workers,omitempty"` +} + +func (x *RoutingInfoResponse) Reset() { + *x = RoutingInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RoutingInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RoutingInfoResponse) ProtoMessage() {} + +func (x *RoutingInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RoutingInfoResponse.ProtoReflect.Descriptor instead. +func (*RoutingInfoResponse) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{17} +} + +func (x *RoutingInfoResponse) GetWorkerId() string { + if x != nil { + return x.WorkerId + } + return "" +} + +func (x *RoutingInfoResponse) GetCalculatedUpstreamAddresses() []string { + if x != nil { + return x.CalculatedUpstreamAddresses + } + return nil +} + +func (x *RoutingInfoResponse) GetAuthorizedDownstreamWorkers() *AuthorizedDownstreamWorkerList { + if x != nil { + return x.AuthorizedDownstreamWorkers + } + return nil +} + +// Session contains information about a worker managed session +type Session struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The ID of the session. + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // The status of the session. + SessionStatus SESSIONSTATUS `protobuf:"varint,2,opt,name=session_status,json=sessionStatus,proto3,enum=controller.servers.services.v1.SESSIONSTATUS" json:"session_status,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // Represents how the worker is processing the session. + SessionType SessionType `protobuf:"varint,3,opt,name=session_type,json=sessionType,proto3,enum=controller.servers.services.v1.SessionType" json:"session_type,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // Connections contains information about the connections managed by the session + Connections []*Connection `protobuf:"bytes,4,rep,name=connections,proto3" json:"connections,omitempty"` +} + +func (x *Session) Reset() { + *x = Session{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Session) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Session) ProtoMessage() {} + +func (x *Session) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Session.ProtoReflect.Descriptor instead. +func (*Session) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{18} +} + +func (x *Session) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *Session) GetSessionStatus() SESSIONSTATUS { + if x != nil { + return x.SessionStatus + } + return SESSIONSTATUS_SESSIONSTATUS_UNSPECIFIED +} + +func (x *Session) GetSessionType() SessionType { + if x != nil { + return x.SessionType + } + return SessionType_SESSION_TYPE_UNSPECIFIED +} + +func (x *Session) GetConnections() []*Connection { + if x != nil { + return x.Connections + } + return nil +} + +// SessionInfoRequest contains a worker session information request. This request +// reports on currently managed sessions on the worker. This includes all ingressed +// and recorded sessions. +type SessionInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The ID of the worker making the request. + WorkerId string `protobuf:"bytes,1,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty" class:"public" eventstream:"observation"` // @gotags: class:"public" eventstream:"observation" + // The sessions which this worker wants to report the status of. + // Includes both ingressed sessions and recorded sessions. + Sessions []*Session `protobuf:"bytes,2,rep,name=sessions,proto3" json:"sessions,omitempty"` +} + +func (x *SessionInfoRequest) Reset() { + *x = SessionInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionInfoRequest) ProtoMessage() {} + +func (x *SessionInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionInfoRequest.ProtoReflect.Descriptor instead. +func (*SessionInfoRequest) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{19} +} + +func (x *SessionInfoRequest) GetWorkerId() string { + if x != nil { + return x.WorkerId + } + return "" +} + +func (x *SessionInfoRequest) GetSessions() []*Session { + if x != nil { + return x.Sessions + } + return nil +} + +// SessionInfoResponse contains a controller session information response. This +// response indicates what, if any, changes should be made to the sessions. +// This also includes any sessions the worker is monitoring but not managing directly. +type SessionInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of sessions that are no longer active and whose status should be updated. + NonActiveSessions []*Session `protobuf:"bytes,1,rep,name=non_active_sessions,json=nonActiveSessions,proto3" json:"non_active_sessions,omitempty"` +} + +func (x *SessionInfoResponse) Reset() { + *x = SessionInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SessionInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionInfoResponse) ProtoMessage() {} + +func (x *SessionInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionInfoResponse.ProtoReflect.Descriptor instead. +func (*SessionInfoResponse) Descriptor() ([]byte, []int) { + return file_controller_servers_services_v1_server_coordination_service_proto_rawDescGZIP(), []int{20} +} + +func (x *SessionInfoResponse) GetNonActiveSessions() []*Session { + if x != nil { + return x.NonActiveSessions + } + return nil +} + var File_controller_servers_services_v1_server_coordination_service_proto protoreflect.FileDescriptor var file_controller_servers_services_v1_server_coordination_service_proto_rawDesc = []byte{ @@ -1263,7 +1756,7 @@ var file_controller_servers_services_v1_server_coordination_service_proto_rawDes 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x55, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x62, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x22, - 0xa7, 0x02, 0x0a, 0x0e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, 0x6e, + 0xab, 0x02, 0x0a, 0x0e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, @@ -1281,43 +1774,44 @@ var file_controller_servers_services_v1_server_coordination_service_proto_rawDes 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, - 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xe0, 0x01, 0x0a, 0x15, 0x4d, 0x6f, - 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x12, 0x45, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x61, 0x0a, 0x10, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x0f, 0x70, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x8e, 0x02, 0x0a, - 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x3b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x12, 0x53, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, - 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x3a, 0x02, 0x18, 0x01, 0x22, 0xe4, 0x01, + 0x0a, 0x15, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x61, 0x0a, + 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x69, 0x0a, 0x14, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, - 0x72, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x12, 0x6d, - 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, - 0x6f, 0x42, 0x0a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x42, 0x0a, - 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, 0x0a, 0x03, 0x6a, 0x6f, - 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, - 0x62, 0x22, 0xb7, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, + 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, + 0x0f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x3a, 0x02, 0x18, 0x01, 0x22, 0x92, 0x02, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x3b, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x4f, 0x42, 0x54, + 0x59, 0x50, 0x45, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x53, 0x0a, 0x0c, 0x73, 0x65, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x48, + 0x00, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x69, + 0x0a, 0x14, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, + 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, + 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x12, 0x6d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x3a, 0x02, 0x18, 0x01, 0x42, 0x0a, 0x0a, + 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x46, 0x0a, 0x09, 0x4a, 0x6f, 0x62, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x3a, 0x02, 0x18, + 0x01, 0x22, 0xbb, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x47, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, @@ -1328,160 +1822,262 @@ var file_controller_servers_services_v1_server_coordination_service_proto_rawDes 0x14, 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x10, 0x02, 0x22, 0xb3, 0x03, 0x0a, 0x0d, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, - 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, + 0x50, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x10, 0x02, 0x3a, 0x02, 0x18, 0x01, 0x22, + 0x92, 0x03, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, + 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, + 0x1e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x67, + 0x73, 0x12, 0x4e, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x28, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x58, 0x0a, 0x29, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x75, + 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, + 0x65, 0x79, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x18, 0x33, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x25, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, + 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4b, 0x65, 0x79, + 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x3d, 0x0a, 0x1b, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, + 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x37, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x18, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x64, 0x73, 0x3a, 0x02, 0x18, 0x01, 0x4a, 0x04, + 0x08, 0x0a, 0x10, 0x0b, 0x4a, 0x04, 0x08, 0x32, 0x10, 0x33, 0x52, 0x06, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x52, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x77, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, + 0x69, 0x65, 0x72, 0x73, 0x22, 0x9c, 0x01, 0x0a, 0x10, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x03, 0x6a, 0x6f, 0x62, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, + 0x12, 0x4d, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, + 0x50, 0x45, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x3a, + 0x02, 0x18, 0x01, 0x22, 0x93, 0x01, 0x0a, 0x1e, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, + 0x65, 0x64, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x1f, 0x75, 0x6e, 0x6d, 0x61, 0x70, 0x70, + 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x1c, 0x75, 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4b, + 0x65, 0x79, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, + 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x69, + 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x64, 0x73, 0x22, 0x9d, 0x03, 0x0a, 0x0e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0d, + 0x6a, 0x6f, 0x62, 0x73, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x14, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x6a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x73, 0x12, 0x61, 0x0a, 0x14, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, + 0x64, 0x5f, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x1e, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x13, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x55, 0x70, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x49, 0x64, 0x12, 0x82, 0x01, 0x0a, 0x1d, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, + 0x65, 0x64, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x77, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x33, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x12, 0x1f, 0x0a, 0x0b, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, 0x1e, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x67, 0x73, 0x12, 0x4e, 0x0a, - 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x28, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x1b, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x3a, 0x02, 0x18, 0x01, 0x4a, 0x04, 0x08, 0x0a, + 0x10, 0x0b, 0x4a, 0x04, 0x08, 0x32, 0x10, 0x33, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x52, 0x12, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, + 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x22, 0x36, 0x0a, 0x0a, 0x57, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x22, 0x18, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5f, 0x0a, 0x17, 0x4c, + 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x22, 0x80, 0x01, 0x0a, + 0x11, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x64, 0x12, 0x4c, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, + 0x7f, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, + 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x4d, 0x0a, 0x08, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x14, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x08, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x22, 0x14, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9e, 0x02, 0x0a, 0x12, 0x52, 0x6f, 0x75, 0x74, 0x69, + 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4e, 0x0a, + 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x4b, 0x0a, - 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, - 0x73, 0x18, 0x32, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x1d, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x49, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x29, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x64, - 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x18, 0x33, 0x20, 0x03, 0x28, 0x09, 0x52, 0x25, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x64, - 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, - 0x69, 0x65, 0x72, 0x73, 0x12, 0x3d, 0x0a, 0x1b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, - 0x69, 0x64, 0x73, 0x18, 0x37, 0x20, 0x03, 0x28, 0x09, 0x52, 0x18, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x49, 0x64, 0x73, 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x06, 0x77, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x22, 0x98, 0x01, 0x0a, 0x10, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x12, 0x4d, 0x0a, - 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x52, - 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x54, 0x0a, 0x14, - 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x16, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, - 0x65, 0x79, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x14, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x4b, 0x65, 0x79, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x3a, 0x02, - 0x18, 0x01, 0x22, 0x93, 0x01, 0x0a, 0x1e, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, - 0x64, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x1f, 0x75, 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, - 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x1c, - 0x75, 0x6e, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4b, 0x65, - 0x79, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, 0x11, - 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x69, 0x64, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x50, - 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x64, 0x73, 0x22, 0xe8, 0x03, 0x0a, 0x0e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0d, 0x6a, - 0x6f, 0x62, 0x73, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x14, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x6a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x73, 0x12, 0x61, 0x0a, 0x14, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, - 0x5f, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x1e, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, + 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x0a, + 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x67, 0x73, 0x12, 0x58, + 0x0a, 0x29, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x6e, 0x6d, 0x61, + 0x70, 0x70, 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x5f, + 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x25, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x55, 0x6e, 0x6d, 0x61, + 0x70, 0x70, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x3d, 0x0a, 0x1b, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, + 0x6c, 0x69, 0x63, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x18, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x50, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x49, 0x64, 0x73, 0x22, 0xfb, 0x01, 0x0a, 0x13, 0x52, 0x6f, 0x75, 0x74, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x12, 0x42, 0x0a, 0x1d, + 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x70, 0x73, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x1b, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x55, + 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, + 0x12, 0x82, 0x01, 0x0a, 0x1d, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x5f, + 0x64, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x7a, 0x65, 0x64, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x1b, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x7a, 0x65, 0x64, 0x44, 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x73, 0x22, 0x9c, 0x02, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, + 0x12, 0x54, 0x0a, 0x0e, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, + 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x52, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x4e, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x76, 0x0a, 0x12, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x12, 0x43, 0x0a, 0x08, 0x73, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x08, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x6e, 0x0a, 0x13, + 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x13, 0x6e, 0x6f, 0x6e, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x76, + 0x65, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, - 0x31, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x52, 0x13, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x55, 0x70, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x67, 0x0a, 0x12, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, - 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x32, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, + 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x6e, 0x6f, 0x6e, 0x41, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2a, 0x92, 0x01, 0x0a, + 0x10, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x12, 0x20, 0x0a, 0x1c, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, + 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, + 0x45, 0x44, 0x10, 0x01, 0x12, 0x1e, 0x0a, 0x1a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, + 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, + 0x45, 0x44, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, + 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x44, 0x10, + 0x03, 0x2a, 0x9e, 0x01, 0x0a, 0x0d, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x18, 0x0a, + 0x14, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, + 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x45, 0x53, 0x53, 0x49, + 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x49, + 0x4e, 0x47, 0x10, 0x03, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, + 0x10, 0x04, 0x2a, 0x71, 0x0a, 0x16, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x24, + 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, + 0x4e, 0x47, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, + 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x29, 0x0a, 0x25, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, + 0x4e, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x45, 0x52, 0x52, + 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x52, 0x45, 0x43, 0x4f, 0x47, 0x4e, 0x49, 0x5a, 0x45, 0x44, 0x10, + 0x01, 0x1a, 0x02, 0x18, 0x01, 0x2a, 0x58, 0x0a, 0x07, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, + 0x12, 0x17, 0x0a, 0x13, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x1b, + 0x0a, 0x17, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, + 0x52, 0x5f, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x02, 0x1a, 0x02, 0x18, 0x01, 0x2a, + 0x49, 0x0a, 0x0a, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x12, 0x1a, 0x0a, + 0x16, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x48, 0x41, + 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x10, 0x01, 0x1a, 0x02, 0x18, 0x01, 0x2a, 0x62, 0x0a, 0x0b, 0x53, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x45, 0x53, + 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x45, 0x53, 0x53, 0x49, + 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x47, 0x52, 0x45, 0x53, 0x53, 0x45, + 0x44, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x43, 0x4f, 0x52, 0x44, 0x45, 0x44, 0x10, 0x02, 0x32, 0xfb, + 0x04, 0x0a, 0x19, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x06, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x88, 0x02, 0x01, 0x12, 0x84, 0x01, 0x0a, 0x0f, 0x4c, + 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, - 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x4c, 0x69, 0x73, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x11, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x69, 0x7a, 0x65, 0x64, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x82, 0x01, 0x0a, 0x1d, - 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x33, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x44, - 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4c, - 0x69, 0x73, 0x74, 0x52, 0x1b, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x44, - 0x6f, 0x77, 0x6e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, - 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x73, 0x22, 0x36, 0x0a, 0x0a, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x4c, - 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5f, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, - 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x44, 0x0a, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, - 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x77, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x2a, 0x92, 0x01, 0x0a, 0x10, 0x43, 0x4f, 0x4e, 0x4e, 0x45, - 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x12, 0x20, 0x0a, 0x1c, 0x43, - 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, - 0x1b, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x45, 0x44, 0x10, 0x01, 0x12, 0x1e, - 0x0a, 0x1a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x1b, - 0x0a, 0x17, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x44, 0x10, 0x03, 0x2a, 0x9e, 0x01, 0x0a, 0x0d, - 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x12, 0x1d, 0x0a, - 0x19, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, - 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, - 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x45, 0x53, 0x53, 0x49, - 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, - 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x1c, - 0x0a, 0x18, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x2a, 0x6d, 0x0a, 0x16, - 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, - 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x24, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, - 0x4e, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x45, 0x52, 0x52, - 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x29, 0x0a, 0x25, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x50, 0x52, 0x4f, 0x43, - 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x52, - 0x45, 0x43, 0x4f, 0x47, 0x4e, 0x49, 0x5a, 0x45, 0x44, 0x10, 0x01, 0x2a, 0x54, 0x0a, 0x07, 0x4a, - 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x12, 0x17, 0x0a, 0x13, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x53, 0x53, 0x49, - 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x5f, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, - 0x02, 0x2a, 0x45, 0x0a, 0x0a, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x12, - 0x1a, 0x0a, 0x16, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x43, - 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x10, 0x01, 0x32, 0x8d, 0x02, 0x0a, 0x19, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x69, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, + 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, + 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x75, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, + 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x84, 0x01, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x78, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, + 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x78, 0x0a, 0x0b, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x51, 0x5a, 0x4f, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x51, 0x5a, 0x4f, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1496,61 +2092,83 @@ func file_controller_servers_services_v1_server_coordination_service_proto_rawDe return file_controller_servers_services_v1_server_coordination_service_proto_rawDescData } -var file_controller_servers_services_v1_server_coordination_service_proto_enumTypes = make([]protoimpl.EnumInfo, 6) -var file_controller_servers_services_v1_server_coordination_service_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_controller_servers_services_v1_server_coordination_service_proto_enumTypes = make([]protoimpl.EnumInfo, 7) +var file_controller_servers_services_v1_server_coordination_service_proto_msgTypes = make([]protoimpl.MessageInfo, 21) var file_controller_servers_services_v1_server_coordination_service_proto_goTypes = []any{ (CONNECTIONSTATUS)(0), // 0: controller.servers.services.v1.CONNECTIONSTATUS (SESSIONSTATUS)(0), // 1: controller.servers.services.v1.SESSIONSTATUS (SessionProcessingError)(0), // 2: controller.servers.services.v1.SessionProcessingError (JOBTYPE)(0), // 3: controller.servers.services.v1.JOBTYPE (CHANGETYPE)(0), // 4: controller.servers.services.v1.CHANGETYPE - (UpstreamServer_TYPE)(0), // 5: controller.servers.services.v1.UpstreamServer.TYPE - (*Connection)(nil), // 6: controller.servers.services.v1.Connection - (*SessionJobInfo)(nil), // 7: controller.servers.services.v1.SessionJobInfo - (*MonitorSessionJobInfo)(nil), // 8: controller.servers.services.v1.MonitorSessionJobInfo - (*Job)(nil), // 9: controller.servers.services.v1.Job - (*JobStatus)(nil), // 10: controller.servers.services.v1.JobStatus - (*UpstreamServer)(nil), // 11: controller.servers.services.v1.UpstreamServer - (*StatusRequest)(nil), // 12: controller.servers.services.v1.StatusRequest - (*JobChangeRequest)(nil), // 13: controller.servers.services.v1.JobChangeRequest - (*AuthorizedWorkerList)(nil), // 14: controller.servers.services.v1.AuthorizedWorkerList + (SessionType)(0), // 5: controller.servers.services.v1.SessionType + (UpstreamServer_TYPE)(0), // 6: controller.servers.services.v1.UpstreamServer.TYPE + (*Connection)(nil), // 7: controller.servers.services.v1.Connection + (*SessionJobInfo)(nil), // 8: controller.servers.services.v1.SessionJobInfo + (*MonitorSessionJobInfo)(nil), // 9: controller.servers.services.v1.MonitorSessionJobInfo + (*Job)(nil), // 10: controller.servers.services.v1.Job + (*JobStatus)(nil), // 11: controller.servers.services.v1.JobStatus + (*UpstreamServer)(nil), // 12: controller.servers.services.v1.UpstreamServer + (*StatusRequest)(nil), // 13: controller.servers.services.v1.StatusRequest + (*JobChangeRequest)(nil), // 14: controller.servers.services.v1.JobChangeRequest (*AuthorizedDownstreamWorkerList)(nil), // 15: controller.servers.services.v1.AuthorizedDownstreamWorkerList (*StatusResponse)(nil), // 16: controller.servers.services.v1.StatusResponse (*WorkerInfo)(nil), // 17: controller.servers.services.v1.WorkerInfo (*ListHcpbWorkersRequest)(nil), // 18: controller.servers.services.v1.ListHcpbWorkersRequest (*ListHcpbWorkersResponse)(nil), // 19: controller.servers.services.v1.ListHcpbWorkersResponse - (*servers.ServerWorkerStatus)(nil), // 20: controller.servers.v1.ServerWorkerStatus + (*SessionStatistics)(nil), // 20: controller.servers.services.v1.SessionStatistics + (*StatisticsRequest)(nil), // 21: controller.servers.services.v1.StatisticsRequest + (*StatisticsResponse)(nil), // 22: controller.servers.services.v1.StatisticsResponse + (*RoutingInfoRequest)(nil), // 23: controller.servers.services.v1.RoutingInfoRequest + (*RoutingInfoResponse)(nil), // 24: controller.servers.services.v1.RoutingInfoResponse + (*Session)(nil), // 25: controller.servers.services.v1.Session + (*SessionInfoRequest)(nil), // 26: controller.servers.services.v1.SessionInfoRequest + (*SessionInfoResponse)(nil), // 27: controller.servers.services.v1.SessionInfoResponse + (*servers.ServerWorkerStatus)(nil), // 28: controller.servers.v1.ServerWorkerStatus } var file_controller_servers_services_v1_server_coordination_service_proto_depIdxs = []int32{ 0, // 0: controller.servers.services.v1.Connection.status:type_name -> controller.servers.services.v1.CONNECTIONSTATUS 1, // 1: controller.servers.services.v1.SessionJobInfo.status:type_name -> controller.servers.services.v1.SESSIONSTATUS - 6, // 2: controller.servers.services.v1.SessionJobInfo.connections:type_name -> controller.servers.services.v1.Connection + 7, // 2: controller.servers.services.v1.SessionJobInfo.connections:type_name -> controller.servers.services.v1.Connection 2, // 3: controller.servers.services.v1.SessionJobInfo.processing_error:type_name -> controller.servers.services.v1.SessionProcessingError 1, // 4: controller.servers.services.v1.MonitorSessionJobInfo.status:type_name -> controller.servers.services.v1.SESSIONSTATUS 2, // 5: controller.servers.services.v1.MonitorSessionJobInfo.processing_error:type_name -> controller.servers.services.v1.SessionProcessingError 3, // 6: controller.servers.services.v1.Job.type:type_name -> controller.servers.services.v1.JOBTYPE - 7, // 7: controller.servers.services.v1.Job.session_info:type_name -> controller.servers.services.v1.SessionJobInfo - 8, // 8: controller.servers.services.v1.Job.monitor_session_info:type_name -> controller.servers.services.v1.MonitorSessionJobInfo - 9, // 9: controller.servers.services.v1.JobStatus.job:type_name -> controller.servers.services.v1.Job - 5, // 10: controller.servers.services.v1.UpstreamServer.type:type_name -> controller.servers.services.v1.UpstreamServer.TYPE - 10, // 11: controller.servers.services.v1.StatusRequest.jobs:type_name -> controller.servers.services.v1.JobStatus - 20, // 12: controller.servers.services.v1.StatusRequest.worker_status:type_name -> controller.servers.v1.ServerWorkerStatus - 9, // 13: controller.servers.services.v1.JobChangeRequest.job:type_name -> controller.servers.services.v1.Job + 8, // 7: controller.servers.services.v1.Job.session_info:type_name -> controller.servers.services.v1.SessionJobInfo + 9, // 8: controller.servers.services.v1.Job.monitor_session_info:type_name -> controller.servers.services.v1.MonitorSessionJobInfo + 10, // 9: controller.servers.services.v1.JobStatus.job:type_name -> controller.servers.services.v1.Job + 6, // 10: controller.servers.services.v1.UpstreamServer.type:type_name -> controller.servers.services.v1.UpstreamServer.TYPE + 11, // 11: controller.servers.services.v1.StatusRequest.jobs:type_name -> controller.servers.services.v1.JobStatus + 28, // 12: controller.servers.services.v1.StatusRequest.worker_status:type_name -> controller.servers.v1.ServerWorkerStatus + 10, // 13: controller.servers.services.v1.JobChangeRequest.job:type_name -> controller.servers.services.v1.Job 4, // 14: controller.servers.services.v1.JobChangeRequest.request_type:type_name -> controller.servers.services.v1.CHANGETYPE - 13, // 15: controller.servers.services.v1.StatusResponse.jobs_requests:type_name -> controller.servers.services.v1.JobChangeRequest - 11, // 16: controller.servers.services.v1.StatusResponse.calculated_upstreams:type_name -> controller.servers.services.v1.UpstreamServer - 14, // 17: controller.servers.services.v1.StatusResponse.authorized_workers:type_name -> controller.servers.services.v1.AuthorizedWorkerList - 15, // 18: controller.servers.services.v1.StatusResponse.authorized_downstream_workers:type_name -> controller.servers.services.v1.AuthorizedDownstreamWorkerList - 17, // 19: controller.servers.services.v1.ListHcpbWorkersResponse.workers:type_name -> controller.servers.services.v1.WorkerInfo - 12, // 20: controller.servers.services.v1.ServerCoordinationService.Status:input_type -> controller.servers.services.v1.StatusRequest - 18, // 21: controller.servers.services.v1.ServerCoordinationService.ListHcpbWorkers:input_type -> controller.servers.services.v1.ListHcpbWorkersRequest - 16, // 22: controller.servers.services.v1.ServerCoordinationService.Status:output_type -> controller.servers.services.v1.StatusResponse - 19, // 23: controller.servers.services.v1.ServerCoordinationService.ListHcpbWorkers:output_type -> controller.servers.services.v1.ListHcpbWorkersResponse - 22, // [22:24] is the sub-list for method output_type - 20, // [20:22] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 14, // 15: controller.servers.services.v1.StatusResponse.jobs_requests:type_name -> controller.servers.services.v1.JobChangeRequest + 12, // 16: controller.servers.services.v1.StatusResponse.calculated_upstreams:type_name -> controller.servers.services.v1.UpstreamServer + 15, // 17: controller.servers.services.v1.StatusResponse.authorized_downstream_workers:type_name -> controller.servers.services.v1.AuthorizedDownstreamWorkerList + 17, // 18: controller.servers.services.v1.ListHcpbWorkersResponse.workers:type_name -> controller.servers.services.v1.WorkerInfo + 7, // 19: controller.servers.services.v1.SessionStatistics.connections:type_name -> controller.servers.services.v1.Connection + 20, // 20: controller.servers.services.v1.StatisticsRequest.sessions:type_name -> controller.servers.services.v1.SessionStatistics + 28, // 21: controller.servers.services.v1.RoutingInfoRequest.worker_status:type_name -> controller.servers.v1.ServerWorkerStatus + 15, // 22: controller.servers.services.v1.RoutingInfoResponse.authorized_downstream_workers:type_name -> controller.servers.services.v1.AuthorizedDownstreamWorkerList + 1, // 23: controller.servers.services.v1.Session.session_status:type_name -> controller.servers.services.v1.SESSIONSTATUS + 5, // 24: controller.servers.services.v1.Session.session_type:type_name -> controller.servers.services.v1.SessionType + 7, // 25: controller.servers.services.v1.Session.connections:type_name -> controller.servers.services.v1.Connection + 25, // 26: controller.servers.services.v1.SessionInfoRequest.sessions:type_name -> controller.servers.services.v1.Session + 25, // 27: controller.servers.services.v1.SessionInfoResponse.non_active_sessions:type_name -> controller.servers.services.v1.Session + 13, // 28: controller.servers.services.v1.ServerCoordinationService.Status:input_type -> controller.servers.services.v1.StatusRequest + 18, // 29: controller.servers.services.v1.ServerCoordinationService.ListHcpbWorkers:input_type -> controller.servers.services.v1.ListHcpbWorkersRequest + 21, // 30: controller.servers.services.v1.ServerCoordinationService.Statistics:input_type -> controller.servers.services.v1.StatisticsRequest + 23, // 31: controller.servers.services.v1.ServerCoordinationService.RoutingInfo:input_type -> controller.servers.services.v1.RoutingInfoRequest + 26, // 32: controller.servers.services.v1.ServerCoordinationService.SessionInfo:input_type -> controller.servers.services.v1.SessionInfoRequest + 16, // 33: controller.servers.services.v1.ServerCoordinationService.Status:output_type -> controller.servers.services.v1.StatusResponse + 19, // 34: controller.servers.services.v1.ServerCoordinationService.ListHcpbWorkers:output_type -> controller.servers.services.v1.ListHcpbWorkersResponse + 22, // 35: controller.servers.services.v1.ServerCoordinationService.Statistics:output_type -> controller.servers.services.v1.StatisticsResponse + 24, // 36: controller.servers.services.v1.ServerCoordinationService.RoutingInfo:output_type -> controller.servers.services.v1.RoutingInfoResponse + 27, // 37: controller.servers.services.v1.ServerCoordinationService.SessionInfo:output_type -> controller.servers.services.v1.SessionInfoResponse + 33, // [33:38] is the sub-list for method output_type + 28, // [28:33] is the sub-list for method input_type + 28, // [28:28] is the sub-list for extension type_name + 28, // [28:28] is the sub-list for extension extendee + 0, // [0:28] is the sub-list for field type_name } func init() { file_controller_servers_services_v1_server_coordination_service_proto_init() } @@ -1656,18 +2274,6 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( } } file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*AuthorizedWorkerList); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*AuthorizedDownstreamWorkerList); i { case 0: return &v.state @@ -1679,7 +2285,7 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( return nil } } - file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[10].Exporter = func(v any, i int) any { + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*StatusResponse); i { case 0: return &v.state @@ -1691,7 +2297,7 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( return nil } } - file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[11].Exporter = func(v any, i int) any { + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*WorkerInfo); i { case 0: return &v.state @@ -1703,7 +2309,7 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( return nil } } - file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[12].Exporter = func(v any, i int) any { + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*ListHcpbWorkersRequest); i { case 0: return &v.state @@ -1715,7 +2321,7 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( return nil } } - file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[13].Exporter = func(v any, i int) any { + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*ListHcpbWorkersResponse); i { case 0: return &v.state @@ -1727,6 +2333,102 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( return nil } } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[13].Exporter = func(v any, i int) any { + switch v := v.(*SessionStatistics); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[14].Exporter = func(v any, i int) any { + switch v := v.(*StatisticsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[15].Exporter = func(v any, i int) any { + switch v := v.(*StatisticsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[16].Exporter = func(v any, i int) any { + switch v := v.(*RoutingInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[17].Exporter = func(v any, i int) any { + switch v := v.(*RoutingInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[18].Exporter = func(v any, i int) any { + switch v := v.(*Session); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[19].Exporter = func(v any, i int) any { + switch v := v.(*SessionInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[20].Exporter = func(v any, i int) any { + switch v := v.(*SessionInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_controller_servers_services_v1_server_coordination_service_proto_msgTypes[3].OneofWrappers = []any{ (*Job_SessionInfo)(nil), @@ -1737,8 +2439,8 @@ func file_controller_servers_services_v1_server_coordination_service_proto_init( File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_controller_servers_services_v1_server_coordination_service_proto_rawDesc, - NumEnums: 6, - NumMessages: 14, + NumEnums: 7, + NumMessages: 21, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/gen/controller/servers/services/server_coordination_service_grpc.pb.go b/internal/gen/controller/servers/services/server_coordination_service_grpc.pb.go index ce1caec4e..c67e96314 100644 --- a/internal/gen/controller/servers/services/server_coordination_service_grpc.pb.go +++ b/internal/gen/controller/servers/services/server_coordination_service_grpc.pb.go @@ -24,18 +24,36 @@ const _ = grpc.SupportPackageIsVersion7 const ( ServerCoordinationService_Status_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/Status" ServerCoordinationService_ListHcpbWorkers_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/ListHcpbWorkers" + ServerCoordinationService_Statistics_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/Statistics" + ServerCoordinationService_RoutingInfo_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/RoutingInfo" + ServerCoordinationService_SessionInfo_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/SessionInfo" ) // ServerCoordinationServiceClient is the client API for ServerCoordinationService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ServerCoordinationServiceClient interface { + // Deprecated: Do not use. // Status gets worker status requests which include the ongoing jobs the worker is handling and // returns the status response which includes the changes the controller would like to make to // jobs as well as provide a list of the controllers in the system. + // This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0. Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) // Returns the addresses of HCP Boundary workers, if any ListHcpbWorkers(ctx context.Context, in *ListHcpbWorkersRequest, opts ...grpc.CallOption) (*ListHcpbWorkersResponse, error) + // Statistics is used by the worker to report non-essential statistics about its sessions and connections. + Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) + // RoutingInfo is used by the worker to inform the controller of information + // required by the controller to make session routing decisions and any startup information. + // The controller may inform the worker of any downstream workers that should be disconnected. + // If the worker fails to successfully report its routing info to the controller, + // it will try again later. + RoutingInfo(ctx context.Context, in *RoutingInfoRequest, opts ...grpc.CallOption) (*RoutingInfoResponse, error) + // SessionInfo is used by the worker to inform the controller of all the sessions + // it is managing. The controller may inform the worker if any sessions need to be changed. + // If the worker repeatedly fails to successfully report its session info to the controller, + // it will tear down any running sessions. + SessionInfo(ctx context.Context, in *SessionInfoRequest, opts ...grpc.CallOption) (*SessionInfoResponse, error) } type serverCoordinationServiceClient struct { @@ -46,6 +64,7 @@ func NewServerCoordinationServiceClient(cc grpc.ClientConnInterface) ServerCoord return &serverCoordinationServiceClient{cc} } +// Deprecated: Do not use. func (c *serverCoordinationServiceClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) { out := new(StatusResponse) err := c.cc.Invoke(ctx, ServerCoordinationService_Status_FullMethodName, in, out, opts...) @@ -64,16 +83,58 @@ func (c *serverCoordinationServiceClient) ListHcpbWorkers(ctx context.Context, i return out, nil } +func (c *serverCoordinationServiceClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) { + out := new(StatisticsResponse) + err := c.cc.Invoke(ctx, ServerCoordinationService_Statistics_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *serverCoordinationServiceClient) RoutingInfo(ctx context.Context, in *RoutingInfoRequest, opts ...grpc.CallOption) (*RoutingInfoResponse, error) { + out := new(RoutingInfoResponse) + err := c.cc.Invoke(ctx, ServerCoordinationService_RoutingInfo_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *serverCoordinationServiceClient) SessionInfo(ctx context.Context, in *SessionInfoRequest, opts ...grpc.CallOption) (*SessionInfoResponse, error) { + out := new(SessionInfoResponse) + err := c.cc.Invoke(ctx, ServerCoordinationService_SessionInfo_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ServerCoordinationServiceServer is the server API for ServerCoordinationService service. // All implementations must embed UnimplementedServerCoordinationServiceServer // for forward compatibility type ServerCoordinationServiceServer interface { + // Deprecated: Do not use. // Status gets worker status requests which include the ongoing jobs the worker is handling and // returns the status response which includes the changes the controller would like to make to // jobs as well as provide a list of the controllers in the system. + // This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0. Status(context.Context, *StatusRequest) (*StatusResponse, error) // Returns the addresses of HCP Boundary workers, if any ListHcpbWorkers(context.Context, *ListHcpbWorkersRequest) (*ListHcpbWorkersResponse, error) + // Statistics is used by the worker to report non-essential statistics about its sessions and connections. + Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) + // RoutingInfo is used by the worker to inform the controller of information + // required by the controller to make session routing decisions and any startup information. + // The controller may inform the worker of any downstream workers that should be disconnected. + // If the worker fails to successfully report its routing info to the controller, + // it will try again later. + RoutingInfo(context.Context, *RoutingInfoRequest) (*RoutingInfoResponse, error) + // SessionInfo is used by the worker to inform the controller of all the sessions + // it is managing. The controller may inform the worker if any sessions need to be changed. + // If the worker repeatedly fails to successfully report its session info to the controller, + // it will tear down any running sessions. + SessionInfo(context.Context, *SessionInfoRequest) (*SessionInfoResponse, error) mustEmbedUnimplementedServerCoordinationServiceServer() } @@ -87,6 +148,15 @@ func (UnimplementedServerCoordinationServiceServer) Status(context.Context, *Sta func (UnimplementedServerCoordinationServiceServer) ListHcpbWorkers(context.Context, *ListHcpbWorkersRequest) (*ListHcpbWorkersResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListHcpbWorkers not implemented") } +func (UnimplementedServerCoordinationServiceServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") +} +func (UnimplementedServerCoordinationServiceServer) RoutingInfo(context.Context, *RoutingInfoRequest) (*RoutingInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RoutingInfo not implemented") +} +func (UnimplementedServerCoordinationServiceServer) SessionInfo(context.Context, *SessionInfoRequest) (*SessionInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SessionInfo not implemented") +} func (UnimplementedServerCoordinationServiceServer) mustEmbedUnimplementedServerCoordinationServiceServer() { } @@ -137,6 +207,60 @@ func _ServerCoordinationService_ListHcpbWorkers_Handler(srv interface{}, ctx con return interceptor(ctx, in, info, handler) } +func _ServerCoordinationService_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatisticsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServerCoordinationServiceServer).Statistics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ServerCoordinationService_Statistics_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServerCoordinationServiceServer).Statistics(ctx, req.(*StatisticsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ServerCoordinationService_RoutingInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RoutingInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServerCoordinationServiceServer).RoutingInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ServerCoordinationService_RoutingInfo_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServerCoordinationServiceServer).RoutingInfo(ctx, req.(*RoutingInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ServerCoordinationService_SessionInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SessionInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServerCoordinationServiceServer).SessionInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ServerCoordinationService_SessionInfo_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServerCoordinationServiceServer).SessionInfo(ctx, req.(*SessionInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ServerCoordinationService_ServiceDesc is the grpc.ServiceDesc for ServerCoordinationService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -152,6 +276,18 @@ var ServerCoordinationService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ListHcpbWorkers", Handler: _ServerCoordinationService_ListHcpbWorkers_Handler, }, + { + MethodName: "Statistics", + Handler: _ServerCoordinationService_Statistics_Handler, + }, + { + MethodName: "RoutingInfo", + Handler: _ServerCoordinationService_RoutingInfo_Handler, + }, + { + MethodName: "SessionInfo", + Handler: _ServerCoordinationService_SessionInfo_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "controller/servers/services/v1/server_coordination_service.proto", diff --git a/internal/proto/controller/servers/services/v1/server_coordination_service.proto b/internal/proto/controller/servers/services/v1/server_coordination_service.proto index 89f215916..2e53ba295 100644 --- a/internal/proto/controller/servers/services/v1/server_coordination_service.proto +++ b/internal/proto/controller/servers/services/v1/server_coordination_service.proto @@ -13,10 +13,29 @@ service ServerCoordinationService { // Status gets worker status requests which include the ongoing jobs the worker is handling and // returns the status response which includes the changes the controller would like to make to // jobs as well as provide a list of the controllers in the system. - rpc Status(StatusRequest) returns (StatusResponse) {} + // This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0. + rpc Status(StatusRequest) returns (StatusResponse) { + option deprecated = true; + } // Returns the addresses of HCP Boundary workers, if any rpc ListHcpbWorkers(ListHcpbWorkersRequest) returns (ListHcpbWorkersResponse) {} + + // Statistics is used by the worker to report non-essential statistics about its sessions and connections. + rpc Statistics(StatisticsRequest) returns (StatisticsResponse) {} + + // RoutingInfo is used by the worker to inform the controller of information + // required by the controller to make session routing decisions and any startup information. + // The controller may inform the worker of any downstream workers that should be disconnected. + // If the worker fails to successfully report its routing info to the controller, + // it will try again later. + rpc RoutingInfo(RoutingInfoRequest) returns (RoutingInfoResponse) {} + + // SessionInfo is used by the worker to inform the controller of all the sessions + // it is managing. The controller may inform the worker if any sessions need to be changed. + // If the worker repeatedly fails to successfully report its session info to the controller, + // it will tear down any running sessions. + rpc SessionInfo(SessionInfoRequest) returns (SessionInfoResponse) {} } enum CONNECTIONSTATUS { @@ -41,31 +60,41 @@ enum SESSIONSTATUS { SESSIONSTATUS_TERMINATED = 4; } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message SessionJobInfo { + option deprecated = true; string session_id = 1; // @gotags: `class:"public" eventstream:"observation"` SESSIONSTATUS status = 2; repeated Connection connections = 3; SessionProcessingError processing_error = 4; // @gotags: `class:"public"` } +// This enum is deprecated and is safe to remove after the release of Boundary v0.20.0. enum SessionProcessingError { + option deprecated = true; SESSION_PROCESSING_ERROR_UNSPECIFIED = 0; SESSION_PROCESSING_ERROR_UNRECOGNIZED = 1; } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message MonitorSessionJobInfo { + option deprecated = true; string session_id = 1; // @gotags: `class:"public" eventstream:"observation"` SESSIONSTATUS status = 2; // @gotags: `class:"public"` SessionProcessingError processing_error = 3; // @gotags: `class:"public"` } +// This enum is deprecated and is safe to remove after the release of Boundary v0.20.0. enum JOBTYPE { + option deprecated = true; JOBTYPE_UNSPECIFIED = 0; JOBTYPE_SESSION = 1; JOBTYPE_MONITOR_SESSION = 2; } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message Job { + option deprecated = true; JOBTYPE type = 1; oneof job_info { // This value is specified when type is JOBTYPE_SESSION. @@ -74,13 +103,17 @@ message Job { } } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message JobStatus { + option deprecated = true; Job job = 1; } // UpstreamServer returns the controller address in the StatusResponse but may be used in the future to return // worker address. +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message UpstreamServer { + option deprecated = true; enum TYPE { TYPE_UNSPECIFIED = 0; TYPE_CONTROLLER = 1; @@ -94,9 +127,11 @@ message UpstreamServer { string address = 20; // @gotags: `class:"public"` } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message StatusRequest { - reserved 10; - reserved "worker"; + option deprecated = true; + reserved 10, 50; + reserved "worker", "connected_worker_key_identifiers"; // Jobs which this worker wants to report the status. repeated JobStatus jobs = 20; @@ -113,12 +148,6 @@ message StatusRequest { // get access to the key and spoof the connection, you're already compromised. servers.v1.ServerWorkerStatus worker_status = 40; - // The worker key identifiers presented by all downstreams connected to this - // worker. - // Deprecated. Should be removed in 0.15.0 at which time - // connected_unmapped_worker_key_identifiers should be used exclusively. - repeated string connected_worker_key_identifiers = 50 [deprecated = true]; - // The worker key identifiers for downstream workers for which there // is not a known worker id mapping for them yet. repeated string connected_unmapped_worker_key_identifiers = 51; @@ -130,36 +159,36 @@ message StatusRequest { repeated string connected_worker_public_ids = 55; } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. enum CHANGETYPE { + option deprecated = true; CHANGETYPE_UNSPECIFIED = 0; // Indicates to the worker to update its knowledge of the state, which may // cause action to be taken. CHANGETYPE_UPDATE_STATE = 1; } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message JobChangeRequest { + option deprecated = true; Job job = 1; CHANGETYPE request_type = 2; } -message AuthorizedWorkerList { - option deprecated = true; - // the key id of authorized workers - repeated string worker_key_identifiers = 1 [deprecated = true]; -} - message AuthorizedDownstreamWorkerList { // the key id of authorized workers which do not have a worker id known yet // to the requester. - repeated string unmapped_worker_key_identifiers = 1; + repeated string unmapped_worker_key_identifiers = 1; // @gotags: class:"public" eventstream:"observation" // the public id of workers that were reported as being connected and which // are known by the controller. - repeated string worker_public_ids = 2; + repeated string worker_public_ids = 2; // @gotags: class:"public" eventstream:"observation" } +// This message is deprecated and is safe to remove after the release of Boundary v0.20.0. message StatusResponse { - reserved 10; - reserved "controllers"; + option deprecated = true; + reserved 10, 50; + reserved "controllers", "authorized_workers"; // List of jobs and the expected state changes. For example, this will // include jobs witch change type of canceled for jobs which are active on a @@ -175,12 +204,6 @@ message StatusResponse { // controller does not need to do a database lookup for the id using the name field. string worker_id = 40; // @gotags: `class:"public" eventstream:"observation"` - // Of the worker key identifiers provided in the request, these are the ones - // which are authorized to remain connected. - // This is deprecated. Use authorized_downstream_workers instead. This - // should be removed in version 0.15.0. - AuthorizedWorkerList authorized_workers = 50 [deprecated = true]; - // Of the downstream workers in the request, these are the ones // which are authorized to remain connected. AuthorizedDownstreamWorkerList authorized_downstream_workers = 51; @@ -202,3 +225,99 @@ message ListHcpbWorkersRequest {} message ListHcpbWorkersResponse { repeated WorkerInfo workers = 1; } + +// SessionStatistics contains statistics about a session and its connections. +message SessionStatistics { + // Session ID holds the ID of the session which the statistics belong to. + string session_id = 1; // @gotags: class:"public" eventstream:"observation" + + // Connections contains statistics about all the existing connections for the given session. + repeated Connection connections = 2; +} + +// StatisticsRequest contains a worker statistics request. This request +// reports on information that is not important either to maintain existing +// sessions or create new ones. +message StatisticsRequest { + // The ID of the worker making the request. + string worker_id = 10; // @gotags: class:"public" eventstream:"observation" + + // The statistics of the sessions managed by this worker + repeated SessionStatistics sessions = 20; +} + +// StatisticsResponse contains a controller statistics response. Empty for now. +message StatisticsResponse {} + +// RoutingIinfoRequest is the request message for the RoutingInfo RPC. +message RoutingInfoRequest { + // The worker status of the worker making the request. + servers.v1.ServerWorkerStatus worker_status = 1; + + // Whether the tags should be updated or not. + bool update_tags = 2; // @gotags: class:"public" eventstream:"observation" + + // The worker key identifiers for downstream workers for which there + // is not a known worker id. + repeated string connected_unmapped_worker_key_identifiers = 3; // @gotags: class:"public" eventstream:"observation" + + // The worker public ids of all downstreams connected to this worker. + repeated string connected_worker_public_ids = 4; // @gotags: class:"public" eventstream:"observation" +} + +// RoutingInfoResponse is the response message for the RoutingInfo RPC. +message RoutingInfoResponse { + // The controller-assigned ID of the worker which made the request. + string worker_id = 1; // @gotags: class:"public" eventstream:"observation" + + // The calculated upstream addresses for the controller cluster. + repeated string calculated_upstream_addresses = 2; // @gotags: class:"public" eventstream:"observation" + + // Of the downstream workers in the request, these are the ones + // which are authorized to remain connected. + AuthorizedDownstreamWorkerList authorized_downstream_workers = 3; +} + +// Session type represents how the worker is processing a session. +enum SessionType { + SESSION_TYPE_UNSPECIFIED = 0; + // A session where the worker is the ingress worker. + SESSION_TYPE_INGRESSED = 1; + // A session where the worker is doing the recording of the session. + SESSION_TYPE_RECORDED = 2; +} + +// Session contains information about a worker managed session +message Session { + // The ID of the session. + string session_id = 1; // @gotags: class:"public" eventstream:"observation" + + // The status of the session. + SESSIONSTATUS session_status = 2; // @gotags: class:"public" eventstream:"observation" + + // Represents how the worker is processing the session. + SessionType session_type = 3; // @gotags: class:"public" eventstream:"observation" + + // Connections contains information about the connections managed by the session + repeated Connection connections = 4; +} + +// SessionInfoRequest contains a worker session information request. This request +// reports on currently managed sessions on the worker. This includes all ingressed +// and recorded sessions. +message SessionInfoRequest { + // The ID of the worker making the request. + string worker_id = 1; // @gotags: class:"public" eventstream:"observation" + + // The sessions which this worker wants to report the status of. + // Includes both ingressed sessions and recorded sessions. + repeated Session sessions = 2; +} + +// SessionInfoResponse contains a controller session information response. This +// response indicates what, if any, changes should be made to the sessions. +// This also includes any sessions the worker is monitoring but not managing directly. +message SessionInfoResponse { + // List of sessions that are no longer active and whose status should be updated. + repeated Session non_active_sessions = 1; +} diff --git a/internal/proto/controller/servers/v1/servers.proto b/internal/proto/controller/servers/v1/servers.proto index c04965387..b678d8094 100644 --- a/internal/proto/controller/servers/v1/servers.proto +++ b/internal/proto/controller/servers/v1/servers.proto @@ -17,22 +17,22 @@ message TagPair { // ServerWorkerStatus is the new message used in place of Server to relay status request info. message ServerWorkerStatus { - // Id of the worker. + // Id of the worker, used after the first request. string public_id = 10; // @gotags: `class:"public" eventstream:"observation"` - // Name of the resource (optional) + // Name of the worker, used to identify workers in the KMS registration flow. string name = 20; // @gotags: `class:"public"` - // Description of the resource (optional) + // Description of the worker (optional). Only used by KMS workers. string description = 25; // @gotags: `class:"public"` - // Address for the worker. This must be set. + // Address for the worker. string address = 30; // @gotags: `class:"public"` // Tags for workers repeated TagPair tags = 40; - // The key id for this worker, if applicable (optional) + // The key id of the worker, used to identify workers in the PKI registration flow. string key_id = 50; // @gotags: `class:"public"` // The version of Boundary the worker binary is running diff --git a/internal/proto/controller/storage/servers/store/v1/worker.proto b/internal/proto/controller/storage/servers/store/v1/worker.proto index ba36fe98a..4fb98e959 100644 --- a/internal/proto/controller/storage/servers/store/v1/worker.proto +++ b/internal/proto/controller/storage/servers/store/v1/worker.proto @@ -159,3 +159,14 @@ message WorkerStorageBucketCredentialState { // @inject_tag: `gorm:"not_null"` timestamp.v1.Timestamp checked_at = 60; } + +// WorkerSessionInfoRequest contains information about a worker and its SessionInfo requests. +message WorkerSessionInfoRequest { + // worker_id is the public id of the worker. + // @inject_tag: `gorm:"primary_key"` + string worker_id = 10; + + // The last_request_time is the last time a worker has reported its session information. + // @inject_tag: `gorm:"default:null"` + timestamp.v1.Timestamp last_request_time = 20; +} diff --git a/internal/server/job/jobs.go b/internal/server/job/jobs.go index 8815076a6..74b11f9f7 100644 --- a/internal/server/job/jobs.go +++ b/internal/server/job/jobs.go @@ -23,7 +23,7 @@ func RegisterJobs( w db.Writer, kms *kms.Kms, controllerExt globals.ControllerExtension, - workerStatusGracePeriod *atomic.Int64, + workerRPCGracePeriod *atomic.Int64, opt ...Option, ) error { const op = "server.(Jobs).RegisterJobs" @@ -40,8 +40,8 @@ func RegisterJobs( if kms == nil { return errors.New(ctx, errors.InvalidParameter, op, "missing kms") } - if isNil(workerStatusGracePeriod) { - return errors.New(ctx, errors.InvalidParameter, op, "missing worker status grace period") + if isNil(workerRPCGracePeriod) { + return errors.New(ctx, errors.InvalidParameter, op, "missing worker RPC grace period") } rotateRootsJob, err := newRotateRootsJob(ctx, r, w, kms, opt...) @@ -52,7 +52,7 @@ func RegisterJobs( return errors.Wrap(ctx, err, op) } - usbJob, err := NewUpsertWorkerStorageBucketJobFn(ctx, r, w, kms, controllerExt, workerStatusGracePeriod, scheduler) + usbJob, err := NewUpsertWorkerStorageBucketJobFn(ctx, r, w, kms, controllerExt, workerRPCGracePeriod, scheduler) if err != nil { return errors.Wrap(ctx, err, op, errors.WithMsg("error creating upsert worker storage bucket job")) } diff --git a/internal/server/repository_worker.go b/internal/server/repository_worker.go index d8d1260aa..f801eeaa7 100644 --- a/internal/server/repository_worker.go +++ b/internal/server/repository_worker.go @@ -284,8 +284,8 @@ func (r *Repository) UpsertWorkerStatus(ctx context.Context, worker *Worker, opt return "", errors.New(ctx, errors.InvalidParameter, op, "scope id is empty") case worker.PublicId != "": return "", errors.New(ctx, errors.InvalidParameter, op, "worker id is not empty") - case worker.GetName() == "" && opts.withKeyId == "": - return "", errors.New(ctx, errors.InvalidParameter, op, "worker keyId and reported name are both empty; one is required") + case worker.GetName() == "" && opts.withKeyId == "" && opts.withPublicId == "": + return "", errors.New(ctx, errors.InvalidParameter, op, "worker keyId, name and public are all empty; one is required") case worker.OperationalState == "": return "", errors.New(ctx, errors.InvalidParameter, op, "worker operational state is empty") case worker.LocalStorageState == "": @@ -898,7 +898,7 @@ func (r *Repository) DeleteWorkerTags(ctx context.Context, workerId string, work // Additionally, it returns the data for the the protocol-aware worker that can // handle recording operations, if enabled. func (r *Repository) SelectSessionWorkers(ctx context.Context, - workerStatusGracePeriod time.Duration, + workerRPCGracePeriod time.Duration, t target.Target, host string, ce globals.ControllerExtension, @@ -907,12 +907,12 @@ func (r *Repository) SelectSessionWorkers(ctx context.Context, ) ([]WorkerAddress, string, error) { const op = "server.(Repository).SelectSessionWorkers" - if workerStatusGracePeriod <= 0 { - workerStatusGracePeriod = DefaultLiveness + if workerRPCGracePeriod <= 0 { + workerRPCGracePeriod = DefaultLiveness } - workerStatusGracePeriod = workerStatusGracePeriod.Truncate(time.Second) + workerRPCGracePeriod = workerRPCGracePeriod.Truncate(time.Second) - query := fmt.Sprintf(listSelectSessionWorkers, uint32(workerStatusGracePeriod.Seconds())) + query := fmt.Sprintf(listSelectSessionWorkers, uint32(workerRPCGracePeriod.Seconds())) var livingWorkers []*Worker _, err := r.writer.DoTx( ctx, diff --git a/internal/server/repository_worker_session_info.go b/internal/server/repository_worker_session_info.go new file mode 100644 index 000000000..c9e707c31 --- /dev/null +++ b/internal/server/repository_worker_session_info.go @@ -0,0 +1,37 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package server + +import ( + "context" + "fmt" + + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/errors" +) + +// UpsertSessionInfo will update the session info latest request time to the current time for the given worker. +func (r *Repository) UpsertSessionInfo(ctx context.Context, workerId string, _ ...Option) error { + const op = "server.(Repository).UpsertSessionInfo" + if workerId == "" { + return errors.New(ctx, errors.InvalidParameter, op, "missing worker id") + } + sessionInfo := NewWorkerSessionInfoRequest(workerId) + _, err := r.writer.DoTx( + ctx, + db.StdRetryCnt, + db.ExpBackoff{}, + func(_ db.Reader, w db.Writer) error { + upsert := db.WithOnConflict(&db.OnConflict{ + Target: db.Columns{"worker_id"}, + Action: db.SetColumns([]string{"last_request_time"}), + }) + return w.Create(ctx, sessionInfo, upsert) + }, + ) + if err != nil { + return errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("failed for %s", workerId))) + } + return nil +} diff --git a/internal/server/repository_worker_session_info_test.go b/internal/server/repository_worker_session_info_test.go new file mode 100644 index 000000000..b082cfde5 --- /dev/null +++ b/internal/server/repository_worker_session_info_test.go @@ -0,0 +1,71 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package server + +import ( + "context" + "testing" + + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/iam" + "github.com/hashicorp/boundary/internal/kms" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUpsertSessionInfo(t *testing.T) { + t.Parallel() + + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + testKms := kms.TestKms(t, conn, wrapper) + testRepo, err := NewRepository(ctx, rw, rw, testKms) + require.NoError(t, err) + iamRepo := iam.TestRepo(t, conn, wrapper) + iam.TestScopes(t, iamRepo) + + tests := []struct { + name string + workerId string + expectedErrMsg string + }{ + { + name: "empty worker id", + expectedErrMsg: "missing worker id", + }, + { + name: "worker does not exist", + workerId: "w_dne", + expectedErrMsg: "server.(Repository).UpsertSessionInfo: failed for w_dne: db.DoTx: db.Create: wt_public_id_check constraint failed: check constraint violated: integrity violation: error #1000", + }, + { + name: "success", + workerId: func() string { + return TestPkiWorker(t, conn, wrapper).PublicId + }(), + }, + { + name: "upsert on conflict", + workerId: func() string { + workerId := TestPkiWorker(t, conn, wrapper).PublicId + require.NoError(t, testRepo.UpsertSessionInfo(context.Background(), workerId)) + return workerId + }(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + err := testRepo.UpsertSessionInfo(context.Background(), tt.workerId) + if tt.expectedErrMsg != "" { + require.Error(err) + assert.ErrorContains(err, tt.expectedErrMsg) + return + } + require.NoError(err) + }) + } +} diff --git a/internal/server/repository_worker_test.go b/internal/server/repository_worker_test.go index be571e38b..13a7815bc 100644 --- a/internal/server/repository_worker_test.go +++ b/internal/server/repository_worker_test.go @@ -1661,7 +1661,7 @@ func TestSelectSessionWorkers(t *testing.T) { require.ErrorContains(t, err, "No workers are available to handle this session.") }) - t.Run("invalidWorkerStatusGracePeriod", func(t *testing.T) { + t.Run("invalidWorkerRPCGracePeriod", func(t *testing.T) { w1 := server.TestKmsWorker(t, conn, wrapper, server.WithWorkerTags(&server.Tag{Key: "type", Value: "worker1"})) require.NotNil(t, w1) t.Cleanup(func() { @@ -1688,7 +1688,7 @@ func TestSelectSessionWorkers(t *testing.T) { require.Equal(t, w1.GetAddress(), was[0].Address) }) - t.Run("validWorkerStatusGracePeriod", func(t *testing.T) { + t.Run("validWorkerRPCGracePeriod", func(t *testing.T) { w1 := server.TestKmsWorker(t, conn, wrapper, server.WithWorkerTags(&server.Tag{Key: "type", Value: "prod"})) require.NotNil(t, w1) t.Cleanup(func() { diff --git a/internal/server/service_verified_workers.go b/internal/server/service_verified_workers.go new file mode 100644 index 000000000..fbe80386b --- /dev/null +++ b/internal/server/service_verified_workers.go @@ -0,0 +1,52 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package server + +import ( + "context" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/util" +) + +// AuthorizedWorkers contain authorized worker +// public ids and unmapped worker key ids +type AuthorizedWorkers struct { + WorkerPublicIds []string + UnmappedWorkerKeyIds []string +} + +// VerifyKnownAndUnmappedWorkers verifies the connected worker ids and unmapped worker key ids are known to the controller +func VerifyKnownAndUnmappedWorkers( + ctx context.Context, + repo *Repository, + workerAuthRepo *WorkerAuthRepositoryStorage, + connectedWorkerIds, + unmappedWorkerKeyIds []string, +) (*AuthorizedWorkers, error) { + const op = "server.VerifyKnownAndUnmappedWorkers" + switch { + case util.IsNil(repo): + return nil, errors.New(ctx, errors.InvalidParameter, op, "repository is required") + case util.IsNil(workerAuthRepo): + return nil, errors.New(ctx, errors.InvalidParameter, op, "worker auth repository is required") + } + var authorizedDownstreams AuthorizedWorkers + if len(connectedWorkerIds) > 0 { + knownConnectedWorkers, err := repo.VerifyKnownWorkers(ctx, connectedWorkerIds) + if err != nil { + return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error getting known connected worker ids")) + } + authorizedDownstreams.WorkerPublicIds = knownConnectedWorkers + } + + if len(unmappedWorkerKeyIds) > 0 { + authorizedKeyIds, err := workerAuthRepo.FilterToAuthorizedWorkerKeyIds(ctx, unmappedWorkerKeyIds) + if err != nil { + return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error getting authorized unmapped worker key ids")) + } + authorizedDownstreams.UnmappedWorkerKeyIds = authorizedKeyIds + } + return &authorizedDownstreams, nil +} diff --git a/internal/server/service_verified_workers_ext_test.go b/internal/server/service_verified_workers_ext_test.go new file mode 100644 index 000000000..5f0b8cd93 --- /dev/null +++ b/internal/server/service_verified_workers_ext_test.go @@ -0,0 +1,61 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package server_test + +import ( + "context" + "crypto/rand" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/kms" + "github.com/hashicorp/boundary/internal/server" + "github.com/hashicorp/boundary/internal/types/scope" + "github.com/stretchr/testify/require" +) + +func TestVerifyUnknownAndUnmappedWorkers(t *testing.T) { + t.Parallel() + + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kmsCache := kms.TestKms(t, conn, wrapper) + require.NoError(t, kmsCache.CreateKeys(context.Background(), scope.Global.String(), kms.WithRandomReader(rand.Reader))) + + serverRepo, err := server.NewRepository(ctx, rw, rw, kmsCache) + require.NoError(t, err) + workerAuthRepo, err := server.NewRepositoryStorage(ctx, rw, rw, kmsCache) + require.NoError(t, err) + + // Set up resources + var w1KeyId, w2KeyId string + w1 := server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w1KeyId)) + _ = server.TestPkiWorker(t, conn, wrapper, server.WithTestPkiWorkerAuthorizedKeyId(&w2KeyId)) + w3 := server.TestKmsWorker(t, conn, wrapper) + + t.Run("With 1 PKI, 1 KMS and 1 unmapped worker", func(t *testing.T) { + t.Parallel() + authzWorkers, err := server.VerifyKnownAndUnmappedWorkers(ctx, serverRepo, workerAuthRepo, []string{w1.PublicId, w3.PublicId, "invalid-worker-id"}, []string{w2KeyId, "invalid-worker-key-id"}) + require.NoError(t, err) + require.Empty(t, cmp.Diff(authzWorkers.WorkerPublicIds, []string{w1.PublicId, w3.PublicId})) + require.Empty(t, cmp.Diff(authzWorkers.UnmappedWorkerKeyIds, []string{w2KeyId})) + }) + + t.Run("missing repository", func(t *testing.T) { + t.Parallel() + authzWorkers, err := server.VerifyKnownAndUnmappedWorkers(ctx, nil, workerAuthRepo, []string{w1.PublicId, w3.PublicId}, []string{w2KeyId}) + require.ErrorContains(t, err, "repository is required") + require.Nil(t, authzWorkers) + }) + + t.Run("missing worker auth repository", func(t *testing.T) { + t.Parallel() + authzWorkers, err := server.VerifyKnownAndUnmappedWorkers(ctx, serverRepo, nil, []string{w1.PublicId, w3.PublicId}, []string{w2KeyId}) + require.ErrorContains(t, err, "worker auth repository is required") + require.Nil(t, authzWorkers) + }) +} diff --git a/internal/server/store/worker.pb.go b/internal/server/store/worker.pb.go index ec900cc01..137c9a209 100644 --- a/internal/server/store/worker.pb.go +++ b/internal/server/store/worker.pb.go @@ -528,6 +528,66 @@ func (x *WorkerStorageBucketCredentialState) GetCheckedAt() *timestamp.Timestamp return nil } +// WorkerSessionInfoRequest contains information about a worker and its SessionInfo requests. +type WorkerSessionInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // worker_id is the public id of the worker. + // @inject_tag: `gorm:"primary_key"` + WorkerId string `protobuf:"bytes,10,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty" gorm:"primary_key"` + // The last_request_time is the last time a worker has reported its session information. + // @inject_tag: `gorm:"default:null"` + LastRequestTime *timestamp.Timestamp `protobuf:"bytes,20,opt,name=last_request_time,json=lastRequestTime,proto3" json:"last_request_time,omitempty" gorm:"default:null"` +} + +func (x *WorkerSessionInfoRequest) Reset() { + *x = WorkerSessionInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_storage_servers_store_v1_worker_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *WorkerSessionInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerSessionInfoRequest) ProtoMessage() {} + +func (x *WorkerSessionInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_controller_storage_servers_store_v1_worker_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerSessionInfoRequest.ProtoReflect.Descriptor instead. +func (*WorkerSessionInfoRequest) Descriptor() ([]byte, []int) { + return file_controller_storage_servers_store_v1_worker_proto_rawDescGZIP(), []int{5} +} + +func (x *WorkerSessionInfoRequest) GetWorkerId() string { + if x != nil { + return x.WorkerId + } + return "" +} + +func (x *WorkerSessionInfoRequest) GetLastRequestTime() *timestamp.Timestamp { + if x != nil { + return x.LastRequestTime + } + return nil +} + var File_controller_storage_servers_store_v1_worker_proto protoreflect.FileDescriptor var file_controller_storage_servers_store_v1_worker_proto_rawDesc = []byte{ @@ -617,11 +677,20 @@ var file_controller_storage_servers_store_v1_worker_proto_rawDesc = []byte{ 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x65, 0x64, 0x41, 0x74, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3b, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x64, 0x41, 0x74, 0x22, 0x8f, 0x01, 0x0a, 0x18, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x12, 0x56, + 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3b, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -636,25 +705,27 @@ func file_controller_storage_servers_store_v1_worker_proto_rawDescGZIP() []byte return file_controller_storage_servers_store_v1_worker_proto_rawDescData } -var file_controller_storage_servers_store_v1_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_controller_storage_servers_store_v1_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_controller_storage_servers_store_v1_worker_proto_goTypes = []any{ (*Worker)(nil), // 0: controller.storage.servers.store.v1.Worker (*WorkerTag)(nil), // 1: controller.storage.servers.store.v1.WorkerTag (*ApiTag)(nil), // 2: controller.storage.servers.store.v1.ApiTag (*ConfigTag)(nil), // 3: controller.storage.servers.store.v1.ConfigTag (*WorkerStorageBucketCredentialState)(nil), // 4: controller.storage.servers.store.v1.WorkerStorageBucketCredentialState - (*timestamp.Timestamp)(nil), // 5: controller.storage.timestamp.v1.Timestamp + (*WorkerSessionInfoRequest)(nil), // 5: controller.storage.servers.store.v1.WorkerSessionInfoRequest + (*timestamp.Timestamp)(nil), // 6: controller.storage.timestamp.v1.Timestamp } var file_controller_storage_servers_store_v1_worker_proto_depIdxs = []int32{ - 5, // 0: controller.storage.servers.store.v1.Worker.create_time:type_name -> controller.storage.timestamp.v1.Timestamp - 5, // 1: controller.storage.servers.store.v1.Worker.update_time:type_name -> controller.storage.timestamp.v1.Timestamp - 5, // 2: controller.storage.servers.store.v1.Worker.last_status_time:type_name -> controller.storage.timestamp.v1.Timestamp - 5, // 3: controller.storage.servers.store.v1.WorkerStorageBucketCredentialState.checked_at:type_name -> controller.storage.timestamp.v1.Timestamp - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 6, // 0: controller.storage.servers.store.v1.Worker.create_time:type_name -> controller.storage.timestamp.v1.Timestamp + 6, // 1: controller.storage.servers.store.v1.Worker.update_time:type_name -> controller.storage.timestamp.v1.Timestamp + 6, // 2: controller.storage.servers.store.v1.Worker.last_status_time:type_name -> controller.storage.timestamp.v1.Timestamp + 6, // 3: controller.storage.servers.store.v1.WorkerStorageBucketCredentialState.checked_at:type_name -> controller.storage.timestamp.v1.Timestamp + 6, // 4: controller.storage.servers.store.v1.WorkerSessionInfoRequest.last_request_time:type_name -> controller.storage.timestamp.v1.Timestamp + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_controller_storage_servers_store_v1_worker_proto_init() } @@ -723,6 +794,18 @@ func file_controller_storage_servers_store_v1_worker_proto_init() { return nil } } + file_controller_storage_servers_store_v1_worker_proto_msgTypes[5].Exporter = func(v any, i int) any { + switch v := v.(*WorkerSessionInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -730,7 +813,7 @@ func file_controller_storage_servers_store_v1_worker_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_controller_storage_servers_store_v1_worker_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 6, NumExtensions: 0, NumServices: 0, }, diff --git a/internal/server/worker_session_info_request.go b/internal/server/worker_session_info_request.go new file mode 100644 index 000000000..f8d7b390e --- /dev/null +++ b/internal/server/worker_session_info_request.go @@ -0,0 +1,81 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package server + +import ( + "context" + + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/timestamp" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/oplog" + "github.com/hashicorp/boundary/internal/server/store" + "google.golang.org/protobuf/proto" +) + +// Ensure WorkerSessionInfoRequest implements interfaces +var ( + _ db.VetForWriter = (*WorkerSessionInfoRequest)(nil) + _ oplog.ReplayableMessage = (*WorkerSessionInfoRequest)(nil) +) + +// WorkerSessionInfoRequest contains the last time a worker daemon sent a status update for session information. +type WorkerSessionInfoRequest struct { + *store.WorkerSessionInfoRequest + tableName string `gorm:"-"` +} + +// NewWorkerSessionInfoRequest create a new instance of a worker session information request. +// The LastStatusTime field is set to the current time. +func NewWorkerSessionInfoRequest(workerId string) *WorkerSessionInfoRequest { + return &WorkerSessionInfoRequest{ + WorkerSessionInfoRequest: &store.WorkerSessionInfoRequest{ + WorkerId: workerId, + LastRequestTime: timestamp.Now(), + }, + } +} + +// WorkerSessionInfoRequest creates an empty in-memory instance of a worker session information request. +func AllocWorkerSessionInfoRequest() *WorkerSessionInfoRequest { + return &WorkerSessionInfoRequest{ + WorkerSessionInfoRequest: &store.WorkerSessionInfoRequest{}, + } +} + +func (w *WorkerSessionInfoRequest) Clone() *WorkerSessionInfoRequest { + cp := proto.Clone(w.WorkerSessionInfoRequest) + return &WorkerSessionInfoRequest{ + WorkerSessionInfoRequest: cp.(*store.WorkerSessionInfoRequest), + } +} + +// VetForWrite implements db.VetForWrite() interface and validates the worker storage bucket credential state +func (w *WorkerSessionInfoRequest) VetForWrite(ctx context.Context, _ db.Reader, opType db.OpType, _ ...db.Option) error { + const op = "server.(WorkerSessionInfoRequest).VetForWrite" + switch { + case w == nil: + return errors.New(ctx, errors.InvalidParameter, op, "missing worker session info request") + case w.WorkerId == "": + return errors.New(ctx, errors.InvalidParameter, op, "missing worker id") + case w.LastRequestTime == nil: + return errors.New(ctx, errors.InvalidParameter, op, "missing last request timestamp") + } + return nil +} + +// TableName returns the tablename to override the default gorm table name +func (w *WorkerSessionInfoRequest) TableName() string { + if w.tableName != "" { + return w.tableName + } + return "server_worker_session_info_request" +} + +// SetTableName sets the tablename and satisfies the ReplayableMessage +// interface. If the caller attempts to set the name to "" the name will be +// reset to the default name. +func (w *WorkerSessionInfoRequest) SetTableName(n string) { + w.tableName = n +} diff --git a/internal/session/job_session_cleanup.go b/internal/session/job_session_cleanup.go index 8ddf88dc7..7ab227b26 100644 --- a/internal/session/job_session_cleanup.go +++ b/internal/session/job_session_cleanup.go @@ -37,7 +37,7 @@ type sessionConnectionCleanupJob struct { // The amount of time to give disconnected workers before marking their // connections as closed. This should be larger than the liveness setting // for the worker. - workerStatusGracePeriod *atomic.Int64 + workerRPCGracePeriod *atomic.Int64 // The total number of connections closed in the last run. totalClosed *atomic.Int64 @@ -47,22 +47,22 @@ type sessionConnectionCleanupJob struct { func newSessionConnectionCleanupJob( ctx context.Context, writer db.Writer, - workerStatusGracePeriod *atomic.Int64, + workerRPCGracePeriod *atomic.Int64, ) (*sessionConnectionCleanupJob, error) { const op = "session.newNewSessionConnectionCleanupJob" switch { case writer == nil: return nil, errors.New(ctx, errors.InvalidParameter, op, "missing db writer") - case workerStatusGracePeriod == nil: + case workerRPCGracePeriod == nil: return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grace period") - case workerStatusGracePeriod.Load() == 0: + case workerRPCGracePeriod.Load() == 0: return nil, errors.New(ctx, errors.InvalidParameter, op, "grace period is zero") } return &sessionConnectionCleanupJob{ - writer: writer, - workerStatusGracePeriod: workerStatusGracePeriod, - totalClosed: new(atomic.Int64), + writer: writer, + workerRPCGracePeriod: workerRPCGracePeriod, + totalClosed: new(atomic.Int64), }, nil } @@ -100,7 +100,7 @@ func (j *sessionConnectionCleanupJob) Run(ctx context.Context, _ time.Duration) j.totalClosed.Store(0) // Run the atomic dead worker cleanup job. - gracePeriod := time.Duration(j.workerStatusGracePeriod.Load()) + gracePeriod := time.Duration(j.workerRPCGracePeriod.Load()) results, err := j.closeConnectionsForDeadWorkers(ctx, gracePeriod) if err != nil { return errors.Wrap(ctx, err, op) diff --git a/internal/session/job_session_cleanup_test.go b/internal/session/job_session_cleanup_test.go index 8de4cd828..b7ede7f26 100644 --- a/internal/session/job_session_cleanup_test.go +++ b/internal/session/job_session_cleanup_test.go @@ -16,7 +16,6 @@ import ( "github.com/hashicorp/boundary/internal/kms" "github.com/hashicorp/boundary/internal/scheduler" "github.com/hashicorp/boundary/internal/server" - "github.com/hashicorp/boundary/internal/types/scope" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -54,6 +53,19 @@ func TestSessionConnectionCleanupJob(t *testing.T) { worker1 := server.TestKmsWorker(t, conn, wrapper) worker2 := server.TestKmsWorker(t, conn, wrapper) + updateServer := func(t *testing.T, w *server.Worker) *server.Worker { + t.Helper() + pubId := w.GetPublicId() + w.PublicId = "" + wkr, err := server.TestUpsertAndReturnWorker(ctx, t, w, serversRepo, server.WithPublicId(pubId)) + require.NoError(err) + err = serversRepo.UpsertSessionInfo(ctx, pubId) + require.NoError(err) + return wkr + } + worker1 = updateServer(t, worker1) + worker2 = updateServer(t, worker2) + // Create a few sessions on each, activate, and authorize a connection var connIds []string connIdsByWorker := make(map[string][]string) @@ -95,7 +107,7 @@ func TestSessionConnectionCleanupJob(t *testing.T) { // Create the job. job, err := newSessionConnectionCleanupJob(ctx, rw, gracePeriod) - job.workerStatusGracePeriod = gracePeriod // by-pass factory assert so we dont have to wait so long + job.workerRPCGracePeriod = gracePeriod // by-pass factory assert so we dont have to wait so long require.NoError(err) // sleep the status grace period. @@ -103,11 +115,7 @@ func TestSessionConnectionCleanupJob(t *testing.T) { // Push an upsert to the first worker so that its status has been // updated. - _, err = serversRepo.UpsertWorkerStatus(ctx, server.NewWorker(scope.Global.String(), - server.WithName(worker1.GetName()), - server.WithAddress(worker1.GetAddress())), - server.WithPublicId(worker1.GetPublicId())) - require.NoError(err) + worker1 = updateServer(t, worker1) // Run the job. require.NoError(job.Run(ctx, 0)) @@ -280,6 +288,8 @@ func TestCloseConnectionsForDeadWorkers(t *testing.T) { w.PublicId = "" wkr, err := server.TestUpsertAndReturnWorker(ctx, t, w, serversRepo, server.WithPublicId(pubId)) require.NoError(err) + err = serversRepo.UpsertSessionInfo(ctx, pubId) + require.NoError(err) return wkr } @@ -372,13 +382,11 @@ func TestCloseConnectionsForDeadWorkers(t *testing.T) { require.NoError(err) // Assert that we have one result with the appropriate ID and // number of connections closed. Due to how things are - require.Equal([]closeConnectionsForDeadWorkersResult{ - { - WorkerId: worker1.PublicId, - LastUpdateTime: timestampPbAsUTC(t, worker1.GetLastStatusTime().AsTime()), - NumberConnectionsClosed: 12, // 18 per server, with 6 closed already - }, - }, result) + require.Len(result, 1) + require.Equal(worker1.PublicId, result[0].WorkerId) + require.Equal(12, result[0].NumberConnectionsClosed) + require.WithinDuration(timestampPbAsUTC(t, worker1.GetLastStatusTime().AsTime()), result[0].LastUpdateTime, time.Second) + // Expect all connections closed on worker1 requireConnectionStatus(t, worker1ConnIds, true) // Expect appropriate split connection state on worker2 @@ -396,18 +404,19 @@ func TestCloseConnectionsForDeadWorkers(t *testing.T) { result, err := job.closeConnectionsForDeadWorkers(ctx, gracePeriod) require.NoError(err) // Assert that we have one result with the appropriate ID and number of connections closed. - require.ElementsMatch([]closeConnectionsForDeadWorkersResult{ - { - WorkerId: worker2.PublicId, - LastUpdateTime: timestampPbAsUTC(t, worker2.GetLastStatusTime().AsTime()), - NumberConnectionsClosed: 12, // 18 per server, with 6 closed already - }, - { - WorkerId: worker3.PublicId, - LastUpdateTime: timestampPbAsUTC(t, worker3.GetLastStatusTime().AsTime()), - NumberConnectionsClosed: 12, // 18 per server, with 6 closed already - }, - }, result) + expectedWorkers := map[string]*server.Worker{ + worker2.PublicId: worker2, + worker3.PublicId: worker3, + } + require.Len(result, len(expectedWorkers)) + for _, r := range result { + expectedWorker, ok := expectedWorkers[r.WorkerId] + require.True(ok) + require.Equal(expectedWorker.PublicId, r.WorkerId) + require.Equal(12, r.NumberConnectionsClosed) + require.WithinDuration(timestampPbAsUTC(t, expectedWorker.GetLastStatusTime().AsTime()), r.LastUpdateTime, time.Second) + } + // Expect all connections closed on worker1 requireConnectionStatus(t, worker1ConnIds, true) // Expect all connections closed on worker2 diff --git a/internal/session/jobs.go b/internal/session/jobs.go index 5b9f9423e..634f613fb 100644 --- a/internal/session/jobs.go +++ b/internal/session/jobs.go @@ -18,14 +18,14 @@ import ( const deleteTerminatedThreshold = time.Hour // RegisterJobs registers session related jobs with the provided scheduler. -func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writer, r db.Reader, k *kms.Kms, workerStatusGracePeriod *atomic.Int64) error { +func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writer, r db.Reader, k *kms.Kms, workerRPCGracePeriod *atomic.Int64) error { const op = "session.RegisterJobs" - if workerStatusGracePeriod == nil { + if workerRPCGracePeriod == nil { return errors.New(ctx, errors.InvalidParameter, op, "nil grace period") } - sessionConnectionCleanupJob, err := newSessionConnectionCleanupJob(ctx, w, workerStatusGracePeriod) + sessionConnectionCleanupJob, err := newSessionConnectionCleanupJob(ctx, w, workerRPCGracePeriod) if err != nil { return fmt.Errorf("error creating session cleanup job: %w", err) } diff --git a/internal/session/query.go b/internal/session/query.go index 5d512dc31..ff21d4a83 100644 --- a/internal/session/query.go +++ b/internal/session/query.go @@ -297,11 +297,11 @@ where with dead_workers (worker_id, last_update_time) as ( select - w.public_id as worker_id, - w.last_status_time as last_update_time - from server_worker w + w.worker_id as worker_id, + w.last_request_time as last_update_time + from server_worker_session_info_request w where - w.last_status_time < wt_sub_seconds_from_now(@grace_period_seconds) + w.last_request_time < wt_sub_seconds_from_now(@grace_period_seconds) ), closed_connections (connection_id, worker_id) as ( update session_connection diff --git a/internal/session/repository_connection.go b/internal/session/repository_connection.go index 339e36dde..e2ab714e4 100644 --- a/internal/session/repository_connection.go +++ b/internal/session/repository_connection.go @@ -108,7 +108,7 @@ func (r *ConnectionRepository) updateBytesUpBytesDown(ctx context.Context, conns // we shouldn't update bytes up and down if the connection // has already been closed. This also guards against // potential data races where a connection closure request - // and a worker status update happen close to each other in + // and a worker statistics update happen close to each other in // terms of timing. db.WithWhere("closed_reason is null"), ) diff --git a/internal/session/service_statistics.go b/internal/session/service_statistics.go new file mode 100644 index 000000000..63924fdfe --- /dev/null +++ b/internal/session/service_statistics.go @@ -0,0 +1,47 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package session + +import ( + "context" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/event" +) + +// CloseOrphanedConnections closes the orphaned connections for a given worker. +func CloseOrphanedConnections(ctx context.Context, repo *ConnectionRepository, workerId string, connections []*Connection) ([]string, error) { + const op = "session.CloseOrphanedConnections" + if repo == nil { + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing connection repository") + } + if workerId == "" { + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing worker id") + } + connectionIds := []string{} + for _, c := range connections { + connectionIds = append(connectionIds, c.GetPublicId()) + } + closed, err := repo.closeOrphanedConnections(ctx, workerId, connectionIds) + if len(closed) > 0 { + event.WriteSysEvent(ctx, op, "marked unclaimed connections as closed", "worker_id", workerId, "count", len(closed)) + } + if err != nil { + return nil, errors.Wrap(ctx, err, op, errors.WithMsg("failed to close orphaned connections for worker %q", workerId)) + } + return closed, nil +} + +// UpdateConnectionBytesUpDown processes the connection statistics for a given worker. +func UpdateConnectionBytesUpDown(ctx context.Context, repo *ConnectionRepository, connections []*Connection) error { + const op = "session.UpdateConnectionBytesUpDown" + if repo == nil { + return errors.New(ctx, errors.InvalidParameter, op, "missing connection repository") + } + err := repo.updateBytesUpBytesDown(ctx, connections...) + if err != nil { + return errors.Wrap(ctx, err, op, errors.WithMsg("failed to update bytes up and down for worker connection statistics")) + } + return nil +} diff --git a/internal/session/service_statistics_test.go b/internal/session/service_statistics_test.go new file mode 100644 index 000000000..0d986b9d1 --- /dev/null +++ b/internal/session/service_statistics_test.go @@ -0,0 +1,534 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package session + +import ( + "context" + "testing" + + "github.com/hashicorp/boundary/internal/authtoken" + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/host/static" + "github.com/hashicorp/boundary/internal/iam" + "github.com/hashicorp/boundary/internal/kms" + "github.com/hashicorp/boundary/internal/server" + "github.com/hashicorp/boundary/internal/server/store" + "github.com/hashicorp/boundary/internal/target" + "github.com/hashicorp/boundary/internal/target/tcp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCloseOrphanedConnections(t *testing.T) { + t.Parallel() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + testKms := kms.TestKms(t, conn, wrapper) + connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms) + require.NoError(t, err) + sessRepo, err := NewRepository(context.Background(), rw, rw, testKms) + require.NoError(t, err) + + serverRepo, _ := server.NewRepository(context.Background(), rw, rw, testKms) + _, err = serverRepo.UpsertController(context.Background(), &store.Controller{ + PrivateId: "test_controller1", + Address: "127.0.0.1", + }) + require.NoError(t, err) + + cases := []struct { + name string + connRepo *ConnectionRepository + setup func() (workerId string, conns []*Connection, expectedClosedConnections []string) + wantErrMsg string + }{ + { + name: "missing connection repository", + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + return + }, + wantErrMsg: "session.CloseOrphanedConnections: missing connection repository: parameter violation: error #100", + }, + { + name: "empty worker id", + connRepo: &ConnectionRepository{}, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + return + }, + wantErrMsg: "session.CloseOrphanedConnections: missing worker id: parameter violation: error #100", + }, + { + name: "nil sessions", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + workerId = w.GetPublicId() + return + }, + }, + { + name: "empty connections", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + workerId = w.GetPublicId() + conns = make([]*Connection, 0) + return + }, + }, + { + name: "ignore unknown connection", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + workerId = w.GetPublicId() + conns = []*Connection{ + { + PublicId: "c_dne", + }, + } + return + }, + }, + { + name: "session already canceled", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(t, err) + require.NoError(t, err) + _, err = connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId) + require.NoError(t, err) + _, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version) + require.NoError(t, err) + workerId = w.GetPublicId() + conns = []*Connection{} + return + }, + }, + { + name: "active session", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(t, err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId()) + require.NoError(t, err) + workerId = w.GetPublicId() + conns = []*Connection{ + { + PublicId: c1.GetPublicId(), + }, + } + return + }, + }, + { + name: "closed session", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(t, err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId) + require.NoError(t, err) + _, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version) + require.NoError(t, err) + + workerId = w.GetPublicId() + conns = []*Connection{ + { + PublicId: c1.GetPublicId(), + }, + } + return + }, + }, + { + name: "multiple closed sessions", + connRepo: connRepo, + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(t, err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId) + require.NoError(t, err) + _, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version) + require.NoError(t, err) + + s2 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t)) + require.NoError(t, err) + c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId) + require.NoError(t, err) + _, err = sessRepo.CancelSession(context.Background(), s2.PublicId, s2.Version) + require.NoError(t, err) + + workerId = w.GetPublicId() + conns = []*Connection{ + { + PublicId: c1.GetPublicId(), + }, + { + PublicId: c2.GetPublicId(), + }, + } + return + }, + }, + { + name: "close orphaned connections", + connRepo: func() *ConnectionRepository { + connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms, WithWorkerStateDelay(0)) + require.NoError(t, err) + return connRepo + }(), + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(t, err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId) + require.NoError(t, err) + s2 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t)) + require.NoError(t, err) + c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId) + require.NoError(t, err) + require.NotEqual(t, c1.PublicId, c2.PublicId) + workerId = w.GetPublicId() + conns = []*Connection{ + { + PublicId: c2.GetPublicId(), + }, + } + expectedClosedConnections = []string{c1.PublicId} + return + }, + }, + { + name: "multiple sessions and orphaned connections", + connRepo: func() *ConnectionRepository { + connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms, WithWorkerStateDelay(0)) + require.NoError(t, err) + return connRepo + }(), + setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) { + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(t, err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId) + require.NoError(t, err) + _, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version) + require.NoError(t, err) + + s2 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t)) + require.NoError(t, err) + c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId) + require.NoError(t, err) + c3, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId) + require.NoError(t, err) + _, err = sessRepo.CancelSession(context.Background(), s2.PublicId, s2.Version) + require.NoError(t, err) + + workerId = w.GetPublicId() + conns = []*Connection{ + { + PublicId: c1.GetPublicId(), + }, + { + PublicId: c2.GetPublicId(), + }, + } + expectedClosedConnections = []string{c3.PublicId} + return + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + require.NotNil(tc.setup) + workerId, stats, expectedClosedConnections := tc.setup() + actualClosedConnections, err := CloseOrphanedConnections(context.Background(), tc.connRepo, workerId, stats) + if tc.wantErrMsg != "" { + require.Error(err) + assert.ErrorContains(err, tc.wantErrMsg) + assert.Empty(actualClosedConnections) + return + } + require.NoError(err) + assert.Equal(expectedClosedConnections, actualClosedConnections) + for _, connectionId := range expectedClosedConnections { + conn, err := connRepo.LookupConnection(context.Background(), connectionId) + require.NoError(err) + assert.Equal(ConnectionSystemError, ClosedReason(conn.ClosedReason)) + assert.Equal(StatusClosed, ConnectionStatusFromString(conn.Status)) + } + }) + } +} + +func TestUpdateConnectionBytesUpDown(t *testing.T) { + t.Parallel() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + testKms := kms.TestKms(t, conn, wrapper) + connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms) + require.NoError(t, err) + sessRepo, err := NewRepository(context.Background(), rw, rw, testKms) + require.NoError(t, err) + + serverRepo, _ := server.NewRepository(context.Background(), rw, rw, testKms) + _, err = serverRepo.UpsertController(context.Background(), &store.Controller{ + PrivateId: "test_controller1", + Address: "127.0.0.1", + }) + require.NoError(t, err) + + t.Run("nil connection repo", func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + err := UpdateConnectionBytesUpDown(context.Background(), nil, nil) + require.Error(err) + assert.ErrorContains(err, "missing connection repository") + }) + + t.Run("bytes up and bytes down", func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + w := server.TestKmsWorker(t, conn, wrapper) + org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId()) + userId := authToken.GetIamUserId() + hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0] + hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0] + host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0] + static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host}) + tar := tcp.TestTarget( + context.Background(), + t, conn, prj.GetPublicId(), "test", + target.WithHostSources([]string{hostSet.GetPublicId()}), + target.WithSessionConnectionLimit(-1), + ) + s1 := TestSession(t, conn, wrapper, ComposedOf{ + UserId: userId, + HostId: host.GetPublicId(), + TargetId: tar.GetPublicId(), + HostSetId: hostSet.GetPublicId(), + AuthTokenId: authToken.GetPublicId(), + ProjectId: prj.GetPublicId(), + Endpoint: "tcp://127.0.0.1:22", + ConnectionLimit: 10, + }) + s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t)) + require.NoError(err) + c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId()) + require.NoError(err) + + // write to active session + var expectedBytesUp int64 = 1024 + var expectedBytesDown int64 = 2048 + conns := []*Connection{ + { + PublicId: c1.GetPublicId(), + BytesUp: expectedBytesUp, + BytesDown: expectedBytesDown, + }, + } + err = UpdateConnectionBytesUpDown(context.Background(), connRepo, conns) + require.NoError(err) + conn, err := connRepo.LookupConnection(context.Background(), c1.PublicId) + require.NoError(err) + assert.Equal(StatusAuthorized, ConnectionStatusFromString(conn.Status)) + assert.Equal(expectedBytesUp, conn.BytesUp) + assert.Equal(expectedBytesDown, conn.BytesDown) + + // write to canceled session + _, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version) + require.NoError(err) + expectedBytesUp = 4096 + expectedBytesDown = 8192 + conns = []*Connection{ + { + PublicId: c1.GetPublicId(), + BytesUp: expectedBytesUp, + BytesDown: expectedBytesDown, + }, + } + err = UpdateConnectionBytesUpDown(context.Background(), connRepo, conns) + require.NoError(err) + conn, err = connRepo.LookupConnection(context.Background(), c1.PublicId) + require.NoError(err) + assert.Equal(expectedBytesUp, conn.BytesUp) + assert.Equal(expectedBytesDown, conn.BytesDown) + }) +} diff --git a/internal/tests/api/credentials/credentials_test.go b/internal/tests/api/credentials/credentials_test.go index 407d8b415..047016896 100644 --- a/internal/tests/api/credentials/credentials_test.go +++ b/internal/tests/api/credentials/credentials_test.go @@ -26,6 +26,7 @@ import ( "github.com/hashicorp/boundary/internal/iam" "github.com/hashicorp/boundary/internal/kms" "github.com/hashicorp/boundary/internal/server" + "github.com/hashicorp/boundary/internal/tests/helper" "github.com/hashicorp/go-hclog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -462,7 +463,7 @@ func TestUpdateAfterKeyRotation(t *testing.T) { WorkerAuthKms: tc.Config().WorkerAuthKms, Name: "worker", }) - require.NoError(w.Worker().WaitForNextSuccessfulStatusUpdate()) + helper.ExpectWorkers(t, tc, w) // Authorize session, requires decrypting json credential _, err = tgClient.AuthorizeSession(ctx, targ.Item.Id) diff --git a/internal/tests/api/proxy/proxy_test.go b/internal/tests/api/proxy/proxy_test.go index cdd15d05e..aa78af56e 100644 --- a/internal/tests/api/proxy/proxy_test.go +++ b/internal/tests/api/proxy/proxy_test.go @@ -48,26 +48,20 @@ func TestConnectionsLeft(t *testing.T) { conf, err := config.DevController() require.NoError(err) c1 := controller.NewTestController(t, &controller.TestControllerOpts{ - Config: conf, - InitialResourcesSuffix: "1234567890", - Logger: logger.Named("c1"), - WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod, + Config: conf, + InitialResourcesSuffix: "1234567890", + Logger: logger.Named("c1"), + WorkerRPCGracePeriod: helper.DefaultControllerRPCGracePeriod, }) - defer c1.Shutdown() helper.ExpectWorkers(t, c1) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w1"), - SuccessfulStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod, - Name: "w1", + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w1"), + SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod, + Name: "w1", }) - defer w1.Shutdown() - err = w1.Worker().WaitForNextSuccessfulStatusUpdate() - require.NoError(err) - err = c1.WaitForNextWorkerStatusUpdate(w1.Name()) - require.NoError(err) helper.ExpectWorkers(t, c1, w1) // Connect target diff --git a/internal/tests/api/targets/target_test.go b/internal/tests/api/targets/target_test.go index d00cc28d9..cf9146b59 100644 --- a/internal/tests/api/targets/target_test.go +++ b/internal/tests/api/targets/target_test.go @@ -8,7 +8,6 @@ import ( "net/http" "strings" "testing" - "time" "github.com/hashicorp/boundary/api" "github.com/hashicorp/boundary/api/aliases" @@ -27,6 +26,7 @@ import ( "github.com/hashicorp/boundary/internal/daemon/worker" "github.com/hashicorp/boundary/internal/db" "github.com/hashicorp/boundary/internal/iam" + "github.com/hashicorp/boundary/internal/tests/helper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -862,10 +862,12 @@ func TestCreateTarget_SlashesInName(t *testing.T) { tc := controller.NewTestController(t, nil) tw, _ := worker.NewAuthorizedPkiTestWorker(t, tc.ServersRepo(), "test", tc.ClusterAddrs()) - require.NotNil(t, tw) + require.NotNil(tw) - // Wait for worker to become ready - time.Sleep(10 * time.Second) + helper.ExpectWorkers(t, tc, tw) + // Wait for an extra routing info update for good measure + // TODO(https://hashicorp.atlassian.net/browse/ICU-16124): why is this necessary? + require.NoError(tw.Worker().WaitForNextSuccessfulRoutingInfoUpdate()) client := tc.Client() token := tc.Token() diff --git a/internal/tests/cluster/parallel/ipv6_listener_test.go b/internal/tests/cluster/parallel/ipv6_listener_test.go index 411bf6b47..1af85c7ff 100644 --- a/internal/tests/cluster/parallel/ipv6_listener_test.go +++ b/internal/tests/cluster/parallel/ipv6_listener_test.go @@ -7,6 +7,7 @@ import ( "context" "sync" "testing" + "time" "github.com/hashicorp/boundary/api" "github.com/hashicorp/boundary/api/scopes" @@ -55,10 +56,11 @@ func TestIPv6Listener(t *testing.T) { require.NoError(err) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - Config: wconf, - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...), - Logger: logger.Named("w1"), + Config: wconf, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...), + Logger: logger.Named("w1"), + WorkerRPCInterval: time.Second, }) wg.Add(2) diff --git a/internal/tests/cluster/parallel/multi_controller_worker_test.go b/internal/tests/cluster/parallel/multi_controller_worker_test.go index 5d271f4ce..aeb47abcd 100644 --- a/internal/tests/cluster/parallel/multi_controller_worker_test.go +++ b/internal/tests/cluster/parallel/multi_controller_worker_test.go @@ -50,9 +50,10 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) { wg.Wait() w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...), - Logger: logger.Named("w1"), + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: append(c1.ClusterAddrs(), c2.ClusterAddrs()...), + Logger: logger.Named("w1"), + WorkerRPCInterval: time.Second, }) wg.Add(2) @@ -95,9 +96,10 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) { wg.Wait() w3 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w3"), + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w3"), + WorkerRPCInterval: time.Second, }) wg.Add(2) @@ -151,22 +153,28 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) { initialUpstreams := append(c1.ClusterAddrs(), "127.0.0.9") w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: initialUpstreams, - Logger: logger.Named("w1"), - SuccessfulStatusGracePeriodDuration: 1 * time.Second, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: initialUpstreams, + Logger: logger.Named("w1"), + SuccessfulControllerRPCGracePeriodDuration: 1 * time.Second, + WorkerRPCInterval: time.Second, }) - // Wait for worker to send status - helper.ExpectWorkers(t, c1, w1) - - // Upstreams should be equivalent to the controller cluster addr after status updates + // Wait for worker to send routing info require.Eventually(func() bool { - if w1.Worker().LastStatusSuccess() == nil { + if err := w1.Worker().WaitForNextSuccessfulRoutingInfoUpdate(); err != nil { return false } - return slices.Equal(c1.ClusterAddrs(), w1.Worker().LastStatusSuccess().LastCalculatedUpstreams) - }, 4*time.Second, 250*time.Millisecond) + return true + }, 30*time.Second, time.Second) + + // Upstreams should be equivalent to the controller cluster addr after routing info updates + require.Eventually(func() bool { + if w1.Worker().LastRoutingInfoSuccess() == nil { + return false + } + return slices.Equal(c1.ClusterAddrs(), w1.Worker().LastRoutingInfoSuccess().LastCalculatedUpstreams) + }, 30*time.Second, time.Second) // Bring down the controller wg := new(sync.WaitGroup) @@ -178,7 +186,7 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) { // Upstreams should now match initial upstreams require.Eventually(func() bool { - return slices.Equal(initialUpstreams, w1.Worker().LastStatusSuccess().LastCalculatedUpstreams) - }, 4*time.Second, 250*time.Millisecond) + return slices.Equal(initialUpstreams, w1.Worker().LastRoutingInfoSuccess().LastCalculatedUpstreams) + }, 30*time.Second, time.Second) wg.Wait() } diff --git a/internal/tests/cluster/parallel/unix_listener_test.go b/internal/tests/cluster/parallel/unix_listener_test.go index 8bd587456..d125b22f0 100644 --- a/internal/tests/cluster/parallel/unix_listener_test.go +++ b/internal/tests/cluster/parallel/unix_listener_test.go @@ -9,6 +9,7 @@ import ( "os" "path" "testing" + "time" "github.com/hashicorp/boundary/api" "github.com/hashicorp/boundary/api/scopes" @@ -36,9 +37,9 @@ func TestUnixListener(t *testing.T) { tempDir, err := os.MkdirTemp("", "boundary-unix-listener-test") require.NoError(err) - defer func() { + t.Cleanup(func() { require.NoError(os.RemoveAll(tempDir)) - }() + }) for _, l := range conf.Listeners { switch l.Purpose[0] { @@ -81,16 +82,16 @@ func TestUnixListener(t *testing.T) { require.NoError(err) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - Config: wconf, - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w1"), + Config: wconf, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w1"), + WorkerRPCInterval: time.Second, }) helper.ExpectWorkers(t, c1, w1) require.NoError(w1.Worker().Shutdown()) - helper.ExpectWorkers(t, c1) require.NoError(c1.Controller().Shutdown()) diff --git a/internal/tests/cluster/sequential/session_cleanup_test.go b/internal/tests/cluster/sequential/session_cleanup_test.go index 8a1f8ef5a..bb9d98ab5 100644 --- a/internal/tests/cluster/sequential/session_cleanup_test.go +++ b/internal/tests/cluster/sequential/session_cleanup_test.go @@ -14,7 +14,6 @@ import ( "github.com/hashicorp/boundary/internal/cmd/config" "github.com/hashicorp/boundary/internal/daemon/controller" "github.com/hashicorp/boundary/internal/daemon/worker" - "github.com/hashicorp/boundary/internal/event" "github.com/hashicorp/boundary/internal/server" "github.com/hashicorp/boundary/internal/session" "github.com/hashicorp/boundary/internal/tests/helper" @@ -51,26 +50,19 @@ var timeoutBurdenCases = []timeoutBurdenType{timeoutBurdenTypeDefault, timeoutBu func controllerGracePeriod(ty timeoutBurdenType) time.Duration { if ty == timeoutBurdenTypeWorker { - return helper.DefaultWorkerStatusGracePeriod * 10 + return helper.DefaultControllerRPCGracePeriod * 10 } - return helper.DefaultWorkerStatusGracePeriod -} - -func workerGracePeriod(ty timeoutBurdenType) time.Duration { - return helper.DefaultWorkerStatusGracePeriod + return helper.DefaultControllerRPCGracePeriod } // TestSessionCleanup is the main test for session cleanup, and // dispatches to the individual subtests. func TestSessionCleanup(t *testing.T) { - for _, burdenCase := range timeoutBurdenCases { - burdenCase := burdenCase - t.Run(string(burdenCase), func(t *testing.T) { - t.Run("single_controller", testWorkerSessionCleanupSingle(burdenCase)) - t.Run("multi_controller", testWorkerSessionCleanupMulti(burdenCase)) - }) - } + t.Run("default/single_controller", testWorkerSessionCleanupSingle("default")) + t.Run("default/multi_controller", testWorkerSessionCleanupMulti("default")) + t.Run("worker/single_controller", testWorkerSessionCleanupSingle("worker")) + t.Run("worker/multi_controller", testWorkerSessionCleanupMulti("worker")) } func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testing.T) { @@ -90,11 +82,13 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin pl, err := net.Listen("tcp", "[::1]:0") require.NoError(err) c1 := controller.NewTestController(t, &controller.TestControllerOpts{ - Config: conf, - InitialResourcesSuffix: "1234567890", - Logger: logger.Named("c1"), - PublicClusterAddr: pl.Addr().String(), - WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase), + Config: conf, + InitialResourcesSuffix: "1234567890", + Logger: logger.Named("c1"), + PublicClusterAddr: pl.Addr().String(), + WorkerRPCGracePeriod: controllerGracePeriod(burdenCase), + // Run the scheduler more often to speed up cleanup of orphaned connections + SchedulerRunJobInterval: time.Second, }) helper.ExpectWorkers(t, c1) @@ -113,10 +107,11 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin require.NotEmpty(t, proxy.ListenerAddr()) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: []string{proxy.ListenerAddr()}, - Logger: logger.Named("w1"), - SuccessfulStatusGracePeriodDuration: workerGracePeriod(burdenCase), + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: []string{proxy.ListenerAddr()}, + Logger: logger.Named("w1"), + SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod, + WorkerRPCInterval: time.Second, }) helper.ExpectWorkers(t, c1, w1) @@ -147,11 +142,15 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin sConn := sess.Connect(ctx, t) // Run initial send/receive test, make sure things are working - event.WriteSysEvent(ctx, op, "running initial send/recv test") + t.Log("running initial send/recv test") sConn.TestSendRecvAll(t) + // Wait for a session info to be sent to the server, so the controller has + // at least one record of the connection. + w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t) + // Kill the link - event.WriteSysEvent(ctx, op, "pausing controller/worker link") + t.Log("pausing controller/worker link") proxy.Pause() // Wait for failure connection state (depends on burden case) @@ -171,15 +170,9 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin sConn.TestSendRecvFail(t) // Resume the connection, and reconnect. - event.WriteSysEvent(ctx, op, "resuming controller/worker link") + t.Log("resuming controller/worker link") proxy.Resume() - require.Eventually(func() bool { - err := w1.Worker().WaitForNextSuccessfulStatusUpdate() - if err != nil { - return false - } - return true - }, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second) + helper.ExpectWorkers(t, c1, w1) // Do something post-reconnect depending on burden case. Note in // the default case, both worker and controller should be @@ -195,7 +188,7 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin } // Proceed with new connection test - event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link") + t.Log("connecting to new session after resuming controller/worker link") sess = helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup() sConn = sess.Connect(ctx, t) sConn.TestSendRecvAll(t) @@ -222,11 +215,11 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing pl1, err := net.Listen("tcp", "[::1]:0") require.NoError(err) c1 := controller.NewTestController(t, &controller.TestControllerOpts{ - Config: conf1, - InitialResourcesSuffix: "1234567890", - Logger: logger.Named("c1"), - PublicClusterAddr: pl1.Addr().String(), - WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase), + Config: conf1, + InitialResourcesSuffix: "1234567890", + Logger: logger.Named("c1"), + PublicClusterAddr: pl1.Addr().String(), + WorkerRPCGracePeriod: controllerGracePeriod(burdenCase), }) // ****************** @@ -235,9 +228,9 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing pl2, err := net.Listen("tcp", "[::1]:0") require.NoError(err) c2 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{ - Logger: logger.Named("c2"), - PublicClusterAddr: pl2.Addr().String(), - WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase), + Logger: logger.Named("c2"), + PublicClusterAddr: pl2.Addr().String(), + WorkerRPCGracePeriod: controllerGracePeriod(burdenCase), }) wg := new(sync.WaitGroup) @@ -286,11 +279,13 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing // ** Worker ** // ************ w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: []string{p1.ListenerAddr(), p2.ListenerAddr()}, - Logger: logger.Named("w1"), - SuccessfulStatusGracePeriodDuration: workerGracePeriod(burdenCase), + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: []string{p1.ListenerAddr(), p2.ListenerAddr()}, + Logger: logger.Named("w1"), + SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod, + WorkerRPCInterval: time.Second, }) + wg.Add(2) go func() { defer wg.Done() @@ -328,39 +323,30 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing sConn := sess.Connect(ctx, t) // Run initial send/receive test, make sure things are working - event.WriteSysEvent(ctx, op, "running initial send/recv test") + t.Log("running initial send/recv test") sConn.TestSendRecvAll(t) + // Wait for a session info to be sent to the server, so the controller has + // at least one record of the connection. + w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t) + // Kill connection to first controller, and run test again, should // pass, deferring to other controller. Wait for the next // successful status report to ensure this. - event.WriteSysEvent(ctx, op, "pausing link to controller #1") + t.Log("pausing link to controller #1") p1.Pause() - require.Eventually(func() bool { - err := w1.Worker().WaitForNextSuccessfulStatusUpdate() - if err != nil { - return false - } - return true - }, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second) + w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t) sConn.TestSendRecvAll(t) // Resume first controller, pause second. This one should work too. - event.WriteSysEvent(ctx, op, "pausing link to controller #2, resuming #1") + t.Log("pausing link to controller #2, resuming #1") p1.Resume() p2.Pause() - require.Eventually(func() bool { - err := w1.Worker().WaitForNextSuccessfulStatusUpdate() - if err != nil { - return false - } - return true - }, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second) - sConn.TestSendRecvAll(t) + w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t) // Kill the first controller connection again. This one should fail // due to lack of any connection. - event.WriteSysEvent(ctx, op, "pausing link to controller #1 again, both connections should be offline") + t.Log("pausing link to controller #1 again, both connections should be offline") p1.Pause() // Wait for failure connection state (depends on burden case) @@ -382,16 +368,10 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing sConn.TestSendRecvFail(t) // Finally resume both, try again. Should behave as per normal. - event.WriteSysEvent(ctx, op, "resuming connections to both controllers") + t.Log("resuming connections to both controllers") p1.Resume() p2.Resume() - require.Eventually(func() bool { - err := w1.Worker().WaitForNextSuccessfulStatusUpdate() - if err != nil { - return false - } - return true - }, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second) + w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t) // Do something post-reconnect depending on burden case. Note in // the default case, both worker and controller should be @@ -407,7 +387,7 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing } // Proceed with new connection test - event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link") + t.Log("connecting to new session after resuming controller/worker link") sess = helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup() sConn = sess.Connect(ctx, t) sConn.TestSendRecvAll(t) diff --git a/internal/tests/cluster/sequential/worker_bytesupdown_test.go b/internal/tests/cluster/sequential/worker_bytesupdown_test.go index 5d3f9a6f5..ff3be16e5 100644 --- a/internal/tests/cluster/sequential/worker_bytesupdown_test.go +++ b/internal/tests/cluster/sequential/worker_bytesupdown_test.go @@ -7,6 +7,7 @@ import ( "context" "net" "testing" + "time" "github.com/hashicorp/boundary/api/sessions" "github.com/hashicorp/boundary/api/targets" @@ -45,10 +46,10 @@ func TestWorkerBytesUpDown(t *testing.T) { } c1 := controller.NewTestController(t, &controller.TestControllerOpts{ - Config: conf, - InitialResourcesSuffix: "1234567890", - Logger: logger.Named("c1"), - WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod, + Config: conf, + InitialResourcesSuffix: "1234567890", + Logger: logger.Named("c1"), + WorkerRPCGracePeriod: helper.DefaultControllerRPCGracePeriod, }) helper.ExpectWorkers(t, c1) @@ -65,11 +66,12 @@ func TestWorkerBytesUpDown(t *testing.T) { t.Cleanup(func() { _ = proxy.Close() }) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: []string{proxy.ListenerAddr()}, - Logger: logger.Named("w1"), - SuccessfulStatusGracePeriodDuration: helper.DefaultSuccessfulStatusGracePeriod, - EnableIPv6: true, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: []string{proxy.ListenerAddr()}, + Logger: logger.Named("w1"), + SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod, + EnableIPv6: true, + WorkerRPCInterval: time.Second, }) helper.ExpectWorkers(t, c1, w1) @@ -105,8 +107,8 @@ func TestWorkerBytesUpDown(t *testing.T) { conn := sess.Connect(ctx, t) conn.TestSendRecvAll(t) - // Wait for next status and then check DB for bytes up and down - helper.ExpectWorkers(t, c1, w1) + // Wait for next statistics update and then check DB for bytes up and down + w1.Worker().TestWaitForNextSuccessfulStatisticsUpdate(t) dbConns1, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId) require.NoError(err) @@ -122,11 +124,11 @@ func TestWorkerBytesUpDown(t *testing.T) { require.NotZero(apiRes1.Item.Connections[0].BytesUp) require.NotZero(apiRes1.Item.Connections[0].BytesDown) - // Send/recv some more traffic, close connection, wait for the next status + // Send/recv some more traffic, close connection, wait for the next statistics // update and check everything again. conn.TestSendRecvAll(t) require.NoError(conn.Close()) - helper.ExpectWorkers(t, c1, w1) + w1.Worker().TestWaitForNextSuccessfulStatisticsUpdate(t) dbConns2, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId) require.NoError(err) diff --git a/internal/tests/cluster/sequential/worker_proxy_test.go b/internal/tests/cluster/sequential/worker_proxy_test.go index 1c1180461..f63e65387 100644 --- a/internal/tests/cluster/sequential/worker_proxy_test.go +++ b/internal/tests/cluster/sequential/worker_proxy_test.go @@ -7,6 +7,7 @@ import ( "context" "net" "testing" + "time" "github.com/hashicorp/boundary/api/targets" "github.com/hashicorp/boundary/internal/cmd/config" @@ -46,10 +47,10 @@ func TestWorkerSessionProxyMultipleConnections(t *testing.T) { } c1 := controller.NewTestController(t, &controller.TestControllerOpts{ - Config: conf, - InitialResourcesSuffix: "1234567890", - Logger: logger.Named("c1"), - WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod, + Config: conf, + InitialResourcesSuffix: "1234567890", + Logger: logger.Named("c1"), + WorkerRPCGracePeriod: helper.DefaultControllerRPCGracePeriod, }) helper.ExpectWorkers(t, c1) @@ -66,11 +67,12 @@ func TestWorkerSessionProxyMultipleConnections(t *testing.T) { require.NotEmpty(t, proxy.ListenerAddr()) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: []string{proxy.ListenerAddr()}, - Logger: logger.Named("w1"), - SuccessfulStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod, - EnableIPv6: true, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: []string{proxy.ListenerAddr()}, + Logger: logger.Named("w1"), + SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod, + EnableIPv6: true, + WorkerRPCInterval: time.Second, }) helper.ExpectWorkers(t, c1, w1) diff --git a/internal/tests/cluster/sequential/worker_tagging_test.go b/internal/tests/cluster/sequential/worker_tagging_test.go index d1d7bbbaf..4a310336b 100644 --- a/internal/tests/cluster/sequential/worker_tagging_test.go +++ b/internal/tests/cluster/sequential/worker_tagging_test.go @@ -5,6 +5,7 @@ package sequential import ( "testing" + "time" "github.com/hashicorp/boundary/api/targets" "github.com/hashicorp/boundary/internal/cmd/config" @@ -57,10 +58,11 @@ func TestWorkerTagging(t *testing.T) { "foo": {"bar"}, } w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - Config: conf, - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w1"), + Config: conf, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w1"), + WorkerRPCInterval: time.Second, }) w1Addr := w1.ProxyAddrs()[0] @@ -73,10 +75,11 @@ func TestWorkerTagging(t *testing.T) { "az": {"one", "two", "three"}, } w2 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - Config: conf, - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w2"), + Config: conf, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w2"), + WorkerRPCInterval: time.Second, }) w2Addr := w2.ProxyAddrs()[0] @@ -89,10 +92,11 @@ func TestWorkerTagging(t *testing.T) { "az": {"one", "three"}, } w3 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - Config: conf, - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w3"), + Config: conf, + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w3"), + WorkerRPCInterval: time.Second, }) w3Addr := w3.ProxyAddrs()[0] diff --git a/internal/tests/cluster/sequential/x509_verification_test.go b/internal/tests/cluster/sequential/x509_verification_test.go index 11c2dcf5c..a73c09d88 100644 --- a/internal/tests/cluster/sequential/x509_verification_test.go +++ b/internal/tests/cluster/sequential/x509_verification_test.go @@ -15,6 +15,7 @@ import ( "net/http" "sync" "testing" + "time" apiproxy "github.com/hashicorp/boundary/api/proxy" "github.com/hashicorp/boundary/api/targets" @@ -43,6 +44,14 @@ func TestCustomX509Verification_Client(t *testing.T) { Name: "test", }) req.NoError(event.InitSysEventer(logger, testLock, "use-TestCustomX509Verification", event.WithEventerConfig(&ec.EventerConfig))) + t.Cleanup(func() { + event.TestResetSystEventer(t) + }) + t.Cleanup(func() { + all, err := io.ReadAll(ec.AllEvents) + require.NoError(t, err) + t.Log(string(all)) + }) conf, err := config.DevController() conf.Eventing = &ec.EventerConfig @@ -57,12 +66,12 @@ func TestCustomX509Verification_Client(t *testing.T) { conf.Eventing = &ec.EventerConfig req.NoError(err) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), - Logger: logger.Named("w1"), + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + Logger: logger.Named("w1"), + WorkerRPCInterval: time.Second, }) - // Give time for it to connect helper.ExpectWorkers(t, c1, w1) // Connect target @@ -221,8 +230,9 @@ func testCustomX509Verification_Server(ec event.TestConfig, certPool *x509.CertP conf.Eventing = &ec.EventerConfig req.NoError(err) w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ - WorkerAuthKms: c1.Config().WorkerAuthKms, - InitialUpstreams: c1.ClusterAddrs(), + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: c1.ClusterAddrs(), + WorkerRPCInterval: time.Second, }) w1.Worker().TestOverrideX509VerifyCertPool = certPool w1.Worker().TestOverrideX509VerifyDnsName = dnsName diff --git a/internal/tests/helper/testing_helper.go b/internal/tests/helper/testing_helper.go index 6788f116e..782be0cc3 100644 --- a/internal/tests/helper/testing_helper.go +++ b/internal/tests/helper/testing_helper.go @@ -27,22 +27,14 @@ import ( ) const ( - DefaultWorkerStatusGracePeriod = time.Second * 15 - DefaultSuccessfulStatusGracePeriod = time.Second * 15 + DefaultControllerRPCGracePeriod = time.Second * 15 expectConnectionStateOnControllerTimeout = time.Minute * 2 - expectConnectionStateOnWorkerTimeout = DefaultWorkerStatusGracePeriod * 3 + expectConnectionStateOnWorkerTimeout = DefaultControllerRPCGracePeriod * 3 // This is the interval that we check states on in the worker. It // needs to be particularly granular to ensure that we allow for // adequate time to catch any edge cases where the connection state // disappears from the worker before we can update the state. - // - // As of this writing, the (hardcoded) status request interval on - // the worker is 2 seconds, and a session is removed from the state - // when it has no connections left on the *next* pass. A one second - // interval would only mean two chances to check with a high - // possibility of skew; while it seems okay I'm still not 100% - // comfortable with this little resolution. expectConnectionStateOnWorkerInterval = time.Millisecond * 100 ) @@ -465,22 +457,27 @@ func NewTestTcpServer(t *testing.T) *TestTcpServer { // ExpectWorkers is a blocking call, where the method validates that the expected workers // can be found in the controllers status update. If the provided list of workers is empty, -// this method will validate that the controller worker status is empty. +// this method will validate that the controller worker routing info is not called. func ExpectWorkers(t *testing.T, c *controller.TestController, workers ...*worker.TestWorker) { + t.Helper() // validate the controller has no reported workers if len(workers) == 0 { - require.Eventually(t, func() bool { + assert.Eventually(t, func() bool { workers := []string{} - c.Controller().WorkerStatusUpdateTimes().Range(func(workerId, lastStatusTime any) bool { - require.NotNil(t, workerId) - require.NotNil(t, lastStatusTime) - if time.Since(lastStatusTime.(time.Time)) < DefaultSuccessfulStatusGracePeriod { + c.Controller().WorkerRoutingInfoUpdateTimes().Range(func(workerId, lastRoutingInfo any) bool { + if assert.NotNil(t, workerId) { + return false + } + if assert.NotNil(t, lastRoutingInfo) { + return false + } + if time.Since(lastRoutingInfo.(time.Time)) < DefaultControllerRPCGracePeriod { workers = append(workers, workerId.(string)) } return true }) return len(workers) == 0 - }, 2*DefaultSuccessfulStatusGracePeriod, 250*time.Millisecond) + }, 2*DefaultControllerRPCGracePeriod, 250*time.Millisecond) return } @@ -490,11 +487,11 @@ func ExpectWorkers(t *testing.T, c *controller.TestController, workers ...*worke wg.Add(1) go func() { defer wg.Done() - require.NoError(t, w.Worker().WaitForNextSuccessfulStatusUpdate()) - require.Eventually(t, func() bool { - _, ok := c.Controller().WorkerStatusUpdateTimes().Load(w.Name()) + assert.NoError(t, w.Worker().WaitForNextSuccessfulRoutingInfoUpdate()) + assert.Eventually(t, func() bool { + _, ok := c.Controller().WorkerRoutingInfoUpdateTimes().Load(w.Name()) return ok - }, 30*time.Second, 100*time.Millisecond) + }, 30*time.Second, time.Second) }() } wg.Wait()