Worker status refactor (#5417)

* oss: feat(proto): statistics rpc added to server coordination service

* oss: feat(session): add update statistics status service

* oss: feat(handlers): add Statistics method to worker service server

* oss: feat(worker): add statistics ticking

* oss: internal/proto: define the new RoutingInfo RPC

* oss: internal/server: repository changes for RoutingInfo RPC

* oss: internal/daemon: add RoutingInfo handler to worker service

The new RoutingInfo RPC is used to get information from the worker
that is used to make routing decisions on the controller. It also
handles the initial request from the worker.

* oss: feat(db): server_worker_session_info table

* oss: feat(proto): session info rpc for server coordination service

* oss: feat(server): worker session info request data type

* oss: feat(worker): session info ticker

* oss: feat(handler): server coordination handler for session info

* oss: feat(session): utilize session info request time for session cleanup

* oss: proto: fix some gotags comments

Some of these fields were missing comments, while others had
incorrectly formatted comments.

* oss: worker: send connection status to server

This is required for the correct tracking of orphaned connections

* oss: internal/daemon/worker: add function to wait for Statistics RPC

* oss: internal/daemon/worker: add routing info RPC

The new RoutingInfo RPC replaces part of the Status RPC,
namely the part which is used by the controller and worker
to perform decisions around the establishment of new sessions
and new worker connections.

* oss: internal/daemon/worker: remove unnecessary error check

* oss: internal/daemon/worker: lock access to config and upstreams

The new confLock protects access to the conf value and to the
addressReceivers slice.

* oss: worker: replace Status RPC with RoutingInfo

* oss: worker: deprecate the Status RPC and related types

The Status RPC and related messages have been replaced
by the RoutingInfo/SessionInfo/Statistics RPCs. The controller
will continue to support the Status RPC until two new minor
versions have been released.

* oss: internal/event: add default deny for new worker RPCs

* oss: internal/daemon/worker: allow RPC interval to be set

The new SessionInfo/RoutingInfo/Statistics RPCs run at
long intervals, which make some of our cluster tests take
a long time to complete. This allows their interval to be
configured in tests.

* oss: all: use TestWorkerRPCInterval in slow tests

This should speed up these tests significantly.

* oss: internal/tests/cluster: reset sys eventer in test

---------

Co-authored-by: Damian Debkowski <damian.debkowski@hashicorp.com>
This commit is contained in:
Johan Brandhorst-SatzkornandDamian Debkowski authored and GitHub committed 2025-01-09 10:26:37 -08:00
1 parent 554195f49f
commit ec0a3f964b
70 files changed
+5492 -1618

No files matched your search

@@ -86,17 +86,17 @@ type ServerWorkerStatus struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Id of the worker.
// Id of the worker, used after the first request.
PublicId string `protobuf:"bytes,10,opt,name=public_id,json=publicId,proto3" json:"public_id,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"`
// Name of the resource (optional)
// Name of the worker, used to identify workers in the KMS registration flow.
Name string `protobuf:"bytes,20,opt,name=name,proto3" json:"name,omitempty" class:"public"` // @gotags: `class:"public"`
// Description of the resource (optional)
// Description of the worker (optional). Only used by KMS workers.
Description string `protobuf:"bytes,25,opt,name=description,proto3" json:"description,omitempty" class:"public"` // @gotags: `class:"public"`
// Address for the worker. This must be set.
// Address for the worker.
Address string `protobuf:"bytes,30,opt,name=address,proto3" json:"address,omitempty" class:"public"` // @gotags: `class:"public"`
// Tags for workers
Tags []*TagPair `protobuf:"bytes,40,rep,name=tags,proto3" json:"tags,omitempty"`
// The key id for this worker, if applicable (optional)
// The key id of the worker, used to identify workers in the PKI registration flow.
KeyId string `protobuf:"bytes,50,opt,name=key_id,json=keyId,proto3" json:"key_id,omitempty" class:"public"` // @gotags: `class:"public"`
// The version of Boundary the worker binary is running
ReleaseVersion string `protobuf:"bytes,60,opt,name=release_version,proto3" json:"release_version,omitempty" class:"public" eventstream:"observation"` // @gotags: `class:"public" eventstream:"observation"`
File diff suppressed because it is too large. Load diff
@@ -24,18 +24,36 @@ const _ = grpc.SupportPackageIsVersion7
const (
ServerCoordinationService_Status_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/Status"
ServerCoordinationService_ListHcpbWorkers_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/ListHcpbWorkers"
ServerCoordinationService_Statistics_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/Statistics"
ServerCoordinationService_RoutingInfo_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/RoutingInfo"
ServerCoordinationService_SessionInfo_FullMethodName = "/controller.servers.services.v1.ServerCoordinationService/SessionInfo"
)
// ServerCoordinationServiceClient is the client API for ServerCoordinationService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type ServerCoordinationServiceClient interface {
// Deprecated: Do not use.
// Status gets worker status requests which include the ongoing jobs the worker is handling and
// returns the status response which includes the changes the controller would like to make to
// jobs as well as provide a list of the controllers in the system.
// This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0.
Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error)
// Returns the addresses of HCP Boundary workers, if any
ListHcpbWorkers(ctx context.Context, in *ListHcpbWorkersRequest, opts ...grpc.CallOption) (*ListHcpbWorkersResponse, error)
// Statistics is used by the worker to report non-essential statistics about its sessions and connections.
Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
// RoutingInfo is used by the worker to inform the controller of information
// required by the controller to make session routing decisions and any startup information.
// The controller may inform the worker of any downstream workers that should be disconnected.
// If the worker fails to successfully report its routing info to the controller,
// it will try again later.
RoutingInfo(ctx context.Context, in *RoutingInfoRequest, opts ...grpc.CallOption) (*RoutingInfoResponse, error)
// SessionInfo is used by the worker to inform the controller of all the sessions
// it is managing. The controller may inform the worker if any sessions need to be changed.
// If the worker repeatedly fails to successfully report its session info to the controller,
// it will tear down any running sessions.
SessionInfo(ctx context.Context, in *SessionInfoRequest, opts ...grpc.CallOption) (*SessionInfoResponse, error)
}
type serverCoordinationServiceClient struct {
@@ -46,6 +64,7 @@ func NewServerCoordinationServiceClient(cc grpc.ClientConnInterface) ServerCoord
return &serverCoordinationServiceClient{cc}
}
// Deprecated: Do not use.
func (c *serverCoordinationServiceClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) {
out := new(StatusResponse)
err := c.cc.Invoke(ctx, ServerCoordinationService_Status_FullMethodName, in, out, opts...)
@@ -64,16 +83,58 @@ func (c *serverCoordinationServiceClient) ListHcpbWorkers(ctx context.Context, i
return out, nil
}
func (c *serverCoordinationServiceClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) {
out := new(StatisticsResponse)
err := c.cc.Invoke(ctx, ServerCoordinationService_Statistics_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *serverCoordinationServiceClient) RoutingInfo(ctx context.Context, in *RoutingInfoRequest, opts ...grpc.CallOption) (*RoutingInfoResponse, error) {
out := new(RoutingInfoResponse)
err := c.cc.Invoke(ctx, ServerCoordinationService_RoutingInfo_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *serverCoordinationServiceClient) SessionInfo(ctx context.Context, in *SessionInfoRequest, opts ...grpc.CallOption) (*SessionInfoResponse, error) {
out := new(SessionInfoResponse)
err := c.cc.Invoke(ctx, ServerCoordinationService_SessionInfo_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// ServerCoordinationServiceServer is the server API for ServerCoordinationService service.
// All implementations must embed UnimplementedServerCoordinationServiceServer
// for forward compatibility
type ServerCoordinationServiceServer interface {
// Deprecated: Do not use.
// Status gets worker status requests which include the ongoing jobs the worker is handling and
// returns the status response which includes the changes the controller would like to make to
// jobs as well as provide a list of the controllers in the system.
// This RPC is deprecated and is safe to remove after the release of Boundary v0.20.0.
Status(context.Context, *StatusRequest) (*StatusResponse, error)
// Returns the addresses of HCP Boundary workers, if any
ListHcpbWorkers(context.Context, *ListHcpbWorkersRequest) (*ListHcpbWorkersResponse, error)
// Statistics is used by the worker to report non-essential statistics about its sessions and connections.
Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
// RoutingInfo is used by the worker to inform the controller of information
// required by the controller to make session routing decisions and any startup information.
// The controller may inform the worker of any downstream workers that should be disconnected.
// If the worker fails to successfully report its routing info to the controller,
// it will try again later.
RoutingInfo(context.Context, *RoutingInfoRequest) (*RoutingInfoResponse, error)
// SessionInfo is used by the worker to inform the controller of all the sessions
// it is managing. The controller may inform the worker if any sessions need to be changed.
// If the worker repeatedly fails to successfully report its session info to the controller,
// it will tear down any running sessions.
SessionInfo(context.Context, *SessionInfoRequest) (*SessionInfoResponse, error)
mustEmbedUnimplementedServerCoordinationServiceServer()
}
@@ -87,6 +148,15 @@ func (UnimplementedServerCoordinationServiceServer) Status(context.Context, *Sta
func (UnimplementedServerCoordinationServiceServer) ListHcpbWorkers(context.Context, *ListHcpbWorkersRequest) (*ListHcpbWorkersResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListHcpbWorkers not implemented")
}
func (UnimplementedServerCoordinationServiceServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented")
}
func (UnimplementedServerCoordinationServiceServer) RoutingInfo(context.Context, *RoutingInfoRequest) (*RoutingInfoResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method RoutingInfo not implemented")
}
func (UnimplementedServerCoordinationServiceServer) SessionInfo(context.Context, *SessionInfoRequest) (*SessionInfoResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method SessionInfo not implemented")
}
func (UnimplementedServerCoordinationServiceServer) mustEmbedUnimplementedServerCoordinationServiceServer() {
}
@@ -137,6 +207,60 @@ func _ServerCoordinationService_ListHcpbWorkers_Handler(srv interface{}, ctx con
return interceptor(ctx, in, info, handler)
}
func _ServerCoordinationService_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(StatisticsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ServerCoordinationServiceServer).Statistics(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ServerCoordinationService_Statistics_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ServerCoordinationServiceServer).Statistics(ctx, req.(*StatisticsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ServerCoordinationService_RoutingInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RoutingInfoRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ServerCoordinationServiceServer).RoutingInfo(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ServerCoordinationService_RoutingInfo_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ServerCoordinationServiceServer).RoutingInfo(ctx, req.(*RoutingInfoRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ServerCoordinationService_SessionInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SessionInfoRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ServerCoordinationServiceServer).SessionInfo(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ServerCoordinationService_SessionInfo_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ServerCoordinationServiceServer).SessionInfo(ctx, req.(*SessionInfoRequest))
}
return interceptor(ctx, in, info, handler)
}
// ServerCoordinationService_ServiceDesc is the grpc.ServiceDesc for ServerCoordinationService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -152,6 +276,18 @@ var ServerCoordinationService_ServiceDesc = grpc.ServiceDesc{
MethodName: "ListHcpbWorkers",
Handler: _ServerCoordinationService_ListHcpbWorkers_Handler,
},
{
MethodName: "Statistics",
Handler: _ServerCoordinationService_Statistics_Handler,
},
{
MethodName: "RoutingInfo",
Handler: _ServerCoordinationService_RoutingInfo_Handler,
},
{
MethodName: "SessionInfo",
Handler: _ServerCoordinationService_SessionInfo_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "controller/servers/services/v1/server_coordination_service.proto",