From c81398cbf346ad911e68f22703c5b8676e9e4665 Mon Sep 17 00:00:00 2001 From: Hugo Vieira Date: Thu, 6 Oct 2022 16:52:26 +0100 Subject: [PATCH] feat(worker): Report bytes up and down on Status updates With this commit, the Worker now sends BytesUp and BytesDown for each of the connections it handles to the Controller in the Status request. The Controller then persists these fields to the database. There is also a new trigger to prevent bytes_up and bytes_down to change after a connection has been closed, as the last update to a session_connection row should be the one that is performed as part of the connection closure process. --- .../daemon/cluster/handlers/worker_service.go | 18 +- internal/daemon/worker/session/session.go | 4 + internal/daemon/worker/status.go | 2 + .../server_coordination_service.pb.go | 308 ++++++++++-------- .../v1/server_coordination_service.proto | 2 + internal/session/repository_connection.go | 47 ++- .../session/repository_connection_test.go | 89 +++++ internal/session/repository_session.go | 10 +- .../session/service_worker_status_report.go | 51 ++- .../service_worker_status_report_test.go | 88 ++--- .../tests/cluster/worker_bytesupdown_test.go | 131 ++++++++ 11 files changed, 538 insertions(+), 212 deletions(-) create mode 100644 internal/tests/cluster/worker_bytesupdown_test.go diff --git a/internal/daemon/cluster/handlers/worker_service.go b/internal/daemon/cluster/handlers/worker_service.go index b370dff6a..ec43ab51a 100644 --- a/internal/daemon/cluster/handlers/worker_service.go +++ b/internal/daemon/cluster/handlers/worker_service.go @@ -140,7 +140,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques WorkerId: wrk.GetPublicId(), } - stateReport := make([]session.StateReport, 0, len(req.GetJobs())) + stateReport := make([]*session.StateReport, 0, len(req.GetJobs())) for _, jobStatus := range req.GetJobs() { switch jobStatus.Job.GetType() { @@ -157,15 +157,19 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques continue } - sr := session.StateReport{ - SessionId: si.GetSessionId(), - ConnectionIds: make([]string, 0, len(si.GetConnections())), + sr := &session.StateReport{ + SessionId: si.GetSessionId(), + Connections: make([]*session.Connection, 0, len(si.GetConnections())), } for _, conn := range si.GetConnections() { switch conn.Status { case pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_AUTHORIZED, pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED: - sr.ConnectionIds = append(sr.ConnectionIds, conn.GetConnectionId()) + sr.Connections = append(sr.Connections, &session.Connection{ + PublicId: conn.GetConnectionId(), + BytesUp: conn.GetBytesUp(), + BytesDown: conn.GetBytesDown(), + }) } } stateReport = append(stateReport, sr) @@ -191,9 +195,9 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques } for _, na := range notActive { var connChanges []*pbs.Connection - for _, connId := range na.ConnectionIds { + for _, conn := range na.Connections { connChanges = append(connChanges, &pbs.Connection{ - ConnectionId: connId, + ConnectionId: conn.GetPublicId(), Status: session.StatusClosed.ProtoVal(), }) } diff --git a/internal/daemon/worker/session/session.go b/internal/daemon/worker/session/session.go index 31369037b..201298034 100644 --- a/internal/daemon/worker/session/session.go +++ b/internal/daemon/worker/session/session.go @@ -386,6 +386,10 @@ func (s *sess) CancelAllLocalConnections() []string { return closedIds } +// ApplyConnectionCounterCallbacks sets a connection's bytes up and bytes +// down callbacks to the provided functions. Both functions must be safe for +// concurrent use. If there is no connection with the provided id, an error +// is returned. func (s *sess) ApplyConnectionCounterCallbacks(connId string, bytesUp func() uint64, bytesDown func() uint64) error { s.lock.Lock() defer s.lock.Unlock() diff --git a/internal/daemon/worker/status.go b/internal/daemon/worker/status.go index f2c27da37..2043b303f 100644 --- a/internal/daemon/worker/status.go +++ b/internal/daemon/worker/status.go @@ -126,6 +126,8 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager sess connections = append(connections, &pbs.Connection{ ConnectionId: k, Status: v.Status, + BytesUp: v.BytesUp(), + BytesDown: v.BytesDown(), }) } jobInfo.SessionId = sessionId diff --git a/internal/gen/controller/servers/services/server_coordination_service.pb.go b/internal/gen/controller/servers/services/server_coordination_service.pb.go index 3e3973b00..bda8c6ceb 100644 --- a/internal/gen/controller/servers/services/server_coordination_service.pb.go +++ b/internal/gen/controller/servers/services/server_coordination_service.pb.go @@ -278,6 +278,8 @@ type Connection struct { ConnectionId string `protobuf:"bytes,1,opt,name=connection_id,json=connectionId,proto3" json:"connection_id,omitempty" class:"public"` // @gotags: `class:"public"` Status CONNECTIONSTATUS `protobuf:"varint,2,opt,name=status,proto3,enum=controller.servers.services.v1.CONNECTIONSTATUS" json:"status,omitempty"` + BytesUp uint64 `protobuf:"varint,3,opt,name=bytes_up,json=bytesUp,proto3" json:"bytes_up,omitempty" class:"public"` // @gotags: `class:"public"` + BytesDown uint64 `protobuf:"varint,4,opt,name=bytes_down,json=bytesDown,proto3" json:"bytes_down,omitempty" class:"public"` // @gotags: `class:"public"` } func (x *Connection) Reset() { @@ -326,6 +328,20 @@ func (x *Connection) GetStatus() CONNECTIONSTATUS { return CONNECTIONSTATUS_CONNECTIONSTATUS_UNSPECIFIED } +func (x *Connection) GetBytesUp() uint64 { + if x != nil { + return x.BytesUp + } + return 0 +} + +func (x *Connection) GetBytesDown() uint64 { + if x != nil { + return x.BytesDown + } + return 0 +} + type SessionJobInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -925,153 +941,157 @@ var file_controller_servers_services_v1_server_coordination_service_proto_rawDes 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x1a, 0x23, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x7b, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x48, 0x0a, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x4f, 0x4e, 0x4e, - 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x52, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x22, 0xc4, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x4c, 0x0a, - 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x03, - 0x4a, 0x6f, 0x62, 0x12, 0x3b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, - 0x76, 0x31, 0x2e, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x12, 0x53, 0x0a, 0x0c, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, - 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x0a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x6e, 0x66, - 0x6f, 0x22, 0x42, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, - 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb5, 0x01, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x48, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, - 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0xb7, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x47, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x54, 0x59, 0x50, 0x45, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x14, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x42, 0x0a, 0x04, 0x54, - 0x59, 0x50, 0x45, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0f, - 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x10, 0x02, 0x22, - 0xcd, 0x01, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, - 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, - 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, - 0x1e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x67, - 0x73, 0x12, 0x4e, 0x0a, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x28, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x06, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x22, - 0x98, 0x01, 0x0a, 0x10, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x4f, 0x4e, + 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x75, + 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x55, 0x70, + 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x62, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x22, + 0xc4, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, + 0x64, 0x12, 0x45, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, - 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x12, 0x4d, 0x0a, 0x0c, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, - 0x76, 0x31, 0x2e, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x52, 0x0b, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xfa, 0x01, 0x0a, 0x0e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, - 0x0d, 0x6a, 0x6f, 0x62, 0x73, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x14, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x6a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x73, 0x12, 0x61, 0x0a, 0x14, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, - 0x65, 0x64, 0x5f, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x1e, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x52, 0x13, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x55, 0x70, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x5f, 0x69, 0x64, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x22, 0x36, 0x0a, 0x0a, 0x57, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, - 0x18, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5f, 0x0a, 0x17, 0x4c, 0x69, 0x73, - 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x2a, 0x92, 0x01, 0x0a, 0x10, 0x43, - 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x12, - 0x20, 0x0a, 0x1c, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x45, 0x44, - 0x10, 0x01, 0x12, 0x1e, 0x0a, 0x1a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, - 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x44, 0x10, 0x03, 0x2a, - 0x9e, 0x01, 0x0a, 0x0d, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x19, 0x0a, 0x15, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, - 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x43, 0x54, - 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x49, 0x4e, 0x47, - 0x10, 0x03, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x5f, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, - 0x2a, 0x37, 0x0a, 0x07, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x12, 0x17, 0x0a, 0x13, 0x4a, - 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x2a, 0x45, 0x0a, 0x0a, 0x43, 0x48, 0x41, - 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x12, 0x1a, 0x0a, 0x16, 0x43, 0x48, 0x41, 0x4e, 0x47, - 0x45, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x10, 0x01, - 0x32, 0x8d, 0x02, 0x0a, 0x19, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6f, 0x72, 0x64, - 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x69, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x84, 0x01, 0x0a, 0x0f, 0x4c, 0x69, - 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x36, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x4c, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x42, 0x51, 0x5a, 0x4f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, - 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x3b, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x4f, + 0x42, 0x54, 0x59, 0x50, 0x45, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x53, 0x0a, 0x0c, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x49, 0x6e, 0x66, + 0x6f, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x42, 0x0a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x42, 0x0a, 0x09, + 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, 0x0a, 0x03, 0x6a, 0x6f, 0x62, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, + 0x22, 0xb7, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x12, 0x47, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x2e, 0x54, 0x59, 0x50, 0x45, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x42, 0x0a, 0x04, 0x54, 0x59, 0x50, 0x45, 0x12, 0x14, + 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, + 0x54, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x45, 0x52, 0x10, 0x02, 0x22, 0xcd, 0x01, 0x0a, 0x0d, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, + 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x67, 0x73, 0x12, 0x4e, 0x0a, 0x0d, + 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x28, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, + 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4a, 0x04, 0x08, 0x0a, + 0x10, 0x0b, 0x52, 0x06, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x22, 0x98, 0x01, 0x0a, 0x10, 0x4a, + 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x35, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, + 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x12, 0x4d, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x48, + 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0xfa, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0d, 0x6a, 0x6f, 0x62, 0x73, + 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x52, 0x0c, 0x6a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, + 0x61, 0x0a, 0x14, 0x63, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x75, 0x70, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x18, 0x1e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x55, + 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x13, 0x63, + 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x4a, + 0x04, 0x08, 0x0a, 0x10, 0x0b, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x73, 0x22, 0x36, 0x0a, 0x0a, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x4c, 0x69, + 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x22, 0x5f, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, + 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x44, 0x0a, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, + 0x31, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x77, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x73, 0x2a, 0x92, 0x01, 0x0a, 0x10, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x12, 0x20, 0x0a, 0x1c, 0x43, 0x4f, + 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, + 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x45, 0x44, 0x10, 0x01, 0x12, 0x1e, 0x0a, + 0x1a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x1b, 0x0a, + 0x17, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x44, 0x10, 0x03, 0x2a, 0x9e, 0x01, 0x0a, 0x0d, 0x53, + 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x12, 0x1d, 0x0a, 0x19, + 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, + 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, + 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, + 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, + 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x1c, 0x0a, + 0x18, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, + 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, 0x10, 0x04, 0x2a, 0x37, 0x0a, 0x07, 0x4a, + 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x12, 0x17, 0x0a, 0x13, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x53, 0x53, 0x49, + 0x4f, 0x4e, 0x10, 0x01, 0x2a, 0x45, 0x0a, 0x0a, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, + 0x50, 0x45, 0x12, 0x1a, 0x0a, 0x16, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, + 0x0a, 0x17, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x10, 0x01, 0x32, 0x8d, 0x02, 0x0a, 0x19, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x69, 0x0a, 0x06, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x84, 0x01, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, + 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, + 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, + 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x63, 0x70, 0x62, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x51, 0x5a, 0x4f, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/proto/controller/servers/services/v1/server_coordination_service.proto b/internal/proto/controller/servers/services/v1/server_coordination_service.proto index ab4181466..c7422d357 100644 --- a/internal/proto/controller/servers/services/v1/server_coordination_service.proto +++ b/internal/proto/controller/servers/services/v1/server_coordination_service.proto @@ -26,6 +26,8 @@ enum CONNECTIONSTATUS { message Connection { string connection_id = 1; // @gotags: `class:"public"` CONNECTIONSTATUS status = 2; + uint64 bytes_up = 3; // @gotags: `class:"public"` + uint64 bytes_down = 4; // @gotags: `class:"public"` } enum SESSIONSTATUS { diff --git a/internal/session/repository_connection.go b/internal/session/repository_connection.go index d6db89124..438febbb1 100644 --- a/internal/session/repository_connection.go +++ b/internal/session/repository_connection.go @@ -7,10 +7,10 @@ import ( "strings" "time" - "github.com/hashicorp/boundary/internal/kms" - "github.com/hashicorp/boundary/internal/db" "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/kms" + "github.com/hashicorp/boundary/internal/observability/event" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -81,6 +81,49 @@ func (r *ConnectionRepository) list(ctx context.Context, resources any, where st return nil } +func (r *ConnectionRepository) updateBytesUpBytesDown(ctx context.Context, conns ...*Connection) error { + const op = "session.(ConnectionRepository).updateBytesUpBytesDown" + if len(conns) == 0 { + return nil + } + + updateMask := []string{"BytesUp", "BytesDown"} + _, err := r.writer.DoTx( + ctx, + db.StdRetryCnt, + db.ExpBackoff{}, + func(_ db.Reader, w db.Writer) error { + for _, c := range conns { + _, err := w.Update( + ctx, + &Connection{PublicId: c.PublicId, BytesUp: c.BytesUp, BytesDown: c.BytesDown}, + updateMask, + nil, + // The last update to these two fields should come from our + // connection closure logic, which does not use this + // function (see the closeConnections func). Therefore, here + // we shouldn't update bytes up and down if the connection + // has already been closed. This also guards against + // potential data races where a connection closure request + // and a worker status update happen close to each other in + // terms of timing. + db.WithWhere("closed_reason is null"), + ) + if err != nil { + // Returning an error will rollback the entire transaction. + // We don't want to bail out of an update batch if just one + // of the connections fails to update, but we still log it. + event.WriteError(ctx, op, fmt.Errorf("failed to update bytes up and down for connection id %q: %w", c.GetPublicId(), err)) + continue + } + } + + return nil + }) + + return err +} + // AuthorizeConnection will check to see if a connection is allowed. Currently, // that authorization checks: // * the hasn't expired based on the session.Expiration diff --git a/internal/session/repository_connection_test.go b/internal/session/repository_connection_test.go index 1c7865721..385b0db04 100644 --- a/internal/session/repository_connection_test.go +++ b/internal/session/repository_connection_test.go @@ -3,6 +3,7 @@ package session import ( "context" "fmt" + "math/rand" "testing" "time" @@ -519,3 +520,91 @@ func TestRepository_CloseConnections(t *testing.T) { }) } } + +func TestUpdateBytesUpDown(t *testing.T) { + t.Parallel() + + conn, _ := db.TestSetup(t, "postgres") + wrapper := db.TestWrapper(t) + iamRepo := iam.TestRepo(t, conn, wrapper) + kms := kms.TestKms(t, conn, wrapper) + + rw := db.New(conn) + ctx := context.Background() + + sessRepo, err := NewRepository(ctx, rw, rw, kms) + require.NoError(t, err) + + connRepo, err := NewConnectionRepository(ctx, rw, rw, kms) + require.NoError(t, err) + + // Create session. + s := TestDefaultSession(t, conn, wrapper, iamRepo) + s, _, err = sessRepo.ActivateSession(context.Background(), s.PublicId, s.Version, TestTofu(t)) + require.NoError(t, err) + + // Create some connections. + connCount := 5 + conns := make([]*Connection, 0, connCount) + for i := 0; i < connCount; i++ { + c := TestConnection(t, conn, s.PublicId, "127.0.0.1", 22, "127.0.0.1", 2222, "127.0.0.1") + c.BytesUp = uint64(rand.Int63()) + c.BytesDown = uint64(rand.Int63()) + conns = append(conns, c) + } + + // Update bytes up and down. + require.NoError(t, connRepo.updateBytesUpBytesDown(ctx, conns...)) + + // Assert that the bytes up and down values have been persisted. + for i := 0; i < len(conns); i++ { + c, _, err := connRepo.LookupConnection(ctx, conns[i].GetPublicId()) + require.NoError(t, err) + + require.Equal(t, conns[i].BytesUp, c.BytesUp) + require.Equal(t, conns[i].BytesDown, c.BytesDown) + } + + // Close all connections + closeReasons := []ClosedReason{ + UnknownReason, + ConnectionTimedOut, + ConnectionClosedByUser, + ConnectionCanceled, + ConnectionNetworkError, + ConnectionSystemError, + } + cws := make([]CloseWith, 0, len(conns)) + for i := 0; i < len(conns); i++ { + conns[i].ClosedReason = closeReasons[rand.Intn(len(closeReasons))].String() + cr, err := convertToClosedReason(conns[i].ClosedReason) + require.NoError(t, err) + + cws = append(cws, CloseWith{ + ConnectionId: conns[i].GetPublicId(), + BytesUp: conns[i].BytesUp, + BytesDown: conns[i].BytesDown, + ClosedReason: cr, + }) + } + _, err = connRepo.closeConnections(ctx, cws) + require.NoError(t, err) + + // Attempt to update bytes up and bytes down. + conns2 := make([]*Connection, len(conns)) + for i := 0; i < len(conns); i++ { + conns2[i] = conns[i].Clone().(*Connection) + conns2[i].BytesUp = uint64(rand.Int63()) + conns2[i].BytesDown = uint64(rand.Int63()) + } + require.NoError(t, connRepo.updateBytesUpBytesDown(ctx, conns2...)) + + // BytesUp and BytesDown values should be set to the old ones. + for i := 0; i < len(conns); i++ { + c, _, err := connRepo.LookupConnection(ctx, conns[i].GetPublicId()) + require.NoError(t, err) + + require.Equal(t, conns[i].BytesUp, c.BytesUp) + require.Equal(t, conns[i].BytesDown, c.BytesDown) + } +} diff --git a/internal/session/repository_session.go b/internal/session/repository_session.go index fee052ba2..58e90e924 100644 --- a/internal/session/repository_session.go +++ b/internal/session/repository_session.go @@ -591,12 +591,12 @@ func (r *Repository) updateState(ctx context.Context, sessionId string, sessionV } // checkIfNoLongerActive checks the given sessions to see if they are in a -// non-active state, i.e. "canceling" or "terminated" -// It returns a []StateReport for each session that is not active, with its current status. -func (r *Repository) checkIfNoLongerActive(ctx context.Context, reportedSessions []string) ([]StateReport, error) { +// non-active state, i.e. "canceling" or "terminated" It returns a *StateReport +// object for each session that is not active, with its current status. +func (r *Repository) checkIfNoLongerActive(ctx context.Context, reportedSessions []string) ([]*StateReport, error) { const op = "session.(Repository).checkIfNotActive" - notActive := make([]StateReport, 0, len(reportedSessions)) + notActive := make([]*StateReport, 0, len(reportedSessions)) args := make([]any, 0, len(reportedSessions)) var inClause string @@ -629,7 +629,7 @@ func (r *Repository) checkIfNoLongerActive(ctx context.Context, reportedSessions if err := rows.Scan(&sessionId, &status); err != nil { return errors.Wrap(ctx, err, op, errors.WithMsg("scan row failed")) } - notActive = append(notActive, StateReport{ + notActive = append(notActive, &StateReport{ SessionId: sessionId, Status: status, }) diff --git a/internal/session/service_worker_status_report.go b/internal/session/service_worker_status_report.go index 0ed4820a3..265e13939 100644 --- a/internal/session/service_worker_status_report.go +++ b/internal/session/service_worker_status_report.go @@ -6,43 +6,60 @@ import ( "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/internal/observability/event" + "github.com/hashicorp/go-multierror" ) // StateReport is used to report on the state of a Session. type StateReport struct { - SessionId string - Status Status - ConnectionIds []string + SessionId string + Status Status + Connections []*Connection } -// WorkerStatusReport is a domain service function that compares the state of -// sessions and connections as reported by a Worker, to the known state in the -// repositories. It returns a []StateReport for each session that is in the -// canceling or terminated state. It also will check for any orphaned -// connections, which is defined as a connection that is in an active state, -// but was not reported by worker. Any orphaned connections will be marked as -// closed. -func WorkerStatusReport(ctx context.Context, repo *Repository, connRepo *ConnectionRepository, workerId string, report []StateReport) ([]StateReport, error) { +// WorkerStatusReport is a domain service function that, given a Worker's +// session state reports, performs a few tasks: +// 1. Updates the bytes up and down statistics for each reported connection. +// 2. Compares the state of sessions and connections as reported by a Worker, +// to the known state in the repositories. It returns a StateReport object +// for each session that is in the canceling or terminated state. +// 3. Checks for any orphaned connections, which is defined as a connection +// that is in an active state, but was not reported by worker. Any orphaned +// connections will be marked as closed. +func WorkerStatusReport(ctx context.Context, repo *Repository, connRepo *ConnectionRepository, workerId string, report []*StateReport) ([]*StateReport, error) { const op = "session.WorkerStatusReport" - reportedConnections := make([]string, 0) + reportedConnectionIds := make([]string, 0) + reportedConnections := make([]*Connection, 0) reportedSessions := make([]string, 0, len(report)) for _, r := range report { reportedSessions = append(reportedSessions, r.SessionId) - reportedConnections = append(reportedConnections, r.ConnectionIds...) + for _, c := range r.Connections { + reportedConnectionIds = append(reportedConnectionIds, c.GetPublicId()) + reportedConnections = append(reportedConnections, c) + } + } + + merr := new(multierror.Error) + err := connRepo.updateBytesUpBytesDown(ctx, reportedConnections...) + if err != nil { + merr = multierror.Append(merr, errors.New(ctx, errors.Internal, op, fmt.Sprintf("failed to update bytes up and down for worker reported connections: %v", err))) } notActive, err := repo.checkIfNoLongerActive(ctx, reportedSessions) if err != nil { - return nil, errors.New(ctx, errors.Internal, op, fmt.Sprintf("Error checking session state for worker %s: %v", workerId, err)) + merr = multierror.Append(merr, errors.New(ctx, errors.Internal, op, fmt.Sprintf("Error checking session state for worker %s: %v", workerId, err))) } - closed, err := connRepo.closeOrphanedConnections(ctx, workerId, reportedConnections) + closed, err := connRepo.closeOrphanedConnections(ctx, workerId, reportedConnectionIds) if err != nil { - return notActive, errors.New(ctx, errors.Internal, op, fmt.Sprintf("Error closing orphaned connections for worker %s: %v", workerId, err)) + merr = multierror.Append(merr, errors.New(ctx, errors.Internal, op, fmt.Sprintf("Error closing orphaned connections for worker %s: %v", workerId, err))) } if len(closed) > 0 { event.WriteSysEvent(ctx, op, "marked unclaimed connections as closed", "controller_id", workerId, "count", len(closed)) } - return notActive, err + + if merr.ErrorOrNil() != nil { + return nil, merr.ErrorOrNil() + } + return notActive, nil } diff --git a/internal/session/service_worker_status_report_test.go b/internal/session/service_worker_status_report_test.go index 5d89a6165..e93822cc5 100644 --- a/internal/session/service_worker_status_report_test.go +++ b/internal/session/service_worker_status_report_test.go @@ -53,8 +53,8 @@ func TestWorkerStatusReport(t *testing.T) { type testCase struct { worker *server.Worker - req []session.StateReport - want []session.StateReport + req []*session.StateReport + want []*session.StateReport orphanedConnections []string } cases := []struct { @@ -67,8 +67,8 @@ func TestWorkerStatusReport(t *testing.T) { worker := server.TestKmsWorker(t, conn, wrapper) return testCase{ worker: worker, - req: []session.StateReport{}, - want: []session.StateReport{}, + req: []*session.StateReport{}, + want: []*session.StateReport{}, } }, }, @@ -99,8 +99,8 @@ func TestWorkerStatusReport(t *testing.T) { return testCase{ worker: worker, - req: []session.StateReport{}, - want: []session.StateReport{}, + req: []*session.StateReport{}, + want: []*session.StateReport{}, } }, }, @@ -127,14 +127,16 @@ func TestWorkerStatusReport(t *testing.T) { require.NoError(t, err) return testCase{ worker: worker, - req: []session.StateReport{ + req: []*session.StateReport{ { - SessionId: sess.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection.PublicId}, + SessionId: sess.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection.PublicId}, + }, }, }, - want: []session.StateReport{}, + want: []*session.StateReport{}, } }, }, @@ -162,14 +164,16 @@ func TestWorkerStatusReport(t *testing.T) { return testCase{ worker: worker, - req: []session.StateReport{ + req: []*session.StateReport{ { - SessionId: sess.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection.PublicId}, + SessionId: sess.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection.PublicId}, + }, }, }, - want: []session.StateReport{ + want: []*session.StateReport{ { SessionId: sess.PublicId, Status: session.StatusCanceling, @@ -220,19 +224,23 @@ func TestWorkerStatusReport(t *testing.T) { return testCase{ worker: worker, - req: []session.StateReport{ + req: []*session.StateReport{ { - SessionId: sess.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection.PublicId}, + SessionId: sess.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection.PublicId}, + }, }, { - SessionId: sess2.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection2.PublicId}, + SessionId: sess2.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection2.PublicId}, + }, }, }, - want: []session.StateReport{ + want: []*session.StateReport{ { SessionId: sess.PublicId, Status: session.StatusCanceling, @@ -284,14 +292,16 @@ func TestWorkerStatusReport(t *testing.T) { return testCase{ worker: worker, - req: []session.StateReport{ + req: []*session.StateReport{ { - SessionId: sess2.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection2.PublicId}, + SessionId: sess2.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection2.PublicId}, + }, }, }, - want: []session.StateReport{}, + want: []*session.StateReport{}, orphanedConnections: []string{connection.PublicId}, } }, @@ -340,19 +350,23 @@ func TestWorkerStatusReport(t *testing.T) { return testCase{ worker: worker, - req: []session.StateReport{ + req: []*session.StateReport{ { - SessionId: sess.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection.PublicId}, + SessionId: sess.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection.PublicId}, + }, }, { - SessionId: sess2.PublicId, - Status: session.StatusActive, - ConnectionIds: []string{connection2.PublicId}, + SessionId: sess2.PublicId, + Status: session.StatusActive, + Connections: []*session.Connection{ + {PublicId: connection2.PublicId}, + }, }, }, - want: []session.StateReport{ + want: []*session.StateReport{ { SessionId: sess.PublicId, Status: session.StatusCanceling, diff --git a/internal/tests/cluster/worker_bytesupdown_test.go b/internal/tests/cluster/worker_bytesupdown_test.go new file mode 100644 index 000000000..13907877b --- /dev/null +++ b/internal/tests/cluster/worker_bytesupdown_test.go @@ -0,0 +1,131 @@ +package cluster + +import ( + "context" + "net" + "testing" + + "github.com/hashicorp/boundary/api/sessions" + "github.com/hashicorp/boundary/api/targets" + "github.com/hashicorp/boundary/internal/cmd/config" + "github.com/hashicorp/boundary/internal/daemon/controller" + "github.com/hashicorp/boundary/internal/daemon/worker" + "github.com/hashicorp/boundary/internal/tests/helper" + "github.com/hashicorp/dawdle" + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/require" +) + +func TestWorkerBytesUpDown(t *testing.T) { + t.Parallel() + require := require.New(t) + + logger := hclog.New(&hclog.LoggerOptions{ + Name: t.Name(), + Level: hclog.Trace, + }) + + conf, err := config.DevController() + require.NoError(err) + + pl, err := net.Listen("tcp", "localhost:0") + require.NoError(err) + c1 := controller.NewTestController(t, &controller.TestControllerOpts{ + Config: conf, + InitialResourcesSuffix: "1234567890", + Logger: logger.Named("c1"), + PublicClusterAddr: pl.Addr().String(), + StatusGracePeriodDuration: helper.DefaultGracePeriod, + }) + t.Cleanup(c1.Shutdown) + + expectWorkers(t, c1) + + // Wire up the testing proxies + require.Len(c1.ClusterAddrs(), 1) + proxy, err := dawdle.NewProxy("tcp", "", c1.ClusterAddrs()[0], + dawdle.WithListener(pl), + dawdle.WithRbufSize(256), + dawdle.WithWbufSize(256), + ) + require.NoError(err) + require.NotEmpty(t, proxy.ListenerAddr()) + t.Cleanup(func() { _ = proxy.Close() }) + + w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{ + WorkerAuthKms: c1.Config().WorkerAuthKms, + InitialUpstreams: []string{proxy.ListenerAddr()}, + Logger: logger.Named("w1"), + StatusGracePeriodDuration: helper.DefaultGracePeriod, + }) + t.Cleanup(w1.Shutdown) + + require.NoError(w1.Worker().WaitForNextSuccessfulStatusUpdate()) + require.NoError(c1.WaitForNextWorkerStatusUpdate(w1.Name())) + expectWorkers(t, c1, w1) + + // Use an independent context for test things that take a context so + // that we aren't tied to any timeouts in the controller, etc. This + // can interfere with some of the test operations. + ctx := context.Background() + + // Connect target + client := c1.Client() + client.SetToken(c1.Token().Token) + tcl := targets.NewClient(client) + tgt, err := tcl.Read(ctx, "ttcp_1234567890") + require.NoError(err) + require.NotNil(tgt) + + // Create test server, update default port on target + ts := helper.NewTestTcpServer(t) + require.NotNil(t, ts) + t.Cleanup(ts.Close) + tgt, err = tcl.Update(ctx, tgt.Item.Id, tgt.Item.Version, targets.WithTcpTargetDefaultPort(ts.Port()), targets.WithSessionConnectionLimit(-1)) + require.NoError(err) + require.NotNil(tgt) + + // Authorize a session, connect and send/recv some traffic + sess := helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") + conn := sess.Connect(ctx, t) + conn.TestSendRecvAll(t) + + // Wait for next status and then check DB for bytes up and down + require.NoError(w1.Worker().WaitForNextSuccessfulStatusUpdate()) + require.NoError(c1.WaitForNextWorkerStatusUpdate(w1.Name())) + + dbConns1, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId) + require.NoError(err) + require.Len(dbConns1, 1) + require.NotZero(dbConns1[0].BytesUp) + require.NotZero(dbConns1[0].BytesDown) + + // Also check the connection data via the API + sc := sessions.NewClient(c1.Client()) + apiRes1, err := sc.Read(ctx, sess.SessionId) + require.NoError(err) + require.Len(apiRes1.Item.Connections, 1) + require.NotZero(apiRes1.Item.Connections[0].BytesUp) + require.NotZero(apiRes1.Item.Connections[0].BytesDown) + + // Send/recv some more traffic, close connection, wait for the next status + // update and check everything again. + conn.TestSendRecvAll(t) + require.NoError(conn.Close()) + require.NoError(w1.Worker().WaitForNextSuccessfulStatusUpdate()) + require.NoError(c1.WaitForNextWorkerStatusUpdate(w1.Name())) + + dbConns2, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId) + require.NoError(err) + require.Len(dbConns2, 1) + require.Greater(dbConns2[0].BytesUp, dbConns1[0].BytesUp) + require.Greater(dbConns2[0].BytesDown, dbConns1[0].BytesDown) + require.NotEmpty(dbConns2[0].ClosedReason) + + apiRes2, err := sc.Read(ctx, sess.SessionId) + require.NoError(err) + require.Len(apiRes2.Item.Connections, 1) + require.Greater(apiRes2.Item.Connections[0].BytesUp, apiRes1.Item.Connections[0].BytesUp) + require.Greater(apiRes2.Item.Connections[0].BytesDown, apiRes1.Item.Connections[0].BytesDown) + require.NotEmpty(apiRes2.Item.Connections[0].ClosedReason) +}