Worker status refactor (#5417)

* oss: feat(proto): statistics rpc added to server coordination service

* oss: feat(session): add update statistics status service

* oss: feat(handlers): add Statistics method to worker service server

* oss: feat(worker): add statistics ticking

* oss: internal/proto: define the new RoutingInfo RPC

* oss: internal/server: repository changes for RoutingInfo RPC

* oss: internal/daemon: add RoutingInfo handler to worker service

The new RoutingInfo RPC is used to get information from the worker
that is used to make routing decisions on the controller. It also
handles the initial request from the worker.

* oss: feat(db): server_worker_session_info table

* oss: feat(proto): session info rpc for server coordination service

* oss: feat(server): worker session info request data type

* oss: feat(worker): session info ticker

* oss: feat(handler): server coordination handler for session info

* oss: feat(session): utilize session info request time for session cleanup

* oss: proto: fix some gotags comments

Some of these fields were missing comments, while others had
incorrectly formatted comments.

* oss: worker: send connection status to server

This is required for the correct tracking of orphaned connections

* oss: internal/daemon/worker: add function to wait for Statistics RPC

* oss: internal/daemon/worker: add routing info RPC

The new RoutingInfo RPC replaces part of the Status RPC,
namely the part which is used by the controller and worker
to perform decisions around the establishment of new sessions
and new worker connections.

* oss: internal/daemon/worker: remove unnecessary error check

* oss: internal/daemon/worker: lock access to config and upstreams

The new confLock protects access to the conf value and to the
addressReceivers slice.

* oss: worker: replace Status RPC with RoutingInfo

* oss: worker: deprecate the Status RPC and related types

The Status RPC and related messages have been replaced
by the RoutingInfo/SessionInfo/Statistics RPCs. The controller
will continue to support the Status RPC until two new minor
versions have been released.

* oss: internal/event: add default deny for new worker RPCs

* oss: internal/daemon/worker: allow RPC interval to be set

The new SessionInfo/RoutingInfo/Statistics RPCs run at
long intervals, which make some of our cluster tests take
a long time to complete. This allows their interval to be
configured in tests.

* oss: all: use TestWorkerRPCInterval in slow tests

This should speed up these tests significantly.

* oss: internal/tests/cluster: reset sys eventer in test

---------

Co-authored-by: Damian Debkowski <damian.debkowski@hashicorp.com>
This commit is contained in:
Johan Brandhorst-SatzkornandDamian Debkowski authored and GitHub committed 2025-01-09 10:26:37 -08:00
1 parent 554195f49f
commit ec0a3f964b
70 files changed
+5492 -1618

No files matched your search

@@ -14,7 +14,6 @@ import (
"github.com/hashicorp/boundary/internal/cmd/config"
"github.com/hashicorp/boundary/internal/daemon/controller"
"github.com/hashicorp/boundary/internal/daemon/worker"
"github.com/hashicorp/boundary/internal/event"
"github.com/hashicorp/boundary/internal/server"
"github.com/hashicorp/boundary/internal/session"
"github.com/hashicorp/boundary/internal/tests/helper"
@@ -51,26 +50,19 @@ var timeoutBurdenCases = []timeoutBurdenType{timeoutBurdenTypeDefault, timeoutBu
func controllerGracePeriod(ty timeoutBurdenType) time.Duration {
if ty == timeoutBurdenTypeWorker {
return helper.DefaultWorkerStatusGracePeriod * 10
return helper.DefaultControllerRPCGracePeriod * 10
}
return helper.DefaultWorkerStatusGracePeriod
}
func workerGracePeriod(ty timeoutBurdenType) time.Duration {
return helper.DefaultWorkerStatusGracePeriod
return helper.DefaultControllerRPCGracePeriod
}
// TestSessionCleanup is the main test for session cleanup, and
// dispatches to the individual subtests.
func TestSessionCleanup(t *testing.T) {
for _, burdenCase := range timeoutBurdenCases {
burdenCase := burdenCase
t.Run(string(burdenCase), func(t *testing.T) {
t.Run("single_controller", testWorkerSessionCleanupSingle(burdenCase))
t.Run("multi_controller", testWorkerSessionCleanupMulti(burdenCase))
})
}
t.Run("default/single_controller", testWorkerSessionCleanupSingle("default"))
t.Run("default/multi_controller", testWorkerSessionCleanupMulti("default"))
t.Run("worker/single_controller", testWorkerSessionCleanupSingle("worker"))
t.Run("worker/multi_controller", testWorkerSessionCleanupMulti("worker"))
}
func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testing.T) {
@@ -90,11 +82,13 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
pl, err := net.Listen("tcp", "[::1]:0")
require.NoError(err)
c1 := controller.NewTestController(t, &controller.TestControllerOpts{
Config: conf,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
PublicClusterAddr: pl.Addr().String(),
WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase),
Config: conf,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
PublicClusterAddr: pl.Addr().String(),
WorkerRPCGracePeriod: controllerGracePeriod(burdenCase),
// Run the scheduler more often to speed up cleanup of orphaned connections
SchedulerRunJobInterval: time.Second,
})
helper.ExpectWorkers(t, c1)
@@ -113,10 +107,11 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
require.NotEmpty(t, proxy.ListenerAddr())
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{proxy.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulStatusGracePeriodDuration: workerGracePeriod(burdenCase),
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{proxy.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
WorkerRPCInterval: time.Second,
})
helper.ExpectWorkers(t, c1, w1)
@@ -147,11 +142,15 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
sConn := sess.Connect(ctx, t)
// Run initial send/receive test, make sure things are working
event.WriteSysEvent(ctx, op, "running initial send/recv test")
t.Log("running initial send/recv test")
sConn.TestSendRecvAll(t)
// Wait for a session info to be sent to the server, so the controller has
// at least one record of the connection.
w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
// Kill the link
event.WriteSysEvent(ctx, op, "pausing controller/worker link")
t.Log("pausing controller/worker link")
proxy.Pause()
// Wait for failure connection state (depends on burden case)
@@ -171,15 +170,9 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
sConn.TestSendRecvFail(t)
// Resume the connection, and reconnect.
event.WriteSysEvent(ctx, op, "resuming controller/worker link")
t.Log("resuming controller/worker link")
proxy.Resume()
require.Eventually(func() bool {
err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
if err != nil {
return false
}
return true
}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
helper.ExpectWorkers(t, c1, w1)
// Do something post-reconnect depending on burden case. Note in
// the default case, both worker and controller should be
@@ -195,7 +188,7 @@ func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testin
}
// Proceed with new connection test
event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link")
t.Log("connecting to new session after resuming controller/worker link")
sess = helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup()
sConn = sess.Connect(ctx, t)
sConn.TestSendRecvAll(t)
@@ -222,11 +215,11 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
pl1, err := net.Listen("tcp", "[::1]:0")
require.NoError(err)
c1 := controller.NewTestController(t, &controller.TestControllerOpts{
Config: conf1,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
PublicClusterAddr: pl1.Addr().String(),
WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase),
Config: conf1,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
PublicClusterAddr: pl1.Addr().String(),
WorkerRPCGracePeriod: controllerGracePeriod(burdenCase),
})
// ******************
@@ -235,9 +228,9 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
pl2, err := net.Listen("tcp", "[::1]:0")
require.NoError(err)
c2 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
Logger: logger.Named("c2"),
PublicClusterAddr: pl2.Addr().String(),
WorkerStatusGracePeriodDuration: controllerGracePeriod(burdenCase),
Logger: logger.Named("c2"),
PublicClusterAddr: pl2.Addr().String(),
WorkerRPCGracePeriod: controllerGracePeriod(burdenCase),
})
wg := new(sync.WaitGroup)
@@ -286,11 +279,13 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
// ** Worker **
// ************
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{p1.ListenerAddr(), p2.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulStatusGracePeriodDuration: workerGracePeriod(burdenCase),
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{p1.ListenerAddr(), p2.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
WorkerRPCInterval: time.Second,
})
wg.Add(2)
go func() {
defer wg.Done()
@@ -328,39 +323,30 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
sConn := sess.Connect(ctx, t)
// Run initial send/receive test, make sure things are working
event.WriteSysEvent(ctx, op, "running initial send/recv test")
t.Log("running initial send/recv test")
sConn.TestSendRecvAll(t)
// Wait for a session info to be sent to the server, so the controller has
// at least one record of the connection.
w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
// Kill connection to first controller, and run test again, should
// pass, deferring to other controller. Wait for the next
// successful status report to ensure this.
event.WriteSysEvent(ctx, op, "pausing link to controller #1")
t.Log("pausing link to controller #1")
p1.Pause()
require.Eventually(func() bool {
err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
if err != nil {
return false
}
return true
}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
sConn.TestSendRecvAll(t)
// Resume first controller, pause second. This one should work too.
event.WriteSysEvent(ctx, op, "pausing link to controller #2, resuming #1")
t.Log("pausing link to controller #2, resuming #1")
p1.Resume()
p2.Pause()
require.Eventually(func() bool {
err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
if err != nil {
return false
}
return true
}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
sConn.TestSendRecvAll(t)
w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
// Kill the first controller connection again. This one should fail
// due to lack of any connection.
event.WriteSysEvent(ctx, op, "pausing link to controller #1 again, both connections should be offline")
t.Log("pausing link to controller #1 again, both connections should be offline")
p1.Pause()
// Wait for failure connection state (depends on burden case)
@@ -382,16 +368,10 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
sConn.TestSendRecvFail(t)
// Finally resume both, try again. Should behave as per normal.
event.WriteSysEvent(ctx, op, "resuming connections to both controllers")
t.Log("resuming connections to both controllers")
p1.Resume()
p2.Resume()
require.Eventually(func() bool {
err := w1.Worker().WaitForNextSuccessfulStatusUpdate()
if err != nil {
return false
}
return true
}, 2*helper.DefaultWorkerStatusGracePeriod, 5*time.Second)
w1.Worker().TestWaitForNextSuccessfulSessionInfoUpdate(t)
// Do something post-reconnect depending on burden case. Note in
// the default case, both worker and controller should be
@@ -407,7 +387,7 @@ func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing
}
// Proceed with new connection test
event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link")
t.Log("connecting to new session after resuming controller/worker link")
sess = helper.NewTestSession(ctx, t, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup()
sConn = sess.Connect(ctx, t)
sConn.TestSendRecvAll(t)
@@ -7,6 +7,7 @@ import (
"context"
"net"
"testing"
"time"
"github.com/hashicorp/boundary/api/sessions"
"github.com/hashicorp/boundary/api/targets"
@@ -45,10 +46,10 @@ func TestWorkerBytesUpDown(t *testing.T) {
}
c1 := controller.NewTestController(t, &controller.TestControllerOpts{
Config: conf,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
Config: conf,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
WorkerRPCGracePeriod: helper.DefaultControllerRPCGracePeriod,
})
helper.ExpectWorkers(t, c1)
@@ -65,11 +66,12 @@ func TestWorkerBytesUpDown(t *testing.T) {
t.Cleanup(func() { _ = proxy.Close() })
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{proxy.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulStatusGracePeriodDuration: helper.DefaultSuccessfulStatusGracePeriod,
EnableIPv6: true,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{proxy.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
EnableIPv6: true,
WorkerRPCInterval: time.Second,
})
helper.ExpectWorkers(t, c1, w1)
@@ -105,8 +107,8 @@ func TestWorkerBytesUpDown(t *testing.T) {
conn := sess.Connect(ctx, t)
conn.TestSendRecvAll(t)
// Wait for next status and then check DB for bytes up and down
helper.ExpectWorkers(t, c1, w1)
// Wait for next statistics update and then check DB for bytes up and down
w1.Worker().TestWaitForNextSuccessfulStatisticsUpdate(t)
dbConns1, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId)
require.NoError(err)
@@ -122,11 +124,11 @@ func TestWorkerBytesUpDown(t *testing.T) {
require.NotZero(apiRes1.Item.Connections[0].BytesUp)
require.NotZero(apiRes1.Item.Connections[0].BytesDown)
// Send/recv some more traffic, close connection, wait for the next status
// Send/recv some more traffic, close connection, wait for the next statistics
// update and check everything again.
conn.TestSendRecvAll(t)
require.NoError(conn.Close())
helper.ExpectWorkers(t, c1, w1)
w1.Worker().TestWaitForNextSuccessfulStatisticsUpdate(t)
dbConns2, err := c1.ConnectionsRepo().ListConnectionsBySessionId(ctx, sess.SessionId)
require.NoError(err)
@@ -7,6 +7,7 @@ import (
"context"
"net"
"testing"
"time"
"github.com/hashicorp/boundary/api/targets"
"github.com/hashicorp/boundary/internal/cmd/config"
@@ -46,10 +47,10 @@ func TestWorkerSessionProxyMultipleConnections(t *testing.T) {
}
c1 := controller.NewTestController(t, &controller.TestControllerOpts{
Config: conf,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
WorkerStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
Config: conf,
InitialResourcesSuffix: "1234567890",
Logger: logger.Named("c1"),
WorkerRPCGracePeriod: helper.DefaultControllerRPCGracePeriod,
})
helper.ExpectWorkers(t, c1)
@@ -66,11 +67,12 @@ func TestWorkerSessionProxyMultipleConnections(t *testing.T) {
require.NotEmpty(t, proxy.ListenerAddr())
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{proxy.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulStatusGracePeriodDuration: helper.DefaultWorkerStatusGracePeriod,
EnableIPv6: true,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: []string{proxy.ListenerAddr()},
Logger: logger.Named("w1"),
SuccessfulControllerRPCGracePeriodDuration: helper.DefaultControllerRPCGracePeriod,
EnableIPv6: true,
WorkerRPCInterval: time.Second,
})
helper.ExpectWorkers(t, c1, w1)
@@ -5,6 +5,7 @@ package sequential
import (
"testing"
"time"
"github.com/hashicorp/boundary/api/targets"
"github.com/hashicorp/boundary/internal/cmd/config"
@@ -57,10 +58,11 @@ func TestWorkerTagging(t *testing.T) {
"foo": {"bar"},
}
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
Config: conf,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w1"),
Config: conf,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w1"),
WorkerRPCInterval: time.Second,
})
w1Addr := w1.ProxyAddrs()[0]
@@ -73,10 +75,11 @@ func TestWorkerTagging(t *testing.T) {
"az": {"one", "two", "three"},
}
w2 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
Config: conf,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w2"),
Config: conf,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w2"),
WorkerRPCInterval: time.Second,
})
w2Addr := w2.ProxyAddrs()[0]
@@ -89,10 +92,11 @@ func TestWorkerTagging(t *testing.T) {
"az": {"one", "three"},
}
w3 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
Config: conf,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w3"),
Config: conf,
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w3"),
WorkerRPCInterval: time.Second,
})
w3Addr := w3.ProxyAddrs()[0]
@@ -15,6 +15,7 @@ import (
"net/http"
"sync"
"testing"
"time"
apiproxy "github.com/hashicorp/boundary/api/proxy"
"github.com/hashicorp/boundary/api/targets"
@@ -43,6 +44,14 @@ func TestCustomX509Verification_Client(t *testing.T) {
Name: "test",
})
req.NoError(event.InitSysEventer(logger, testLock, "use-TestCustomX509Verification", event.WithEventerConfig(&ec.EventerConfig)))
t.Cleanup(func() {
event.TestResetSystEventer(t)
})
t.Cleanup(func() {
all, err := io.ReadAll(ec.AllEvents)
require.NoError(t, err)
t.Log(string(all))
})
conf, err := config.DevController()
conf.Eventing = &ec.EventerConfig
@@ -57,12 +66,12 @@ func TestCustomX509Verification_Client(t *testing.T) {
conf.Eventing = &ec.EventerConfig
req.NoError(err)
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w1"),
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
Logger: logger.Named("w1"),
WorkerRPCInterval: time.Second,
})
// Give time for it to connect
helper.ExpectWorkers(t, c1, w1)
// Connect target
@@ -221,8 +230,9 @@ func testCustomX509Verification_Server(ec event.TestConfig, certPool *x509.CertP
conf.Eventing = &ec.EventerConfig
req.NoError(err)
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
WorkerAuthKms: c1.Config().WorkerAuthKms,
InitialUpstreams: c1.ClusterAddrs(),
WorkerRPCInterval: time.Second,
})
w1.Worker().TestOverrideX509VerifyCertPool = certPool
w1.Worker().TestOverrideX509VerifyDnsName = dnsName