Chris Marchesi
a15961e461
This adds functionality to ensure that connections are closed on a worker when the server is shut down or reloaded. Connections are closed, and then the shutdown process waits for the next report to the controller to go through, waiting the grace period that was added in #1330. The rest of shutdown is then proceeded with. Listener shutdown has been moved to the start of the process to ensure that new connections can't be made while we are shutting down. Additionally, functionality has been added to force shut down servers after a second interrupt has been received after the first one. This takes the form of code in both the server and dev mode code paths that just performs an os.Exit if another shutdown is received during the normal shutdown process. Additionally, the commit also includes some refactoring to tests so that we could use some of the new E2E testing code to test session connections during shutdown and reload. Additionally, this fixes a panic in worker startup when there are no controllers defined. Looks like the shutdown call was attempting to call into the controller, not the worker. Finally, this also removes the extra shutdown within the outer worker startup. Shutdown is actually handled within the inner StartWorker(), attempting shutdown on errors was actually leading to potential use of an uninitialized worker, which is leading to panics in testing.
432 lines
15 KiB
Go
432 lines
15 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"net"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/boundary/api/targets"
|
|
"github.com/hashicorp/boundary/internal/cmd/config"
|
|
"github.com/hashicorp/boundary/internal/observability/event"
|
|
"github.com/hashicorp/boundary/internal/servers/controller"
|
|
"github.com/hashicorp/boundary/internal/servers/worker"
|
|
"github.com/hashicorp/boundary/internal/session"
|
|
"github.com/hashicorp/boundary/internal/tests/helper"
|
|
"github.com/hashicorp/dawdle"
|
|
"github.com/hashicorp/go-hclog"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// timeoutBurdenType details our "burden cases" for the session
|
|
// cleanup tests.
|
|
//
|
|
// There are 3 burden cases:
|
|
//
|
|
// * default: This case simulates normal default operation where both
|
|
// worker and controller generally are timing out connections at
|
|
// generally the same interval. In reality, this is not necessarily
|
|
// going to be the case, but it's hard to test individual cases when
|
|
// both settings are the same.
|
|
//
|
|
// * worker: This case assumes the worker is the source of truth for
|
|
// controller state. Here, the controller's grace period is
|
|
// increased to a high factor over the default to ensure that the
|
|
// worker is managing the lifecycle of a connection and will properly
|
|
// unclaim it closed once the connection resumes, ensuring the
|
|
// connection is marked as closed on the worker.
|
|
//
|
|
// * controller: Here, the controller is the one doing the work. The
|
|
// connection will be open on the worker until status checks resume
|
|
// from the worker. At this point, the controller will request the
|
|
// status change on the worker, physically closing the connection
|
|
// there.
|
|
type timeoutBurdenType string
|
|
|
|
const (
|
|
timeoutBurdenTypeDefault timeoutBurdenType = "default"
|
|
timeoutBurdenTypeWorker timeoutBurdenType = "worker"
|
|
timeoutBurdenTypeController timeoutBurdenType = "controller"
|
|
)
|
|
|
|
var timeoutBurdenCases = []timeoutBurdenType{timeoutBurdenTypeDefault, timeoutBurdenTypeWorker, timeoutBurdenTypeController}
|
|
|
|
func controllerGracePeriod(ty timeoutBurdenType) time.Duration {
|
|
if ty == timeoutBurdenTypeWorker {
|
|
return helper.DefaultGracePeriod * 10
|
|
}
|
|
|
|
return helper.DefaultGracePeriod
|
|
}
|
|
|
|
func workerGracePeriod(ty timeoutBurdenType) time.Duration {
|
|
if ty == timeoutBurdenTypeController {
|
|
return helper.DefaultGracePeriod * 10
|
|
}
|
|
|
|
return helper.DefaultGracePeriod
|
|
}
|
|
|
|
// TestSessionCleanup is the main test for session cleanup, and
|
|
// dispatches to the individual subtests.
|
|
func TestSessionCleanup(t *testing.T) {
|
|
t.Parallel()
|
|
for _, burdenCase := range timeoutBurdenCases {
|
|
burdenCase := burdenCase
|
|
t.Run(string(burdenCase), func(t *testing.T) {
|
|
t.Parallel()
|
|
t.Run("single_controller", testWorkerSessionCleanupSingle(burdenCase))
|
|
t.Run("multi_controller", testWorkerSessionCleanupMulti(burdenCase))
|
|
})
|
|
}
|
|
}
|
|
|
|
func testWorkerSessionCleanupSingle(burdenCase timeoutBurdenType) func(t *testing.T) {
|
|
const op = "cluster.testWorkerSessionCleanupSingle"
|
|
return func(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
logger := hclog.New(&hclog.LoggerOptions{
|
|
Name: t.Name(),
|
|
Level: hclog.Trace,
|
|
})
|
|
|
|
conf, err := config.DevController()
|
|
require.NoError(err)
|
|
|
|
pl, err := net.Listen("tcp", "localhost:0")
|
|
require.NoError(err)
|
|
c1 := controller.NewTestController(t, &controller.TestControllerOpts{
|
|
Config: conf,
|
|
InitialResourcesSuffix: "1234567890",
|
|
Logger: logger.Named("c1"),
|
|
PublicClusterAddr: pl.Addr().String(),
|
|
StatusGracePeriodDuration: controllerGracePeriod(burdenCase),
|
|
})
|
|
defer c1.Shutdown()
|
|
|
|
expectWorkers(t, c1)
|
|
|
|
// Wire up the testing proxies
|
|
require.Len(c1.ClusterAddrs(), 1)
|
|
proxy, err := dawdle.NewProxy("tcp", "", c1.ClusterAddrs()[0],
|
|
dawdle.WithListener(pl),
|
|
dawdle.WithRbufSize(256),
|
|
dawdle.WithWbufSize(256),
|
|
)
|
|
require.NoError(err)
|
|
defer proxy.Close()
|
|
require.NotEmpty(t, proxy.ListenerAddr())
|
|
|
|
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
|
|
WorkerAuthKms: c1.Config().WorkerAuthKms,
|
|
InitialControllers: []string{proxy.ListenerAddr()},
|
|
Logger: logger.Named("w1"),
|
|
StatusGracePeriodDuration: workerGracePeriod(burdenCase),
|
|
})
|
|
defer w1.Shutdown()
|
|
|
|
err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
|
|
require.NoError(err)
|
|
err = c1.WaitForNextWorkerStatusUpdate(w1.Name())
|
|
require.NoError(err)
|
|
expectWorkers(t, c1, w1)
|
|
|
|
// Use an independent context for test things that take a context so
|
|
// that we aren't tied to any timeouts in the controller, etc. This
|
|
// can interfere with some of the test operations.
|
|
ctx := context.Background()
|
|
|
|
// Connect target
|
|
client := c1.Client()
|
|
client.SetToken(c1.Token().Token)
|
|
tcl := targets.NewClient(client)
|
|
tgt, err := tcl.Read(ctx, "ttcp_1234567890")
|
|
require.NoError(err)
|
|
require.NotNil(tgt)
|
|
|
|
// Create test server, update default port on target
|
|
ts := helper.NewTestTcpServer(t, logger)
|
|
require.NotNil(t, ts)
|
|
defer ts.Close()
|
|
tgt, err = tcl.Update(ctx, tgt.Item.Id, tgt.Item.Version, targets.WithTcpTargetDefaultPort(ts.Port()), targets.WithSessionConnectionLimit(-1))
|
|
require.NoError(err)
|
|
require.NotNil(tgt)
|
|
|
|
// Authorize and connect
|
|
sess := helper.NewTestSession(ctx, t, logger, tcl, "ttcp_1234567890")
|
|
sConn := sess.Connect(ctx, t)
|
|
|
|
// Run initial send/receive test, make sure things are working
|
|
event.WriteSysEvent(ctx, op, "running initial send/recv test")
|
|
sConn.TestSendRecvAll(t)
|
|
|
|
// Kill the link
|
|
event.WriteSysEvent(ctx, op, "pausing controller/worker link")
|
|
proxy.Pause()
|
|
|
|
// Wait for failure connection state (depends on burden case)
|
|
switch burdenCase {
|
|
case timeoutBurdenTypeWorker:
|
|
// Wait on worker, then check controller
|
|
sess.ExpectConnectionStateOnWorker(ctx, t, w1, session.StatusClosed)
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusConnected)
|
|
|
|
case timeoutBurdenTypeController:
|
|
// Wait on controller, then check worker
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusClosed)
|
|
sess.ExpectConnectionStateOnWorker(ctx, t, w1, session.StatusConnected)
|
|
|
|
default:
|
|
// Should be closed on both worker and controller. Wait on
|
|
// worker then check controller.
|
|
sess.ExpectConnectionStateOnWorker(ctx, t, w1, session.StatusClosed)
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusClosed)
|
|
}
|
|
|
|
// Run send/receive test again to check expected connection-level
|
|
// behavior
|
|
if burdenCase == timeoutBurdenTypeController {
|
|
// Burden on controller, should be successful until connection
|
|
// resumes
|
|
sConn.TestSendRecvAll(t)
|
|
} else {
|
|
// Connection should die in other cases
|
|
sConn.TestSendRecvFail(t)
|
|
}
|
|
|
|
// Resume the connection, and reconnect.
|
|
event.WriteSysEvent(ctx, op, "resuming controller/worker link")
|
|
proxy.Resume()
|
|
err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
|
|
require.NoError(err)
|
|
|
|
// Do something post-reconnect depending on burden case. Note in
|
|
// the default case, both worker and controller should be
|
|
// relatively in sync, so we don't worry about these
|
|
// post-reconnection assertions.
|
|
switch burdenCase {
|
|
case timeoutBurdenTypeWorker:
|
|
// If we are expecting the worker to be the source of truth of
|
|
// a connection status, ensure that our old session's
|
|
// connections are actually closed now that the worker is
|
|
// properly reporting in again.
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusClosed)
|
|
|
|
case timeoutBurdenTypeController:
|
|
// If we are expecting the controller to be the source of
|
|
// truth, the connection should now be forcibly closed after
|
|
// the worker gets a status change request back.
|
|
sConn.TestSendRecvFail(t)
|
|
}
|
|
|
|
// Proceed with new connection test
|
|
event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link")
|
|
sess = helper.NewTestSession(ctx, t, logger, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup()
|
|
sConn = sess.Connect(ctx, t)
|
|
sConn.TestSendRecvAll(t)
|
|
}
|
|
}
|
|
|
|
func testWorkerSessionCleanupMulti(burdenCase timeoutBurdenType) func(t *testing.T) {
|
|
const op = "cluster.testWorkerSessionCleanupMulti"
|
|
return func(t *testing.T) {
|
|
t.Parallel()
|
|
require := require.New(t)
|
|
logger := hclog.New(&hclog.LoggerOptions{
|
|
Name: t.Name(),
|
|
Level: hclog.Trace,
|
|
})
|
|
|
|
// ******************
|
|
// ** Controller 1 **
|
|
// ******************
|
|
conf1, err := config.DevController()
|
|
require.NoError(err)
|
|
|
|
pl1, err := net.Listen("tcp", "localhost:0")
|
|
require.NoError(err)
|
|
c1 := controller.NewTestController(t, &controller.TestControllerOpts{
|
|
Config: conf1,
|
|
InitialResourcesSuffix: "1234567890",
|
|
Logger: logger.Named("c1"),
|
|
PublicClusterAddr: pl1.Addr().String(),
|
|
StatusGracePeriodDuration: controllerGracePeriod(burdenCase),
|
|
})
|
|
defer c1.Shutdown()
|
|
|
|
// ******************
|
|
// ** Controller 2 **
|
|
// ******************
|
|
pl2, err := net.Listen("tcp", "localhost:0")
|
|
require.NoError(err)
|
|
c2 := c1.AddClusterControllerMember(t, &controller.TestControllerOpts{
|
|
Logger: logger.Named("c2"),
|
|
PublicClusterAddr: pl2.Addr().String(),
|
|
StatusGracePeriodDuration: controllerGracePeriod(burdenCase),
|
|
})
|
|
defer c2.Shutdown()
|
|
expectWorkers(t, c1)
|
|
expectWorkers(t, c2)
|
|
|
|
// *************
|
|
// ** Proxy 1 **
|
|
// *************
|
|
require.Len(c1.ClusterAddrs(), 1)
|
|
p1, err := dawdle.NewProxy("tcp", "", c1.ClusterAddrs()[0],
|
|
dawdle.WithListener(pl1),
|
|
dawdle.WithRbufSize(256),
|
|
dawdle.WithWbufSize(256),
|
|
)
|
|
require.NoError(err)
|
|
defer p1.Close()
|
|
require.NotEmpty(t, p1.ListenerAddr())
|
|
|
|
// *************
|
|
// ** Proxy 2 **
|
|
// *************
|
|
require.Len(c2.ClusterAddrs(), 1)
|
|
p2, err := dawdle.NewProxy("tcp", "", c2.ClusterAddrs()[0],
|
|
dawdle.WithListener(pl2),
|
|
dawdle.WithRbufSize(256),
|
|
dawdle.WithWbufSize(256),
|
|
)
|
|
require.NoError(err)
|
|
defer p2.Close()
|
|
require.NotEmpty(t, p2.ListenerAddr())
|
|
|
|
// ************
|
|
// ** Worker **
|
|
// ************
|
|
w1 := worker.NewTestWorker(t, &worker.TestWorkerOpts{
|
|
WorkerAuthKms: c1.Config().WorkerAuthKms,
|
|
InitialControllers: []string{p1.ListenerAddr(), p2.ListenerAddr()},
|
|
Logger: logger.Named("w1"),
|
|
StatusGracePeriodDuration: workerGracePeriod(burdenCase),
|
|
})
|
|
defer w1.Shutdown()
|
|
|
|
err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
|
|
require.NoError(err)
|
|
err = c1.WaitForNextWorkerStatusUpdate(w1.Name())
|
|
require.NoError(err)
|
|
err = c2.WaitForNextWorkerStatusUpdate(w1.Name())
|
|
require.NoError(err)
|
|
expectWorkers(t, c1, w1)
|
|
expectWorkers(t, c2, w1)
|
|
|
|
// Use an independent context for test things that take a context so
|
|
// that we aren't tied to any timeouts in the controller, etc. This
|
|
// can interfere with some of the test operations.
|
|
ctx := context.Background()
|
|
|
|
// Connect target
|
|
client := c1.Client()
|
|
client.SetToken(c1.Token().Token)
|
|
tcl := targets.NewClient(client)
|
|
tgt, err := tcl.Read(ctx, "ttcp_1234567890")
|
|
require.NoError(err)
|
|
require.NotNil(tgt)
|
|
|
|
// Create test server, update default port on target
|
|
ts := helper.NewTestTcpServer(t, logger)
|
|
require.NotNil(ts)
|
|
defer ts.Close()
|
|
tgt, err = tcl.Update(ctx, tgt.Item.Id, tgt.Item.Version, targets.WithTcpTargetDefaultPort(ts.Port()), targets.WithSessionConnectionLimit(-1))
|
|
require.NoError(err)
|
|
require.NotNil(tgt)
|
|
|
|
// Authorize and connect
|
|
sess := helper.NewTestSession(ctx, t, logger, tcl, "ttcp_1234567890")
|
|
sConn := sess.Connect(ctx, t)
|
|
|
|
// Run initial send/receive test, make sure things are working
|
|
event.WriteSysEvent(ctx, op, "running initial send/recv test")
|
|
sConn.TestSendRecvAll(t)
|
|
|
|
// Kill connection to first controller, and run test again, should
|
|
// pass, deferring to other controller. Wait for the next
|
|
// successful status report to ensure this.
|
|
event.WriteSysEvent(ctx, op, "pausing link to controller #1")
|
|
p1.Pause()
|
|
err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
|
|
require.NoError(err)
|
|
sConn.TestSendRecvAll(t)
|
|
|
|
// Resume first controller, pause second. This one should work too.
|
|
event.WriteSysEvent(ctx, op, "pausing link to controller #2, resuming #1")
|
|
p1.Resume()
|
|
p2.Pause()
|
|
err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
|
|
require.NoError(err)
|
|
sConn.TestSendRecvAll(t)
|
|
|
|
// Kill the first controller connection again. This one should fail
|
|
// due to lack of any connection.
|
|
event.WriteSysEvent(ctx, op, "pausing link to controller #1 again, both connections should be offline")
|
|
p1.Pause()
|
|
|
|
// Wait for failure connection state (depends on burden case)
|
|
switch burdenCase {
|
|
case timeoutBurdenTypeWorker:
|
|
// Wait on worker, then check controller
|
|
sess.ExpectConnectionStateOnWorker(ctx, t, w1, session.StatusClosed)
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusConnected)
|
|
|
|
case timeoutBurdenTypeController:
|
|
// Wait on controller, then check worker
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusClosed)
|
|
sess.ExpectConnectionStateOnWorker(ctx, t, w1, session.StatusConnected)
|
|
|
|
default:
|
|
// Should be closed on both worker and controller. Wait on
|
|
// worker then check controller.
|
|
sess.ExpectConnectionStateOnWorker(ctx, t, w1, session.StatusClosed)
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusClosed)
|
|
}
|
|
|
|
// Run send/receive test again to check expected connection-level
|
|
// behavior
|
|
if burdenCase == timeoutBurdenTypeController {
|
|
// Burden on controller, should be successful until connection
|
|
// resumes
|
|
sConn.TestSendRecvAll(t)
|
|
} else {
|
|
// Connection should die in other cases
|
|
sConn.TestSendRecvFail(t)
|
|
}
|
|
|
|
// Finally resume both, try again. Should behave as per normal.
|
|
event.WriteSysEvent(ctx, op, "resuming connections to both controllers")
|
|
p1.Resume()
|
|
p2.Resume()
|
|
err = w1.Worker().WaitForNextSuccessfulStatusUpdate()
|
|
require.NoError(err)
|
|
|
|
// Do something post-reconnect depending on burden case. Note in
|
|
// the default case, both worker and controller should be
|
|
// relatively in sync, so we don't worry about these
|
|
// post-reconnection assertions.
|
|
switch burdenCase {
|
|
case timeoutBurdenTypeWorker:
|
|
// If we are expecting the worker to be the source of truth of
|
|
// a connection status, ensure that our old session's
|
|
// connections are actually closed now that the worker is
|
|
// properly reporting in again.
|
|
sess.ExpectConnectionStateOnController(ctx, t, c1.Controller().SessionRepoFn, session.StatusClosed)
|
|
|
|
case timeoutBurdenTypeController:
|
|
// If we are expecting the controller to be the source of
|
|
// truth, the connection should now be forcibly closed after
|
|
// the worker gets a status change request back.
|
|
sConn.TestSendRecvFail(t)
|
|
}
|
|
|
|
// Proceed with new connection test
|
|
event.WriteSysEvent(ctx, op, "connecting to new session after resuming controller/worker link")
|
|
sess = helper.NewTestSession(ctx, t, logger, tcl, "ttcp_1234567890") // re-assign, other connection will close in t.Cleanup()
|
|
sConn = sess.Connect(ctx, t)
|
|
sConn.TestSendRecvAll(t)
|
|
}
|
|
}
|