internal/servers/controller: Worker failure connection cleanup (#1340)
This commit adds the support to do the following: * Mark connections for non-reporting workers as closed. This is the controller counterpart to the worker functionality (see #1330). This is written as a scheduled job that does the work DB-side in a single atomic query. * Works to reconcile states if such a broken controller-worker connection resumes and a worker reports a connection as connected that should be disconnected. In this case, the controller will send an update request, and the worker will honor it and terminate the connection. * Further refinement of the grace period setting has been added here. We have converged on the current server "liveness" setting as our default here, which is half of the previous 30s (15 seconds, in other words). Additionally, this is now configurable on the controller and worker side, with the caveat that it's currently impossible to do so in config as the setting has been untagged in HCL. This is exposed so that we can run some sophisticated testing scenarios where we skew the grace period to either the controller or worker to ensure the aforementioned reconciliation works. * Some repository functions have been added to support the new functionality, in addition to some test code to the worker to allow querying of session state while testing. * Finally, we've added some add timestamp subtraction functions as well, basically serving as the opposite of the addition functions.
This commit is contained in:
1 parent
462306fea5
commit
5a70875726
23 files changed
+2072
-297
No files matched your search
@@ -38,7 +38,7 @@ type Controller struct {
|
||||
|
||||
workerAuthCache *cache.Cache
|
||||
|
||||
// Used for testing
|
||||
// Used for testing and tracking worker health
|
||||
workerStatusUpdateTimes *sync.Map
|
||||
|
||||
// Repo factory methods
|
||||
@@ -117,7 +117,9 @@ func New(conf *Config) (*Controller, error) {
|
||||
jobRepoFn := func() (*job.Repository, error) {
|
||||
return job.NewRepository(dbase, dbase, c.kms)
|
||||
}
|
||||
c.scheduler, err = scheduler.New(c.conf.RawConfig.Controller.Name, jobRepoFn, c.logger)
|
||||
// TODO: the RunJobsLimit is temporary until a better fix gets in. This
|
||||
// currently caps the scheduler at running 10 jobs per interval.
|
||||
c.scheduler, err = scheduler.New(c.conf.RawConfig.Controller.Name, jobRepoFn, c.logger, scheduler.WithRunJobsLimit(10))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating new scheduler: %w", err)
|
||||
}
|
||||
@@ -182,7 +184,29 @@ func (c *Controller) Start() error {
|
||||
|
||||
func (c *Controller) registerJobs() error {
|
||||
rw := db.New(c.conf.Database)
|
||||
return vault.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.logger)
|
||||
if err := vault.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.logger); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := c.registerSessionCleanupJob(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerSessionCleanupJob is a helper method to abstract
|
||||
// registering the session cleanup job specifically.
|
||||
func (c *Controller) registerSessionCleanupJob() error {
|
||||
sessionCleanupJob, err := newSessionCleanupJob(c.logger, c.SessionRepoFn, int(c.conf.StatusGracePeriodDuration.Seconds()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating session cleanup job: %w", err)
|
||||
}
|
||||
if err = c.scheduler.RegisterJob(c.baseContext, sessionCleanupJob); err != nil {
|
||||
return fmt.Errorf("error registering session cleanup job: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Controller) Shutdown(serversOnly bool) error {
|
||||
|
||||
@@ -76,12 +76,19 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
|
||||
Controllers: controllers,
|
||||
}
|
||||
|
||||
// TODO (jeff, 05-2021): We should possibly list all connections here with
|
||||
// their statuses, and check those against the found connections below. If
|
||||
// any we think should be closed are marked open by the worker, the worker
|
||||
// should be told to close them.
|
||||
var (
|
||||
// For tracking the reported open connections.
|
||||
reportedOpenConns []string
|
||||
// For tracking the session IDs we've already requested
|
||||
// cancellation for. We won't need to add connection cancel
|
||||
// requests for these because canceling the session terminates the
|
||||
// connections.
|
||||
requestedSessionCancelIds []string
|
||||
)
|
||||
|
||||
var foundConns []string
|
||||
// This is a map of all sessions and their statuses. We keep track of
|
||||
// this for easy lookup if we need to make change requests.
|
||||
sessionStatuses := make(map[string]pbs.SESSIONSTATUS)
|
||||
|
||||
for _, jobStatus := range req.GetJobs() {
|
||||
switch jobStatus.Job.GetType() {
|
||||
@@ -92,6 +99,9 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
|
||||
return nil, status.Error(codes.Internal, "Error getting session info at status time")
|
||||
}
|
||||
|
||||
// Record status.
|
||||
sessionStatuses[si.GetSessionId()] = si.Status
|
||||
|
||||
// Check connections before potentially bypassing the rest of the
|
||||
// logic in the switch on si.Status.
|
||||
sessConns := si.GetConnections()
|
||||
@@ -103,7 +113,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
|
||||
// report as found, so that we should attempt to close it.
|
||||
// Note that unspecified is the default state for the enum
|
||||
// but it's not ever explicitly set by us.
|
||||
foundConns = append(foundConns, conn.GetConnectionId())
|
||||
reportedOpenConns = append(reportedOpenConns, conn.GetConnectionId())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,7 +137,7 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
|
||||
}
|
||||
// If the session from the DB is in canceling status, and we're
|
||||
// here, it means the job is in pending or active; cancel it. If
|
||||
// it's in termianted status something went wrong and we're
|
||||
// it's in terminated status something went wrong and we're
|
||||
// mismatched, so ensure we cancel it also.
|
||||
currState := sessionInfo.States[0].Status
|
||||
if currState.ProtoVal() != si.Status {
|
||||
@@ -148,13 +158,53 @@ func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusReques
|
||||
},
|
||||
RequestType: pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE,
|
||||
})
|
||||
// Log the session ID so we don't add a duplicate change
|
||||
// request on connection normalization.
|
||||
requestedSessionCancelIds = append(requestedSessionCancelIds, sessionId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run our connection cleanup function
|
||||
closedConns, err := sessRepo.CloseDeadConnectionsOnWorkerReport(ctx, req.Worker.PrivateId, foundConns)
|
||||
// Normalize the current state of connections on the worker side
|
||||
// with the data from the controller. In other words, if one of our
|
||||
// found connections isn't supposed to be alive still, kill it.
|
||||
//
|
||||
// This is separate from the above session normalization and is
|
||||
// additive to it, we don't add sessions that have already been
|
||||
// added there as canceling sessions already closes the
|
||||
// connections.
|
||||
shouldCloseConnections, err := sessRepo.ShouldCloseConnectionsOnWorker(ctx, reportedOpenConns, requestedSessionCancelIds)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "Error fetching connections that should be closed: %v", err)
|
||||
}
|
||||
|
||||
for sessionId, connIds := range shouldCloseConnections {
|
||||
var connChanges []*pbs.Connection
|
||||
for _, connId := range connIds {
|
||||
connChanges = append(connChanges, &pbs.Connection{
|
||||
ConnectionId: connId,
|
||||
Status: session.StatusClosed.ProtoVal(),
|
||||
})
|
||||
}
|
||||
|
||||
ret.JobsRequests = append(ret.JobsRequests, &pbs.JobChangeRequest{
|
||||
Job: &pbs.Job{
|
||||
Type: pbs.JOBTYPE_JOBTYPE_SESSION,
|
||||
JobInfo: &pbs.Job_SessionInfo{
|
||||
SessionInfo: &pbs.SessionJobInfo{
|
||||
SessionId: sessionId,
|
||||
Status: sessionStatuses[sessionId],
|
||||
Connections: connChanges,
|
||||
},
|
||||
},
|
||||
},
|
||||
RequestType: pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE,
|
||||
})
|
||||
}
|
||||
|
||||
// Run our controller-side cleanup function.
|
||||
closedConns, err := sessRepo.CloseDeadConnectionsForWorker(ctx, req.Worker.PrivateId, reportedOpenConns)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "Error closing dead conns for worker %s: %v", req.Worker.PrivateId, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/boundary/internal/errors"
|
||||
"github.com/hashicorp/boundary/internal/scheduler"
|
||||
"github.com/hashicorp/boundary/internal/servers/controller/common"
|
||||
"github.com/hashicorp/boundary/internal/session"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
)
|
||||
|
||||
// sessionCleanupJob defines a periodic job that monitors workers for
|
||||
// loss of connection and terminates connections on workers that have
|
||||
// not sent a heartbeat in a significant period of time.
|
||||
//
|
||||
// Relevant connections are simply marked as disconnected in the
|
||||
// database. Connections will be independently terminated by the
|
||||
// worker, or the event of a synchronization issue between the two,
|
||||
// the controller will win out and order that the connections be
|
||||
// closed on the worker.
|
||||
type sessionCleanupJob struct {
|
||||
logger hclog.Logger
|
||||
sessionRepoFn common.SessionRepoFactory
|
||||
|
||||
// The amount of time to give disconnected workers before marking
|
||||
// their connections as closed.
|
||||
gracePeriod int
|
||||
|
||||
// The total number of connections closed in the last run.
|
||||
totalClosed int
|
||||
}
|
||||
|
||||
// newSessionCleanupJob instantiates the session cleanup job.
|
||||
func newSessionCleanupJob(
|
||||
logger hclog.Logger,
|
||||
sessionRepoFn common.SessionRepoFactory,
|
||||
gracePeriod int,
|
||||
) (*sessionCleanupJob, error) {
|
||||
const op = "controller.newNewSessionCleanupJob"
|
||||
switch {
|
||||
case logger == nil:
|
||||
return nil, errors.New(errors.InvalidParameter, op, "missing logger")
|
||||
case sessionRepoFn == nil:
|
||||
return nil, errors.New(errors.InvalidParameter, op, "missing sessionRepoFn")
|
||||
case gracePeriod < session.DeadWorkerConnCloseMinGrace:
|
||||
return nil, errors.New(
|
||||
errors.InvalidParameter, op, fmt.Sprintf("invalid gracePeriod, must be greater than %d", session.DeadWorkerConnCloseMinGrace))
|
||||
}
|
||||
|
||||
return &sessionCleanupJob{
|
||||
logger: logger,
|
||||
sessionRepoFn: sessionRepoFn,
|
||||
gracePeriod: gracePeriod,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns a short, unique name for the job.
|
||||
func (j *sessionCleanupJob) Name() string { return "session_cleanup" }
|
||||
|
||||
// Description returns the description for the job.
|
||||
func (j *sessionCleanupJob) Description() string {
|
||||
return "Clean up session connections from disconnected workers"
|
||||
}
|
||||
|
||||
// NextRunIn returns the next run time after a job is completed.
|
||||
//
|
||||
// The next run time is defined for sessionCleanupJob as one second.
|
||||
// This is because the job should run continuously to terminate
|
||||
// connections as soon as a worker has not reported in for a long
|
||||
// enough time. Only one job will ever run at once, so there is no
|
||||
// reason why it cannot run again immediately.
|
||||
func (j *sessionCleanupJob) NextRunIn() (time.Duration, error) { return time.Second, nil }
|
||||
|
||||
// Status returns the status of the running job.
|
||||
func (j *sessionCleanupJob) Status() scheduler.JobStatus {
|
||||
return scheduler.JobStatus{
|
||||
Completed: j.totalClosed,
|
||||
Total: j.totalClosed,
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the job.
|
||||
func (j *sessionCleanupJob) Run(ctx context.Context) error {
|
||||
const op = "controller.(sessionCleanupJob).Run"
|
||||
j.logger.Debug(
|
||||
"starting job",
|
||||
"op", op,
|
||||
)
|
||||
j.totalClosed = 0
|
||||
|
||||
// Load repos.
|
||||
sessionRepo, err := j.sessionRepoFn()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, op, errors.WithMsg("error getting session repo"))
|
||||
}
|
||||
|
||||
// Run the atomic dead worker cleanup job.
|
||||
results, err := sessionRepo.CloseConnectionsForDeadWorkers(ctx, j.gracePeriod)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, op)
|
||||
}
|
||||
|
||||
if len(results) < 1 {
|
||||
j.logger.Debug(
|
||||
"all workers OK, no connections to close",
|
||||
"op", op,
|
||||
)
|
||||
} else {
|
||||
for _, result := range results {
|
||||
// Log a closed connection message for each worker as a warning
|
||||
j.logger.Warn(
|
||||
"worker has not reported status within acceptable grace period, all connections closed",
|
||||
"op", op,
|
||||
"private_id", result.ServerId,
|
||||
"update_time", result.LastUpdateTime,
|
||||
"grace_period_seconds", j.gracePeriod,
|
||||
"number_connections_closed", result.NumberConnectionsClosed,
|
||||
)
|
||||
|
||||
j.totalClosed += result.NumberConnectionsClosed
|
||||
}
|
||||
}
|
||||
|
||||
j.logger.Debug(
|
||||
"job finished",
|
||||
"op", op,
|
||||
"total_connections_closed", j.totalClosed,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/boundary/internal/db"
|
||||
"github.com/hashicorp/boundary/internal/errors"
|
||||
"github.com/hashicorp/boundary/internal/iam"
|
||||
"github.com/hashicorp/boundary/internal/kms"
|
||||
"github.com/hashicorp/boundary/internal/scheduler"
|
||||
"github.com/hashicorp/boundary/internal/servers"
|
||||
"github.com/hashicorp/boundary/internal/session"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// assert the interface
|
||||
var _ = scheduler.Job(new(sessionCleanupJob))
|
||||
|
||||
// This test has been largely adapted from
|
||||
// TestRepository_CloseDeadConnectionsOnWorker in
|
||||
// internal/session/repository_connection_test.go.
|
||||
func TestSessionCleanupJob(t *testing.T) {
|
||||
t.Parallel()
|
||||
require, assert := require.New(t), assert.New(t)
|
||||
conn, _ := db.TestSetup(t, "postgres")
|
||||
rw := db.New(conn)
|
||||
wrapper := db.TestWrapper(t)
|
||||
iamRepo := iam.TestRepo(t, conn, wrapper)
|
||||
kms := kms.TestKms(t, conn, wrapper)
|
||||
serversRepo, err := servers.NewRepository(rw, rw, kms)
|
||||
require.NoError(err)
|
||||
sessionRepo, err := session.NewRepository(rw, rw, kms)
|
||||
require.NoError(err)
|
||||
ctx := context.Background()
|
||||
numConns := 12
|
||||
|
||||
// Create two "workers". One will remain untouched while the other "goes
|
||||
// away and comes back" (worker 2).
|
||||
worker1 := session.TestWorker(t, conn, wrapper, session.WithServerId("worker1"))
|
||||
worker2 := session.TestWorker(t, conn, wrapper, session.WithServerId("worker2"))
|
||||
|
||||
// Create a few sessions on each, activate, and authorize a connection
|
||||
var connIds []string
|
||||
connIdsByWorker := make(map[string][]string)
|
||||
for i := 0; i < numConns; i++ {
|
||||
serverId := worker1.PrivateId
|
||||
if i%2 == 0 {
|
||||
serverId = worker2.PrivateId
|
||||
}
|
||||
sess := session.TestDefaultSession(t, conn, wrapper, iamRepo, session.WithServerId(serverId), session.WithDbOpts(db.WithSkipVetForWrite(true)))
|
||||
sess, _, err = sessionRepo.ActivateSession(ctx, sess.GetPublicId(), sess.Version, serverId, "worker", []byte("foo"))
|
||||
require.NoError(err)
|
||||
c, cs, _, err := sessionRepo.AuthorizeConnection(ctx, sess.GetPublicId(), serverId)
|
||||
require.NoError(err)
|
||||
require.Len(cs, 1)
|
||||
require.Equal(session.StatusAuthorized, cs[0].Status)
|
||||
connIds = append(connIds, c.GetPublicId())
|
||||
if i%2 == 0 {
|
||||
connIdsByWorker[worker2.PrivateId] = append(connIdsByWorker[worker2.PrivateId], c.GetPublicId())
|
||||
} else {
|
||||
connIdsByWorker[worker1.PrivateId] = append(connIdsByWorker[worker1.PrivateId], c.GetPublicId())
|
||||
}
|
||||
}
|
||||
|
||||
// Mark half of the connections connected and leave the others authorized.
|
||||
// This is just to ensure we have a spread when we test it out.
|
||||
for i, connId := range connIds {
|
||||
if i%2 == 0 {
|
||||
_, cs, err := sessionRepo.ConnectConnection(ctx, session.ConnectWith{
|
||||
ConnectionId: connId,
|
||||
ClientTcpAddress: "127.0.0.1",
|
||||
ClientTcpPort: 22,
|
||||
EndpointTcpAddress: "127.0.0.1",
|
||||
EndpointTcpPort: 22,
|
||||
})
|
||||
require.NoError(err)
|
||||
require.Len(cs, 2)
|
||||
var foundAuthorized, foundConnected bool
|
||||
for _, status := range cs {
|
||||
if status.Status == session.StatusAuthorized {
|
||||
foundAuthorized = true
|
||||
}
|
||||
if status.Status == session.StatusConnected {
|
||||
foundConnected = true
|
||||
}
|
||||
}
|
||||
require.True(foundAuthorized)
|
||||
require.True(foundConnected)
|
||||
}
|
||||
}
|
||||
|
||||
// Create the job.
|
||||
job, err := newSessionCleanupJob(
|
||||
hclog.New(&hclog.LoggerOptions{Level: hclog.Trace}),
|
||||
func() (*session.Repository, error) { return sessionRepo, nil },
|
||||
session.DeadWorkerConnCloseMinGrace,
|
||||
)
|
||||
require.NoError(err)
|
||||
|
||||
// sleep the status grace period.
|
||||
time.Sleep(time.Second * time.Duration(session.DeadWorkerConnCloseMinGrace))
|
||||
|
||||
// Push an upsert to the first worker so that its status has been
|
||||
// updated.
|
||||
_, rowsUpdated, err := serversRepo.UpsertServer(ctx, worker1, []servers.Option{}...)
|
||||
require.NoError(err)
|
||||
require.Equal(1, rowsUpdated)
|
||||
|
||||
// Run the job.
|
||||
require.NoError(job.Run(ctx))
|
||||
|
||||
// Assert connection state on both workers.
|
||||
assertConnections := func(workerId string, closed bool) {
|
||||
connIds, ok := connIdsByWorker[workerId]
|
||||
require.True(ok)
|
||||
require.Len(connIds, 6)
|
||||
for _, connId := range connIds {
|
||||
_, states, err := sessionRepo.LookupConnection(ctx, connId, nil)
|
||||
require.NoError(err)
|
||||
var foundClosed bool
|
||||
for _, state := range states {
|
||||
if state.Status == session.StatusClosed {
|
||||
foundClosed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.Equal(closed, foundClosed)
|
||||
}
|
||||
}
|
||||
|
||||
// Assert that all connections on the second worker are closed
|
||||
assertConnections(worker2.PrivateId, true)
|
||||
// Assert that all connections on the first worker are still open
|
||||
assertConnections(worker1.PrivateId, false)
|
||||
}
|
||||
|
||||
func TestSessionCleanupJobNewJobErr(t *testing.T) {
|
||||
t.Parallel()
|
||||
const op = "controller.newNewSessionCleanupJob"
|
||||
require := require.New(t)
|
||||
|
||||
job, err := newSessionCleanupJob(nil, nil, 0)
|
||||
require.Equal(err, errors.E(
|
||||
errors.WithCode(errors.InvalidParameter),
|
||||
errors.WithOp(op),
|
||||
errors.WithMsg("missing logger"),
|
||||
))
|
||||
require.Nil(job)
|
||||
|
||||
job, err = newSessionCleanupJob(hclog.New(nil), nil, 0)
|
||||
require.Equal(err, errors.E(
|
||||
errors.WithCode(errors.InvalidParameter),
|
||||
errors.WithOp(op),
|
||||
errors.WithMsg("missing sessionRepoFn"),
|
||||
))
|
||||
require.Nil(job)
|
||||
|
||||
job, err = newSessionCleanupJob(hclog.New(nil), func() (*session.Repository, error) { return nil, nil }, 0)
|
||||
require.Equal(err, errors.E(
|
||||
errors.WithCode(errors.InvalidParameter),
|
||||
errors.WithOp(op),
|
||||
errors.WithMsg(fmt.Sprintf("invalid gracePeriod, must be greater than %d", session.DeadWorkerConnCloseMinGrace)),
|
||||
))
|
||||
require.Nil(job)
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/boundary/api"
|
||||
"github.com/hashicorp/boundary/api/authmethods"
|
||||
@@ -371,6 +372,10 @@ type TestControllerOpts struct {
|
||||
// A cluster address for overriding the advertised controller listener
|
||||
// (overrides address provided in config, if any)
|
||||
PublicClusterAddr string
|
||||
|
||||
// The amount of time to wait before marking connections as closed when a
|
||||
// worker has not reported in
|
||||
StatusGracePeriodDuration time.Duration
|
||||
}
|
||||
|
||||
func NewTestController(t *testing.T, opts *TestControllerOpts) *TestController {
|
||||
@@ -456,6 +461,9 @@ func NewTestController(t *testing.T, opts *TestControllerOpts) *TestController {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Initialize status grace period
|
||||
tc.b.SetStatusGracePeriodDuration(opts.StatusGracePeriodDuration)
|
||||
|
||||
if opts.Config.Controller == nil {
|
||||
opts.Config.Controller = new(config.Controller)
|
||||
}
|
||||
@@ -615,6 +623,7 @@ func (tc *TestController) AddClusterControllerMember(t *testing.T, opts *TestCon
|
||||
DisableKmsKeyCreation: true,
|
||||
DisableAuthMethodCreation: true,
|
||||
PublicClusterAddr: opts.PublicClusterAddr,
|
||||
StatusGracePeriodDuration: opts.StatusGracePeriodDuration,
|
||||
}
|
||||
if opts.Logger != nil {
|
||||
nextOpts.Logger = opts.Logger
|
||||
@@ -629,3 +638,71 @@ func (tc *TestController) AddClusterControllerMember(t *testing.T, opts *TestCon
|
||||
}
|
||||
return NewTestController(t, nextOpts)
|
||||
}
|
||||
|
||||
// WaitForNextWorkerStatusUpdate waits for the next status check from a worker to
|
||||
// come in. If it does not come in within the default status grace
|
||||
// period, this function returns an error.
|
||||
func (tc *TestController) WaitForNextWorkerStatusUpdate(workerId string) error {
|
||||
tc.Logger().Debug("waiting for next status report from worker", "worker", workerId)
|
||||
|
||||
if err := tc.waitForNextWorkerStatusUpdate(workerId); err != nil {
|
||||
tc.Logger().Error("error waiting for next status report from worker", "worker", workerId, "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
tc.Logger().Debug("waiting for next status report from worker received successfully", "worker", workerId)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc *TestController) waitForNextWorkerStatusUpdate(workerId string) error {
|
||||
waitStatusStart := time.Now()
|
||||
ctx, cancel := context.WithTimeout(tc.ctx, tc.b.StatusGracePeriodDuration)
|
||||
defer cancel()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
|
||||
case <-time.After(time.Second):
|
||||
// pass
|
||||
}
|
||||
|
||||
var waitStatusCurrent time.Time
|
||||
var err error
|
||||
tc.Controller().WorkerStatusUpdateTimes().Range(func(k, v interface{}) bool {
|
||||
if k == nil || v == nil {
|
||||
err = fmt.Errorf("nil key or value on entry: key=%#v value=%#v", k, v)
|
||||
return false
|
||||
}
|
||||
|
||||
workerStatusUpdateId, ok := k.(string)
|
||||
if !ok {
|
||||
err = fmt.Errorf("unexpected type %T for key: key=%#v value=%#v", k, k, v)
|
||||
return false
|
||||
}
|
||||
|
||||
workerStatusUpdateTime, ok := v.(time.Time)
|
||||
if !ok {
|
||||
err = fmt.Errorf("unexpected type %T for value: key=%#v value=%#v", k, k, v)
|
||||
return false
|
||||
}
|
||||
|
||||
if workerStatusUpdateId == workerId {
|
||||
waitStatusCurrent = workerStatusUpdateTime
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if waitStatusCurrent.Sub(waitStatusStart) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user