Disconnect PKI workers from upstreams when they are no longer authorized (#2515)

internal/daemon/controller/controller.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	credstatic "github.com/hashicorp/boundary/internal/credential/static"
 	"github.com/hashicorp/boundary/internal/credential/vault"
+	"github.com/hashicorp/boundary/internal/daemon/cluster"
 	"github.com/hashicorp/boundary/internal/daemon/controller/common"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/health"
 	"github.com/hashicorp/boundary/internal/daemon/controller/internal/metric"
@@ -132,6 +133,8 @@ type Controller struct {
 	// Used to signal the Health Service to start
 	// replying to queries with "503 Service Unavailable".
 	HealthService *health.Service
+
+	pkiConnManager *cluster.DownstreamManager
 }
 
 func New(ctx context.Context, conf *Config) (*Controller, error) {
@@ -146,6 +149,7 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 		workerStatusUpdateTimes: new(sync.Map),
 		enabledPlugins:          conf.Server.EnabledPlugins,
 		apiListeners:            make([]*base.ServerListener, 0),
+		pkiConnManager:          cluster.NewDownstreamManager(),
 	}
 
 	if downstreamRouterFactory != nil {
@@ -385,7 +389,7 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 
 func (c *Controller) Start() error {
 	const op = "controller.(Controller).Start"
-	if c.started.Load() {
+	if c.started.Swap(true) {
 		event.WriteSysEvent(context.TODO(), op, "already started, skipping")
 		return nil
 	}
@@ -424,10 +428,9 @@ func (c *Controller) Start() error {
 		defer c.tickerWg.Done()
 		c.startCloseExpiredPendingTokens(c.baseContext)
 	}()
-	go func() {
-		defer c.tickerWg.Done()
-		c.started.Store(true)
-	}()
+	if err := c.startWorkerConnectionMaintenanceTicking(c.baseContext, c.tickerWg, c.pkiConnManager); err != nil {
+		return errors.Wrap(c.baseContext, err, op)
+	}
 
 	if c.downstreamRoutes != nil {
 		c.tickerWg.Add(2)