Downstream worker connections are tracked by worker id (#2949)

When possible downstream workers will include their worker id information in
the connections they establish with the upstream using the nodeenrollment
library.  The tracking listener now tracks connections that are connected in
this same way instead of just connections of type *tls.Conn.

internal/daemon/controller/listeners.go

@@ -21,11 +21,18 @@ import (
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/go-multierror"
+	nodee "github.com/hashicorp/nodeenrollment"
 	nodeenet "github.com/hashicorp/nodeenrollment/net"
 	"github.com/hashicorp/nodeenrollment/protocol"
 	"google.golang.org/grpc"
 )
 
+const (
+	// the purpose strings used to identify listeners
+	reverseGrpcListenerPurpose = "reverse-grpc"
+	grpcListenerPurpose        = "grpc"
+)
+
 // the function that handles a secondary connection over a provided listener
 var handleSecondaryConnection = closeListener
 
@@ -151,7 +158,7 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 	// This handles connections coming in on the cluster port that are
 	// authenticated via nodeenrollment but not with any extra purpose; these
 	// are normal worker connections
-	nodeeAuthedListener, err := splitListener.GetListener(nodeenet.AuthenticatedNonSpecificNextProto)
+	nodeeAuthedListener, err := splitListener.GetListener(nodeenet.AuthenticatedNonSpecificNextProto, nodee.WithNativeConns(true))
 	if err != nil {
 		return nil, fmt.Errorf("error instantiating node enrollment authed split listener: %w", err)
 	}
@@ -164,7 +171,7 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 
 	// This wraps the normal pki worker connections with a listener which adds
 	// the worker key id of the connections to the controller's pkiConnManager.
-	pkiWorkerTrackingListener, err := cluster.NewTrackingListener(c.baseContext, eventingAuthedListener, c.pkiConnManager)
+	pkiWorkerTrackingListener, err := cluster.NewTrackingListener(c.baseContext, eventingAuthedListener, c.pkiConnManager, sourcePurpose(grpcListenerPurpose))
 	if err != nil {
 		return nil, fmt.Errorf("%s: error creating pki worker tracking listener: %w", op, err)
 	}
@@ -186,14 +193,14 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 
 	// Connections coming in here are authed by nodeenrollment and are for the
 	// reverse grpc purpose
-	reverseGrpcListener, err := splitListener.GetListener(common.ReverseGrpcConnectionAlpnValue)
+	reverseGrpcListener, err := splitListener.GetListener(common.ReverseGrpcConnectionAlpnValue, nodee.WithNativeConns(true))
 	if err != nil {
 		return nil, fmt.Errorf("error instantiating reverse gprc connection split listener: %w", err)
 	}
 
 	// This wraps the reverse grpc pki worker connections with a listener which
 	// adds the worker key id of the connections to the pkiConnManager.
-	revPkiWorkerTrackingListener, err := cluster.NewTrackingListener(c.baseContext, reverseGrpcListener, c.pkiConnManager)
+	revPkiWorkerTrackingListener, err := cluster.NewTrackingListener(c.baseContext, reverseGrpcListener, c.pkiConnManager, sourcePurpose(reverseGrpcListenerPurpose))
 	if err != nil {
 		return nil, fmt.Errorf("%s: error creating reverse grpc pki worker tracking listener: %w", op, err)
 	}
@@ -246,7 +253,7 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 	ln.GrpcServer = workerServer
 
 	return func() {
-		err := handleSecondaryConnection(c.baseContext, metric.InstrumentClusterTrackingListener(multiplexingReverseGrpcListener, "reverse-grpc"),
+		err := handleSecondaryConnection(c.baseContext, metric.InstrumentClusterTrackingListener(multiplexingReverseGrpcListener, reverseGrpcListenerPurpose),
 			c.downstreamRoutes)
 		if err != nil {
 			event.WriteError(c.baseContext, op, err, event.WithInfoMsg("handleSecondaryConnection error"))
@@ -258,7 +265,7 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 			}
 		}()
 		go func() {
-			err := ln.GrpcServer.Serve(metric.InstrumentClusterTrackingListener(multiplexingAuthedListener, "grpc"))
+			err := ln.GrpcServer.Serve(metric.InstrumentClusterTrackingListener(multiplexingAuthedListener, grpcListenerPurpose))
 			if err != nil {
 				event.WriteError(c.baseContext, op, err, event.WithInfoMsg("multiplexingAuthedListener error"))
 			}
@@ -372,3 +379,7 @@ func listenerCloseErrorCheck(lnType string, err error) error {
 
 	return err
 }
+
+func sourcePurpose(purpose string) string {
+	return fmt.Sprintf("controller %s", purpose)
+}

internal/daemon/controller/tickers.go

@@ -10,10 +10,12 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/daemon/cluster"
-	"github.com/hashicorp/boundary/internal/server/store"
-
+	"github.com/hashicorp/boundary/internal/daemon/common"
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/boundary/internal/server/store"
+	"github.com/hashicorp/boundary/internal/types/scope"
 )
 
 // In the future we could make this configurable
@@ -190,18 +192,34 @@ func (c *Controller) startWorkerConnectionMaintenanceTicking(cancelCtx context.C
 				return
 
 			case <-timer.C:
-				repo, err := c.WorkerAuthRepoStorageFn()
-				if err != nil {
-					event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error fetching repository for cluster connection maintenance"))
-					break
+				connectionState := m.Connected()
+				if len(connectionState.WorkerIds()) > 0 {
+					serverRepo, err := c.ServersRepoFn()
+					if err != nil {
+						event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error fetching server repository for cluster connection maintenance"))
+						break
+					}
+					knownWorker, err := serverRepo.ListWorkers(cancelCtx, []string{scope.Global.String()}, server.WithWorkerPool(connectionState.WorkerIds()), server.WithLiveness(-1))
+					if err != nil {
+						event.WriteError(cancelCtx, op, err, event.WithInfoMsg("couldn't get known workers from repo"))
+						break
+					}
+					connectionState.DisconnectMissingWorkers(common.WorkerList(knownWorker).PublicIds())
 				}
-				wKeyIds := m.Connected()
-				authorized, err := repo.FilterToAuthorizedWorkerKeyIds(cancelCtx, wKeyIds)
-				if err != nil {
-					event.WriteError(cancelCtx, op, err, event.WithInfoMsg("couldn't get authorized workers from repo"))
-					break
+
+				if len(connectionState.UnmappedKeyIds()) > 0 {
+					repo, err := c.WorkerAuthRepoStorageFn()
+					if err != nil {
+						event.WriteError(cancelCtx, op, err, event.WithInfoMsg("error fetching worker auth repository for cluster connection maintenance"))
+						break
+					}
+					authorized, err := repo.FilterToAuthorizedWorkerKeyIds(cancelCtx, connectionState.UnmappedKeyIds())
+					if err != nil {
+						event.WriteError(cancelCtx, op, err, event.WithInfoMsg("couldn't get authorized workers from repo"))
+						break
+					}
+					connectionState.DisconnectMissingKeyIds(authorized)
 				}
-				cluster.DisconnectUnauthorized(m, wKeyIds, authorized)
 			}
 
 			timer.Reset(getRandomInterval())