feat(metric): adds accepted/closed connections counters for controller and worker cluster connections (#2656)

same contents as PR https://github.com/hashicorp/boundary/pull/2593 but this time actually push to main

internal/daemon/controller/internal/metric/cluster.go

@@ -2,6 +2,7 @@ package metric
 
 import (
 	"context"
+	"net"
 
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/daemon/metric"
@@ -33,6 +34,28 @@ var grpcRequestLatency prometheus.ObserverVec = prometheus.NewHistogramVec(
 	metric.ListGrpcLabels,
 )
 
+// acceptedConnsTotal keeps a count of the total accepted connections to a controller.
+var acceptedConnsTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: globals.MetricNamespace,
+		Subsystem: clusterSubSystem,
+		Name:      "accepted_connections_total",
+		Help:      "Count of total accepted network connections to this controller.",
+	},
+	[]string{metric.LabelConnectionPurpose},
+)
+
+// closedConnsTotal keeps a count of the total closed connections to a controller.
+var closedConnsTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: globals.MetricNamespace,
+		Subsystem: clusterSubSystem,
+		Name:      "closed_connections_total",
+		Help:      "Count of total closed network connections to this controller.",
+	},
+	[]string{metric.LabelConnectionPurpose},
+)
+
 // All the codes expected to be returned by boundary or the grpc framework to
 // requests to the cluster server.
 var expectedGrpcCodes = []codes.Code{
@@ -45,6 +68,11 @@ var expectedGrpcCodes = []codes.Code{
 	codes.Unavailable, codes.Unauthenticated,
 }
 
+func InstrumentClusterTrackingListener(l net.Listener, purpose string) net.Listener {
+	p := prometheus.Labels{metric.LabelConnectionPurpose: purpose}
+	return metric.NewConnectionTrackingListener(l, acceptedConnsTotal.With(p), closedConnsTotal.With(p))
+}
+
 // InstrumentClusterStatsHandler returns a gRPC stats.Handler which observes
 // cluster specific metrics. Use with the cluster gRPC server.
 func InstrumentClusterStatsHandler(ctx context.Context) (stats.Handler, error) {
@@ -57,3 +85,7 @@ func InstrumentClusterStatsHandler(ctx context.Context) (stats.Handler, error) {
 func InitializeClusterCollectors(r prometheus.Registerer, server *grpc.Server) {
 	metric.InitializeGrpcCollectorsFromServer(r, grpcRequestLatency, server, expectedGrpcCodes)
 }
+
+func InitializeConnectionCounters(r prometheus.Registerer) {
+	metric.InitializeConnectionCounters(r, []prometheus.CounterVec{*acceptedConnsTotal, *closedConnsTotal})
+}

internal/daemon/controller/listeners.go

@@ -237,6 +237,7 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 		}
 	}
 
+	metric.InitializeConnectionCounters(c.conf.PrometheusRegisterer)
 	metric.InitializeClusterCollectors(c.conf.PrometheusRegisterer, workerServer)
 
 	ln.GrpcServer = workerServer
@@ -249,13 +250,13 @@ func (c *Controller) configureForCluster(ln *base.ServerListener) (func(), error
 			}
 		}()
 		go func() {
-			err := handleSecondaryConnection(c.baseContext, multiplexingReverseGrpcListener, c.downstreamRoutes, -1)
+			err := handleSecondaryConnection(c.baseContext, metric.InstrumentClusterTrackingListener(revPkiWorkerTrackingListener, "reverse-grpc"), c.downstreamRoutes, -1)
 			if err != nil {
 				event.WriteError(c.baseContext, op, err, event.WithInfoMsg("handleSecondaryConnection error"))
 			}
 		}()
 		go func() {
-			err := ln.GrpcServer.Serve(multiplexingAuthedListener)
+			err := ln.GrpcServer.Serve(metric.InstrumentClusterTrackingListener(multiplexingAuthedListener, "grpc"))
 			if err != nil {
 				event.WriteError(c.baseContext, op, err, event.WithInfoMsg("multiplexingAuthedListener error"))
 			}