feat(ratelimit): Expose metrics on quota usage

This provides two prometheus metrics to monitor the rate limiter:

boundary_controller_api_ratelimiter_quota_storage_capacity:

This is a gague that shows the total number of quotas that can be stored
at a given moment by the rate limiter.

boundary_controller_api_ratelimiter_quota_storage_usage:

This is a gague that shows the number of quotas currently being stored
by the limiter. If this becomes equal to the capacity, requests that
would result in a new quota being created will get limited and receive a
429 status code.

internal/daemon/controller/controller.go

@@ -160,6 +160,7 @@ type Controller struct {
 func New(ctx context.Context, conf *Config) (*Controller, error) {
 	const op = "controller.New"
 	metric.InitializeApiCollectors(conf.PrometheusRegisterer)
+	ratelimit.InitializeMetrics(conf.PrometheusRegisterer)
 	c := &Controller{
 		conf:                    conf,
 		logger:                  conf.Logger.Named("controller"),
@@ -255,7 +256,10 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 		return nil, fmt.Errorf("error parsing rate limit configuration: %w", err)
 	}
 
-	rateLimiter, err := rate.NewLimiter(rateLimits, conf.RawConfig.Controller.ApiRateLimiterMaxEntries)
+	rateLimiter, err := ratelimit.NewLimiter(
+		rateLimits,
+		conf.RawConfig.Controller.ApiRateLimiterMaxEntries,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("error initializing rate limiter: %w", err)
 	}
@@ -656,7 +660,7 @@ func (c *Controller) ReloadRateLimiter(newLimitConfigs ratelimit.Configs, newMax
 		return fmt.Errorf("error parsing rate limit configuration: %w", err)
 	}
 
-	limiter, err := rate.NewLimiter(ratelimits, newMaxEntries)
+	limiter, err := ratelimit.NewLimiter(ratelimits, newMaxEntries)
 	if err != nil {
 		return err
 	}

internal/ratelimit/limiter.go

@@ -0,0 +1,16 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package ratelimit
+
+import "github.com/hashicorp/go-rate"
+
+// NewLimiter creates a rate.Limiter.
+func NewLimiter(limits []*rate.Limit, maxEntries int) (*rate.Limiter, error) {
+	return rate.NewLimiter(
+		limits,
+		maxEntries,
+		rate.WithQuotaStorageUsageMetric(rateLimitQuotaUsage),
+		rate.WithQuotaStorageCapacityMetric(rateLimitQuotaStorageCapacity),
+	)
+}

internal/ratelimit/metrics.go

@@ -0,0 +1,43 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package ratelimit
+
+import (
+	"github.com/hashicorp/boundary/globals"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	subsystem = "controller_api_ratelimiter"
+)
+
+var (
+	rateLimitQuotaStorageCapacity prometheus.Gauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: globals.MetricNamespace,
+			Subsystem: subsystem,
+			Name:      "quota_storage_capacity",
+			Help:      "Guague of the number if quotas that can be stored by the rate limiter",
+		},
+	)
+	rateLimitQuotaUsage prometheus.Gauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: globals.MetricNamespace,
+			Subsystem: subsystem,
+			Name:      "quota_storage_usage",
+			Help:      "Guague of the number if quotas that are currently being stored by the rate limiter",
+		},
+	)
+)
+
+// InitializeMetrics initializes the metrics for visibility into the rate limiter.
+func InitializeMetrics(r prometheus.Registerer) {
+	if r == nil {
+		return
+	}
+	r.MustRegister(
+		rateLimitQuotaStorageCapacity,
+		rateLimitQuotaUsage,
+	)
+}