feat(controller): Emit sys event detailing the rate limit configuration

A sys event will be emitted when the controller is constructed, and if
the rate limits are changed via a SIGHUP. The event contains the details
of the rate limits for each combination of resource and action. This
event can be used by operators to confirm the limits are set as they
would expect based on the controller config.

internal/daemon/controller/rate_limiter.go

@@ -8,10 +8,29 @@ import (
 
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/event"
 	"github.com/hashicorp/boundary/internal/ratelimit"
 	"github.com/hashicorp/go-rate"
 )
 
+// limit is a representation of a rate.Limit that is used when emitting a sys
+// event to report the rate limit configuration.
+type limit struct {
+	Resource  string `json:"resource"`
+	Action    string `json:"action"`
+	Per       string `json:"per"`
+	Unlimited bool   `json:"unlimited"`
+	Limit     uint64 `json:"limit"`
+	Period    string `json:"period"`
+}
+
+// Types used for the nested structure of the sys event.
+type (
+	actionLimits    []limit
+	resourceActions map[string]actionLimits
+	resources       map[string]resourceActions
+)
+
 type rateLimiterConfig struct {
 	maxSize  int
 	disabled bool
@@ -44,6 +63,69 @@ func newRateLimiterConfig(ctx context.Context, configs ratelimit.Configs, maxSiz
 	}, nil
 }
 
+// writeSysEvent writes a sys event for c
+func (c *rateLimiterConfig) writeSysEvent(ctx context.Context) {
+	const op = "controller.(rateLimiterConfig).writeSysEvent"
+
+	if c.disabled {
+		event.WriteSysEvent(
+			ctx,
+			op,
+			"controller api rate limiter",
+			"disabled",
+			true,
+		)
+		return
+	}
+
+	e := make(resources)
+
+	for _, l := range c.limits {
+		var r resourceActions
+		var a actionLimits
+		var ok bool
+		r, ok = e[l.GetResource()]
+		if !ok {
+			r = make(resourceActions)
+			e[l.GetResource()] = r
+		}
+
+		a, ok = r[l.GetAction()]
+		if !ok {
+			a = make(actionLimits, 0, 3)
+		}
+
+		switch ll := l.(type) {
+		case *rate.Limited:
+			a = append(a, limit{
+				Resource:  ll.Resource,
+				Action:    ll.Action,
+				Per:       ll.Per.String(),
+				Unlimited: false,
+				Limit:     ll.MaxRequests,
+				Period:    ll.Period.String(),
+			})
+		case *rate.Unlimited:
+			a = append(a, limit{
+				Resource:  ll.Resource,
+				Action:    ll.Action,
+				Per:       ll.Per.String(),
+				Unlimited: true,
+			})
+		}
+		r[l.GetAction()] = a
+	}
+	event.WriteSysEvent(
+		ctx,
+		op,
+		"controller api rate limiter",
+		"limits",
+		e,
+		"max_size",
+		c.maxSize,
+	)
+}
+
 func (c *Controller) initializeRateLimiter(conf *config.Config) error {
 	const op = "controller.(Controller).initializeRateLimiter"
 	switch {
@@ -78,6 +160,7 @@ func (c *Controller) initializeRateLimiter(conf *config.Config) error {
 
 	c.conf.rateLimiterConfig = rlConfig
 
+	rlConfig.writeSysEvent(c.baseContext)
 	return nil
 }
 
@@ -136,6 +219,7 @@ func (c *Controller) ReloadRateLimiter(newConfig *config.Config) error {
 	c.rateLimiter = limiter
 	c.conf.rateLimiterConfig = rlConfig
 	c.rateLimiterMu.Unlock()
+	rlConfig.writeSysEvent(c.baseContext)
 
 	return old.Shutdown()
 }

internal/daemon/controller/rate_limiter_test.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -19,11 +18,22 @@ import (
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/event"
 	"github.com/hashicorp/boundary/internal/ratelimit"
+	"github.com/hashicorp/eventlogger/formatter_filters/cloudevents"
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-rate"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func testLogger(t *testing.T, testLock hclog.Locker) hclog.Logger {
+	t.Helper()
+	return hclog.New(&hclog.LoggerOptions{
+		Mutex:      testLock,
+		Name:       "test",
+		JSONFormat: true,
+	})
+}
+
 func Test_newRateLimiterConfig(t *testing.T) {
 	ctx := context.Background()
 
@@ -479,3 +489,183 @@ func TestControllerReloadRateLimiter(t *testing.T) {
 		})
 	}
 }
+
+func Test_rateLimiterConfig_writeSysEvent(t *testing.T) {
+	c := event.TestEventerConfig(t, t.Name())
+
+	testLock := &sync.Mutex{}
+	testLogger := testLogger(t, testLock)
+	e, err := event.NewEventer(testLogger, testLock, t.Name(), c.EventerConfig)
+	require.NoError(t, err)
+
+	info := &event.RequestInfo{Id: "867-5309", EventId: "411"}
+
+	testCtx, err := event.NewEventerContext(context.Background(), e)
+	require.NoError(t, err)
+	testCtx, err = event.NewRequestInfoContext(testCtx, info)
+	require.NoError(t, err)
+
+	cases := []struct {
+		name         string
+		setup        func(n string) error
+		cleanup      func()
+		sinkFileName string
+		configs      ratelimit.Configs
+		maxSize      int
+		disabled     bool
+	}{
+		{
+			"defaults",
+			func(n string) error {
+				return event.InitSysEventer(testLogger, testLock, n, event.WithEventerConfig(&c.EventerConfig))
+			},
+			func() { event.TestResetSystEventer(t) },
+			c.AllEvents.Name(),
+			nil,
+			ratelimit.DefaultLimiterMaxEntries(),
+			false,
+		},
+		{
+			"override",
+			func(n string) error {
+				return event.InitSysEventer(testLogger, testLock, n, event.WithEventerConfig(&c.EventerConfig))
+			},
+			func() { event.TestResetSystEventer(t) },
+			c.AllEvents.Name(),
+			ratelimit.Configs{
+				{
+					Resources: []string{"*"},
+					Actions:   []string{"*"},
+					Per:       rate.LimitPerTotal.String(),
+					Limit:     100,
+					Period:    time.Minute,
+					Unlimited: false,
+				},
+				{
+					Resources: []string{"*"},
+					Actions:   []string{"*"},
+					Per:       rate.LimitPerIPAddress.String(),
+					Limit:     100,
+					Period:    time.Minute,
+					Unlimited: false,
+				},
+				{
+					Resources: []string{"*"},
+					Actions:   []string{"*"},
+					Per:       rate.LimitPerAuthToken.String(),
+					Limit:     100,
+					Period:    time.Minute,
+					Unlimited: false,
+				},
+			},
+			ratelimit.DefaultLimiterMaxEntries(),
+			false,
+		},
+		{
+			"max_size",
+			func(n string) error {
+				return event.InitSysEventer(testLogger, testLock, n, event.WithEventerConfig(&c.EventerConfig))
+			},
+			func() { event.TestResetSystEventer(t) },
+			c.AllEvents.Name(),
+			nil,
+			3000,
+			false,
+		},
+		{
+			"disabled",
+			func(n string) error {
+				return event.InitSysEventer(testLogger, testLock, n, event.WithEventerConfig(&c.EventerConfig))
+			},
+			func() { event.TestResetSystEventer(t) },
+			c.AllEvents.Name(),
+			nil,
+			0,
+			true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.setup != nil {
+				require.NoError(t, tc.setup(t.Name()))
+			}
+			if tc.cleanup != nil {
+				defer tc.cleanup()
+			}
+
+			wantFile, err := os.Open(filepath.Join("testdata", t.Name()+".json"))
+			require.NoError(t, err)
+			defer wantFile.Close()
+			want := &cloudevents.Event{}
+			wantDecoder := json.NewDecoder(wantFile)
+			err = wantDecoder.Decode(want)
+			require.NoError(t, err)
+
+			rlc, err := newRateLimiterConfig(testCtx, tc.configs, tc.maxSize, tc.disabled)
+			require.NoError(t, err)
+
+			rlc.writeSysEvent(testCtx)
+
+			defer func() { _ = os.WriteFile(tc.sinkFileName, nil, 0o666) }()
+			b, err := os.ReadFile(tc.sinkFileName)
+			require.NoError(t, err)
+
+			got := &cloudevents.Event{}
+			err = json.Unmarshal(b, got)
+			require.NoErrorf(t, err, "json: %s", string(b))
+
+			assert.Empty(t, cmp.Diff(
+				got,
+				want,
+				cmpopts.IgnoreFields(cloudevents.Event{}, "ID", "Time", "Data"),
+			))
+
+			gotData := got.Data.(map[string]interface{})
+			wantData := want.Data.(map[string]interface{})
+			assert.Equal(t, len(gotData), len(wantData))
+
+			for k, v := range wantData {
+				switch k {
+				case "data":
+					wantDataData := v.(map[string]interface{})
+					gotDataData := gotData[k].(map[string]interface{})
+					assert.Equal(t, len(gotDataData), len(wantDataData))
+					for k, v := range wantDataData {
+						switch k {
+						case "limits":
+							wantResources := v.(map[string]interface{})
+							gotResources := gotDataData[k].(map[string]interface{})
+							for k, v := range wantResources {
+								gotv, ok := gotResources[k]
+								require.True(t, ok)
+
+								wantResourceLimits := v.(map[string]interface{})
+								gotResourceLimits := gotv.(map[string]interface{})
+								require.Equal(t, len(gotResourceLimits), len(wantResourceLimits))
+
+								for k, v := range wantResourceLimits {
+									gotv, ok := gotResourceLimits[k]
+									require.True(t, ok)
+									gotActionLimits := v.([]interface{})
+									wantActionLimits := gotv.([]interface{})
+									require.Equal(t, len(gotActionLimits), len(wantActionLimits))
+
+									assert.ElementsMatch(t, gotActionLimits, wantActionLimits)
+								}
+							}
+						case "max_size", "msg", "disabled":
+							assert.Equal(t, v, gotDataData[k])
+						default:
+							require.Fail(t, "unexpected key %s", k)
+						}
+					}
+				case "op", "version":
+					assert.Equal(t, v, gotData[k])
+				default:
+					require.Fail(t, "unexpected key %s", k)
+				}
+			}
+		})
+	}
+}

internal/daemon/controller/testdata/Test_rateLimiterConfig_writeSysEvent/disabled.json

@@ -0,0 +1,15 @@
+{
+  "id": "",
+  "source": "https://hashicorp.com/boundary/Test_rateLimiterConfig_writeSysEvent",
+  "specversion": "1.0",
+  "type": "system",
+  "data": {
+    "data": {
+      "disabled": true,
+      "msg": "controller api rate limiter"
+    },
+    "op": "controller.(rateLimiterConfig).writeSysEvent",
+    "version": "v0.1"
+  },
+  "datacontentype": "application/cloudevents"
+}