internal/event: avoid reusing canceled context (#5304)

Don't error in ReleaseGate if the context used for the original
event has been cancelled. Any error in ReleaseGate causes the
controller to terminate its startup. If a user was trying to send
a request to the Boundary controller as it started up, the context
tied to the request would be reused to attempt logging the
observation associated with the request after the logging gate
was released. This would always fail, as the context associated
with the request was canceled.

We now use a new context timeout for events logged after the release
gate is released, if the original context was canceled.

internal/event/eventer.go

@@ -794,14 +794,18 @@ func (e *Eventer) ReleaseGate() error {
 		if qe == nil {
 			continue // we may have already sent this but gotten errors later
 		}
+		ctx, cancel := newSendCtx(qe.ctx)
+		if cancel != nil {
+			defer cancel()
+		}
 		var queuedOp string
 		switch t := qe.event.(type) {
 		case *sysEvent:
 			queuedOp = "system"
-			writeErr = e.writeSysEvent(qe.ctx, t, WithNoGateLocking(true))
+			writeErr = e.writeSysEvent(ctx, t, WithNoGateLocking(true))
 		case *err:
 			queuedOp = "error"
-			writeErr = e.writeError(qe.ctx, t, WithNoGateLocking(true))
+			writeErr = e.writeError(ctx, t, WithNoGateLocking(true))
 		default:
 			// Have no idea what this is and shouldn't have gotten in here to
 			// begin with, so just continue, and log it

internal/event/eventer_gate_test.go

@@ -159,3 +159,48 @@ func TestEventer_Gating(t *testing.T) {
 		})
 	}
 }
+
+func TestReleaseGate_NoError_CanceledContext(t *testing.T) {
+	require := require.New(t)
+
+	buffer := new(bytes.Buffer)
+	eventerConfig := EventerConfig{
+		AuditEnabled:        true,
+		ObservationsEnabled: true,
+		SysEventsEnabled:    true,
+		Sinks: []*SinkConfig{
+			{
+				Name:       "test-sink",
+				EventTypes: []Type{EveryType},
+				Format:     TextHclogSinkFormat,
+				Type:       WriterSink,
+				WriterConfig: &WriterSinkTypeConfig{
+					Writer: buffer,
+				},
+			},
+		},
+	}
+	testLock := &sync.Mutex{}
+	testLogger := testLogger(t, testLock)
+
+	eventer, err := NewEventer(
+		testLogger,
+		testLock,
+		"TestEventer_Gating",
+		eventerConfig,
+		WithGating(true),
+	)
+	require.NoError(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	ctx, err = NewEventerContext(ctx, eventer)
+	require.NoError(err)
+
+	WriteError(ctx, "error-1", fmt.Errorf("error-1"))
+	_ = WriteObservation(ctx, "observation-1", WithId("observation-1"), WithHeader("foo", "bar"))
+
+	cancel()
+
+	require.NoError(eventer.ReleaseGate())
+}