fix(controller): Use max request duration cluster listener (#4805)

The cluster listener supports a `max_request_duration` option, and
defaults to 90 seconds. However, this configuration was not being use
when processing grpc requests from the listener. This meant that
requests from worker to controller could hang indefinitely and
contributed to the recently fixed issue with database connections being
stuck in `idle in transaction`.

This commit adds a grpc interceptor that ensures each request has a
deadline set to the max request duration for the listener.

Refs: 29cd7bc9d6
This commit is contained in:
Timothy Messier authored and GitHub committed 2024-05-17 11:59:50 -04:00
1 parent 7954ad42fd
commit a2c529b9d1
3 files changed
+29

No files matched your search

+10
View File
@@ -9,6 +9,7 @@ import (
"net/http"
"reflect"
"runtime/debug"
"time"
grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
"github.com/hashicorp/boundary/globals"
@@ -496,6 +497,15 @@ func eventsResponseInterceptor(
}
}
func requestMaxDurationInterceptor(_ context.Context, maxRequestDuration time.Duration) grpc.UnaryServerInterceptor {
const op = "controller.requestMaxDurationInterceptor"
return func(interceptorCtx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
withTimeout, cancel := context.WithTimeout(interceptorCtx, maxRequestDuration)
defer cancel()
return handler(withTimeout, req)
}
}
func workerRequestInfoInterceptor(ctx context.Context, eventer *event.Eventer) (grpc.UnaryServerInterceptor, error) {
const op = "worker.requestInfoInterceptor"
if eventer == nil {