From 7e214026af349a2029774075df63a9d5134e099b Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Fri, 15 Mar 2024 15:31:07 -0400 Subject: [PATCH] Fix issue with workers connecting in high latency conditions (#4535) At some point we migrated to grpc ConnectParams to control how often retries were happening. One value that we didn't set was the MinConnectTimeout. It turns out that if you don't set this, it won't use the default; instead if will actually use a value of zero, which gRPC internally ups to something that seems to be around 25ms to meet a minimum threshold. This is not documented in the function comments. This commit simply sets the value of MinConnectTimeout to the unexported gRPC default. --- internal/daemon/worker/controller_connection.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/daemon/worker/controller_connection.go b/internal/daemon/worker/controller_connection.go index b3aeb4ee2..33bd766f4 100644 --- a/internal/daemon/worker/controller_connection.go +++ b/internal/daemon/worker/controller_connection.go @@ -220,6 +220,11 @@ func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(co } `, defaultTimeout) + // minConnectTimeout replaces the unexported gRPC default value of 20 + // seconds with the WithConnectParams option below; otherwise it sets the + // value to zero instead of unset, which causes it to be very small. + const minConnectTimeout = 20 * time.Second + dialOpts := []grpc.DialOption{ grpc.WithResolvers(res), grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()), @@ -238,6 +243,7 @@ func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(co Jitter: 0.2, MaxDelay: 3 * time.Second, }, + MinConnectTimeout: minConnectTimeout, }), }