Fix issue with workers connecting in high latency conditions (#4535)

At some point we migrated to grpc ConnectParams to control how often
retries were happening. One value that we didn't set was the
MinConnectTimeout. It turns out that if you don't set this, it won't use
the default; instead if will actually use a value of zero, which gRPC
internally ups to something that seems to be around 25ms to meet a
minimum threshold. This is not documented in the function comments.

This commit simply sets the value of MinConnectTimeout to the unexported
gRPC default.

internal/daemon/worker/controller_connection.go

@@ -220,6 +220,11 @@ func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(co
 	  }
 	  `, defaultTimeout)
 
+	// minConnectTimeout replaces the unexported gRPC default value of 20
+	// seconds with the WithConnectParams option below; otherwise it sets the
+	// value to zero instead of unset, which causes it to be very small.
+	const minConnectTimeout = 20 * time.Second
+
 	dialOpts := []grpc.DialOption{
 		grpc.WithResolvers(res),
 		grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
@@ -238,6 +243,7 @@ func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(co
 				Jitter:     0.2,
 				MaxDelay:   3 * time.Second,
 			},
+			MinConnectTimeout: minConnectTimeout,
 		}),
 	}