feat: Worker Connectivity Health Check (#3650)

* feat: Worker Connectivity Health Check

Add worker upstream connection state. API calls to the worker's ops `/heath?worker_info=1` endpoint should now return a new filed called `upstream_connection_state`

```
{
   "worker_process_info":{
      "state":"active",
      "active_session_count":0,
      "upstream_connection_state":"READY"
   }
}
```

The states returned will be GRPC connection states: https://grpc.github.io/grpc/core/md_doc_connectivity-semantics-and-api.html

There could be multiple upstream addresses but GRPC manual resolver determines the state of the connection.
This commit is contained in:
Elim Tsiagbey authored and GitHub committed 2023-09-01 13:42:41 -04:00
1 parent c55df52d44
commit fabd4093de
8 files changed
+272 -59

No files matched your search

+69 -33
View File
@@ -19,6 +19,7 @@ import (
"os"
"path/filepath"
"strings"
"sync/atomic"
"time"
"github.com/hashicorp/boundary/globals"
@@ -30,6 +31,7 @@ import (
pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
"github.com/hashicorp/boundary/internal/observability/event"
"github.com/hashicorp/boundary/internal/server"
"github.com/hashicorp/boundary/internal/util"
"github.com/hashicorp/boundary/version"
"github.com/hashicorp/go-secure-stdlib/base62"
"github.com/hashicorp/nodeenrollment"
@@ -38,6 +40,7 @@ import (
"github.com/hashicorp/nodeenrollment/util/toggledlogger"
"google.golang.org/grpc"
"google.golang.org/grpc/backoff"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
"google.golang.org/protobuf/proto"
@@ -211,19 +214,7 @@ func (w *Worker) v1KmsAuthDialFn(ctx context.Context, addr string, extraAlpnProt
func (w *Worker) createClientConn(addr string) error {
const op = "worker.(Worker).createClientConn"
defaultTimeout := (time.Second + time.Nanosecond).String()
defServiceConfig := fmt.Sprintf(`
{
"loadBalancingConfig": [ { "round_robin": {} } ],
"methodConfig": [
{
"name": [],
"timeout": %q,
"waitForReady": true
}
]
}
`, defaultTimeout)
var res resolver.Builder
for _, v := range w.addressReceivers {
if rec, ok := v.(*grpcResolverReceiver); ok {
@@ -233,26 +224,9 @@ func (w *Worker) createClientConn(addr string) error {
if res == nil {
return errors.New(w.baseContext, errors.Internal, op, "unable to find a resolver.Builder amongst the address receivers")
}
dialOpts := []grpc.DialOption{
grpc.WithResolvers(res),
grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
grpc.WithContextDialer(w.upstreamDialerFunc()),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultServiceConfig(defServiceConfig),
// Don't have the resolver reach out for a service config from the
// resolver, use the one specified as default
grpc.WithDisableServiceConfig(),
grpc.WithConnectParams(grpc.ConnectParams{
Backoff: backoff.Config{
BaseDelay: time.Second,
Multiplier: 1.2,
Jitter: 0.2,
MaxDelay: 3 * time.Second,
},
}),
}
dialOpts := createDefaultGRPCDialOptions(res, w.upstreamDialerFunc())
cc, err := grpc.DialContext(w.baseContext,
fmt.Sprintf("%s:///%s", res.Scheme(), addr),
dialOpts...,
@@ -271,9 +245,51 @@ func (w *Worker) createClientConn(addr string) error {
w.controllerUpstreamMsgConn.Store(&producer)
go monitorUpstreamConnectionState(w.baseContext, cc, w.upstreamConnectionState)
return nil
}
// createDefaultGRPCDialOptions creates grpc.DialOption using default options
func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(context.Context, string) (net.Conn, error)) []grpc.DialOption {
defaultTimeout := (time.Second + time.Nanosecond).String()
defServiceConfig := fmt.Sprintf(`
{
"loadBalancingConfig": [ { "round_robin": {} } ],
"methodConfig": [
{
"name": [],
"timeout": %q,
"waitForReady": true
}
]
}
`, defaultTimeout)
dialOpts := []grpc.DialOption{
grpc.WithResolvers(res),
grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
grpc.WithContextDialer(upstreamDialerFn),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultServiceConfig(defServiceConfig),
// Don't have the resolver reach out for a service config from the
// resolver, use the one specified as default
grpc.WithDisableServiceConfig(),
grpc.WithConnectParams(grpc.ConnectParams{
Backoff: backoff.Config{
BaseDelay: time.Second,
Multiplier: 1.2,
Jitter: 0.2,
MaxDelay: 3 * time.Second,
},
}),
}
return dialOpts
}
func (w *Worker) workerAuthTLSConfig(extraAlpnProtos ...string) (*tls.Config, *base.WorkerAuthInfo, error) {
var err error
info := &base.WorkerAuthInfo{
@@ -403,3 +419,23 @@ func (w *Worker) workerConnectionInfo(addr string) (*structpb.Struct, error) {
}
return st, nil
}
// monitorUpstreamConnectionState listens for new state changes from grpc client
// connection and updates the state
func monitorUpstreamConnectionState(ctx context.Context, cc *grpc.ClientConn, connectionState *atomic.Value) {
var state connectivity.State
if v := connectionState.Load(); !util.IsNil(v) {
state = v.(connectivity.State)
}
for cc.WaitForStateChange(ctx, state) {
newState := cc.GetState()
// if the client is shutdown, exit function
if newState == connectivity.Shutdown {
return
}
connectionState.Store(newState)
}
}