feat: Worker Connectivity Health Check (#3650)
* feat: Worker Connectivity Health Check
Add worker upstream connection state. API calls to the worker's ops `/heath?worker_info=1` endpoint should now return a new filed called `upstream_connection_state`
```
{
"worker_process_info":{
"state":"active",
"active_session_count":0,
"upstream_connection_state":"READY"
}
}
```
The states returned will be GRPC connection states: https://grpc.github.io/grpc/core/md_doc_connectivity-semantics-and-api.html
There could be multiple upstream addresses but GRPC manual resolver determines the state of the connection.
This commit is contained in:
1 parent
c55df52d44
commit
fabd4093de
8 files changed
+272
-59
No files matched your search
@@ -19,6 +19,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/boundary/globals"
|
||||
@@ -30,6 +31,7 @@ import (
|
||||
pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
|
||||
"github.com/hashicorp/boundary/internal/observability/event"
|
||||
"github.com/hashicorp/boundary/internal/server"
|
||||
"github.com/hashicorp/boundary/internal/util"
|
||||
"github.com/hashicorp/boundary/version"
|
||||
"github.com/hashicorp/go-secure-stdlib/base62"
|
||||
"github.com/hashicorp/nodeenrollment"
|
||||
@@ -38,6 +40,7 @@ import (
|
||||
"github.com/hashicorp/nodeenrollment/util/toggledlogger"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/backoff"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/resolver"
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -211,19 +214,7 @@ func (w *Worker) v1KmsAuthDialFn(ctx context.Context, addr string, extraAlpnProt
|
||||
|
||||
func (w *Worker) createClientConn(addr string) error {
|
||||
const op = "worker.(Worker).createClientConn"
|
||||
defaultTimeout := (time.Second + time.Nanosecond).String()
|
||||
defServiceConfig := fmt.Sprintf(`
|
||||
{
|
||||
"loadBalancingConfig": [ { "round_robin": {} } ],
|
||||
"methodConfig": [
|
||||
{
|
||||
"name": [],
|
||||
"timeout": %q,
|
||||
"waitForReady": true
|
||||
}
|
||||
]
|
||||
}
|
||||
`, defaultTimeout)
|
||||
|
||||
var res resolver.Builder
|
||||
for _, v := range w.addressReceivers {
|
||||
if rec, ok := v.(*grpcResolverReceiver); ok {
|
||||
@@ -233,26 +224,9 @@ func (w *Worker) createClientConn(addr string) error {
|
||||
if res == nil {
|
||||
return errors.New(w.baseContext, errors.Internal, op, "unable to find a resolver.Builder amongst the address receivers")
|
||||
}
|
||||
dialOpts := []grpc.DialOption{
|
||||
grpc.WithResolvers(res),
|
||||
grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
|
||||
grpc.WithContextDialer(w.upstreamDialerFunc()),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(defServiceConfig),
|
||||
// Don't have the resolver reach out for a service config from the
|
||||
// resolver, use the one specified as default
|
||||
grpc.WithDisableServiceConfig(),
|
||||
grpc.WithConnectParams(grpc.ConnectParams{
|
||||
Backoff: backoff.Config{
|
||||
BaseDelay: time.Second,
|
||||
Multiplier: 1.2,
|
||||
Jitter: 0.2,
|
||||
MaxDelay: 3 * time.Second,
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
dialOpts := createDefaultGRPCDialOptions(res, w.upstreamDialerFunc())
|
||||
|
||||
cc, err := grpc.DialContext(w.baseContext,
|
||||
fmt.Sprintf("%s:///%s", res.Scheme(), addr),
|
||||
dialOpts...,
|
||||
@@ -271,9 +245,51 @@ func (w *Worker) createClientConn(addr string) error {
|
||||
|
||||
w.controllerUpstreamMsgConn.Store(&producer)
|
||||
|
||||
go monitorUpstreamConnectionState(w.baseContext, cc, w.upstreamConnectionState)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createDefaultGRPCDialOptions creates grpc.DialOption using default options
|
||||
func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(context.Context, string) (net.Conn, error)) []grpc.DialOption {
|
||||
defaultTimeout := (time.Second + time.Nanosecond).String()
|
||||
defServiceConfig := fmt.Sprintf(`
|
||||
{
|
||||
"loadBalancingConfig": [ { "round_robin": {} } ],
|
||||
"methodConfig": [
|
||||
{
|
||||
"name": [],
|
||||
"timeout": %q,
|
||||
"waitForReady": true
|
||||
}
|
||||
]
|
||||
}
|
||||
`, defaultTimeout)
|
||||
|
||||
dialOpts := []grpc.DialOption{
|
||||
grpc.WithResolvers(res),
|
||||
grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
|
||||
grpc.WithContextDialer(upstreamDialerFn),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(defServiceConfig),
|
||||
// Don't have the resolver reach out for a service config from the
|
||||
// resolver, use the one specified as default
|
||||
grpc.WithDisableServiceConfig(),
|
||||
grpc.WithConnectParams(grpc.ConnectParams{
|
||||
Backoff: backoff.Config{
|
||||
BaseDelay: time.Second,
|
||||
Multiplier: 1.2,
|
||||
Jitter: 0.2,
|
||||
MaxDelay: 3 * time.Second,
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
return dialOpts
|
||||
}
|
||||
|
||||
func (w *Worker) workerAuthTLSConfig(extraAlpnProtos ...string) (*tls.Config, *base.WorkerAuthInfo, error) {
|
||||
var err error
|
||||
info := &base.WorkerAuthInfo{
|
||||
@@ -403,3 +419,23 @@ func (w *Worker) workerConnectionInfo(addr string) (*structpb.Struct, error) {
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// monitorUpstreamConnectionState listens for new state changes from grpc client
|
||||
// connection and updates the state
|
||||
func monitorUpstreamConnectionState(ctx context.Context, cc *grpc.ClientConn, connectionState *atomic.Value) {
|
||||
var state connectivity.State
|
||||
if v := connectionState.Load(); !util.IsNil(v) {
|
||||
state = v.(connectivity.State)
|
||||
}
|
||||
|
||||
for cc.WaitForStateChange(ctx, state) {
|
||||
newState := cc.GetState()
|
||||
|
||||
// if the client is shutdown, exit function
|
||||
if newState == connectivity.Shutdown {
|
||||
return
|
||||
}
|
||||
|
||||
connectionState.Store(newState)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user