feat: Worker Connectivity Health Check (#3650)
* feat: Worker Connectivity Health Check
Add worker upstream connection state. API calls to the worker's ops `/heath?worker_info=1` endpoint should now return a new filed called `upstream_connection_state`
```
{
"worker_process_info":{
"state":"active",
"active_session_count":0,
"upstream_connection_state":"READY"
}
}
```
The states returned will be GRPC connection states: https://grpc.github.io/grpc/core/md_doc_connectivity-semantics-and-api.html
There could be multiple upstream addresses but GRPC manual resolver determines the state of the connection.
This commit is contained in:
1 parent
c55df52d44
commit
fabd4093de
8 files changed
+272
-59
No files matched your search
@@ -19,6 +19,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/boundary/globals"
|
||||
@@ -30,6 +31,7 @@ import (
|
||||
pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
|
||||
"github.com/hashicorp/boundary/internal/observability/event"
|
||||
"github.com/hashicorp/boundary/internal/server"
|
||||
"github.com/hashicorp/boundary/internal/util"
|
||||
"github.com/hashicorp/boundary/version"
|
||||
"github.com/hashicorp/go-secure-stdlib/base62"
|
||||
"github.com/hashicorp/nodeenrollment"
|
||||
@@ -38,6 +40,7 @@ import (
|
||||
"github.com/hashicorp/nodeenrollment/util/toggledlogger"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/backoff"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/resolver"
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -211,19 +214,7 @@ func (w *Worker) v1KmsAuthDialFn(ctx context.Context, addr string, extraAlpnProt
|
||||
|
||||
func (w *Worker) createClientConn(addr string) error {
|
||||
const op = "worker.(Worker).createClientConn"
|
||||
defaultTimeout := (time.Second + time.Nanosecond).String()
|
||||
defServiceConfig := fmt.Sprintf(`
|
||||
{
|
||||
"loadBalancingConfig": [ { "round_robin": {} } ],
|
||||
"methodConfig": [
|
||||
{
|
||||
"name": [],
|
||||
"timeout": %q,
|
||||
"waitForReady": true
|
||||
}
|
||||
]
|
||||
}
|
||||
`, defaultTimeout)
|
||||
|
||||
var res resolver.Builder
|
||||
for _, v := range w.addressReceivers {
|
||||
if rec, ok := v.(*grpcResolverReceiver); ok {
|
||||
@@ -233,26 +224,9 @@ func (w *Worker) createClientConn(addr string) error {
|
||||
if res == nil {
|
||||
return errors.New(w.baseContext, errors.Internal, op, "unable to find a resolver.Builder amongst the address receivers")
|
||||
}
|
||||
dialOpts := []grpc.DialOption{
|
||||
grpc.WithResolvers(res),
|
||||
grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
|
||||
grpc.WithContextDialer(w.upstreamDialerFunc()),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(defServiceConfig),
|
||||
// Don't have the resolver reach out for a service config from the
|
||||
// resolver, use the one specified as default
|
||||
grpc.WithDisableServiceConfig(),
|
||||
grpc.WithConnectParams(grpc.ConnectParams{
|
||||
Backoff: backoff.Config{
|
||||
BaseDelay: time.Second,
|
||||
Multiplier: 1.2,
|
||||
Jitter: 0.2,
|
||||
MaxDelay: 3 * time.Second,
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
dialOpts := createDefaultGRPCDialOptions(res, w.upstreamDialerFunc())
|
||||
|
||||
cc, err := grpc.DialContext(w.baseContext,
|
||||
fmt.Sprintf("%s:///%s", res.Scheme(), addr),
|
||||
dialOpts...,
|
||||
@@ -271,9 +245,51 @@ func (w *Worker) createClientConn(addr string) error {
|
||||
|
||||
w.controllerUpstreamMsgConn.Store(&producer)
|
||||
|
||||
go monitorUpstreamConnectionState(w.baseContext, cc, w.upstreamConnectionState)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createDefaultGRPCDialOptions creates grpc.DialOption using default options
|
||||
func createDefaultGRPCDialOptions(res resolver.Builder, upstreamDialerFn func(context.Context, string) (net.Conn, error)) []grpc.DialOption {
|
||||
defaultTimeout := (time.Second + time.Nanosecond).String()
|
||||
defServiceConfig := fmt.Sprintf(`
|
||||
{
|
||||
"loadBalancingConfig": [ { "round_robin": {} } ],
|
||||
"methodConfig": [
|
||||
{
|
||||
"name": [],
|
||||
"timeout": %q,
|
||||
"waitForReady": true
|
||||
}
|
||||
]
|
||||
}
|
||||
`, defaultTimeout)
|
||||
|
||||
dialOpts := []grpc.DialOption{
|
||||
grpc.WithResolvers(res),
|
||||
grpc.WithUnaryInterceptor(metric.InstrumentClusterClient()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(math.MaxInt32)),
|
||||
grpc.WithContextDialer(upstreamDialerFn),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(defServiceConfig),
|
||||
// Don't have the resolver reach out for a service config from the
|
||||
// resolver, use the one specified as default
|
||||
grpc.WithDisableServiceConfig(),
|
||||
grpc.WithConnectParams(grpc.ConnectParams{
|
||||
Backoff: backoff.Config{
|
||||
BaseDelay: time.Second,
|
||||
Multiplier: 1.2,
|
||||
Jitter: 0.2,
|
||||
MaxDelay: 3 * time.Second,
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
return dialOpts
|
||||
}
|
||||
|
||||
func (w *Worker) workerAuthTLSConfig(extraAlpnProtos ...string) (*tls.Config, *base.WorkerAuthInfo, error) {
|
||||
var err error
|
||||
info := &base.WorkerAuthInfo{
|
||||
@@ -403,3 +419,23 @@ func (w *Worker) workerConnectionInfo(addr string) (*structpb.Struct, error) {
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// monitorUpstreamConnectionState listens for new state changes from grpc client
|
||||
// connection and updates the state
|
||||
func monitorUpstreamConnectionState(ctx context.Context, cc *grpc.ClientConn, connectionState *atomic.Value) {
|
||||
var state connectivity.State
|
||||
if v := connectionState.Load(); !util.IsNil(v) {
|
||||
state = v.(connectivity.State)
|
||||
}
|
||||
|
||||
for cc.WaitForStateChange(ctx, state) {
|
||||
newState := cc.GetState()
|
||||
|
||||
// if the client is shutdown, exit function
|
||||
if newState == connectivity.Shutdown {
|
||||
return
|
||||
}
|
||||
|
||||
connectionState.Store(newState)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
opsservices "github.com/hashicorp/boundary/internal/gen/ops/services"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/nettest"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
"google.golang.org/grpc/resolver/manual"
|
||||
)
|
||||
|
||||
func TestMonitorUpstreamConnectionState(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
stateCtx, cancelStateCtx := context.WithCancel(ctx)
|
||||
|
||||
upstreamConnectionState := new(atomic.Value)
|
||||
|
||||
servers, err := createTestServers(t)
|
||||
require.NoError(t, err)
|
||||
|
||||
scheme := strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
res := manual.NewBuilderWithScheme(scheme)
|
||||
grpcResolver := &grpcResolverReceiver{res}
|
||||
|
||||
dialOpts := createDefaultGRPCDialOptions(res, nil)
|
||||
|
||||
cc, err := grpc.Dial(
|
||||
fmt.Sprintf("%s:///%s", res.Scheme(), servers[0].address),
|
||||
dialOpts...,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// track GRPC state changes
|
||||
go monitorUpstreamConnectionState(stateCtx, cc, upstreamConnectionState)
|
||||
|
||||
grpcResolver.InitialAddresses([]string{servers[0].address})
|
||||
|
||||
t.Cleanup(func() {
|
||||
cc.Close()
|
||||
cancelStateCtx()
|
||||
|
||||
assert.Equal(t, connectivity.Shutdown, cc.GetState())
|
||||
|
||||
for _, s := range servers {
|
||||
s.srv.GracefulStop()
|
||||
}
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
expectedResponse *opsservices.GetHealthResponse
|
||||
addresses []string
|
||||
expectedState connectivity.State
|
||||
}{
|
||||
{
|
||||
name: "connection with 1 good address",
|
||||
addresses: []string{servers[0].address},
|
||||
expectedState: connectivity.Ready,
|
||||
},
|
||||
{
|
||||
name: "connection with multiple good addresses",
|
||||
addresses: []string{servers[1].address, servers[2].address},
|
||||
expectedState: connectivity.Ready,
|
||||
},
|
||||
{
|
||||
name: "connection with bad address",
|
||||
addresses: []string{"bad_address"},
|
||||
expectedState: connectivity.TransientFailure,
|
||||
},
|
||||
{
|
||||
name: "connection with 1 bad address and 1 good address",
|
||||
addresses: []string{servers[0].address, "bad_address"},
|
||||
expectedState: connectivity.Ready,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
doneWait := make(chan struct{})
|
||||
grpcResolver.SetAddresses(tt.addresses)
|
||||
|
||||
go waitForConnectionStateCondition(upstreamConnectionState, tt.expectedState, doneWait)
|
||||
|
||||
select {
|
||||
case <-doneWait:
|
||||
// The connection condition was met, Proceed with assertions
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Error("Time out waiting for condition")
|
||||
}
|
||||
|
||||
got := upstreamConnectionState.Load()
|
||||
assert.Equal(t, tt.expectedState, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func waitForConnectionStateCondition(upstreamConnectionState *atomic.Value, expectedValue connectivity.State, ch chan<- struct{}) {
|
||||
for {
|
||||
currentValue := upstreamConnectionState.Load()
|
||||
if expectedValue == currentValue {
|
||||
ch <- struct{}{}
|
||||
return
|
||||
}
|
||||
// Small delay for checking state
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type serverTestInfo struct {
|
||||
srv *grpc.Server
|
||||
address string
|
||||
}
|
||||
|
||||
func createTestServers(t *testing.T) ([]*serverTestInfo, error) {
|
||||
serverCount := 4
|
||||
|
||||
servers := make([]*serverTestInfo, 0, serverCount)
|
||||
for i := 0; i < serverCount; i++ {
|
||||
listener, err := nettest.NewLocalListener("tcp")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
lInfo := &serverTestInfo{srv: srv, address: listener.Addr().String()}
|
||||
servers = append(servers, lInfo)
|
||||
go func(i int) {
|
||||
servers[i].srv.Serve(listener)
|
||||
}(i)
|
||||
}
|
||||
|
||||
return servers, nil
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
pbhealth "github.com/hashicorp/boundary/internal/gen/worker/health"
|
||||
"github.com/hashicorp/boundary/internal/server"
|
||||
"github.com/hashicorp/boundary/internal/util"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/types/known/wrapperspb"
|
||||
)
|
||||
@@ -50,12 +51,19 @@ func (w workerHealthServer) GetHealth(ctx context.Context, req *opsservices.GetH
|
||||
|
||||
// HealthInformation returns the current worker process health information.
|
||||
func (w *Worker) HealthInformation() *pbhealth.HealthInfo {
|
||||
state := server.UnknownOperationalState
|
||||
operationalState := server.UnknownOperationalState
|
||||
if v := w.operationalState.Load(); !util.IsNil(v) {
|
||||
state = v.(server.OperationalState)
|
||||
operationalState = v.(server.OperationalState)
|
||||
}
|
||||
|
||||
var upstreamConnectionState connectivity.State
|
||||
if v := w.upstreamConnectionState.Load(); !util.IsNil(v) {
|
||||
upstreamConnectionState = v.(connectivity.State)
|
||||
}
|
||||
|
||||
healthInfo := &pbhealth.HealthInfo{
|
||||
State: state.String(),
|
||||
State: operationalState.String(),
|
||||
UpstreamConnectionState: upstreamConnectionState.String(),
|
||||
}
|
||||
|
||||
if w.sessionManager == nil {
|
||||
|
||||
@@ -79,7 +79,14 @@ func TestGetHealth(t *testing.T) {
|
||||
resp := &opsservices.GetHealthResponse{}
|
||||
require.NoError(t, healthCheckMarshaler.Unmarshal(b, resp))
|
||||
|
||||
assert.Empty(t, cmp.Diff(tt.expectedResponse, resp, protocmp.Transform()))
|
||||
assert.Empty(t,
|
||||
cmp.Diff(
|
||||
tt.expectedResponse,
|
||||
resp,
|
||||
protocmp.Transform(),
|
||||
protocmp.IgnoreFields(&pbhealth.HealthInfo{}, "upstream_connection_state"),
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -127,10 +127,11 @@ type Worker struct {
|
||||
|
||||
recorderManager recorderManager
|
||||
|
||||
everAuthenticated *ua.Uint32
|
||||
lastStatusSuccess *atomic.Value
|
||||
workerStartTime time.Time
|
||||
operationalState *atomic.Value
|
||||
everAuthenticated *ua.Uint32
|
||||
lastStatusSuccess *atomic.Value
|
||||
workerStartTime time.Time
|
||||
operationalState *atomic.Value
|
||||
upstreamConnectionState *atomic.Value
|
||||
|
||||
controllerMultihopConn *atomic.Value
|
||||
|
||||
@@ -207,6 +208,7 @@ func New(ctx context.Context, conf *Config) (*Worker, error) {
|
||||
pkiConnManager: cluster.NewDownstreamManager(),
|
||||
successfulStatusGracePeriod: new(atomic.Int64),
|
||||
statusCallTimeoutDuration: new(atomic.Int64),
|
||||
upstreamConnectionState: new(atomic.Value),
|
||||
}
|
||||
|
||||
w.operationalState.Store(server.UnknownOperationalState)
|
||||
|
||||
Reference in new issue
Block a user