Append initial worker upstreams if current upstreams are unresponsive (#2698)

* Append initial worker upstreams if current upstreams are unresponsive
This commit is contained in:
Irena Rindos authored and GitHub committed 2022-12-12 13:48:28 -05:00
1 parent 24c3d6824d
commit 173e1587d0
4 files changed
+121 -32

No files matched your search

+63 -30
View File
@@ -16,6 +16,7 @@ import (
"github.com/hashicorp/boundary/internal/server"
"github.com/hashicorp/boundary/version"
"github.com/hashicorp/go-secure-stdlib/strutil"
"google.golang.org/grpc/connectivity"
)
var firstStatusCheckPostHooks []func(context.Context, *Worker) error
@@ -206,6 +207,31 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager sess
return true
})
// In the case that the control plane has gone down and come up with different IPs,
// append initial upstreams/ cluster addr to the resolver to try
if w.GrpcClientConn.GetState() == connectivity.TransientFailure {
lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
addrs := lastStatus.LastCalculatedUpstreams
if len(w.conf.RawConfig.Worker.InitialUpstreams) > 0 {
addrs = append(addrs, w.conf.RawConfig.Worker.InitialUpstreams...)
}
if len(w.conf.RawConfig.HcpbClusterId) > 0 {
clusterAddress := fmt.Sprintf("%s%s", w.conf.RawConfig.HcpbClusterId, hcpbUrlSuffix)
addrs = append(addrs, clusterAddress)
}
addrs = strutil.RemoveDuplicates(addrs, false)
if strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) {
// Nothing to update
return
}
w.updateAddresses(cancelCtx, addrs, addressReceivers)
lastStatus.LastCalculatedUpstreams = addrs
w.lastStatusSuccess.Store(lastStatus)
}
// Exit out of status function; our work here is done and we don't need to create closeConnection requests
return
}
@@ -243,36 +269,7 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager sess
}
}
if len(addrs) > 0 {
lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
// Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses
if lastStatus != nil && !strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) {
upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastStatus.LastCalculatedUpstreams, addrs)
event.WriteSysEvent(cancelCtx, op, upstreamsMessage)
for _, as := range *addressReceivers {
as.SetAddresses(addrs)
}
} else if lastStatus == nil {
for _, as := range *addressReceivers {
as.SetAddresses(addrs)
}
event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first status set to: %s", addrs))
}
}
// regardless of whether or not it's a new address, we need to set
// them for dialingListeners
for _, as := range *addressReceivers {
switch {
case as.Type() == dialingListenerReceiverType:
tmpAddrs := make([]string, len(addrs))
copy(tmpAddrs, addrs)
if len(tmpAddrs) == 0 {
tmpAddrs = append(tmpAddrs, w.conf.RawConfig.Worker.InitialUpstreams...)
}
as.SetAddresses(tmpAddrs)
}
}
w.updateAddresses(cancelCtx, addrs, addressReceivers)
w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now(), LastCalculatedUpstreams: addrs})
@@ -330,6 +327,42 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context, sessionManager sess
}
}
// Update address receivers and dialing listeners with new addrs
func (w *Worker) updateAddresses(cancelCtx context.Context, addrs []string, addressReceivers *[]addressReceiver) {
const op = "worker.(Worker).updateAddrs"
if len(addrs) > 0 {
lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
// Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses
if lastStatus != nil && !strutil.EquivalentSlices(lastStatus.LastCalculatedUpstreams, addrs) {
upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", lastStatus.LastCalculatedUpstreams, addrs)
event.WriteSysEvent(cancelCtx, op, upstreamsMessage)
for _, as := range *addressReceivers {
as.SetAddresses(addrs)
}
} else if lastStatus == nil {
for _, as := range *addressReceivers {
as.SetAddresses(addrs)
}
event.WriteSysEvent(cancelCtx, op, fmt.Sprintf("Upstreams after first status set to: %s", addrs))
}
}
// regardless of whether or not it's a new address, we need to set
// them for dialingListeners
for _, as := range *addressReceivers {
switch {
case as.Type() == dialingListenerReceiverType:
tmpAddrs := make([]string, len(addrs))
copy(tmpAddrs, addrs)
if len(tmpAddrs) == 0 {
tmpAddrs = append(tmpAddrs, w.conf.RawConfig.Worker.InitialUpstreams...)
}
as.SetAddresses(tmpAddrs)
}
}
}
// cleanupConnections walks all sessions and shuts down all proxy connections.
// After the local connections are terminated, they are requested to be marked
// close on the controller.
+1 -1
View File
@@ -275,7 +275,7 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
tw.name = opts.Config.Worker.Name
if opts.SuccessfulStatusGracePeriodDuration != 0 {
opts.Config.Worker.SuccessfulStatusGracePeriod = opts.SuccessfulStatusGracePeriodDuration
opts.Config.Worker.SuccessfulStatusGracePeriodDuration = opts.SuccessfulStatusGracePeriodDuration
}
serverName, err := os.Hostname()
-1
View File
@@ -203,7 +203,6 @@ func New(conf *Config) (*Worker, error) {
err)
}
}
switch conf.RawConfig.Worker.SuccessfulStatusGracePeriodDuration {
case 0:
w.successfulStatusGracePeriod.Store(int64(server.DefaultLiveness))