feat(worker): Map cluster ID in config to upstream address

internal/cmd/commands/server/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
+	"github.com/hashicorp/go-uuid"
 	"net"
 	"os"
 	"os/signal"
@@ -330,6 +331,14 @@ func (c *Command) Run(args []string) int {
 				}
 			}
 		}
+
+		if c.Config.Worker.HCPBClusterId != "" {
+			_, err := uuid.ParseUUID(c.Config.Worker.HCPBClusterId)
+			if err != nil {
+				c.UI.Error(fmt.Errorf("Invalid HCPB cluster id %q: %w", c.Config.Worker.HCPBClusterId, err).Error())
+				return base.CommandUserError
+			}
+		}
 	}
 	if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy", "ops"}); err != nil {
 		c.UI.Error(err.Error())

internal/cmd/config/config.go

@@ -198,6 +198,9 @@ type Worker struct {
 
 	// AuthStoragePath represents the location a worker stores its node credentials, if set
 	AuthStoragePath string `hcl:"auth_storage_path"`
+
+	// Internal field for use with HCP deployments. Will be used if controllers/ initial_upstreams is not set
+	HCPBClusterId string `hcl:"hcp_boundary_cluster_id"`
 }
 
 func (w *Worker) InitNameIfEmpty() (string, error) {

internal/daemon/worker/controller_connection.go

@@ -35,6 +35,8 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const hcpb_url_suffix = ".proxy.boundary.hashicorp.cloud"
+
 // StartControllerConnections starts up the resolver and initiates controller
 // connection client creation
 func (w *Worker) StartControllerConnections() error {
@@ -57,7 +59,13 @@ func (w *Worker) StartControllerConnections() error {
 	}
 
 	if len(initialAddrs) == 0 {
-		return errors.New("no initial controller addresses found")
+		if w.conf.RawConfig.Worker.HCPBClusterId != "" {
+			clusterAddress := fmt.Sprintf("%s%s", w.conf.RawConfig.Worker.HCPBClusterId, hcpb_url_suffix)
+			initialAddrs = append(initialAddrs, resolver.Address{Addr: clusterAddress})
+			event.WriteSysEvent(context.TODO(), op, fmt.Sprintf("Setting HCPB Cluster address %s as upstream address", clusterAddress))
+		} else {
+			return errors.New("no initial controller addresses found")
+		}
 	}
 
 	w.Resolver().InitialState(resolver.State{
@@ -101,8 +109,10 @@ func (w *Worker) controllerDialerFunc() func(context.Context, string) (net.Conn,
 			event.WriteError(ctx, op, err)
 		}
 
-		if !w.everAuthenticated.Load() && err == nil && conn != nil {
-			w.everAuthenticated.Store(true)
+		if err == nil && conn != nil {
+			if !w.everAuthenticated.Load() {
+				w.everAuthenticated.Store(true)
+			}
 			event.WriteSysEvent(ctx, op, "worker has successfully authenticated")
 		}
 

internal/daemon/worker/status.go

@@ -3,21 +3,24 @@ package worker
 import (
 	"context"
 	"errors"
+	"fmt"
 	"math/rand"
+	"sort"
+	"strings"
 	"time"
 
-	"github.com/hashicorp/boundary/internal/servers"
-
 	"github.com/hashicorp/boundary/internal/daemon/worker/common"
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/hashicorp/boundary/internal/servers"
 	"google.golang.org/grpc/resolver"
 )
 
 type LastStatusInformation struct {
 	*pbs.StatusResponse
-	StatusTime time.Time
+	StatusTime              time.Time
+	LastCalculatedUpstreams []resolver.Address
 }
 
 func (w *Worker) startStatusTicking(cancelCtx context.Context) {
@@ -191,14 +194,29 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context) {
 	} else {
 		w.updateTags.Store(false)
 		// This may be nil if we are in a multiple hop scenario
+		var addrs []resolver.Address
 		if len(result.CalculatedUpstreams) > 0 {
-			addrs := make([]resolver.Address, 0, len(result.CalculatedUpstreams))
+			addrs = make([]resolver.Address, 0, len(result.CalculatedUpstreams))
 			for _, v := range result.CalculatedUpstreams {
 				addrs = append(addrs, resolver.Address{Addr: v.Address})
 			}
-			w.Resolver().UpdateState(resolver.State{Addresses: addrs})
+			lastStatus := w.lastStatusSuccess.Load().(*LastStatusInformation)
+			// Compare upstreams; update resolver if there is a difference, and emit an event with old and new addresses
+			if lastStatus != nil && upstreamsHasChanged(lastStatus.LastCalculatedUpstreams, addrs) {
+				var oldUpstreams []string
+				for _, v := range lastStatus.LastCalculatedUpstreams {
+					oldUpstreams = append(oldUpstreams, v.Addr)
+				}
+				var newUpstreams []string
+				for _, v := range addrs {
+					newUpstreams = append(newUpstreams, v.Addr)
+				}
+				upstreamsMessage := fmt.Sprintf("Upstreams has changed; old upstreams were: %s, new upstreams are: %s", oldUpstreams, newUpstreams)
+				event.WriteSysEvent(context.TODO(), op, upstreamsMessage)
+				w.Resolver().UpdateState(resolver.State{Addresses: addrs})
+			}
 		}
-		w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now()})
+		w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now(), LastCalculatedUpstreams: addrs})
 
 		for _, request := range result.GetJobsRequests() {
 			switch request.GetRequestType() {
@@ -239,6 +257,28 @@ func (w *Worker) sendWorkerStatus(cancelCtx context.Context) {
 	w.cleanupConnections(cancelCtx, false)
 }
 
+func upstreamsHasChanged(oldUpstreams, newUpstreams []resolver.Address) bool {
+	if len(oldUpstreams) != len(newUpstreams) {
+		return true
+	}
+
+	// Sort both upstreams
+	sort.Slice(oldUpstreams, func(i, j int) bool {
+		return strings.Compare(oldUpstreams[i].Addr, oldUpstreams[j].Addr) < 0
+	})
+	sort.Slice(newUpstreams, func(i, j int) bool {
+		return strings.Compare(newUpstreams[i].Addr, newUpstreams[j].Addr) < 0
+	})
+
+	// Compare and return true if change detected
+	for i, _ := range oldUpstreams {
+		if oldUpstreams[i].Addr != newUpstreams[i].Addr {
+			return true
+		}
+	}
+	return false
+}
+
 // cleanupConnections walks all sessions and shuts down connections.
 // Additionally, sessions without connections are cleaned up from the
 // local worker's state.