Add worker tagging (#862)

This allows workers to have tags assigned to them, which can then be filtered to control which targets they are allowed to serve.

internal/servers/controller/handlers/targets/target_service.go

@@ -28,6 +28,7 @@ import (
 	"github.com/hashicorp/boundary/internal/types/resource"
 	"github.com/hashicorp/boundary/internal/types/scope"
 	"github.com/hashicorp/boundary/sdk/strutil"
+	"github.com/hashicorp/go-bexpr"
 	"github.com/mr-tron/base58"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/protobuf/proto"
@@ -418,6 +419,7 @@ HostSetIterationLoop:
 		Endpoint:        endpointUrl.String(),
 		ExpirationTime:  &timestamp.Timestamp{Timestamp: expTime},
 		ConnectionLimit: t.GetSessionConnectionLimit(),
+		WorkerFilter:    t.GetWorkerFilter(),
 	}
 
 	sess, err := session.New(sessionComposition)
@@ -433,15 +435,77 @@ HostSetIterationLoop:
 		return nil, err
 	}
 
+	// WorkerInfo only contains the address; worker IDs below is used to contain
+	// their IDs in the same order. This is used to fetch tags for filtering.
+	// But we avoid allocation unless we actually need it.
 	var workers []*pb.WorkerInfo
+	var workerIds []string
+	hasWorkerFilter := len(t.GetWorkerFilter()) > 0
 	servers, err := serversRepo.ListServers(ctx, servers.ServerTypeWorker)
 	if err != nil {
 		return nil, err
 	}
 	for _, v := range servers {
+		if hasWorkerFilter {
+			workerIds = append(workerIds, v.GetPrivateId())
+		}
 		workers = append(workers, &pb.WorkerInfo{Address: v.Address})
 	}
 
+	if hasWorkerFilter && len(workerIds) > 0 {
+		finalWorkers := make([]*pb.WorkerInfo, 0, len(workers))
+		// Fetch the tags for the given worker IDs
+		tags, err := serversRepo.ListTagsForServers(ctx, workerIds)
+		if err != nil {
+			return nil, err
+		}
+		// Build the map for filtering. This is similar to the filter map we
+		// built from the worker config, but with one extra level: a map of the
+		// worker's ID to its filter map.
+		tagMap := make(map[string]map[string][]string)
+		for _, tag := range tags {
+			currWorkerMap := tagMap[tag.ServerId]
+			if currWorkerMap == nil {
+				currWorkerMap = make(map[string][]string)
+				tagMap[tag.ServerId] = currWorkerMap
+			}
+			currWorkerMap[tag.Key] = append(currWorkerMap[tag.Key], tag.Value)
+			// We don't need to reinsert after the fact because maps are
+			// reference types, so we don't need to re-insert into tagMap
+		}
+
+		// Create the evaluator
+		eval, err := bexpr.CreateEvaluator(t.GetWorkerFilter())
+		if err != nil {
+			return nil, err
+		}
+
+		// Iterate through the known worker IDs, and evaluate. If evaluation
+		// returns true, add to the final worker slice, which is assigned back
+		// to workers after this.
+		for i, worker := range workerIds {
+			filterInput := map[string]interface{}{
+				"name": worker,
+				"tags": tagMap[worker],
+			}
+			ok, err := eval.Evaluate(filterInput)
+			if err != nil {
+				return nil, handlers.ApiErrorWithCodeAndMessage(
+					codes.FailedPrecondition,
+					fmt.Sprintf("Worker filter expression evaluation resulted in error: %s", err))
+			}
+			if ok {
+				finalWorkers = append(finalWorkers, workers[i])
+			}
+		}
+		workers = finalWorkers
+	}
+	if len(workers) == 0 {
+		return nil, handlers.ApiErrorWithCodeAndMessage(
+			codes.FailedPrecondition,
+			"No workers are available to handle this session, or all have been filtered.")
+	}
+
 	sad := &pb.SessionAuthorizationData{
 		SessionId:       sess.PublicId,
 		TargetId:        t.GetPublicId(),
@@ -505,6 +569,9 @@ func (s Service) createInRepo(ctx context.Context, item *pb.Target) (*pb.Target,
 	if item.GetSessionConnectionLimit() != nil {
 		opts = append(opts, target.WithSessionConnectionLimit(item.GetSessionConnectionLimit().GetValue()))
 	}
+	if item.GetWorkerFilter() != nil {
+		opts = append(opts, target.WithWorkerFilter(item.GetWorkerFilter().GetValue()))
+	}
 	tcpAttrs := &pb.TcpTargetAttributes{}
 	if err := handlers.StructToProto(item.GetAttributes(), tcpAttrs); err != nil {
 		return nil, handlers.ApiErrorWithCodeAndMessage(codes.InvalidArgument, "Provided attributes don't match expected format.")
@@ -544,6 +611,9 @@ func (s Service) updateInRepo(ctx context.Context, scopeId, id string, mask []st
 	if item.GetSessionConnectionLimit() != nil {
 		opts = append(opts, target.WithSessionConnectionLimit(item.GetSessionConnectionLimit().GetValue()))
 	}
+	if filter := item.GetWorkerFilter(); filter != nil {
+		opts = append(opts, target.WithWorkerFilter(item.GetWorkerFilter().GetValue()))
+	}
 	tcpAttrs := &pb.TcpTargetAttributes{}
 	if err := handlers.StructToProto(item.GetAttributes(), tcpAttrs); err != nil {
 		return nil, handlers.ApiErrorWithCodeAndMessage(codes.InvalidArgument, "Provided attributes don't match expected format.")
@@ -737,6 +807,9 @@ func toProto(in target.Target, m []*target.TargetSet) (*pb.Target, error) {
 	if in.GetName() != "" {
 		out.Name = wrapperspb.String(in.GetName())
 	}
+	if in.GetWorkerFilter() != "" {
+		out.WorkerFilter = wrapperspb.String(in.GetWorkerFilter())
+	}
 	attrs := &pb.TcpTargetAttributes{}
 	if in.GetDefaultPort() > 0 {
 		attrs.DefaultPort = &wrappers.UInt32Value{Value: in.GetDefaultPort()}
@@ -803,6 +876,11 @@ func validateCreateRequest(req *pbs.CreateTargetRequest) error {
 		default:
 			badFields["type"] = "Unknown type provided."
 		}
+		if filter := req.GetItem().GetWorkerFilter(); filter != nil {
+			if _, err := bexpr.CreateEvaluator(filter.GetValue()); err != nil {
+				badFields["worker_filter"] = "Unable to successfully parse filter expression."
+			}
+		}
 		return badFields
 	})
 }
@@ -838,6 +916,11 @@ func validateUpdateRequest(req *pbs.UpdateTargetRequest) error {
 				badFields["attributes.default_port"] = "This optional field cannot be set to 0."
 			}
 		}
+		if filter := req.GetItem().GetWorkerFilter(); filter != nil {
+			if _, err := bexpr.CreateEvaluator(filter.GetValue()); err != nil {
+				badFields["worker_filter"] = "Unable to successfully parse filter expression."
+			}
+		}
 		return badFields
 	})
 }

internal/servers/controller/handlers/workers/worker_service.go

@@ -2,15 +2,19 @@ package workers
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	"github.com/hashicorp/boundary/internal/gen/controller/api/resources/targets"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/servers"
 	"github.com/hashicorp/boundary/internal/servers/controller/common"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/boundary/internal/types/resource"
+	"github.com/hashicorp/go-bexpr"
 	"github.com/hashicorp/go-hclog"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -48,15 +52,15 @@ var (
 )
 
 func (ws *workerServiceServer) Status(ctx context.Context, req *pbs.StatusRequest) (*pbs.StatusResponse, error) {
-	ws.logger.Trace("got status request from worker", "name", req.Worker.Name, "address", req.Worker.Address, "jobs", req.GetJobs())
-	ws.updateTimes.Store(req.Worker.Name, time.Now())
+	ws.logger.Trace("got status request from worker", "name", req.Worker.PrivateId, "address", req.Worker.Address, "jobs", req.GetJobs())
+	ws.updateTimes.Store(req.Worker.PrivateId, time.Now())
 	repo, err := ws.serversRepoFn()
 	if err != nil {
 		ws.logger.Error("error getting servers repo", "error", err)
 		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error aqcuiring repo to store worker status: %v", err)
 	}
 	req.Worker.Type = resource.Worker.String()
-	controllers, _, err := repo.UpsertServer(ctx, req.Worker)
+	controllers, _, err := repo.UpsertServer(ctx, req.Worker, servers.WithUpdateTags(req.GetUpdateTags()))
 	if err != nil {
 		ws.logger.Error("error storing worker status", "error", err)
 		return &pbs.StatusResponse{}, status.Errorf(codes.Internal, "Error storing worker status: %v", err)
@@ -149,6 +153,51 @@ func (ws *workerServiceServer) LookupSession(ctx context.Context, req *pbs.Looku
 		return nil, status.Error(codes.Internal, "Empty session states during lookup.")
 	}
 
+	if sessionInfo.WorkerFilter != "" {
+		if req.ServerId == "" {
+			ws.logger.Error("worker filter enabled for session but got no server ID from worker")
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Did not receive server ID when looking up session but filtering is enabled: %v", err)
+		}
+		serversRepo, err := ws.serversRepoFn()
+		if err != nil {
+			ws.logger.Error("error getting servers repo", "error", err)
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error acquiring server repo when looking up session: %v", err)
+		}
+		tags, err := serversRepo.ListTagsForServers(ctx, []string{req.ServerId})
+		if err != nil {
+			ws.logger.Error("error looking up tags for server", "error", err, "server_id", req.ServerId)
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error looking up tags for server: %v", err)
+		}
+		// Build the map for filtering.
+		tagMap := make(map[string][]string)
+		for _, tag := range tags {
+			tagMap[tag.Key] = append(tagMap[tag.Key], tag.Value)
+			// We don't need to reinsert after the fact because maps are
+			// reference types, so we don't need to re-insert into tagMap
+		}
+
+		// Create the evaluator
+		eval, err := bexpr.CreateEvaluator(sessionInfo.WorkerFilter)
+		if err != nil {
+			ws.logger.Error("error creating worker filter evaluator", "error", err, "server_id", req.ServerId)
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal, "Error creating worker filter evaluator: %v", err)
+		}
+		filterInput := map[string]interface{}{
+			"name": req.ServerId,
+			"tags": tagMap,
+		}
+		ok, err := eval.Evaluate(filterInput)
+		if err != nil {
+			return &pbs.LookupSessionResponse{}, status.Errorf(codes.Internal,
+				fmt.Sprintf("Worker filter expression evaluation resulted in error: %s", err))
+		}
+		if !ok {
+			return nil, handlers.ApiErrorWithCodeAndMessage(
+				codes.FailedPrecondition,
+				"Worker filter expression precludes this worker from serving this session")
+		}
+	}
+
 	resp := &pbs.LookupSessionResponse{
 		Authorization: &targets.SessionAuthorizationData{
 			SessionId:   sessionInfo.GetPublicId(),

internal/servers/controller/testing.go

@@ -305,6 +305,9 @@ type TestControllerOpts struct {
 	// generated, unless provided in a non-nil Config
 	Name string
 
+	// The suffix to use for initial resources
+	InitialResourcesSuffix string
+
 	// The logger to use, or one will be created
 	Logger hclog.Logger
 }
@@ -383,6 +386,18 @@ func NewTestController(t *testing.T, opts *TestControllerOpts) *TestController {
 	}
 	tc.name = opts.Config.Controller.Name
 
+	if opts.InitialResourcesSuffix != "" {
+		suffix := opts.InitialResourcesSuffix
+		tc.b.DevAuthMethodId = "ampw_" + suffix
+		tc.b.DevHostCatalogId = "hcst_" + suffix
+		tc.b.DevHostId = "hst_" + suffix
+		tc.b.DevHostSetId = "hsst_" + suffix
+		tc.b.DevOrgId = "o_" + suffix
+		tc.b.DevProjectId = "p_" + suffix
+		tc.b.DevTargetId = "ttcp_" + suffix
+		tc.b.DevUserId = "u_" + suffix
+	}
+
 	// Set up KMSes
 	switch {
 	case opts.RootKms != nil && opts.WorkerAuthKms != nil:

internal/servers/options.go

@@ -16,8 +16,9 @@ type Option func(*options)
 
 // options = how options are represented
 type options struct {
-	withLimit    int
-	withLiveness time.Duration
+	withLimit      int
+	withLiveness   time.Duration
+	withUpdateTags bool
 }
 
 func getDefaultOptions() options {
@@ -43,3 +44,12 @@ func WithLiveness(liveness time.Duration) Option {
 		o.withLiveness = liveness
 	}
 }
+
+// WithUpdateTags indicates that we should perform tag updates in the DB.
+// This would happen on first sync from a worker after startup or (eventually,
+// perhaps), after a SIGHUP.
+func WithUpdateTags(updateTags bool) Option {
+	return func(o *options) {
+		o.withUpdateTags = updateTags
+	}
+}

internal/servers/options_test.go

@@ -0,0 +1,27 @@
+package servers
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Test_GetOpts provides unit tests for GetOpts and all the options
+func Test_GetOpts(t *testing.T) {
+	t.Parallel()
+	opts := getOpts()
+	assert.Equal(t, options{}, opts)
+
+	opts = getOpts(
+		WithLimit(5),
+		WithLiveness(time.Hour),
+		WithUpdateTags(true),
+	)
+	exp := options{
+		withLimit:      5,
+		withLiveness:   time.Hour,
+		withUpdateTags: true,
+	}
+	assert.Equal(t, exp, opts)
+}

internal/servers/query.go

@@ -1,5 +1,18 @@
 package servers
 
 const (
-	deleteWhereSql = `create_time < $1`
+	serverUpsertQuery = `
+		insert into server
+			(private_id, type, description, address, update_time)
+		values
+			($1, $2, $3, $4, $5)
+		on conflict on constraint server_pkey
+		do update set
+			type = $2,
+			description = $3,
+			address = $4,
+			update_time = $5;
+	`
+	deleteWhereCreateTimeSql = `create_time < $1`
+	deleteTagsSql            = `server_id = $1`
 )

internal/servers/repository.go

@@ -2,12 +2,12 @@ package servers
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/types/resource"
 )
@@ -37,14 +37,15 @@ type Repository struct {
 // NewRepository creates a new servers Repository. Supports the options: WithLimit
 // which sets a default limit on results returned by repo operations.
 func NewRepository(r db.Reader, w db.Writer, kms *kms.Kms) (*Repository, error) {
+	const op = "servers.NewRepository"
 	if r == nil {
-		return nil, errors.New("error creating server repository with nil reader")
+		return nil, errors.New(errors.InvalidParameter, op, "nil reader")
 	}
 	if w == nil {
-		return nil, errors.New("error creating server repository with nil writer")
+		return nil, errors.New(errors.InvalidParameter, op, "nil writer")
 	}
 	if kms == nil {
-		return nil, errors.New("error creating server repository with nil kms")
+		return nil, errors.New(errors.InvalidParameter, op, "nil kms")
 	}
 	return &Repository{
 		reader: r,
@@ -53,9 +54,16 @@ func NewRepository(r db.Reader, w db.Writer, kms *kms.Kms) (*Repository, error)
 	}, nil
 }
 
-// list will return a listing of resources and honor the WithLimit option or the
-// repo defaultLimit
+// ListServers is a passthrough to listServersWithReader that uses the repo's
+// normal reader.
 func (r *Repository) ListServers(ctx context.Context, serverType ServerType, opt ...Option) ([]*Server, error) {
+	return r.listServersWithReader(ctx, r.reader, serverType, opt...)
+}
+
+// listServersWithReader will return a listing of resources and honor the
+// WithLimit option or the repo defaultLimit. It accepts a reader, allowing it
+// to be used within a transaction or without.
+func (r *Repository) listServersWithReader(ctx context.Context, reader db.Reader, serverType ServerType, opt ...Option) ([]*Server, error) {
 	opts := getOpts(opt...)
 	liveness := opts.withLiveness
 	if liveness == 0 {
@@ -63,58 +71,128 @@ func (r *Repository) ListServers(ctx context.Context, serverType ServerType, opt
 	}
 	updateTime := time.Now().Add(-1 * liveness)
 	var servers []*Server
-	if err := r.reader.SearchWhere(
+	if err := reader.SearchWhere(
 		ctx,
 		&servers,
 		"type = $1 and update_time > $2",
 		[]interface{}{serverType, updateTime.Format(time.RFC3339)},
 		db.WithLimit(-1),
 	); err != nil {
-		return nil, fmt.Errorf("error listing servers: %w", err)
+		return nil, errors.Wrap(err, "servers.listServersWithReader")
 	}
 	return servers, nil
 }
 
+// ServerTag holds the information for the server_tag table for Gorm.
+type ServerTag struct {
+	ServerId string
+	Key      string
+	Value    string
+}
+
+// TableName overrides the table name used by ServerTag to `server_tag`
+func (ServerTag) TableName() string {
+	return "server_tag"
+}
+
+// ListTagsForServers pulls out tag tuples into ServerTag structs for the
+// given server ID values.
+func (r *Repository) ListTagsForServers(ctx context.Context, serverIds []string, opt ...Option) ([]*ServerTag, error) {
+	var serverTags []*ServerTag
+	if err := r.reader.SearchWhere(
+		ctx,
+		&serverTags,
+		"server_id in (?)",
+		[]interface{}{serverIds},
+		db.WithLimit(-1),
+	); err != nil {
+		return nil, errors.Wrap(err, "servers.ListTagsForServers", errors.WithMsg(fmt.Sprintf("server IDs %v", serverIds)))
+	}
+	return serverTags, nil
+}
+
 // UpsertServer adds or updates a server in the DB
 func (r *Repository) UpsertServer(ctx context.Context, server *Server, opt ...Option) ([]*Server, int, error) {
-	if server == nil {
-		return nil, db.NoRowsAffected, errors.New("cannot update server that is nil")
-	}
-	// Ensure, for now at least, the private ID is always equivalent to the name
-	server.PrivateId = server.Name
-	// Build query
-	q := `
-	insert into server
-		(private_id, type, name, description, address, update_time)
-	values
-		($1, $2, $3, $4, $5, $6)
-	on conflict on constraint server_pkey
-	do update set
-		name = $3,
-		description = $4,
-		address = $5,
-		update_time = $6;
-	`
+	const op = "servers.UpsertServer"
 
-	rowsAffected, err := r.writer.Exec(ctx, q,
-		[]interface{}{
-			server.PrivateId,
-			server.Type,
-			server.Name,
-			server.Description,
-			server.Address,
-			time.Now().Format(time.RFC3339),
-		})
+	if server == nil {
+		return nil, db.NoRowsAffected, errors.New(errors.InvalidParameter, op, "server is nil")
+	}
+
+	opts := getOpts(opt...)
+
+	var rowsUpdated int
+	var controllers []*Server
+	_, err := r.writer.DoTx(
+		ctx,
+		db.StdRetryCnt,
+		db.ExpBackoff{},
+		func(read db.Reader, w db.Writer) error {
+			var err error
+			rowsUpdated, err = w.Exec(ctx,
+				serverUpsertQuery,
+				[]interface{}{
+					server.PrivateId,
+					server.Type,
+					server.Description,
+					server.Address,
+					time.Now().Format(time.RFC3339),
+				})
+			if err != nil {
+				return errors.Wrap(err, op+":Upsert")
+			}
+
+			// If it's a worker, fetch the current controllers to feed to them
+			if server.Type == resource.Worker.String() {
+				// Fetch current controllers to feed to the workers
+				controllers, err = r.listServersWithReader(ctx, read, ServerTypeController)
+				if err != nil {
+					return errors.Wrap(err, op+":ListServer")
+				}
+			}
+
+			// If we've been told to update tags, we need to clean out old
+			// ones and add new ones. Within the current transaction, simply
+			// delete all tags for the given worker, then add the new ones
+			// we've been sent.
+			if opts.withUpdateTags {
+				_, err = w.Delete(ctx, &ServerTag{}, db.WithWhere(deleteTagsSql, server.PrivateId))
+				if err != nil {
+					return errors.Wrap(err, op+":DeleteTags", errors.WithMsg(server.PrivateId))
+				}
+
+				// If tags were cleared out entirely, then we'll have nothing
+				// to do here, e.g., it will result in deletion of all tags.
+				// Otherwise, go through and stage each tuple for insertion
+				// below.
+				if len(server.Tags) > 0 {
+					tags := make([]interface{}, 0, len(server.Tags))
+					for k, v := range server.Tags {
+						if v == nil {
+							return errors.New(errors.InvalidParameter, op+":RangeTags", fmt.Sprintf("found nil tag value for worker %s and key %s", server.PrivateId, k))
+						}
+						for _, val := range v.Values {
+							tags = append(tags, ServerTag{
+								ServerId: server.PrivateId,
+								Key:      k,
+								Value:    val,
+							})
+						}
+					}
+					if err = w.CreateItems(ctx, tags); err != nil {
+						return errors.Wrap(err, op+":CreateTags", errors.WithMsg(server.PrivateId))
+					}
+				}
+			}
+
+			return nil
+		},
+	)
 	if err != nil {
-		return nil, db.NoRowsAffected, fmt.Errorf("error performing status upsert: %w", err)
+		return nil, db.NoRowsAffected, err
 	}
-	// If updating a controller, done
-	if server.Type == resource.Controller.String() {
-		return nil, int(rowsAffected), nil
-	}
-	// Fetch current controllers to feed to the workers
-	controllers, err := r.ListServers(ctx, ServerTypeController)
-	return controllers, len(controllers), err
+
+	return controllers, rowsUpdated, nil
 }
 
 type RecoveryNonce struct {
@@ -123,12 +201,13 @@ type RecoveryNonce struct {
 
 // AddRecoveryNonce adds a nonce
 func (r *Repository) AddRecoveryNonce(ctx context.Context, nonce string, opt ...Option) error {
+	const op = "servers.AddRecoveryNonce"
 	if nonce == "" {
-		return errors.New("empty nonce provided")
+		return errors.New(errors.InvalidParameter, op, "empty nonce")
 	}
 	rn := &RecoveryNonce{Nonce: nonce}
 	if err := r.writer.Create(ctx, rn); err != nil {
-		return fmt.Errorf("error performing nonce insertion: %w", err)
+		return errors.Wrap(err, op+":Insertion")
 	}
 	return nil
 }
@@ -138,9 +217,9 @@ func (r *Repository) CleanupNonces(ctx context.Context, opt ...Option) (int, err
 	// If something was inserted before 3x the actual validity period, clean it out
 	endTime := time.Now().Add(-3 * globals.RecoveryTokenValidityPeriod)
 
-	rows, err := r.writer.Delete(ctx, &RecoveryNonce{}, db.WithWhere(deleteWhereSql, endTime))
+	rows, err := r.writer.Delete(ctx, &RecoveryNonce{}, db.WithWhere(deleteWhereCreateTimeSql, endTime))
 	if err != nil {
-		return db.NoRowsAffected, fmt.Errorf("error performing nonce cleanup: %w", err)
+		return db.NoRowsAffected, errors.Wrap(err, "servers.CleanupNonces")
 	}
 	return rows, nil
 }
@@ -149,7 +228,7 @@ func (r *Repository) CleanupNonces(ctx context.Context, opt ...Option) (int, err
 func (r *Repository) ListNonces(ctx context.Context, opt ...Option) ([]*RecoveryNonce, error) {
 	var nonces []*RecoveryNonce
 	if err := r.reader.SearchWhere(ctx, &nonces, "", nil, db.WithLimit(-1)); err != nil {
-		return nil, fmt.Errorf("error listing nonces: %w", err)
+		return nil, errors.Wrap(err, "servers.ListNonces")
 	}
 	return nonces, nil
 }

internal/servers/repository_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/hashicorp/boundary/api/roles"
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/servers"
 	"github.com/hashicorp/boundary/internal/servers/controller"
 	"github.com/hashicorp/boundary/internal/types/scope"
 	"github.com/hashicorp/boundary/sdk/recovery"
@@ -78,3 +79,104 @@ func TestRecoveryNonces(t *testing.T) {
 		assert.Len(nonces, 0)
 	}
 }
+
+func TestTagUpdatingListing(t *testing.T) {
+	require := require.New(t)
+
+	wrapper := db.TestWrapper(t)
+	tc := controller.NewTestController(t, &controller.TestControllerOpts{
+		RecoveryKms: wrapper,
+	})
+	defer tc.Shutdown()
+
+	repo := tc.ServersRepo()
+	srv := &servers.Server{
+		PrivateId: "test1",
+		Type:      "worker",
+		Address:   "127.0.0.1",
+		Tags: map[string]*servers.TagValues{
+			"tag1": {
+				Values: []string{"value1", "value2"},
+			},
+		},
+	}
+	_, _, err := repo.UpsertServer(tc.Context(), srv, servers.WithUpdateTags(true))
+	require.NoError(err)
+
+	srv = &servers.Server{
+		PrivateId: "test2",
+		Type:      "worker",
+		Address:   "127.0.0.1",
+		Tags: map[string]*servers.TagValues{
+			"tag2": {
+				Values: []string{"value1", "value2"},
+			},
+		},
+	}
+	_, _, err = repo.UpsertServer(tc.Context(), srv, servers.WithUpdateTags(true))
+	require.NoError(err)
+
+	tags, err := repo.ListTagsForServers(tc.Context(), []string{"test1", "test2"})
+	require.NoError(err)
+
+	// Base case
+	exp := []*servers.ServerTag{
+		{
+			ServerId: "test1",
+			Key:      "tag1",
+			Value:    "value1",
+		},
+		{
+			ServerId: "test1",
+			Key:      "tag1",
+			Value:    "value2",
+		},
+		{
+			ServerId: "test2",
+			Key:      "tag2",
+			Value:    "value1",
+		},
+		{
+			ServerId: "test2",
+			Key:      "tag2",
+			Value:    "value2",
+		},
+	}
+	require.Equal(exp, tags)
+
+	// Update without saying to update tags
+	srv = &servers.Server{
+		PrivateId: "test2",
+		Type:      "worker",
+		Address:   "192.168.1.1",
+		Tags: map[string]*servers.TagValues{
+			"tag22": {
+				Values: []string{"value21", "value22"},
+			},
+		},
+	}
+	_, _, err = repo.UpsertServer(tc.Context(), srv)
+	require.NoError(err)
+	tags, err = repo.ListTagsForServers(tc.Context(), []string{"test1", "test2"})
+	require.NoError(err)
+	require.Equal(exp, tags)
+
+	// Update tags and test again
+	_, _, err = repo.UpsertServer(tc.Context(), srv, servers.WithUpdateTags(true))
+	require.NoError(err)
+	tags, err = repo.ListTagsForServers(tc.Context(), []string{"test1", "test2"})
+	require.NoError(err)
+	require.NotEqual(exp, tags)
+	// Update and try again
+	exp[2] = &servers.ServerTag{
+		ServerId: "test2",
+		Key:      "tag22",
+		Value:    "value21",
+	}
+	exp[3] = &servers.ServerTag{
+		ServerId: "test2",
+		Key:      "tag22",
+		Value:    "value22",
+	}
+	require.Equal(exp, tags)
+}

internal/servers/servers.pb.go

@@ -31,7 +31,12 @@ type Server struct {
 	PrivateId string `protobuf:"bytes,10,opt,name=private_id,json=privateId,proto3" json:"private_id,omitempty"`
 	// Type of the resource (controller, worker)
 	Type string `protobuf:"bytes,20,opt,name=type,proto3" json:"type,omitempty"`
-	// Name of the resource
+	// Name of the resource. This is deprecated because we ended up setting the
+	// private ID and name to be identical, and there is currently no reason we
+	// can think of to allow a given server's name to match another within the
+	// same cluster. So we simply use the private ID and deprecate this.
+	//
+	// Deprecated: Do not use.
 	Name string `protobuf:"bytes,30,opt,name=name,proto3" json:"name,omitempty"`
 	// Description of the resource
 	Description string `protobuf:"bytes,40,opt,name=description,proto3" json:"description,omitempty"`
@@ -41,6 +46,8 @@ type Server struct {
 	CreateTime *timestamp.Timestamp `protobuf:"bytes,60,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"`
 	// Last time there was an update
 	UpdateTime *timestamp.Timestamp `protobuf:"bytes,70,opt,name=update_time,json=updateTime,proto3" json:"update_time,omitempty"`
+	// Tags for workers
+	Tags map[string]*TagValues `protobuf:"bytes,80,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (x *Server) Reset() {
@@ -89,6 +96,7 @@ func (x *Server) GetType() string {
 	return ""
 }
 
+// Deprecated: Do not use.
 func (x *Server) GetName() string {
 	if x != nil {
 		return x.Name
@@ -124,6 +132,62 @@ func (x *Server) GetUpdateTime() *timestamp.Timestamp {
 	return nil
 }
 
+func (x *Server) GetTags() map[string]*TagValues {
+	if x != nil {
+		return x.Tags
+	}
+	return nil
+}
+
+// TagValues is used because map fields cannot be repeated but can be a
+// message
+type TagValues struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Values []string `protobuf:"bytes,10,rep,name=values,proto3" json:"values,omitempty"`
+}
+
+func (x *TagValues) Reset() {
+	*x = TagValues{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_controller_servers_v1_servers_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *TagValues) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*TagValues) ProtoMessage() {}
+
+func (x *TagValues) ProtoReflect() protoreflect.Message {
+	mi := &file_controller_servers_v1_servers_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use TagValues.ProtoReflect.Descriptor instead.
+func (*TagValues) Descriptor() ([]byte, []int) {
+	return file_controller_servers_v1_servers_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *TagValues) GetValues() []string {
+	if x != nil {
+		return x.Values
+	}
+	return nil
+}
+
 var File_controller_servers_v1_servers_proto protoreflect.FileDescriptor
 
 var file_controller_servers_v1_servers_proto_rawDesc = []byte{
@@ -133,30 +197,42 @@ var file_controller_servers_v1_servers_proto_rawDesc = []byte{
 	0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, 0x1a, 0x2f, 0x63, 0x6f,
 	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
 	0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2f, 0x76, 0x31, 0x2f, 0x74, 0x69,
-	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa5, 0x02,
+	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x03,
 	0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x69, 0x76,
 	0x61, 0x74, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72,
 	0x69, 0x76, 0x61, 0x74, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18,
-	0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e,
-	0x61, 0x6d, 0x65, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
-	0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x28,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f,
-	0x6e, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x32, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x4b, 0x0a, 0x0b, 0x63,
-	0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x3c, 0x20, 0x01, 0x28, 0x0b,
-	0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x74,
-	0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e,
-	0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72,
-	0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61,
-	0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x46, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e,
-	0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61,
-	0x67, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x76, 0x31, 0x2e,
-	0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74,
-	0x65, 0x54, 0x69, 0x6d, 0x65, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
-	0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f,
-	0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f,
-	0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x62,
-	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x04, 0x6e,
+	0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69,
+	0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
+	0x18, 0x32, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12,
+	0x4b, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x3c,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65,
+	0x72, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74,
+	0x61, 0x6d, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x0b,
+	0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x46, 0x20, 0x01, 0x28,
+	0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73,
+	0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2e, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x75,
+	0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x74, 0x61, 0x67,
+	0x73, 0x18, 0x50, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
+	0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, 0x2e,
+	0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
+	0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x1a, 0x59, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65,
+	0x72, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67,
+	0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
+	0x01, 0x22, 0x23, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x16,
+	0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06,
+	0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62,
+	0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
+	0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -171,19 +247,23 @@ func file_controller_servers_v1_servers_proto_rawDescGZIP() []byte {
 	return file_controller_servers_v1_servers_proto_rawDescData
 }
 
-var file_controller_servers_v1_servers_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_controller_servers_v1_servers_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
 var file_controller_servers_v1_servers_proto_goTypes = []interface{}{
 	(*Server)(nil),              // 0: controller.servers.v1.Server
-	(*timestamp.Timestamp)(nil), // 1: controller.storage.timestamp.v1.Timestamp
+	(*TagValues)(nil),           // 1: controller.servers.v1.TagValues
+	nil,                         // 2: controller.servers.v1.Server.TagsEntry
+	(*timestamp.Timestamp)(nil), // 3: controller.storage.timestamp.v1.Timestamp
 }
 var file_controller_servers_v1_servers_proto_depIdxs = []int32{
-	1, // 0: controller.servers.v1.Server.create_time:type_name -> controller.storage.timestamp.v1.Timestamp
-	1, // 1: controller.servers.v1.Server.update_time:type_name -> controller.storage.timestamp.v1.Timestamp
-	2, // [2:2] is the sub-list for method output_type
-	2, // [2:2] is the sub-list for method input_type
-	2, // [2:2] is the sub-list for extension type_name
-	2, // [2:2] is the sub-list for extension extendee
-	0, // [0:2] is the sub-list for field type_name
+	3, // 0: controller.servers.v1.Server.create_time:type_name -> controller.storage.timestamp.v1.Timestamp
+	3, // 1: controller.servers.v1.Server.update_time:type_name -> controller.storage.timestamp.v1.Timestamp
+	2, // 2: controller.servers.v1.Server.tags:type_name -> controller.servers.v1.Server.TagsEntry
+	1, // 3: controller.servers.v1.Server.TagsEntry.value:type_name -> controller.servers.v1.TagValues
+	4, // [4:4] is the sub-list for method output_type
+	4, // [4:4] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
 }
 
 func init() { file_controller_servers_v1_servers_proto_init() }
@@ -204,6 +284,18 @@ func file_controller_servers_v1_servers_proto_init() {
 				return nil
 			}
 		}
+		file_controller_servers_v1_servers_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*TagValues); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -211,7 +303,7 @@ func file_controller_servers_v1_servers_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_controller_servers_v1_servers_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   1,
+			NumMessages:   3,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

internal/servers/worker/session.go

@@ -67,6 +67,7 @@ func (w *Worker) getSessionTls(hello *tls.ClientHelloInfo) (*tls.Config, error)
 
 	w.logger.Trace("looking up session", "session_id", sessionId)
 	resp, err := conn.LookupSession(timeoutContext, &pbs.LookupSessionRequest{
+		ServerId:  w.conf.RawConfig.Worker.Name,
 		SessionId: sessionId,
 	})
 	if err != nil {

internal/servers/worker/status.go

@@ -45,143 +45,160 @@ func (w *Worker) startStatusTicking(cancelCtx context.Context) {
 				return
 
 			case <-timer.C:
-				// First send info as-is. We'll perform cleanup duties after we
-				// get cancel/job change info back.
-				var activeJobs []*pbs.JobStatus
-				w.sessionInfoMap.Range(func(key, value interface{}) bool {
-					var jobInfo pbs.SessionJobInfo
-					sessionId := key.(string)
-					si := value.(*sessionInfo)
-					si.RLock()
-					status := si.status
-					connections := make([]*pbs.Connection, 0, len(si.connInfoMap))
-					for k, v := range si.connInfoMap {
-						connections = append(connections, &pbs.Connection{
-							ConnectionId: k,
-							Status:       v.status,
-						})
-					}
-					si.RUnlock()
-					jobInfo.SessionId = sessionId
-					activeJobs = append(activeJobs, &pbs.JobStatus{
-						Job: &pbs.Job{
-							Type: pbs.JOBTYPE_JOBTYPE_SESSION,
-							JobInfo: &pbs.Job_SessionInfo{
-								SessionInfo: &pbs.SessionJobInfo{
-									SessionId:   sessionId,
-									Status:      status,
-									Connections: connections,
-								},
-							},
-						},
-					})
-					return true
-				})
-				client := w.controllerStatusConn.Load().(pbs.ServerCoordinationServiceClient)
-				result, err := client.Status(cancelCtx, &pbs.StatusRequest{
-					Jobs: activeJobs,
-					Worker: &servers.Server{
-						PrivateId:   w.conf.RawConfig.Worker.Name,
-						Name:        w.conf.RawConfig.Worker.Name,
-						Type:        resource.Worker.String(),
-						Description: w.conf.RawConfig.Worker.Description,
-						Address:     w.conf.RawConfig.Worker.PublicAddr,
-					},
-				})
-				if err != nil {
-					w.logger.Error("error making status request to controller", "error", err)
-				} else {
-					w.logger.Trace("successfully sent status to controller")
-					addrs := make([]resolver.Address, 0, len(result.Controllers))
-					strAddrs := make([]string, 0, len(result.Controllers))
-					for _, v := range result.Controllers {
-						addrs = append(addrs, resolver.Address{Addr: v.Address})
-						strAddrs = append(strAddrs, v.Address)
-					}
-					w.logger.Trace("found controllers", "addresses", strAddrs)
-					switch len(strAddrs) {
-					case 0:
-						w.logger.Warn("got no controller addresses from controller; possibly prior to first status save, not persisting")
-					default:
-						w.Resolver().UpdateState(resolver.State{Addresses: addrs})
-					}
-					w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now()})
-
-					for _, request := range result.GetJobsRequests() {
-						switch request.GetRequestType() {
-						case pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE:
-							switch request.GetJob().GetType() {
-							case pbs.JOBTYPE_JOBTYPE_SESSION:
-								sessInfo := request.GetJob().GetSessionInfo()
-								sessionId := sessInfo.GetSessionId()
-								siRaw, ok := w.sessionInfoMap.Load(sessionId)
-								if !ok {
-									w.logger.Warn("asked to cancel session but could not find a local information for it", "session_id", sessionId)
-									continue
-								}
-								si := siRaw.(*sessionInfo)
-								si.Lock()
-								si.status = sessInfo.GetStatus()
-								si.Unlock()
-							}
-						}
-					}
-				}
-
-				// Cleanup: Run through current jobs. Cancel connections for any
-				// canceling session or any session that is expired. Clear out
-				// sessions that are canceled or expired with all connections
-				// marked as closed. Close any that aren't marked as such.
-				closeInfo := make(map[string]string)
-				cleanSessionIds := make([]string, 0)
-				w.sessionInfoMap.Range(func(key, value interface{}) bool {
-					si := value.(*sessionInfo)
-					si.Lock()
-					switch {
-					case si.status == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
-						si.status == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
-						time.Until(si.lookupSessionResponse.Expiration.AsTime()) < 0:
-						var toClose int
-						for k, v := range si.connInfoMap {
-							if v.closeTime.IsZero() {
-								toClose++
-								v.connCancel()
-								w.logger.Info("terminated connection due to cancelation or expiration", "session_id", si.id, "connection_id", k)
-								closeInfo[k] = si.id
-							}
-						}
-						// closeTime is marked by closeConnections iff the
-						// status is returned for that connection as closed. If
-						// the session is no longer valid and all connections
-						// are marked closed, clean up the session.
-						if toClose == 0 {
-							cleanSessionIds = append(cleanSessionIds, si.id)
-						}
-					}
-					si.Unlock()
-					return true
-				})
-
-				// Note that we won't clean these from the info map until the
-				// next time we run this function
-				if len(closeInfo) > 0 {
-					if err := w.closeConnections(cancelCtx, closeInfo); err != nil {
-						w.logger.Error("error marking connections closed", "error", err)
-					}
-				}
-
-				// Forget sessions where the session is expired/canceled and all
-				// connections are canceled and marked closed
-				for _, v := range cleanSessionIds {
-					w.sessionInfoMap.Delete(v)
-				}
-
+				w.sendWorkerStatus(cancelCtx)
 				timer.Reset(getRandomInterval())
 			}
 		}
 	}()
 }
 
+// LastStatusSuccess is used in tests (it's exported for tests in other
+// packages) to verify the last time we sent status
 func (w *Worker) LastStatusSuccess() *LastStatusInformation {
 	return w.lastStatusSuccess.Load().(*LastStatusInformation)
 }
+
+func (w *Worker) sendWorkerStatus(cancelCtx context.Context) {
+	// First send info as-is. We'll perform cleanup duties after we
+	// get cancel/job change info back.
+	var activeJobs []*pbs.JobStatus
+
+	// Range over known sessions and collect info
+	w.sessionInfoMap.Range(func(key, value interface{}) bool {
+		var jobInfo pbs.SessionJobInfo
+		sessionId := key.(string)
+		si := value.(*sessionInfo)
+		si.RLock()
+		status := si.status
+		connections := make([]*pbs.Connection, 0, len(si.connInfoMap))
+		for k, v := range si.connInfoMap {
+			connections = append(connections, &pbs.Connection{
+				ConnectionId: k,
+				Status:       v.status,
+			})
+		}
+		si.RUnlock()
+		jobInfo.SessionId = sessionId
+		activeJobs = append(activeJobs, &pbs.JobStatus{
+			Job: &pbs.Job{
+				Type: pbs.JOBTYPE_JOBTYPE_SESSION,
+				JobInfo: &pbs.Job_SessionInfo{
+					SessionInfo: &pbs.SessionJobInfo{
+						SessionId:   sessionId,
+						Status:      status,
+						Connections: connections,
+					},
+				},
+			},
+		})
+		return true
+	})
+
+	// Send status information
+	client := w.controllerStatusConn.Load().(pbs.ServerCoordinationServiceClient)
+	var tags map[string]*servers.TagValues
+	// If we're not going to request a tag update, no reason to have these
+	// marshaled on every status call.
+	if w.updateTags.Load() {
+		tags = w.tags.Load().(map[string]*servers.TagValues)
+	}
+	result, err := client.Status(cancelCtx, &pbs.StatusRequest{
+		Jobs: activeJobs,
+		Worker: &servers.Server{
+			PrivateId:   w.conf.RawConfig.Worker.Name,
+			Type:        resource.Worker.String(),
+			Description: w.conf.RawConfig.Worker.Description,
+			Address:     w.conf.RawConfig.Worker.PublicAddr,
+			Tags:        tags,
+		},
+		UpdateTags: w.updateTags.Load(),
+	})
+	if err != nil {
+		w.logger.Error("error making status request to controller", "error", err)
+	} else {
+		w.logger.Trace("successfully sent status to controller")
+		w.updateTags.Store(false)
+		addrs := make([]resolver.Address, 0, len(result.Controllers))
+		strAddrs := make([]string, 0, len(result.Controllers))
+		for _, v := range result.Controllers {
+			addrs = append(addrs, resolver.Address{Addr: v.Address})
+			strAddrs = append(strAddrs, v.Address)
+		}
+		w.logger.Trace("found controllers", "addresses", strAddrs)
+		switch len(strAddrs) {
+		case 0:
+			w.logger.Warn("got no controller addresses from controller; possibly prior to first status save, not persisting")
+		default:
+			w.Resolver().UpdateState(resolver.State{Addresses: addrs})
+		}
+		w.lastStatusSuccess.Store(&LastStatusInformation{StatusResponse: result, StatusTime: time.Now()})
+
+		for _, request := range result.GetJobsRequests() {
+			switch request.GetRequestType() {
+			case pbs.CHANGETYPE_CHANGETYPE_UPDATE_STATE:
+				switch request.GetJob().GetType() {
+				case pbs.JOBTYPE_JOBTYPE_SESSION:
+					sessInfo := request.GetJob().GetSessionInfo()
+					sessionId := sessInfo.GetSessionId()
+					siRaw, ok := w.sessionInfoMap.Load(sessionId)
+					if !ok {
+						w.logger.Warn("asked to cancel session but could not find a local information for it", "session_id", sessionId)
+						continue
+					}
+					si := siRaw.(*sessionInfo)
+					si.Lock()
+					si.status = sessInfo.GetStatus()
+					si.Unlock()
+				}
+			}
+		}
+	}
+
+	// Cleanup: Run through current jobs. Cancel connections for any
+	// canceling session or any session that is expired. Clear out
+	// sessions that are canceled or expired with all connections
+	// marked as closed. Close any that aren't marked as such.
+	closeInfo := make(map[string]string)
+	cleanSessionIds := make([]string, 0)
+	w.sessionInfoMap.Range(func(key, value interface{}) bool {
+		si := value.(*sessionInfo)
+		si.Lock()
+		switch {
+		case si.status == pbs.SESSIONSTATUS_SESSIONSTATUS_CANCELING,
+			si.status == pbs.SESSIONSTATUS_SESSIONSTATUS_TERMINATED,
+			time.Until(si.lookupSessionResponse.Expiration.AsTime()) < 0:
+			var toClose int
+			for k, v := range si.connInfoMap {
+				if v.closeTime.IsZero() {
+					toClose++
+					v.connCancel()
+					w.logger.Info("terminated connection due to cancelation or expiration", "session_id", si.id, "connection_id", k)
+					closeInfo[k] = si.id
+				}
+			}
+			// closeTime is marked by closeConnections iff the
+			// status is returned for that connection as closed. If
+			// the session is no longer valid and all connections
+			// are marked closed, clean up the session.
+			if toClose == 0 {
+				cleanSessionIds = append(cleanSessionIds, si.id)
+			}
+		}
+		si.Unlock()
+		return true
+	})
+
+	// Note that we won't clean these from the info map until the
+	// next time we run this function
+	if len(closeInfo) > 0 {
+		if err := w.closeConnections(cancelCtx, closeInfo); err != nil {
+			w.logger.Error("error marking connections closed", "error", err)
+		}
+	}
+
+	// Forget sessions where the session is expired/canceled and all
+	// connections are canceled and marked closed
+	for _, v := range cleanSessionIds {
+		w.sessionInfoMap.Delete(v)
+	}
+}

internal/servers/worker/worker.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/config"
+	"github.com/hashicorp/boundary/internal/servers"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/sdk/helper/base62"
 	"github.com/hashicorp/vault/sdk/helper/mlock"
@@ -33,6 +34,14 @@ type Worker struct {
 
 	controllerSessionConn *atomic.Value
 	sessionInfoMap        *sync.Map
+
+	// We store the current set in an atomic value so that we can add
+	// reload-on-sighup behavior later
+	tags *atomic.Value
+	// This stores whether or not to send updated tags on the next status
+	// request. It can be set via startup in New below, or (eventually) via
+	// SIGHUP.
+	updateTags ua.Bool
 }
 
 func New(conf *Config) (*Worker, error) {
@@ -44,12 +53,15 @@ func New(conf *Config) (*Worker, error) {
 		controllerResolver:    new(atomic.Value),
 		controllerSessionConn: new(atomic.Value),
 		sessionInfoMap:        new(sync.Map),
+		tags:                  new(atomic.Value),
 	}
 
 	w.lastStatusSuccess.Store((*LastStatusInformation)(nil))
-	w.started.Store(false)
 	w.controllerResolver.Store((*manual.Resolver)(nil))
 
+	w.parseAndStoreTags(conf.RawConfig.Worker.Tags)
+	w.updateTags.Store(true)
+
 	if conf.SecureRandomReader == nil {
 		conf.SecureRandomReader = rand.Reader
 	}
@@ -135,3 +147,17 @@ func (w *Worker) Resolver() *manual.Resolver {
 	}
 	return raw.(*manual.Resolver)
 }
+
+func (w *Worker) parseAndStoreTags(incoming map[string][]string) {
+	if len(incoming) == 0 {
+		w.tags.Store(map[string]*servers.TagValues{})
+		return
+	}
+	tags := make(map[string]*servers.TagValues, len(incoming))
+	for k, v := range incoming {
+		tags[k] = &servers.TagValues{
+			Values: append(make([]string, 0, len(v)), v...),
+		}
+	}
+	w.tags.Store(tags)
+}