Add controller-led worker auth flow (#2413)

This adds an action to the API that allows pulling out an activation
token that can be given to a worker in its config (directly or via env
or file). The worker can pass this along with its normal request in
place of its normal nonce and the server will automatically accept it.

Co-authored-by: Irena Rindos <irenarindos@users.noreply.github.com>

internal/cmd/commands.go

@@ -1114,6 +1114,12 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 				Func:    "create",
 			}, nil
 		},
+		"workers create controller-led": func() (cli.Command, error) {
+			return &workerscmd.ControllerLedCommand{
+				Command: base.NewCommand(ui),
+				Func:    "create",
+			}, nil
+		},
 		"workers read": func() (cli.Command, error) {
 			return &workerscmd.Command{
 				Command: base.NewCommand(ui),

internal/cmd/commands/dev/dev.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"os/signal"
@@ -90,7 +91,7 @@ type Command struct {
 	flagEveryEventDenyFilters        []string
 	flagCreateLoopbackHostPlugin     bool
 	flagPluginExecutionDir           string
-	flagUseKmsWorkerAuthMethod       bool
+	flagWorkerAuthMethod             string
 	flagWorkerAuthStorageDir         string
 	flagWorkerAuthStorageSkipCleanup bool
 	flagWorkerAuthRotationInterval   time.Duration
@@ -337,10 +338,11 @@ func (c *Command) Flags() *base.FlagSets {
 		EnvVar: "BOUNDARY_DEV_PLUGIN_EXECUTION_DIR",
 		Usage:  "Specifies where Boundary should write plugins that it is executing; if not set defaults to system temp directory.",
 	})
-	f.BoolVar(&base.BoolVar{
-		Name:   "use-kms-worker-auth-method",
-		Target: &c.flagUseKmsWorkerAuthMethod,
-		Usage:  "If set, the original KMS-based method of worker auth will be used to connect the initial dev worker to the controller.",
+	f.StringVar(&base.StringVar{
+		Name:    "worker-auth-method",
+		Target:  &c.flagWorkerAuthMethod,
+		Default: "random-pki",
+		Usage:   `Allows specifying how the generated worker will authenticate to the controller. Valid values are "kms" for the KMS-based mechanism; "pki-controller-led" for the PKI mechanism via the server-led authorization flow; "pki-worker-led" for the PKI mechanism via the worker-led authorization flow; and "random-pki" to randomly choose one of the PKI methods.`,
 	})
 
 	f.StringVar(&base.StringVar{
@@ -598,7 +600,7 @@ func (c *Command) Run(args []string) int {
 	c.InfoKeys = append(c.InfoKeys, "[Worker-Auth] AEAD Key Bytes")
 	c.Info["[Worker-Auth] AEAD Key Bytes"] = c.Config.DevWorkerAuthKey
 
-	if !c.flagUseKmsWorkerAuthMethod {
+	if c.flagWorkerAuthMethod != "kms" {
 		c.DevUsePkiForUpstream = true
 		// These must be unset for PKI
 		c.Config.Worker.Name = ""
@@ -701,6 +703,54 @@ func (c *Command) Run(args []string) int {
 			c.worker.TestOverrideAuthRotationPeriod = c.flagWorkerAuthRotationInterval
 		}
 
+		// Note: this should be done before starting the worker so that the
+		// activation token is populated
+		var useWorkerLed bool
+		if c.flagWorkerAuthMethod != "kms" {
+			// Flip a coin. Use one method or the other; it's transparent to
+			// users, but keeps both exercised.
+			if c.flagWorkerAuthMethod == "random-pki" {
+				randPki := rand.New(rand.NewSource(time.Now().UnixMicro())).Intn(2)
+				if randPki == 0 {
+					c.flagWorkerAuthMethod = "pki-controller-led"
+				} else {
+					c.flagWorkerAuthMethod = "pki-worker-led"
+				}
+			}
+			switch c.flagWorkerAuthMethod {
+			case "pki-controller-led":
+				// Controller-led
+				serversRepo, err := c.controller.ServersRepoFn()
+				if err != nil {
+					c.UI.Error(fmt.Errorf("Error instantiating server repo: %w", err).Error())
+					if err := c.controller.Shutdown(); err != nil {
+						c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error())
+					}
+					return base.CommandCliError
+				}
+
+				// Create the worker in the database and fetch an activation token
+				worker, err := serversRepo.CreateWorker(c.Context, &server.Worker{
+					Worker: &store.Worker{
+						ScopeId: scope.Global.String(),
+					},
+				}, server.WithCreateControllerLedActivationToken(true))
+				if err != nil {
+					c.UI.Error(fmt.Errorf("Error creating worker in database: %w", err).Error())
+					if err := c.controller.Shutdown(); err != nil {
+						c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error())
+					}
+					return base.CommandCliError
+				}
+
+				// Set the activation token in the config
+				conf.RawConfig.Worker.ControllerGeneratedActivationToken = worker.ControllerGeneratedActivationToken
+
+			default:
+				useWorkerLed = true
+			}
+		}
+
 		if err := c.worker.Start(); err != nil {
 			retErr := fmt.Errorf("Error starting worker: %w", err)
 			if err := c.worker.Shutdown(); err != nil {
@@ -714,46 +764,48 @@ func (c *Command) Run(args []string) int {
 			return base.CommandCliError
 		}
 
-		if !c.flagUseKmsWorkerAuthMethod {
-			req := c.worker.WorkerAuthRegistrationRequest
-			if req == "" {
-				c.UI.Error("No worker auth registration request found at worker start time")
-				return base.CommandCliError
-			}
-			c.InfoKeys = append(c.InfoKeys, "worker auth registration request")
-			c.Info["worker auth registration request"] = req
+		if c.flagWorkerAuthMethod != "kms" {
 			c.InfoKeys = append(c.InfoKeys, "worker auth current key id")
 			c.Info["worker auth current key id"] = c.worker.WorkerAuthCurrentKeyId.Load()
 			c.InfoKeys = append(c.InfoKeys, "worker auth storage path")
 			c.Info["worker auth storage path"] = c.worker.WorkerAuthStorage.BaseDir()
-			if err := c.StoreWorkerAuthReq(c.worker.WorkerAuthRegistrationRequest, c.worker.WorkerAuthStorage.BaseDir()); err != nil {
-				// Shutdown on failure
-				retErr := fmt.Errorf("Error storing worker auth request: %w", err)
-				if err := c.worker.Shutdown(); err != nil {
+
+			if useWorkerLed {
+				req := c.worker.WorkerAuthRegistrationRequest
+				if req == "" {
+					c.UI.Error("No worker auth registration request found at worker start time")
+					return base.CommandCliError
+				}
+
+				if err := c.StoreWorkerAuthReq(c.worker.WorkerAuthRegistrationRequest, c.worker.WorkerAuthStorage.BaseDir()); err != nil {
+					// Shutdown on failure
+					retErr := fmt.Errorf("Error storing worker auth request: %w", err)
+					if err := c.worker.Shutdown(); err != nil {
+						c.UI.Error(retErr.Error())
+						retErr = fmt.Errorf("Error shutting down worker: %w", err)
+					}
 					c.UI.Error(retErr.Error())
-					retErr = fmt.Errorf("Error shutting down worker: %w", err)
+					if err := c.controller.Shutdown(); err != nil {
+						c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error())
+					}
+					return base.CommandCliError
 				}
-				c.UI.Error(retErr.Error())
-				if err := c.controller.Shutdown(); err != nil {
-					c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error())
-				}
-				return base.CommandCliError
-			}
-			go func() {
-				for {
-					select {
-					case <-c.Context.Done():
-						return
-					case <-time.After(time.Second):
-						if err := authorizeWorker(c.Context, c.controller, req); err != nil {
-							c.UI.Error(fmt.Errorf("Error authorizing node: %w", err).Error())
-							errorEncountered.Store(true)
+				go func() {
+					for {
+						select {
+						case <-c.Context.Done():
+							return
+						case <-time.After(time.Second):
+							if err := authorizeWorker(c.Context, c.controller, req); err != nil {
+								c.UI.Error(fmt.Errorf("Error authorizing node: %w", err).Error())
+								errorEncountered.Store(true)
+								return
+							}
 							return
 						}
-						return
 					}
-				}
-			}()
+				}()
+			}
 		}
 	}
 

internal/cmd/commands/dev/flags_test.go

@@ -64,7 +64,7 @@ func TestCommand_Flags(t *testing.T) {
 	assert.Contains(completions, "-event-allow-filter")
 	assert.Contains(completions, "-event-deny-filter")
 	assert.Contains(completions, "-plugin-execution-dir")
-	assert.Contains(completions, "-use-kms-worker-auth-method")
+	assert.Contains(completions, "-worker-auth-method")
 	assert.Contains(completions, "-worker-auth-storage-dir")
 	assert.Contains(completions, "-worker-auth-storage-skip-cleanup")
 

internal/cmd/commands/server/server.go

@@ -210,6 +210,10 @@ func (c *Command) Run(args []string) int {
 				c.UI.Error("Worker is using KMS auth but has no name set. It must be the unique name of this instance.")
 				return base.CommandUserError
 			}
+			if c.Config.Worker.ControllerGeneratedActivationToken != "" {
+				c.UI.Error("Worker has KMS auth info but also has a controller-generated activation token set, which is incompatible.")
+				return base.CommandUserError
+			}
 		}
 	}
 

internal/cmd/commands/workerscmd/controller-led_funcs.go

@@ -0,0 +1,40 @@
+package workerscmd
+
+import (
+	"github.com/hashicorp/boundary/api"
+	"github.com/hashicorp/boundary/api/workers"
+	"github.com/hashicorp/boundary/internal/cmd/base"
+)
+
+func init() {
+	executeExtraControllerLedActions = executeExtraControllerLedActionsImpl
+}
+
+func (c *ControllerLedCommand) extraControllerLedHelpFunc(helpMap map[string]func() string) string {
+	var helpStr string
+	switch c.Func {
+	case "create":
+		helpStr = base.WrapForHelpText([]string{
+			"Usage: boundary workers create controller-led [options] [args]",
+			"",
+			"  Create a worker using the controller-led approach, receiving an activation token for a worker. Example:",
+			"",
+			`    $ boundary workers create controller-led -name us-east-1-1 "`,
+			"",
+			"",
+		})
+	}
+	return helpStr + c.Flags().Help()
+}
+
+func executeExtraControllerLedActionsImpl(c *ControllerLedCommand, origResp *api.Response, origItem *workers.Worker, origError error, workerClient *workers.Client, version uint32, opts []workers.Option) (*api.Response, *workers.Worker, error) {
+	switch c.Func {
+	case "create":
+		result, err := workerClient.CreateControllerLed(c.Context, c.FlagScopeId, opts...)
+		if err != nil {
+			return nil, nil, err
+		}
+		return result.GetResponse(), result.GetItem(), nil
+	}
+	return origResp, origItem, origError
+}

internal/cmd/commands/workerscmd/controller-led_workers.gen.go

@@ -0,0 +1,247 @@
+// Code generated by "make cli"; DO NOT EDIT.
+package workerscmd
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/hashicorp/boundary/api"
+	"github.com/hashicorp/boundary/api/workers"
+	"github.com/hashicorp/boundary/internal/cmd/base"
+	"github.com/hashicorp/boundary/internal/cmd/common"
+	"github.com/hashicorp/go-secure-stdlib/strutil"
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+)
+
+func initControllerLedFlags() {
+	flagsOnce.Do(func() {
+		extraFlags := extraControllerLedActionsFlagsMapFunc()
+		for k, v := range extraFlags {
+			flagsControllerLedMap[k] = append(flagsControllerLedMap[k], v...)
+		}
+	})
+}
+
+var (
+	_ cli.Command             = (*ControllerLedCommand)(nil)
+	_ cli.CommandAutocomplete = (*ControllerLedCommand)(nil)
+)
+
+type ControllerLedCommand struct {
+	*base.Command
+
+	Func string
+
+	plural string
+}
+
+func (c *ControllerLedCommand) AutocompleteArgs() complete.Predictor {
+	initControllerLedFlags()
+	return complete.PredictAnything
+}
+
+func (c *ControllerLedCommand) AutocompleteFlags() complete.Flags {
+	initControllerLedFlags()
+	return c.Flags().Completions()
+}
+
+func (c *ControllerLedCommand) Synopsis() string {
+	if extra := extraControllerLedSynopsisFunc(c); extra != "" {
+		return extra
+	}
+
+	synopsisStr := "worker"
+
+	synopsisStr = fmt.Sprintf("%s %s", "controller-led-type", synopsisStr)
+
+	return common.SynopsisFunc(c.Func, synopsisStr)
+}
+
+func (c *ControllerLedCommand) Help() string {
+	initControllerLedFlags()
+
+	var helpStr string
+	helpMap := common.HelpMap("worker")
+
+	switch c.Func {
+
+	default:
+
+		helpStr = c.extraControllerLedHelpFunc(helpMap)
+
+	}
+
+	// Keep linter from complaining if we don't actually generate code using it
+	_ = helpMap
+	return helpStr
+}
+
+var flagsControllerLedMap = map[string][]string{
+
+	"create": {"scope-id", "name", "description"},
+}
+
+func (c *ControllerLedCommand) Flags() *base.FlagSets {
+	if len(flagsControllerLedMap[c.Func]) == 0 {
+		return c.FlagSet(base.FlagSetNone)
+	}
+
+	set := c.FlagSet(base.FlagSetHTTP | base.FlagSetClient | base.FlagSetOutputFormat)
+	f := set.NewFlagSet("Command Options")
+	common.PopulateCommonFlags(c.Command, f, "controller-led-type worker", flagsControllerLedMap, c.Func)
+
+	extraControllerLedFlagsFunc(c, set, f)
+
+	return set
+}
+
+func (c *ControllerLedCommand) Run(args []string) int {
+	initControllerLedFlags()
+
+	switch c.Func {
+	case "":
+		return cli.RunResultHelp
+
+	case "update":
+		return cli.RunResultHelp
+
+	}
+
+	c.plural = "controller-led-type worker"
+	switch c.Func {
+	case "list":
+		c.plural = "controller-led-type workers"
+	}
+
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.PrintCliError(err)
+		return base.CommandUserError
+	}
+
+	if strutil.StrListContains(flagsControllerLedMap[c.Func], "id") && c.FlagId == "" {
+		c.PrintCliError(errors.New("ID is required but not passed in via -id"))
+		return base.CommandUserError
+	}
+
+	var opts []workers.Option
+
+	if strutil.StrListContains(flagsControllerLedMap[c.Func], "scope-id") {
+		switch c.Func {
+
+		case "create":
+			if c.FlagScopeId == "" {
+				c.PrintCliError(errors.New("Scope ID must be passed in via -scope-id or BOUNDARY_SCOPE_ID"))
+				return base.CommandUserError
+			}
+
+		}
+	}
+
+	client, err := c.Client()
+	if c.WrapperCleanupFunc != nil {
+		defer func() {
+			if err := c.WrapperCleanupFunc(); err != nil {
+				c.PrintCliError(fmt.Errorf("Error cleaning kms wrapper: %w", err))
+			}
+		}()
+	}
+	if err != nil {
+		c.PrintCliError(fmt.Errorf("Error creating API client: %w", err))
+		return base.CommandCliError
+	}
+	workersClient := workers.NewClient(client)
+
+	switch c.FlagName {
+	case "":
+	case "null":
+		opts = append(opts, workers.DefaultName())
+	default:
+		opts = append(opts, workers.WithName(c.FlagName))
+	}
+
+	switch c.FlagDescription {
+	case "":
+	case "null":
+		opts = append(opts, workers.DefaultDescription())
+	default:
+		opts = append(opts, workers.WithDescription(c.FlagDescription))
+	}
+
+	switch c.FlagRecursive {
+	case true:
+		opts = append(opts, workers.WithRecursive(true))
+	}
+
+	if c.FlagFilter != "" {
+		opts = append(opts, workers.WithFilter(c.FlagFilter))
+	}
+
+	var version uint32
+
+	if ok := extraControllerLedFlagsHandlingFunc(c, f, &opts); !ok {
+		return base.CommandUserError
+	}
+
+	var resp *api.Response
+	var item *workers.Worker
+
+	switch c.Func {
+
+	}
+
+	resp, item, err = executeExtraControllerLedActions(c, resp, item, err, workersClient, version, opts)
+	if exitCode := c.checkFuncError(err); exitCode > 0 {
+		return exitCode
+	}
+
+	output, err := printCustomControllerLedActionOutput(c)
+	if err != nil {
+		c.PrintCliError(err)
+		return base.CommandUserError
+	}
+	if output {
+		return base.CommandSuccess
+	}
+
+	switch c.Func {
+
+	}
+
+	switch base.Format(c.UI) {
+	case "table":
+		c.UI.Output(printItemTable(item, resp))
+
+	case "json":
+		if ok := c.PrintJsonItem(resp); !ok {
+			return base.CommandCliError
+		}
+	}
+
+	return base.CommandSuccess
+}
+
+func (c *ControllerLedCommand) checkFuncError(err error) int {
+	if err == nil {
+		return 0
+	}
+	if apiErr := api.AsServerError(err); apiErr != nil {
+		c.PrintApiError(apiErr, fmt.Sprintf("Error from controller when performing %s on %s", c.Func, c.plural))
+		return base.CommandApiError
+	}
+	c.PrintCliError(fmt.Errorf("Error trying to %s %s: %s", c.Func, c.plural, err.Error()))
+	return base.CommandCliError
+}
+
+var (
+	extraControllerLedActionsFlagsMapFunc = func() map[string][]string { return nil }
+	extraControllerLedSynopsisFunc        = func(*ControllerLedCommand) string { return "" }
+	extraControllerLedFlagsFunc           = func(*ControllerLedCommand, *base.FlagSets, *base.FlagSet) {}
+	extraControllerLedFlagsHandlingFunc   = func(*ControllerLedCommand, *base.FlagSets, *[]workers.Option) bool { return true }
+	executeExtraControllerLedActions      = func(_ *ControllerLedCommand, inResp *api.Response, inItem *workers.Worker, inErr error, _ *workers.Client, _ uint32, _ []workers.Option) (*api.Response, *workers.Worker, error) {
+		return inResp, inItem, inErr
+	}
+	printCustomControllerLedActionOutput = func(*ControllerLedCommand) (bool, error) { return false, nil }
+)

internal/cmd/commands/workerscmd/funcs.go

@@ -261,6 +261,9 @@ func printItemTable(item *workers.Worker, resp *api.Response) string {
 	if !item.LastStatusTime.IsZero() {
 		nonAttributeMap["Last Status Time"] = item.LastStatusTime
 	}
+	if item.ControllerGeneratedActivationToken != "" {
+		nonAttributeMap["Controller-Generated Activation Token"] = item.ControllerGeneratedActivationToken
+	}
 
 	resultMap := resp.Map
 	if count, ok := resultMap[globals.ActiveConnectionCountField]; ok {

internal/cmd/config/config.go

@@ -212,6 +212,11 @@ type Worker struct {
 
 	// AuthStoragePath represents the location a worker stores its node credentials, if set
 	AuthStoragePath string `hcl:"auth_storage_path"`
+
+	// ControllerGeneratedActivationToken is a controller-generated activation
+	// token used to register this worker to the cluster. It can be a path, env
+	// var, or direct value.
+	ControllerGeneratedActivationToken string `hcl:"controller_generated_activation_token"`
 }
 
 type Database struct {
@@ -454,6 +459,11 @@ func Parse(d string) (*Config, error) {
 			return nil, errors.New("Worker description contains non-printable characters")
 		}
 
+		result.Worker.ControllerGeneratedActivationToken, err = parseutil.ParsePath(result.Worker.ControllerGeneratedActivationToken)
+		if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
+			return nil, fmt.Errorf("Error parsing worker activation token: %w", err)
+		}
+
 		if result.Worker.TagsRaw != nil {
 			switch t := result.Worker.TagsRaw.(type) {
 			// We allow `tags` to be a simple string containing a URL with schema.

internal/cmd/config/config_test.go

@@ -301,6 +301,24 @@ func TestDevWorker(t *testing.T) {
 
 	_, err = Parse(devConfig + devWorkerKeyValueConfig)
 	assert.Error(t, err)
+
+	// Check activation token parsing
+	devWorkerActivationTokenConfig := `
+		listener "tcp" {
+			purpose = "proxy"
+		}
+	
+		worker {
+			name = "dev-worker"
+			description = "A default worker created in dev mode"
+			initial_upstreams = ["127.0.0.1"]
+			controller_generated_activation_token = "foobar"
+		}
+		`
+
+	actual, err = Parse(devConfig + devWorkerActivationTokenConfig)
+	require.NoError(t, err)
+	assert.Equal(t, "foobar", actual.Worker.ControllerGeneratedActivationToken)
 }
 
 func TestDevCombined(t *testing.T) {

internal/cmd/gencli/input.go

@@ -552,5 +552,19 @@ var inputStructs = map[string][]*cmdInfo{
 			NeedsSubtypeInCreate:  true,
 			SkipClientCallActions: []string{"create"},
 		},
+		{
+			ResourceType:          resource.Worker.String(),
+			Pkg:                   "workers",
+			StdActions:            []string{"create"},
+			SubActionPrefix:       "controller-led",
+			SkipNormalHelp:        true,
+			HasExtraHelpFunc:      true,
+			HasId:                 true,
+			HasName:               true,
+			Container:             "Scope",
+			HasDescription:        true,
+			NeedsSubtypeInCreate:  true,
+			SkipClientCallActions: []string{"create"},
+		},
 	},
 }