feat(worker): implements WorkerAuthStorage wrapper for worker credential encryption (#2076)

* Adds WorkerAuthStorage wrapper for worker credential encryption using nodeenrollment library.

* Adds AuthStoragePath field in worker config and dev mode to specify directory to store credentials.

globals/kms.go

@@ -1,8 +1,9 @@
 package globals
 
 const (
-	KmsPurposeRoot       = "root"
-	KmsPurposeWorkerAuth = "worker-auth"
-	KmsPurposeRecovery   = "recovery"
-	KmsPurposeConfig     = "config"
+	KmsPurposeRoot              = "root"
+	KmsPurposeWorkerAuth        = "worker-auth"
+	KmsPurposeWorkerAuthStorage = "worker-auth-storage"
+	KmsPurposeRecovery          = "recovery"
+	KmsPurposeConfig            = "config"
 )

internal/cmd/base/server_test.go

@@ -55,8 +55,11 @@ func TestServer_SetupKMSes(t *testing.T) {
 			purposes: []string{globals.KmsPurposeRoot},
 		},
 		{
-			name:     "multi purpose",
-			purposes: []string{globals.KmsPurposeRoot, globals.KmsPurposeRecovery, globals.KmsPurposeWorkerAuth, globals.KmsPurposeConfig},
+			name: "multi purpose",
+			purposes: []string{
+				globals.KmsPurposeRoot, globals.KmsPurposeRecovery, globals.KmsPurposeWorkerAuth,
+				globals.KmsPurposeWorkerAuthStorage, globals.KmsPurposeConfig,
+			},
 		},
 	}
 	logger := hclog.Default()
@@ -89,6 +92,8 @@ func TestServer_SetupKMSes(t *testing.T) {
 					assert.NotNil(s.RootKms)
 				case globals.KmsPurposeWorkerAuth:
 					assert.NotNil(s.WorkerAuthKms)
+				case globals.KmsPurposeWorkerAuthStorage:
+					assert.NotNil(s.WorkerAuthStorageKms)
 				case globals.KmsPurposeRecovery:
 					assert.NotNil(s.RecoveryKms)
 				}

internal/cmd/base/servers.go

@@ -82,11 +82,12 @@ type Server struct {
 	StderrLock *sync.Mutex
 	Eventer    *event.Eventer
 
-	RootKms            wrapping.Wrapper
-	WorkerAuthKms      wrapping.Wrapper
-	RecoveryKms        wrapping.Wrapper
-	Kms                *kms.Kms
-	SecureRandomReader io.Reader
+	RootKms              wrapping.Wrapper
+	WorkerAuthKms        wrapping.Wrapper
+	WorkerAuthStorageKms wrapping.Wrapper
+	RecoveryKms          wrapping.Wrapper
+	Kms                  *kms.Kms
+	SecureRandomReader   io.Reader
 
 	PrometheusRegisterer prometheus.Registerer
 
@@ -533,7 +534,7 @@ func (b *Server) SetupKMSes(ctx context.Context, ui cli.Ui, config *config.Confi
 				if opts.withSkipWorkerAuthKmsInstantiation {
 					continue
 				}
-			case globals.KmsPurposeRoot, globals.KmsPurposeConfig:
+			case globals.KmsPurposeRoot, globals.KmsPurposeConfig, globals.KmsPurposeWorkerAuthStorage:
 			case globals.KmsPurposeRecovery:
 				if config.Controller != nil && config.DevRecoveryKey != "" {
 					kms.Config["key"] = config.DevRecoveryKey
@@ -597,6 +598,8 @@ func (b *Server) SetupKMSes(ctx context.Context, ui cli.Ui, config *config.Confi
 				b.RootKms = wrapper
 			case globals.KmsPurposeWorkerAuth:
 				b.WorkerAuthKms = wrapper
+			case globals.KmsPurposeWorkerAuthStorage:
+				b.WorkerAuthStorageKms = wrapper
 			case globals.KmsPurposeRecovery:
 				b.RecoveryKms = wrapper
 			case globals.KmsPurposeConfig:

internal/cmd/commands/dev/dev.go

@@ -86,6 +86,8 @@ type Command struct {
 	flagCreateLoopbackHostPlugin        bool
 	flagPluginExecutionDir              string
 	flagUseEphemeralKmsWorkerAuthMethod bool
+	flagWorkerAuthStorageDir            string
+	flagWorkerAuthStorageSkipCleanup    bool
 }
 
 func (c *Command) Synopsis() string {
@@ -335,6 +337,18 @@ func (c *Command) Flags() *base.FlagSets {
 		Usage:  "If set, the original \"ephemeral\" method of worker auth will be used to connect the initial dev worker to the controller.",
 	})
 
+	f.StringVar(&base.StringVar{
+		Name:   "worker-auth-storage-dir",
+		Target: &c.flagWorkerAuthStorageDir,
+		Usage:  "Specifies the directory to store worker authentication credentials in dev mode.",
+	})
+
+	f.BoolVar(&base.BoolVar{
+		Name:   "worker-auth-storage-skip-cleanup",
+		Target: &c.flagWorkerAuthStorageSkipCleanup,
+		Usage:  "Prevents deletion of temp worker credential storage directory if set",
+	})
+
 	f.BoolVar(&base.BoolVar{
 		Name:   "create-loopback-host-plugin",
 		Target: &c.flagCreateLoopbackHostPlugin,
@@ -391,6 +405,7 @@ func (c *Command) Run(args []string) int {
 	c.DevUnprivilegedPassword = c.flagUnprivilegedPassword
 	c.DevTargetDefaultPort = c.flagTargetDefaultPort
 	c.Config.Plugins.ExecutionDir = c.flagPluginExecutionDir
+	c.Config.Worker.AuthStoragePath = c.flagWorkerAuthStorageDir
 	if c.flagIdSuffix != "" {
 		if len(c.flagIdSuffix) != 10 {
 			c.UI.Error("Invalid ID suffix, must be exactly 10 characters")
@@ -756,6 +771,9 @@ func (c *Command) Run(args []string) int {
 			}
 
 			if !c.flagControllerOnly {
+				if !c.flagWorkerAuthStorageSkipCleanup {
+					c.worker.NodeeFileStorage.Cleanup()
+				}
 				if err := c.worker.Shutdown(); err != nil {
 					c.UI.Error(fmt.Errorf("Error shutting down worker: %w", err).Error())
 				}

internal/cmd/commands/dev/flags_test.go

@@ -0,0 +1,72 @@
+package dev
+
+import (
+	"bufio"
+	"os"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/cmd/base"
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCommand_Flags(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+
+	serverCmdUi := &base.BoundaryUI{
+		Ui: &cli.ColoredUi{
+			ErrorColor: cli.UiColorRed,
+			WarnColor:  cli.UiColorYellow,
+			Ui: &cli.BasicUi{
+				Reader: bufio.NewReader(os.Stdin),
+				Writer: os.Stdout,
+			},
+		},
+		Format: "table",
+	}
+	base.NewServer(base.NewCommand(serverCmdUi))
+
+	cmd := &Command{Server: base.NewServer(base.NewCommand(serverCmdUi))}
+	got := cmd.Flags()
+	completions := got.Completions()
+
+	assert.Contains(completions, "-log-level")
+	assert.Contains(completions, "-log-format")
+	assert.Contains(completions, "-id-suffix")
+	assert.Contains(completions, "-password")
+	assert.Contains(completions, "-login-name")
+	assert.Contains(completions, "-unprivileged-password")
+	assert.Contains(completions, "-unprivileged-login-name")
+	assert.Contains(completions, "-api-listen-address")
+	assert.Contains(completions, "-host-address")
+	assert.Contains(completions, "-target-default-port")
+	assert.Contains(completions, "-target-session-connection-limit")
+	assert.Contains(completions, "-target-session-max-seconds")
+	assert.Contains(completions, "-cluster-listen-address")
+	assert.Contains(completions, "-controller-public-cluster-address")
+	assert.Contains(completions, "-ops-listen-address")
+	assert.Contains(completions, "-controller-only")
+	assert.Contains(completions, "-proxy-listen-address")
+	assert.Contains(completions, "-worker-public-address")
+	assert.Contains(completions, "-worker-auth-key")
+	assert.Contains(completions, "-disable-database-destruction")
+	assert.Contains(completions, "-combine-logs")
+	assert.Contains(completions, "-ui-passthrough-dir")
+	assert.Contains(completions, "-recovery-key")
+	assert.Contains(completions, "-database-url")
+	assert.Contains(completions, "-container-image")
+	assert.Contains(completions, "-event-format")
+	assert.Contains(completions, "-observation-events")
+	assert.Contains(completions, "-audit-events")
+	assert.Contains(completions, "-system-events")
+	assert.Contains(completions, "-audit-events")
+	assert.Contains(completions, "-event-allow-filter")
+	assert.Contains(completions, "-event-deny-filter")
+	assert.Contains(completions, "-plugin-execution-dir")
+	assert.Contains(completions, "-use-ephemeral-kms-worker-auth-method")
+	assert.Contains(completions, "-worker-auth-storage-dir")
+	assert.Contains(completions, "-worker-auth-storage-skip-cleanup")
+
+	// keep adding assertions for other flags which should be set as a result of cmd.Flags()
+}

internal/cmd/commands/server/listener_reload_test.go

@@ -94,7 +94,9 @@ func TestServer_ReloadListener(t *testing.T) {
 	}
 	defer os.RemoveAll(td)
 
-	controllerKey, workerAuthKey, recoveryKey := config.DevKeyGeneration()
+	controllerKey := config.DevKeyGeneration()
+	workerAuthKey := config.DevKeyGeneration()
+	recoveryKey := config.DevKeyGeneration()
 
 	cmd := testServerCommand(t, testServerCommandOpts{
 		CreateDevDatabase: true,

internal/cmd/config/config.go

@@ -57,6 +57,13 @@ kms "aead" {
 	key_id = "global_worker-auth"
 }
 
+kms "aead" {
+    purpose = "worker-auth-storage"
+	aead_type = "aes-gcm"
+	key = "%s"
+	key_id = "global_worker-auth-storage"
+}
+
 kms "aead" {
 	purpose = "recovery"
 	aead_type = "aes-gcm"
@@ -105,11 +112,12 @@ type Config struct {
 	Controller *Controller `hcl:"controller"`
 
 	// Dev-related options
-	DevController       bool   `hcl:"-"`
-	DevUiPassthroughDir string `hcl:"-"`
-	DevControllerKey    string `hcl:"-"`
-	DevWorkerAuthKey    string `hcl:"-"`
-	DevRecoveryKey      string `hcl:"-"`
+	DevController           bool   `hcl:"-"`
+	DevUiPassthroughDir     string `hcl:"-"`
+	DevControllerKey        string `hcl:"-"`
+	DevWorkerAuthKey        string `hcl:"-"`
+	DevWorkerAuthStorageKey string `hcl:"-"`
+	DevRecoveryKey          string `hcl:"-"`
 
 	// Eventing configuration for the controller
 	Eventing *event.EventerConfig `hcl:"events"`
@@ -189,7 +197,7 @@ type Worker struct {
 	StatusGracePeriodDuration time.Duration `hcl:"-"`
 
 	// StoragePath represents the location a pki worker stores its node credentials, if set.
-	StoragePath string `hcl:"storage_path"`
+	AuthStoragePath string `hcl:"storage_path"`
 }
 
 func (w *Worker) InitNameIfEmpty() (string, error) {
@@ -234,8 +242,8 @@ func DevWorker() (*Config, error) {
 	return parsed, nil
 }
 
-func DevKeyGeneration() (string, string, string) {
-	var numBytes int64 = 96
+func DevKeyGeneration() string {
+	var numBytes int64 = 32
 	randBuf := new(bytes.Buffer)
 	n, err := randBuf.ReadFrom(&io.LimitedReader{
 		R: rand.Reader,
@@ -245,21 +253,21 @@ func DevKeyGeneration() (string, string, string) {
 		panic(err)
 	}
 	if n != numBytes {
-		panic(fmt.Errorf("expected to read 64 bytes, read %d", n))
+		panic(fmt.Errorf("expected to read 32 bytes, read %d", n))
 	}
-	controllerKey := base64.StdEncoding.EncodeToString(randBuf.Bytes()[0:32])
-	workerAuthKey := base64.StdEncoding.EncodeToString(randBuf.Bytes()[32:64])
-	recoveryKey := base64.StdEncoding.EncodeToString(randBuf.Bytes()[64:numBytes])
-
-	return controllerKey, workerAuthKey, recoveryKey
+	devKey := base64.StdEncoding.EncodeToString(randBuf.Bytes())[:numBytes]
+	return devKey
 }
 
 // DevController is a Config that is used for dev mode of Boundary
 // controllers
 func DevController() (*Config, error) {
-	controllerKey, workerAuthKey, recoveryKey := DevKeyGeneration()
+	controllerKey := DevKeyGeneration()
+	workerAuthKey := DevKeyGeneration()
+	workerAuthStorageKey := DevKeyGeneration()
+	recoveryKey := DevKeyGeneration()
 
-	hclStr := fmt.Sprintf(devConfig+devControllerExtraConfig, controllerKey, workerAuthKey, recoveryKey)
+	hclStr := fmt.Sprintf(devConfig+devControllerExtraConfig, controllerKey, workerAuthKey, workerAuthStorageKey, recoveryKey)
 	parsed, err := Parse(hclStr)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing dev config: %w", err)
@@ -267,13 +275,17 @@ func DevController() (*Config, error) {
 	parsed.DevController = true
 	parsed.DevControllerKey = controllerKey
 	parsed.DevWorkerAuthKey = workerAuthKey
+	parsed.DevWorkerAuthStorageKey = workerAuthStorageKey
 	parsed.DevRecoveryKey = recoveryKey
 	return parsed, nil
 }
 
 func DevCombined() (*Config, error) {
-	controllerKey, workerAuthKey, recoveryKey := DevKeyGeneration()
-	hclStr := fmt.Sprintf(devConfig+devControllerExtraConfig+devWorkerExtraConfig, controllerKey, workerAuthKey, recoveryKey)
+	controllerKey := DevKeyGeneration()
+	workerAuthKey := DevKeyGeneration()
+	workerAuthStorageKey := DevKeyGeneration()
+	recoveryKey := DevKeyGeneration()
+	hclStr := fmt.Sprintf(devConfig+devControllerExtraConfig+devWorkerExtraConfig, controllerKey, workerAuthKey, workerAuthStorageKey, recoveryKey)
 	parsed, err := Parse(hclStr)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing dev config: %w", err)
@@ -281,6 +293,7 @@ func DevCombined() (*Config, error) {
 	parsed.DevController = true
 	parsed.DevControllerKey = controllerKey
 	parsed.DevWorkerAuthKey = workerAuthKey
+	parsed.DevWorkerAuthStorageKey = workerAuthStorageKey
 	parsed.DevRecoveryKey = recoveryKey
 	return parsed, nil
 }

internal/cmd/config/config_test.go

@@ -61,6 +61,14 @@ func TestDevController(t *testing.T) {
 						"key_id":    "global_worker-auth",
 					},
 				},
+				{
+					Type:    "aead",
+					Purpose: []string{"worker-auth-storage"},
+					Config: map[string]string{
+						"aead_type": "aes-gcm",
+						"key_id":    "global_worker-auth-storage",
+					},
+				},
 				{
 					Type:    "aead",
 					Purpose: []string{"recovery"},
@@ -84,9 +92,11 @@ func TestDevController(t *testing.T) {
 	exp.Seals[0].Config["key"] = actual.Seals[0].Config["key"]
 	exp.Seals[1].Config["key"] = actual.Seals[1].Config["key"]
 	exp.Seals[2].Config["key"] = actual.Seals[2].Config["key"]
+	exp.Seals[3].Config["key"] = actual.Seals[3].Config["key"]
 	exp.DevControllerKey = actual.Seals[0].Config["key"]
 	exp.DevWorkerAuthKey = actual.Seals[1].Config["key"]
-	exp.DevRecoveryKey = actual.Seals[2].Config["key"]
+	exp.DevWorkerAuthStorageKey = actual.Seals[2].Config["key"]
+	exp.DevRecoveryKey = actual.Seals[3].Config["key"]
 
 	assert.Equal(t, exp, actual)
 
@@ -291,6 +301,170 @@ func TestDevWorker(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestDevCombined(t *testing.T) {
+	actual, err := DevCombined()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	truePointer := new(bool)
+	*truePointer = true
+	exp := &Config{
+		Eventing: event.DefaultEventerConfig(),
+		SharedConfig: &configutil.SharedConfig{
+			DisableMlock: true,
+			Listeners: []*listenerutil.ListenerConfig{
+				{
+					Type:               "tcp",
+					Purpose:            []string{"api"},
+					TLSDisable:         true,
+					CorsEnabled:        truePointer,
+					CorsAllowedOrigins: []string{"*"},
+				},
+				{
+					Type:    "tcp",
+					Purpose: []string{"cluster"},
+				},
+				{
+					Type:       "tcp",
+					Purpose:    []string{"ops"},
+					TLSDisable: true,
+				},
+				{
+					Type:    "tcp",
+					Purpose: []string{"proxy"},
+				},
+			},
+			Seals: []*configutil.KMS{
+				{
+					Type:    "aead",
+					Purpose: []string{"root"},
+					Config: map[string]string{
+						"aead_type": "aes-gcm",
+						"key_id":    "global_root",
+					},
+				},
+				{
+					Type:    "aead",
+					Purpose: []string{"worker-auth"},
+					Config: map[string]string{
+						"aead_type": "aes-gcm",
+						"key_id":    "global_worker-auth",
+					},
+				},
+				{
+					Type:    "aead",
+					Purpose: []string{"worker-auth-storage"},
+					Config: map[string]string{
+						"aead_type": "aes-gcm",
+						"key_id":    "global_worker-auth-storage",
+					},
+				},
+				{
+					Type:    "aead",
+					Purpose: []string{"recovery"},
+					Config: map[string]string{
+						"aead_type": "aes-gcm",
+						"key_id":    "global_recovery",
+					},
+				},
+			},
+		},
+		Controller: &Controller{
+			Name:        "dev-controller",
+			Description: "A default controller created in dev mode",
+		},
+		DevController: true,
+		Worker: &Worker{
+			Name:           "w_1234567890",
+			Description:    "A default worker created in dev mode",
+			Controllers:    []string{"127.0.0.1"},
+			ControllersRaw: []interface{}{"127.0.0.1"},
+			Tags: map[string][]string{
+				"type": {"dev", "local"},
+			},
+		},
+	}
+
+	exp.Listeners[0].RawConfig = actual.Listeners[0].RawConfig
+	exp.Listeners[1].RawConfig = actual.Listeners[1].RawConfig
+	exp.Listeners[2].RawConfig = actual.Listeners[2].RawConfig
+	exp.Listeners[3].RawConfig = actual.Listeners[3].RawConfig
+	exp.Seals[0].Config["key"] = actual.Seals[0].Config["key"]
+	exp.Seals[1].Config["key"] = actual.Seals[1].Config["key"]
+	exp.Seals[2].Config["key"] = actual.Seals[2].Config["key"]
+	exp.Seals[3].Config["key"] = actual.Seals[3].Config["key"]
+	exp.DevControllerKey = actual.Seals[0].Config["key"]
+	exp.DevWorkerAuthKey = actual.Seals[1].Config["key"]
+	exp.DevWorkerAuthStorageKey = actual.Seals[2].Config["key"]
+	exp.DevRecoveryKey = actual.Seals[3].Config["key"]
+	exp.Worker.TagsRaw = actual.Worker.TagsRaw
+	assert.Equal(t, exp, actual)
+}
+
+func TestDevWorkerCredentialStorageDir(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name            string
+		devWorkerConfig string
+		storagePath     string
+	}{
+		{
+			name: "Relative Storage Directory",
+			devWorkerConfig: `
+			listener "tcp" {
+				purpose = "proxy"
+			}
+
+			worker {
+				name = "w_1234567890"
+				description = "A default worker created in dev mode"
+				controllers = ["127.0.0.1"]
+				tags {
+					type = ["dev", "local"]
+				}
+				storage_path = ".."
+			}
+			`,
+			storagePath: "..",
+		},
+		{
+			name: "Nonexistent Storage Directory",
+			devWorkerConfig: `
+			listener "tcp" {
+				purpose = "proxy"
+			}
+
+			worker {
+				name = "w_1234567890"
+				description = "A default worker created in dev mode"
+				controllers = ["127.0.0.1"]
+				tags {
+					type = ["dev", "local"]
+				}
+				storage_path = "nonexistent_dir/here"
+			}
+			`,
+			storagePath: "nonexistent_dir/here",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			parsed, err := Parse(devConfig + tt.devWorkerConfig)
+			require.NoError(t, err)
+			require.Equal(t, tt.storagePath, parsed.Worker.AuthStoragePath)
+		})
+	}
+}
+
+func TestDevKeyGeneration(t *testing.T) {
+	t.Parallel()
+	dk := DevKeyGeneration()
+	numBytes := 32
+	require.Equal(t, numBytes, len(dk))
+	require.NotEqual(t, dk, DevKeyGeneration())
+}
+
 func TestParsingName(t *testing.T) {
 	t.Parallel()
 	config := `

internal/daemon/worker/controller_connection.go

@@ -23,6 +23,7 @@ import (
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/go-secure-stdlib/base62"
+	"github.com/hashicorp/nodeenrollment"
 	"github.com/hashicorp/nodeenrollment/multihop"
 	"github.com/hashicorp/nodeenrollment/protocol"
 	"google.golang.org/grpc"
@@ -73,7 +74,7 @@ func (w *Worker) controllerDialerFunc() func(context.Context, string) (net.Conn,
 		case w.conf.WorkerAuthKms != nil:
 			conn, err = w.v1KmsAuthDialFn(ctx, addr)
 		default:
-			conn, err = protocol.Dial(ctx, w.NodeeFileStorage, addr)
+			conn, err = protocol.Dial(ctx, w.NodeeFileStorage, addr, nodeenrollment.WithWrapper(w.conf.WorkerAuthStorageKms))
 		}
 
 		if !w.everAuthenticated.Load() && err == nil && conn != nil {

internal/daemon/worker/testing.go

@@ -174,6 +174,9 @@ type TestWorkerOpts struct {
 	// The worker auth KMS to use, or one will be created
 	WorkerAuthKms wrapping.Wrapper
 
+	// The worker credential storage KMS to use, or one will be created
+	WorkerAuthStorageKms wrapping.Wrapper
+
 	// The name to use for the worker, otherwise one will be randomly
 	// generated, unless provided in a non-nil Config
 	Name string
@@ -261,6 +264,8 @@ func NewTestWorker(t testing.TB, opts *TestWorkerOpts) *TestWorker {
 	switch {
 	case opts.WorkerAuthKms != nil:
 		tw.b.WorkerAuthKms = opts.WorkerAuthKms
+	case opts.WorkerAuthStorageKms != nil:
+		tw.b.WorkerAuthStorageKms = opts.WorkerAuthStorageKms
 	default:
 		if err := tw.b.SetupKMSes(tw.b.Context, nil, opts.Config); err != nil {
 			t.Fatal(err)
@@ -310,6 +315,7 @@ func (tw *TestWorker) AddClusterWorkerMember(t testing.TB, opts *TestWorkerOpts)
 	}
 	nextOpts := &TestWorkerOpts{
 		WorkerAuthKms:             tw.w.conf.WorkerAuthKms,
+		WorkerAuthStorageKms:      tw.w.conf.WorkerAuthStorageKms,
 		Name:                      opts.Name,
 		InitialControllers:        tw.ControllerAddrs(),
 		Logger:                    tw.w.conf.Logger,

internal/daemon/worker/testing_test.go

@@ -6,7 +6,9 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/daemon/worker/session"
+	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
+	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
 	"github.com/stretchr/testify/require"
 )
 
@@ -60,3 +62,28 @@ func TestTestWorkerLookupSessionMissing(t *testing.T) {
 	require.False(ok)
 	require.Equal(TestSessionInfo{}, actual)
 }
+
+func TestTestWorker_WorkerAuthStorageKms(t *testing.T) {
+	tests := []struct {
+		name    string
+		wrapper wrapping.Wrapper
+	}{
+		{
+			name:    "Nil Wrapper",
+			wrapper: nil,
+		},
+		{
+			name:    "Valid Wrapper",
+			wrapper: db.TestWrapper(t),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require := require.New(t)
+			tw := NewTestWorker(t, &TestWorkerOpts{
+				WorkerAuthStorageKms: tt.wrapper,
+			})
+			require.Equal(tt.wrapper, tw.Config().WorkerAuthStorageKms)
+		})
+	}
+}

internal/daemon/worker/worker.go

@@ -162,7 +162,7 @@ func New(conf *Config) (*Worker, error) {
 	}
 
 	w.NodeeFileStorage, err = nodeefile.NewFileStorage(w.baseContext,
-		nodeefile.WithBaseDirectory(w.conf.RawConfig.Worker.StoragePath))
+		nodeefile.WithBaseDirectory(w.conf.RawConfig.Worker.AuthStoragePath))
 	if err != nil {
 		return nil, err
 	}
@@ -184,11 +184,18 @@ func (w *Worker) Start() error {
 	controllerResolver := manual.NewBuilderWithScheme(scheme)
 	w.controllerResolver.Store(controllerResolver)
 
+	var err error
+	w.NodeeFileStorage, err = nodeefile.NewFileStorage(w.baseContext,
+		nodeefile.WithBaseDirectory(w.conf.RawConfig.Worker.AuthStoragePath))
+	if err != nil {
+		return err
+	}
+
 	if err := w.startListeners(); err != nil {
 		return fmt.Errorf("error starting worker listeners: %w", err)
 	}
 
-	nodeCreds, err := types.NewNodeCredentials(w.baseContext, w.NodeeFileStorage)
+	nodeCreds, err := types.NewNodeCredentials(w.baseContext, w.NodeeFileStorage, nodeenrollment.WithWrapper(w.conf.WorkerAuthStorageKms))
 	if err != nil {
 		return fmt.Errorf("error generating new node creds: %w", err)
 	}

internal/daemon/worker/worker_test.go

@@ -1,13 +1,16 @@
 package worker
 
 import (
+	"context"
 	"testing"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
+	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/configutil/v2"
 	"github.com/hashicorp/go-secure-stdlib/listenerutil"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -123,3 +126,24 @@ func TestWorkerNew(t *testing.T) {
 		})
 	}
 }
+
+func TestSetupWorkerCredentialStorage(t *testing.T) {
+	ctx := context.Background()
+	assert, require := assert.New(t), require.New(t)
+	ts := db.TestWrapper(t)
+	keyId, err := ts.KeyId(ctx)
+
+	tw := NewTestWorker(t, &TestWorkerOpts{
+		WorkerAuthStorageKms: ts,
+		DisableAutoStart:     true,
+	})
+
+	err = tw.Worker().Start()
+	require.NoError(err)
+	wKeyId, err := tw.Config().WorkerAuthStorageKms.KeyId(ctx)
+	require.NoError(err)
+	assert.Equal(keyId, wKeyId)
+
+	err = tw.Worker().Shutdown()
+	require.NoError(err)
+}

internal/kms/const.go

@@ -27,6 +27,9 @@ const (
 	// KeyPurposeWorkerAuth is used for worker auth
 	KeyPurposeWorkerAuth
 
+	// KeyPurposeWorkerAuthStorage is used for worker credential storage
+	KeyPurposeWorkerAuthStorage
+
 	// KeyPurposeTokens is used for token encryption
 	KeyPurposeTokens
 
@@ -56,6 +59,8 @@ func (k KeyPurpose) String() string {
 		return "recovery"
 	case KeyPurposeWorkerAuth:
 		return "workerauth"
+	case KeyPurposeWorkerAuthStorage:
+		return "workerauthstorage"
 	case KeyPurposeTokens:
 		return "tokens"
 	case KeyPurposeSessions:

internal/kms/external_wrappers_test.go

@@ -20,9 +20,10 @@ func TestExternalWrappers(t *testing.T) {
 	rootWrapper := db.TestWrapper(t)
 	recoveryWrapper := db.TestWrapper(t)
 	workerAuthWrapper := db.TestWrapper(t)
+	workerAuthStorageWrapper := db.TestWrapper(t)
 
 	k := TestKms(t, conn, rootWrapper)
-	err := k.AddExternalWrappers(testCtx, WithRecoveryWrapper(recoveryWrapper), WithWorkerAuthWrapper(workerAuthWrapper))
+	err := k.AddExternalWrappers(testCtx, WithRecoveryWrapper(recoveryWrapper), WithWorkerAuthWrapper(workerAuthWrapper), WithWorkerAuthStorageWrapper(workerAuthStorageWrapper))
 	require.NoError(err)
 
 	assert.Equal(rootWrapper, k.GetExternalWrappers(testCtx).Root())

internal/kms/kms.go

@@ -258,7 +258,8 @@ func stdNewKmsPurposes() []wrappingKms.KeyPurpose {
 	for _, p := range ValidDekPurposes() {
 		purposes = append(purposes, wrappingKms.KeyPurpose(p.String()))
 	}
-	purposes = append(purposes, wrappingKms.KeyPurpose(KeyPurposeWorkerAuth.String()), wrappingKms.KeyPurpose(KeyPurposeRecovery.String()))
+	purposes = append(purposes, wrappingKms.KeyPurpose(KeyPurposeWorkerAuth.String()), wrappingKms.KeyPurpose(KeyPurposeWorkerAuthStorage.String()),
+		wrappingKms.KeyPurpose(KeyPurposeRecovery.String()))
 	return purposes
 }
 

internal/kms/kms_test.go

@@ -52,7 +52,8 @@ func Test_New(t *testing.T) {
 					for _, p := range ValidDekPurposes() {
 						purposes = append(purposes, wrappingKms.KeyPurpose(p.String()))
 					}
-					purposes = append(purposes, wrappingKms.KeyPurpose(KeyPurposeWorkerAuth.String()), wrappingKms.KeyPurpose(KeyPurposeRecovery.String()))
+					purposes = append(purposes, wrappingKms.KeyPurpose(KeyPurposeWorkerAuth.String()),
+						wrappingKms.KeyPurpose(KeyPurposeWorkerAuthStorage.String()), wrappingKms.KeyPurpose(KeyPurposeRecovery.String()))
 
 					r := dbw.New(rw.UnderlyingDB())
 					w := dbw.New(rw.UnderlyingDB())

internal/kms/options.go

@@ -21,16 +21,17 @@ type Option func(*options)
 
 // options = how options are represented
 type options struct {
-	withLimit             int
-	withRootWrapper       wrapping.Wrapper
-	withWorkerAuthWrapper wrapping.Wrapper
-	withRecoveryWrapper   wrapping.Wrapper
-	withOrderByVersion    db.OrderBy
-	withKeyId             string
-	withScopeIds          []string
-	withRandomReader      io.Reader
-	withReader            db.Reader
-	withWriter            db.Writer
+	withLimit                    int
+	withRootWrapper              wrapping.Wrapper
+	withWorkerAuthWrapper        wrapping.Wrapper
+	withWorkerAuthStorageWrapper wrapping.Wrapper
+	withRecoveryWrapper          wrapping.Wrapper
+	withOrderByVersion           db.OrderBy
+	withKeyId                    string
+	withScopeIds                 []string
+	withRandomReader             io.Reader
+	withReader                   db.Reader
+	withWriter                   db.Writer
 }
 
 func getDefaultOptions() options {
@@ -61,6 +62,14 @@ func WithWorkerAuthWrapper(w wrapping.Wrapper) Option {
 	}
 }
 
+// WithWorkerAuthStorageWrapper sets the external pki worker storage wrapper for a
+// given scope
+func WithWorkerAuthStorageWrapper(w wrapping.Wrapper) Option {
+	return func(o *options) {
+		o.withWorkerAuthStorageWrapper = w
+	}
+}
+
 // WithRecoveryWrapper sets the recovery wrapper for a given scope
 func WithRecoveryWrapper(w wrapping.Wrapper) Option {
 	return func(o *options) {