Allow a worker to use KMS auth but accept PKI for proxy (#2206)

This essentially worked for non-dev mode before but would create storage
directories that weren't actually used or needed, and it didn't work in
dev mode because it made assumptions about the presence of a worker auth
kms or not. This fixes those behaviors.

internal/cmd/base/servers.go

@@ -126,6 +126,10 @@ type Server struct {
 	DevTargetSessionConnectionLimit  int
 	DevLoopbackHostPluginId          string
 
+	// DevUsePkiForUpstream is a hint that we are in dev mode and have a worker
+	// auth KMS but want to use PKI for upstream connections
+	DevUsePkiForUpstream bool
+
 	EnabledPlugins []EnabledPlugin
 	HostPlugins    map[string]plgpb.HostPluginServiceClient
 

internal/cmd/commands/dev/dev.go

@@ -577,7 +577,7 @@ func (c *Command) Run(args []string) int {
 	if c.flagRecoveryKey != "" {
 		c.Config.DevRecoveryKey = c.flagRecoveryKey
 	}
-	if err := c.SetupKMSes(c.Context, c.UI, c.Config, base.WithSkipWorkerAuthKmsInstantiation(!c.flagUseKmsWorkerAuthMethod)); err != nil {
+	if err := c.SetupKMSes(c.Context, c.UI, c.Config); err != nil {
 		c.UI.Error(err.Error())
 		return base.CommandUserError
 	}
@@ -585,14 +585,15 @@ func (c *Command) Run(args []string) int {
 		c.UI.Error("Controller KMS not found after parsing KMS blocks")
 		return base.CommandUserError
 	}
-	if c.flagUseKmsWorkerAuthMethod {
-		if c.WorkerAuthKms == nil {
-			c.UI.Error("Worker Auth KMS not found after parsing KMS blocks")
-			return base.CommandUserError
-		}
-		c.InfoKeys = append(c.InfoKeys, "[Worker-Auth] AEAD Key Bytes")
-		c.Info["[Worker-Auth] AEAD Key Bytes"] = c.Config.DevWorkerAuthKey
-	} else {
+	if c.WorkerAuthKms == nil {
+		c.UI.Error("Worker Auth KMS not found after parsing KMS blocks")
+		return base.CommandUserError
+	}
+	c.InfoKeys = append(c.InfoKeys, "[Worker-Auth] AEAD Key Bytes")
+	c.Info["[Worker-Auth] AEAD Key Bytes"] = c.Config.DevWorkerAuthKey
+
+	if !c.flagUseKmsWorkerAuthMethod {
+		c.DevUsePkiForUpstream = true
 		// These must be unset for PKI
 		c.Config.Worker.Name = ""
 		c.Config.Worker.Description = ""
@@ -809,7 +810,7 @@ func (c *Command) Run(args []string) int {
 			}
 
 			if !c.flagControllerOnly {
-				if !c.flagWorkerAuthStorageSkipCleanup {
+				if !c.flagWorkerAuthStorageSkipCleanup && c.worker.WorkerAuthStorage != nil {
 					c.worker.WorkerAuthStorage.Cleanup()
 				}
 				if err := c.worker.Shutdown(); err != nil {

internal/cmd/commands/server/server.go

@@ -494,7 +494,7 @@ func (c *Command) Run(args []string) int {
 			return base.CommandCliError
 		}
 
-		if c.WorkerAuthKms == nil && c.worker.WorkerAuthRegistrationRequest != "" {
+		if c.worker.WorkerAuthRegistrationRequest != "" {
 			c.InfoKeys = append(c.InfoKeys, "worker auth registration request")
 			c.Info["worker auth registration request"] = c.worker.WorkerAuthRegistrationRequest
 			c.InfoKeys = append(c.InfoKeys, "worker auth current key id")
@@ -677,14 +677,6 @@ func (c *Command) StartWorker() error {
 		return fmt.Errorf("Error initializing worker: %w", err)
 	}
 
-	if c.WorkerAuthKms == nil {
-		if c.worker.WorkerAuthStorage == nil {
-			return fmt.Errorf("No worker auth KMS specified and no worker auth storage found")
-		}
-		c.InfoKeys = append(c.InfoKeys, "worker auth storage path")
-		c.Info["worker auth storage path"] = c.worker.WorkerAuthStorage.BaseDir()
-	}
-
 	if err := c.worker.Start(); err != nil {
 		retErr := fmt.Errorf("Error starting worker: %w", err)
 		if err := c.worker.Shutdown(); err != nil {
@@ -694,6 +686,14 @@ func (c *Command) StartWorker() error {
 		return retErr
 	}
 
+	if c.WorkerAuthKms == nil || c.DevUsePkiForUpstream {
+		if c.worker.WorkerAuthStorage == nil {
+			return fmt.Errorf("No worker auth storage found")
+		}
+		c.InfoKeys = append(c.InfoKeys, "worker auth storage path")
+		c.Info["worker auth storage path"] = c.worker.WorkerAuthStorage.BaseDir()
+	}
+
 	return nil
 }