KMS-PKI Workers (#3101)

This adds a new mechanism for worker authentication that uses the
third-party arbitration of the original KMS flow by leveraging
recently-added capabilities in nodeenrollment to support a registration
wrapper. This allows for workers to be registered to the system with
just a name and a matching wrapper, while at the same time allowing
these workers to be used for private Vault access and multi-hop as they
contain per-worker encryption keys and support rotation.

Nodes using this mode will generate and rotate credentials in-memory
(using another recently added nodeenrollment capability) so will have a
new set of keys on every startup. This has the side effect of ensuring
that there is never a conflict or re-use of these credentials.

Note that the normal test worker flow was using the now-deprecated KMS
method; there is still a test that uses this (explicitly) but all other
tests using test workers that did not specify a specific PKI flow now
actually use this mechanism. This means there is fairly decent test
coverage in a general sense, plus the specific tests updated/added for
this feature.

This also adds a new KMS type that can be used to have
distinct KMSes for upstream authentication vs. accepting from
downstream.

internal/daemon/controller/testing.go

@@ -37,6 +37,7 @@ import (
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/go-hclog"
 	wrapping "github.com/hashicorp/go-kms-wrapping/v2"
+	"github.com/hashicorp/go-kms-wrapping/v2/extras/multi"
 	"github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 	"github.com/prometheus/client_golang/prometheus"
@@ -424,6 +425,9 @@ type TestControllerOpts struct {
 	// The worker auth KMS to use, or one will be created
 	WorkerAuthKms wrapping.Wrapper
 
+	// The downstream worker auth KMS to use, or one will be created
+	DownstreamWorkerAuthKms *multi.PooledWrapper
+
 	// The recovery KMS to use, or one will be created
 	RecoveryKms wrapping.Wrapper
 
@@ -670,16 +674,17 @@ func TestControllerConfig(t testing.TB, ctx context.Context, tc *TestController,
 	tc.b.DevUnprivilegedUserId = "u_" + strutil.Reverse(strings.TrimPrefix(tc.b.DevUserId, "u_"))
 
 	// Set up KMSes
-	switch {
-	case opts.RootKms != nil && opts.WorkerAuthKms != nil:
+	if err := tc.b.SetupKMSes(tc.b.Context, nil, opts.Config); err != nil {
+		t.Fatal(err)
+	}
+	if opts.RootKms != nil {
 		tc.b.RootKms = opts.RootKms
+	}
+	if opts.WorkerAuthKms != nil {
 		tc.b.WorkerAuthKms = opts.WorkerAuthKms
-	case opts.RootKms == nil && opts.WorkerAuthKms == nil:
-		if err := tc.b.SetupKMSes(tc.b.Context, nil, opts.Config); err != nil {
-			t.Fatal(err)
-		}
-	default:
-		t.Fatal("either controller and worker auth KMS must both be set, or neither")
+	}
+	if opts.DownstreamWorkerAuthKms != nil {
+		tc.b.DownstreamWorkerAuthKms = opts.DownstreamWorkerAuthKms
 	}
 	if opts.RecoveryKms != nil {
 		tc.b.RecoveryKms = opts.RecoveryKms
@@ -778,6 +783,7 @@ func (tc *TestController) AddClusterControllerMember(t testing.TB, opts *TestCon
 		DefaultOidcAuthMethodId:         tc.c.conf.DevOidcAuthMethodId,
 		RootKms:                         tc.c.conf.RootKms,
 		WorkerAuthKms:                   tc.c.conf.WorkerAuthKms,
+		DownstreamWorkerAuthKms:         tc.c.conf.DownstreamWorkerAuthKms,
 		RecoveryKms:                     tc.c.conf.RecoveryKms,
 		Name:                            opts.Name,
 		Logger:                          tc.c.conf.Logger,