Jeff Mitchell
163ce184b8
This adds a new mechanism for worker authentication that uses the third-party arbitration of the original KMS flow by leveraging recently-added capabilities in nodeenrollment to support a registration wrapper. This allows for workers to be registered to the system with just a name and a matching wrapper, while at the same time allowing these workers to be used for private Vault access and multi-hop as they contain per-worker encryption keys and support rotation. Nodes using this mode will generate and rotate credentials in-memory (using another recently added nodeenrollment capability) so will have a new set of keys on every startup. This has the side effect of ensuring that there is never a conflict or re-use of these credentials. Note that the normal test worker flow was using the now-deprecated KMS method; there is still a test that uses this (explicitly) but all other tests using test workers that did not specify a specific PKI flow now actually use this mechanism. This means there is fairly decent test coverage in a general sense, plus the specific tests updated/added for this feature. This also adds a new KMS type that can be used to have distinct KMSes for upstream authentication vs. accepting from downstream.
102 lines
3.1 KiB
Go
102 lines
3.1 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package worker
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/boundary/internal/cmd/config"
|
|
"github.com/hashicorp/boundary/internal/daemon/controller"
|
|
"github.com/hashicorp/boundary/internal/server"
|
|
"github.com/hashicorp/go-hclog"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc/connectivity"
|
|
)
|
|
|
|
func TestDeleteConnectedWorkers(t *testing.T) {
|
|
ctx := context.Background()
|
|
logger := hclog.New(&hclog.LoggerOptions{
|
|
Level: hclog.Trace,
|
|
})
|
|
conf, err := config.DevController()
|
|
require.NoError(t, err)
|
|
c := controller.NewTestController(t, &controller.TestControllerOpts{
|
|
Config: conf,
|
|
Logger: logger.Named("controller"),
|
|
})
|
|
t.Cleanup(c.Shutdown)
|
|
_, directPkiWorker, multiHoppedPkiWorker, multiHoppedKmsWorker := NewTestMultihopWorkers(t, logger, c.Context(), c.ClusterAddrs(),
|
|
c.Config().WorkerAuthKms, c.Controller().ServersRepoFn, nil, nil, nil, nil)
|
|
|
|
serverRepo, err := c.Controller().ServersRepoFn()
|
|
require.NoError(t, err)
|
|
workers, err := serverRepo.ListWorkers(ctx, []string{"global"}, server.WithLiveness(-1))
|
|
require.NoError(t, err)
|
|
var pkiWorker, childPkiWorker, childKmsWorker *server.Worker
|
|
for _, w := range workers {
|
|
if w.Type == "pki" && w.GetAddress() == multiHoppedPkiWorker.ProxyAddrs()[0] {
|
|
childPkiWorker = w
|
|
}
|
|
if w.Type == "pki" && w.GetAddress() == multiHoppedKmsWorker.ProxyAddrs()[0] {
|
|
childKmsWorker = w
|
|
}
|
|
if w.Type == "pki" && w.GetAddress() == directPkiWorker.ProxyAddrs()[0] {
|
|
pkiWorker = w
|
|
}
|
|
}
|
|
require.NotNil(t, childKmsWorker)
|
|
require.NotNil(t, childPkiWorker)
|
|
require.NotNil(t, pkiWorker)
|
|
|
|
cases := []struct {
|
|
name string
|
|
workerId string
|
|
testWorker *TestWorker
|
|
}{
|
|
{
|
|
name: "multi hop pki worker",
|
|
workerId: childPkiWorker.GetPublicId(),
|
|
testWorker: multiHoppedPkiWorker,
|
|
},
|
|
{
|
|
name: "multi hop kms worker",
|
|
workerId: childKmsWorker.GetPublicId(),
|
|
testWorker: multiHoppedKmsWorker,
|
|
},
|
|
{
|
|
name: "directly connected worker",
|
|
workerId: pkiWorker.GetPublicId(),
|
|
testWorker: directPkiWorker,
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
prevState := tc.testWorker.Worker().GrpcClientConn.GetState()
|
|
require.NotEqual(t, connectivity.TransientFailure, prevState)
|
|
require.NotEqual(t, connectivity.Shutdown, prevState)
|
|
_, err = serverRepo.DeleteWorker(ctx, tc.workerId)
|
|
require.NoError(t, err)
|
|
stateChangeCtx, cancel := context.WithTimeout(ctx, 4*time.Second)
|
|
defer cancel()
|
|
for {
|
|
tc.testWorker.Worker().GrpcClientConn.ResetConnectBackoff()
|
|
if !tc.testWorker.Worker().GrpcClientConn.WaitForStateChange(stateChangeCtx, prevState) {
|
|
assert.Fail(t, "State didn't change before context timed out")
|
|
break
|
|
}
|
|
newState := tc.testWorker.Worker().GrpcClientConn.GetState()
|
|
t.Logf("Changed from previous state: %s to new state: %s", prevState, newState)
|
|
if newState == connectivity.Shutdown || newState == connectivity.TransientFailure {
|
|
break
|
|
}
|
|
prevState = newState
|
|
}
|
|
})
|
|
}
|
|
}
|