threshold/internal/daemon/worker/auth_rotation_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package worker_test

import (
	"bytes"
	"context"
	"sync/atomic"
	"testing"
	"time"

	"github.com/hashicorp/boundary/internal/cmd/config"
	"github.com/hashicorp/boundary/internal/daemon/controller"
	"github.com/hashicorp/boundary/internal/daemon/worker"
	"github.com/hashicorp/boundary/internal/server"
	"github.com/hashicorp/boundary/internal/server/store"
	"github.com/hashicorp/boundary/internal/types/scope"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/nodeenrollment"
	"github.com/hashicorp/nodeenrollment/types"
	"github.com/mr-tron/base58"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"google.golang.org/protobuf/proto"
)

// TestRotationTicking ensures that we see new credential information for a
// worker on both the controller side and worker side in a periodic fashion
func TestRotationTicking(t *testing.T) {
	t.Parallel()

	require, assert := require.New(t), assert.New(t)
	logger := hclog.New(&hclog.LoggerOptions{
		Level: hclog.Trace,
	})

	const rotationPeriod = 20 * time.Second

	conf, err := config.DevController()
	require.NoError(err)

	workerAuthDebugEnabled := new(atomic.Bool)
	workerAuthDebugEnabled.Store(true)
	c := controller.NewTestController(t, &controller.TestControllerOpts{
		Config:                          conf,
		Logger:                          logger.Named("controller"),
		WorkerAuthCaCertificateLifetime: rotationPeriod,
		WorkerAuthDebuggingEnabled:      workerAuthDebugEnabled,
	})

	// names should not be set when using pki workers
	wConf, err := config.DevWorker()
	require.NoError(err)
	wConf.Worker.Name = ""
	wConf.Worker.InitialUpstreams = c.ClusterAddrs()
	w := worker.NewTestWorker(t, &worker.TestWorkerOpts{
		InitialUpstreams:           c.ClusterAddrs(),
		Logger:                     logger.Named("worker"),
		Config:                     wConf,
		WorkerAuthDebuggingEnabled: workerAuthDebugEnabled,
	})

	// Get a server repo and worker auth repo
	serversRepo, err := c.Controller().ServersRepoFn()
	require.NoError(err)
	workerAuthRepo, err := c.Controller().WorkerAuthRepoStorageFn()
	require.NoError(err)

	// Give time for the job to ensure that roots are created and offset us from
	// the rotation period
	time.Sleep(rotationPeriod / 2)

	// Verify that there are no nodes authorized yet
	auths, err := workerAuthRepo.List(c.Context(), (*types.NodeInformation)(nil))
	require.NoError(err)
	assert.Len(auths, 0)

	// Perform initial authentication of worker to controller
	reqBytes, err := base58.FastBase58Decoding(w.Worker().WorkerAuthRegistrationRequest)
	require.NoError(err)

	// Decode the proto into the request and create the worker
	req := new(types.FetchNodeCredentialsRequest)
	require.NoError(proto.Unmarshal(reqBytes, req))
	newWorker, err := serversRepo.CreateWorker(c.Context(), &server.Worker{
		Worker: &store.Worker{
			ScopeId: scope.Global.String(),
		},
	}, server.WithFetchNodeCredentialsRequest(req))
	require.NoError(err)

	// Wait for a short while; there will be an initial rotation of credentials
	// after authentication
	time.Sleep(rotationPeriod / 2)

	// Verify we see authorized credentials now
	auths, err = workerAuthRepo.List(c.Context(), (*types.NodeInformation)(nil))
	require.NoError(err)
	require.Len(auths, 2)
	// Fetch creds and store current key
	currNodeCreds, err := types.LoadNodeCredentials(
		w.Context(),
		w.Worker().WorkerAuthStorage,
		nodeenrollment.CurrentId,
		nodeenrollment.WithStorageWrapper(w.Config().WorkerAuthStorageKms),
	)
	require.NoError(err)
	currKey := currNodeCreds.CertificatePublicKeyPkix

	// Now we wait and check that we see new values in the DB and different
	// creds on the worker after each rotation period

	// Make sure we have a failsafe in case the below loop never finds what it's
	// looking for; at expiration the List will fail and we'll hit the Require,
	// exiting
	testTimeout := 10 * time.Minute
	testDeadline, ok := t.Deadline()
	if ok && time.Until(testDeadline) < testTimeout {
		// If the test deadline is less than the default timeout, use the deadline
		// Give a minute buffer for shutdowns and other test cleanup
		testTimeout = time.Until(testDeadline) - time.Minute
	}
	deadlineCtx, deadlineCtxCancel := context.WithTimeout(c.Context(), testTimeout)
	defer deadlineCtxCancel()

	rotationCount := 2
	for {
		if rotationCount > 3 {
			break
		}
		nextRotation := w.Worker().AuthRotationNextRotation.Load()
		// Sleep until 5 seconds after next rotation
		time.Sleep(time.Until(*nextRotation) + 5*time.Second)

		// Verify we see 2- after credentials have rotated, we should see current and previous
		auths, err = workerAuthRepo.List(deadlineCtx, (*types.NodeInformation)(nil))
		require.NoError(err)
		require.Len(auths, 2)

		// Fetch creds and compare current key
		currNodeCreds, err := types.LoadNodeCredentials(
			w.Context(),
			w.Worker().WorkerAuthStorage,
			nodeenrollment.CurrentId,
			nodeenrollment.WithStorageWrapper(w.Config().WorkerAuthStorageKms),
		)
		require.NoError(err)
		if bytes.Equal(currKey, currNodeCreds.CertificatePublicKeyPkix) {
			// Load due to parallel tests may mean that we didn't hit rotation
			// yet, loop back to evaluate again
			continue
		}
		assert.NotEqual(currKey, currNodeCreds.CertificatePublicKeyPkix)
		currKey = currNodeCreds.CertificatePublicKeyPkix
		currKeyId, err := nodeenrollment.KeyIdFromPkix(currNodeCreds.CertificatePublicKeyPkix)
		require.NoError(err)
		assert.Equal(currKeyId, w.Worker().WorkerAuthCurrentKeyId.Load())

		// Get workerAuthSet for this worker id and compare keys
		workerAuthSet, err := workerAuthRepo.FindWorkerAuthByWorkerId(c.Context(), newWorker.PublicId)
		require.NoError(err)
		assert.NotNil(workerAuthSet)
		assert.NotNil(workerAuthSet.Previous)
		assert.NotNil(workerAuthSet.Current)
		assert.Equal(workerAuthSet.Current.WorkerKeyIdentifier, currKeyId)

		// Stop and start the client connections to ensure the new credentials
		// are valid; if not, we won't establish a new connection and rotation
		// will fail
		require.NotNil(w.Worker().GrpcClientConn.Load())
		require.NoError(w.Worker().GrpcClientConn.Load().Close())
		require.NoError(w.Worker().StartControllerConnections())
		rotationCount++
	}
}