Add more logging to worker auth rotation and test flags

This makes it much easier to figure out when and why rotation is
happening and control it for tests/debugging.

Remove cert skew

Cherry-picked from release/0.12.x

internal/cmd/commands/dev/dev.go

@@ -65,45 +65,46 @@ type Command struct {
 	controller *controller.Controller
 	worker     *worker.Worker
 
-	flagLogLevel                     string
-	flagLogFormat                    string
-	flagCombineLogs                  bool
-	flagLoginName                    string
-	flagPassword                     string
-	flagUnprivilegedLoginName        string
-	flagUnprivilegedPassword         string
-	flagIdSuffix                     string
-	flagSecondaryIdSuffix            string
-	flagHostAddress                  string
-	flagTargetDefaultPort            int
-	flagTargetSessionMaxSeconds      int
-	flagTargetSessionConnectionLimit int
-	flagControllerAPIListenAddr      string
-	flagControllerClusterListenAddr  string
-	flagControllerPublicClusterAddr  string
-	flagControllerOnly               bool
-	flagWorkerAuthKey                string
-	flagWorkerProxyListenAddr        string
-	flagWorkerPublicAddr             string
-	flagOpsListenAddr                string
-	flagUiPassthroughDir             string
-	flagRecoveryKey                  string
-	flagDatabaseUrl                  string
-	flagContainerImage               string
-	flagDisableDatabaseDestruction   bool
-	flagEventFormat                  string
-	flagAudit                        string
-	flagObservations                 string
-	flagSysEvents                    string
-	flagEveryEventAllowFilters       []string
-	flagEveryEventDenyFilters        []string
-	flagCreateLoopbackHostPlugin     bool
-	flagPluginExecutionDir           string
-	flagWorkerAuthMethod             string
-	flagWorkerAuthStorageDir         string
-	flagWorkerAuthStorageSkipCleanup bool
-	flagWorkerAuthRotationInterval   time.Duration
-	flagWorkerAuthDebuggingEnabled   bool
+	flagLogLevel                         string
+	flagLogFormat                        string
+	flagCombineLogs                      bool
+	flagLoginName                        string
+	flagPassword                         string
+	flagUnprivilegedLoginName            string
+	flagUnprivilegedPassword             string
+	flagIdSuffix                         string
+	flagSecondaryIdSuffix                string
+	flagHostAddress                      string
+	flagTargetDefaultPort                int
+	flagTargetSessionMaxSeconds          int
+	flagTargetSessionConnectionLimit     int
+	flagControllerAPIListenAddr          string
+	flagControllerClusterListenAddr      string
+	flagControllerPublicClusterAddr      string
+	flagControllerOnly                   bool
+	flagWorkerAuthKey                    string
+	flagWorkerProxyListenAddr            string
+	flagWorkerPublicAddr                 string
+	flagOpsListenAddr                    string
+	flagUiPassthroughDir                 string
+	flagRecoveryKey                      string
+	flagDatabaseUrl                      string
+	flagContainerImage                   string
+	flagDisableDatabaseDestruction       bool
+	flagEventFormat                      string
+	flagAudit                            string
+	flagObservations                     string
+	flagSysEvents                        string
+	flagEveryEventAllowFilters           []string
+	flagEveryEventDenyFilters            []string
+	flagCreateLoopbackHostPlugin         bool
+	flagPluginExecutionDir               string
+	flagWorkerAuthMethod                 string
+	flagWorkerAuthStorageDir             string
+	flagWorkerAuthStorageSkipCleanup     bool
+	flagWorkerAuthWorkerRotationInterval time.Duration
+	flagWorkerAuthCaCertificateLifetime  time.Duration
+	flagWorkerAuthDebuggingEnabled       bool
 }
 
 func (c *Command) Synopsis() string {
@@ -385,8 +386,13 @@ func (c *Command) Flags() *base.FlagSets {
 	})
 
 	f.DurationVar(&base.DurationVar{
-		Name:   "worker-auth-rotation-interval",
-		Target: &c.flagWorkerAuthRotationInterval,
+		Name:   "worker-auth-worker-rotation-interval",
+		Target: &c.flagWorkerAuthWorkerRotationInterval,
+		Hidden: true,
+	})
+	f.DurationVar(&base.DurationVar{
+		Name:   "worker-auth-ca-certificate-lifetime",
+		Target: &c.flagWorkerAuthCaCertificateLifetime,
 		Hidden: true,
 	})
 
@@ -708,6 +714,7 @@ func (c *Command) Run(args []string) int {
 		conf := &controller.Config{
 			RawConfig: c.Config,
 			Server:    c.Server,
+			TestOverrideWorkerAuthCaCertificateLifetime: c.flagWorkerAuthCaCertificateLifetime,
 		}
 
 		var err error
@@ -742,10 +749,7 @@ func (c *Command) Run(args []string) int {
 			c.UI.Error(fmt.Errorf("Error initializing worker: %w", err).Error())
 			return base.CommandCliError
 		}
-
-		if c.flagWorkerAuthRotationInterval > 0 {
-			c.worker.TestOverrideAuthRotationPeriod = c.flagWorkerAuthRotationInterval
-		}
+		c.worker.TestOverrideAuthRotationPeriod = c.flagWorkerAuthWorkerRotationInterval
 
 		if c.flagWorkerAuthMethod == PkiRandomWorkerAuthMechanism {
 			// Flip a coin. Use one method or the other; it's transparent to

internal/cmd/commands/server/server.go

@@ -62,9 +62,11 @@ type Command struct {
 	flagLogFormat   string
 	flagCombineLogs bool
 
-	reloadedCh   chan struct{}  // for tests
-	startedCh    chan struct{}  // for tests
-	presetConfig *atomic.String // for tests
+	reloadedCh                           chan struct{}  // for tests
+	startedCh                            chan struct{}  // for tests
+	presetConfig                         *atomic.String // for tests
+	flagWorkerAuthWorkerRotationInterval time.Duration  // for tests
+	flagWorkerAuthCaCertificateLifetime  time.Duration  // for tests
 }
 
 func (c *Command) Synopsis() string {
@@ -126,6 +128,17 @@ func (c *Command) Flags() *base.FlagSets {
 		Usage:      `Log format, mostly as a fallback for events. Supported values are "standard" and "json".`,
 	})
 
+	f.DurationVar(&base.DurationVar{
+		Name:   "worker-auth-worker-rotation-interval",
+		Target: &c.flagWorkerAuthWorkerRotationInterval,
+		Hidden: true,
+	})
+	f.DurationVar(&base.DurationVar{
+		Name:   "worker-auth-ca-certificate-lifetime",
+		Target: &c.flagWorkerAuthCaCertificateLifetime,
+		Hidden: true,
+	})
+
 	return set
 }
 
@@ -583,6 +596,7 @@ func (c *Command) StartController(ctx context.Context) error {
 	conf := &controller.Config{
 		RawConfig: c.Config,
 		Server:    c.Server,
+		TestOverrideWorkerAuthCaCertificateLifetime: c.flagWorkerAuthCaCertificateLifetime,
 	}
 
 	var err error
@@ -614,6 +628,7 @@ func (c *Command) StartWorker() error {
 	if err != nil {
 		return fmt.Errorf("Error initializing worker: %w", err)
 	}
+	c.worker.TestOverrideAuthRotationPeriod = c.flagWorkerAuthWorkerRotationInterval
 
 	if err := c.worker.Start(); err != nil {
 		retErr := fmt.Errorf("Error starting worker: %w", err)

internal/daemon/controller/config.go

@@ -4,6 +4,8 @@
 package controller
 
 import (
+	"time"
+
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
 )
@@ -17,4 +19,6 @@ type Config struct {
 	RawConfig *config.Config
 	// If set, authorization checking occurrs but failures are ignored
 	DisableAuthorizationFailures bool
+	// Override worker auth CA certificate lifetime for testing
+	TestOverrideWorkerAuthCaCertificateLifetime time.Duration
 }

internal/daemon/controller/controller.go

@@ -47,6 +47,7 @@ import (
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-secure-stdlib/mlock"
 	"github.com/hashicorp/go-secure-stdlib/pluginutil/v2"
+	"github.com/hashicorp/nodeenrollment"
 	ua "go.uber.org/atomic"
 	"google.golang.org/grpc"
 )
@@ -392,7 +393,7 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 	if err != nil {
 		return nil, fmt.Errorf("unable to instantiate worker auth repository: %w", err)
 	}
-	_, err = server.RotateRoots(ctx, serversRepo)
+	_, err = server.RotateRoots(ctx, serversRepo, nodeenrollment.WithCertificateLifetime(conf.TestOverrideWorkerAuthCaCertificateLifetime))
 	if err != nil {
 		return nil, fmt.Errorf("unable to ensure worker auth roots exist: %w", err)
 	}
@@ -518,7 +519,14 @@ func (c *Controller) registerJobs() error {
 	if err := session.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, c.workerStatusGracePeriod); err != nil {
 		return err
 	}
-	if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms); err != nil {
+	var serverJobOpts []serversjob.Option
+	if c.conf.TestOverrideWorkerAuthCaCertificateLifetime > 0 {
+		serverJobOpts = append(serverJobOpts,
+			serversjob.WithCertificateLifetime(c.conf.TestOverrideWorkerAuthCaCertificateLifetime),
+			serversjob.WithRotationFrequency(c.conf.TestOverrideWorkerAuthCaCertificateLifetime/2),
+		)
+	}
+	if err := serversjob.RegisterJobs(c.baseContext, c.scheduler, rw, rw, c.kms, serverJobOpts...); err != nil {
 		return err
 	}
 	if err := kmsjob.RegisterJobs(c.baseContext, c.scheduler, c.kms); err != nil {

internal/daemon/worker/auth_rotation.go

@@ -5,7 +5,9 @@ package worker
 
 import (
 	"context"
+	"crypto/x509"
 	"errors"
+	"fmt"
 	"time"
 
 	berrors "github.com/hashicorp/boundary/internal/errors"
@@ -70,24 +72,77 @@ func (w *Worker) startAuthRotationTicking(cancelCtx context.Context) {
 
 			// Check the certificates to see if it's time
 			if len(currentNodeCreds.CertificateBundles) != 2 {
-				// Likely this means we haven't been authorized yet, and
-				// unclear what to do in any other situation
+				// Likely this means we haven't been authorized yet, and unclear
+				// what to do in any other situation. However, we should try
+				// again fairly quickly because this may also be prior to the
+				// first rotation and we want to ensure the validity period is
+				// checked soon.
+				resetDuration = 5 * time.Second
 				continue
 			}
 
 			var earliestValid, latestValid time.Time
 			// Go through the bundles and find the full range of time that we
 			// have valid certificates
-			for _, bundle := range currentNodeCreds.CertificateBundles {
+			var args []any
+			for i, bundle := range currentNodeCreds.CertificateBundles {
 				if curr := bundle.CertificateNotBefore.AsTime(); earliestValid.IsZero() || curr.Before(earliestValid) {
 					earliestValid = curr
 				}
 				if curr := bundle.CertificateNotAfter.AsTime(); latestValid.IsZero() || curr.After(latestValid) {
 					latestValid = curr
 				}
+
+				cert, err := x509.ParseCertificate(bundle.CertificateDer)
+				if err != nil {
+					event.WriteError(cancelCtx, op, fmt.Errorf("error parsing leaf certificate in current worker auth bundle: %w", err))
+					continue
+				}
+				certPkixBytes, err := x509.MarshalPKIXPublicKey(cert.PublicKey)
+				if err != nil {
+					event.WriteError(cancelCtx, op, fmt.Errorf("error marshaling leaf certificate public key in current worker auth bundle: %w", err))
+					continue
+				}
+				certKeyId, err := nodeenrollment.KeyIdFromPkix(certPkixBytes)
+				if err != nil {
+					event.WriteError(cancelCtx, op, fmt.Errorf("error deriving pkix string from leaf certificate public key in current worker auth bundle: %w", err))
+					continue
+				}
+				args = append(args,
+					fmt.Sprintf("leaf_cert_%d_id", i), certKeyId,
+					fmt.Sprintf("leaf_cert_%d_not_before", i), cert.NotBefore.Format(time.RFC3339),
+					fmt.Sprintf("leaf_cert_%d_not_after", i), cert.NotAfter.Format(time.RFC3339),
+				)
+				caCert, err := x509.ParseCertificate(bundle.CaCertificateDer)
+				if err != nil {
+					event.WriteError(cancelCtx, op, fmt.Errorf("error parsing CA certificate in current worker auth bundle: %w", err))
+					continue
+				}
+				caCertPkixBytes, err := x509.MarshalPKIXPublicKey(caCert.PublicKey)
+				if err != nil {
+					event.WriteError(cancelCtx, op, fmt.Errorf("error marshaling CA certificate public key in current worker auth bundle: %w", err))
+					continue
+				}
+				caCertKeyId, err := nodeenrollment.KeyIdFromPkix(caCertPkixBytes)
+				if err != nil {
+					event.WriteError(cancelCtx, op, fmt.Errorf("error deriving pkix string from CA certificate public key in current worker auth bundle: %w", err))
+					continue
+				}
+				args = append(args,
+					fmt.Sprintf("ca_cert_%d_id", i), caCertKeyId,
+					fmt.Sprintf("ca_cert_%d_not_before", i), caCert.NotBefore.Format(time.RFC3339),
+					fmt.Sprintf("ca_cert_%d_not_after", i), caCert.NotAfter.Format(time.RFC3339),
+				)
 			}
 			// Ensure that our time periods aren't somehow quite messed up
 			now := time.Now()
+			event.WriteSysEvent(cancelCtx, op, "checking if worker auth should rotate", append(
+				args,
+				"now", now.Format(time.RFC3339),
+				"earliest_valid", earliestValid.Format(time.RFC3339),
+				"latest_valid", latestValid.Format(time.RFC3339),
+			)...,
+			)
 			if earliestValid.After(now) || latestValid.Before(now) || earliestValid.After(latestValid) {
 				// We basically have no valid creds so we can't rotate.
 				// TODO (maybe): Have this trigger a new set of creds and request?
@@ -97,15 +152,21 @@ func (w *Worker) startAuthRotationTicking(cancelCtx context.Context) {
 
 			// Figure out the midpoint; if we're after it, try to rotate
 			var shouldRotate bool
+			delta := latestValid.Sub(earliestValid)
 			switch {
 			case w.TestOverrideAuthRotationPeriod != 0:
 				shouldRotate = true
 			default:
-				delta := latestValid.Sub(earliestValid)
+				resetDuration = latestValid.Sub(now) / 2
 				shouldRotate = now.After(earliestValid.Add(delta / 2))
 			}
 
 			if !shouldRotate {
+				event.WriteSysEvent(cancelCtx, op, "not time to rotate yet",
+					"now", now.Format(time.RFC3339),
+					"midpoint", earliestValid.Add(delta/2).Format(time.RFC3339),
+					"next_check", now.Add(resetDuration).Format(time.RFC3339),
+				)
 				continue
 			}
 
@@ -113,7 +174,7 @@ func (w *Worker) startAuthRotationTicking(cancelCtx context.Context) {
 				event.WriteError(cancelCtx, op, err)
 				continue
 			}
-			event.WriteSysEvent(cancelCtx, op, "worker credentials rotated")
+			event.WriteSysEvent(cancelCtx, op, "worker credentials rotated", "next_check", now.Add(resetDuration).Format(time.RFC3339))
 
 			// TODO (maybe): Calculate new delta and set a custom retry time on the timer?
 		}
@@ -207,5 +268,50 @@ func rotateWorkerAuth(ctx context.Context, w *Worker, currentNodeCreds *types.No
 	}
 	w.WorkerAuthCurrentKeyId.Store(newKeyId)
 
+	var args []any
+	for i, bundle := range newNodeCreds.GetCertificateBundles() {
+		cert, err := x509.ParseCertificate(bundle.CertificateDer)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error parsing leaf certificate in new worker auth bundle: %w", err))
+			continue
+		}
+		certPkixBytes, err := x509.MarshalPKIXPublicKey(cert.PublicKey)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error marshaling leaf certificate public key in new worker auth bundle: %w", err))
+			continue
+		}
+		certKeyId, err := nodeenrollment.KeyIdFromPkix(certPkixBytes)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error deriving pkix string from leaf certificate public key in new worker auth bundle: %w", err))
+			continue
+		}
+		args = append(args,
+			fmt.Sprintf("leaf_cert_%d_id", i), certKeyId,
+			fmt.Sprintf("leaf_cert_%d_not_before", i), cert.NotBefore.Format(time.RFC3339),
+			fmt.Sprintf("leaf_cert_%d_not_after", i), cert.NotAfter.Format(time.RFC3339),
+		)
+		caCert, err := x509.ParseCertificate(bundle.CaCertificateDer)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error parsing CA certificate in new worker auth bundle: %w", err))
+			continue
+		}
+		caCertPkixBytes, err := x509.MarshalPKIXPublicKey(caCert.PublicKey)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error marshaling CA certificate public key in new worker auth bundle: %w", err))
+			continue
+		}
+		caCertKeyId, err := nodeenrollment.KeyIdFromPkix(caCertPkixBytes)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error deriving pkix string from CA certificate public key in new worker auth bundle: %w", err))
+			continue
+		}
+		args = append(args,
+			fmt.Sprintf("ca_cert_%d_id", i), caCertKeyId,
+			fmt.Sprintf("ca_cert_%d_not_before", i), caCert.NotBefore.Format(time.RFC3339),
+			fmt.Sprintf("ca_cert_%d_not_after", i), caCert.NotAfter.Format(time.RFC3339),
+		)
+	}
+	event.WriteSysEvent(ctx, op, "rotate worker auth job finished successfully", args...)
+
 	return nil
 }

internal/server/job/jobs.go

@@ -14,7 +14,7 @@ import (
 )
 
 // RegisterJobs registers the rotate roots job with the provided scheduler.
-func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, r db.Reader, w db.Writer, kms *kms.Kms) error {
+func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, r db.Reader, w db.Writer, kms *kms.Kms, opt ...Option) error {
 	const op = "server.(Jobs).RegisterJobs"
 
 	if isNil(scheduler) {
@@ -30,7 +30,7 @@ func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, r db.Read
 		return errors.New(ctx, errors.InvalidParameter, op, "missing kms")
 	}
 
-	rotateRootsJob, err := newRotateRootsJob(ctx, r, w, kms)
+	rotateRootsJob, err := newRotateRootsJob(ctx, r, w, kms, opt...)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}

internal/server/job/options.go

@@ -0,0 +1,48 @@
+package servers
+
+import (
+	"time"
+)
+
+const defaultRotationFrequency = time.Hour
+
+// getOpts - iterate the inbound Options and return a struct
+func getOpts(opt ...Option) options {
+	opts := getDefaultOptions()
+	for _, o := range opt {
+		o(&opts)
+	}
+	return opts
+}
+
+// Option - how Options are passed as arguments
+type Option func(*options)
+
+// options = how options are represented
+type options struct {
+	withRotationFrequency   time.Duration
+	withCertificateLifetime time.Duration
+}
+
+func getDefaultOptions() options {
+	return options{
+		withRotationFrequency: defaultRotationFrequency,
+	}
+}
+
+// WithRotationFrequency provides an frequency for running a job. If the passed-in
+// frequency is zero, it will not update the value, so that the default is used
+// instead.
+func WithRotationFrequency(with time.Duration) Option {
+	return func(o *options) {
+		o.withRotationFrequency = with
+	}
+}
+
+// WithCertificateLifetime provides a way to specify the lifetime of generated
+// roots
+func WithCertificateLifetime(with time.Duration) Option {
+	return func(o *options) {
+		o.withCertificateLifetime = with
+	}
+}

internal/server/job/options_test.go

@@ -0,0 +1,24 @@
+package servers
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_GetOpts(t *testing.T) {
+	t.Parallel()
+	t.Run("WithRotationFrequency", func(t *testing.T) {
+		opts := getOpts(WithRotationFrequency(time.Minute))
+		testOpts := getDefaultOptions()
+		assert.Equal(t, testOpts.withRotationFrequency, defaultRotationFrequency)
+		assert.Equal(t, time.Minute, opts.withRotationFrequency)
+	})
+	t.Run("WithCertificateLifetime", func(t *testing.T) {
+		opts := getOpts(WithCertificateLifetime(time.Minute))
+		testOpts := getDefaultOptions()
+		assert.Equal(t, testOpts.withCertificateLifetime, defaultRotationFrequency)
+		assert.Equal(t, time.Minute, opts.withCertificateLifetime)
+	})
+}

internal/server/job/rotate_roots_job.go

@@ -12,10 +12,9 @@ import (
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/scheduler"
 	"github.com/hashicorp/boundary/internal/server"
+	"github.com/hashicorp/nodeenrollment"
 )
 
-const rotateFrequency = time.Hour
-
 // rotateRootsJob defines a periodic job that initiates root certificate rotation
 // It runs every hour; the root rotation function in the library is designed to not
 // do anything if it's not time to rotate (roots are within their valid ranges)
@@ -23,10 +22,14 @@ type rotateRootsJob struct {
 	workerAuthRepo *server.WorkerAuthRepositoryStorage
 
 	totalRotates int
+
+	rotateFrequency time.Duration
+
+	certificateLifetime time.Duration
 }
 
 // newRotateRootsJob instantiates the rotate roots job.
-func newRotateRootsJob(ctx context.Context, r db.Reader, w db.Writer, kms *kms.Kms) (*rotateRootsJob, error) {
+func newRotateRootsJob(ctx context.Context, r db.Reader, w db.Writer, kms *kms.Kms, opt ...Option) (*rotateRootsJob, error) {
 	const op = "server.newRotateRootsJob"
 	switch {
 	case isNil(r):
@@ -42,9 +45,13 @@ func newRotateRootsJob(ctx context.Context, r db.Reader, w db.Writer, kms *kms.K
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
+	opts := getOpts(opt...)
+
 	return &rotateRootsJob{
-		workerAuthRepo: workerAuthRepo,
-		totalRotates:   0,
+		workerAuthRepo:      workerAuthRepo,
+		totalRotates:        0,
+		rotateFrequency:     opts.withRotationFrequency,
+		certificateLifetime: opts.withCertificateLifetime,
 	}, nil
 }
 
@@ -59,7 +66,7 @@ func (r *rotateRootsJob) Description() string {
 // NextRunIn returns the next run time after a job is completed.
 // This is represented by RotateFrequency
 func (r *rotateRootsJob) NextRunIn(_ context.Context) (time.Duration, error) {
-	return rotateFrequency, nil
+	return r.rotateFrequency, nil
 }
 
 // Status returns the status of the running job.
@@ -74,7 +81,7 @@ func (r *rotateRootsJob) Status() scheduler.JobStatus {
 func (r *rotateRootsJob) Run(ctx context.Context) error {
 	const op = "server.(rotateRootsJob).Run"
 
-	_, err := server.RotateRoots(ctx, r.workerAuthRepo)
+	_, err := server.RotateRoots(ctx, r.workerAuthRepo, nodeenrollment.WithCertificateLifetime(r.certificateLifetime))
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}

internal/server/service_rotate_roots.go

@@ -5,17 +5,21 @@ package server
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/nodeenrollment"
 	"github.com/hashicorp/nodeenrollment/rotation"
 	"github.com/hashicorp/nodeenrollment/types"
 )
 
-// RotateRoots is a domain service function that initiates a rotation of root certificates
-// via a call to the nodenenrollment RotateRootCertificates function
-// Accepts the nodeenrollment option, WithCertificateLifetime(time.Duration) to specify the lifetime
-// of the generated cert(s)
+// RotateRoots is a domain service function that initiates a rotation of root
+// certificates via a call to the nodenenrollment RotateRootCertificates
+// function. Accepts the nodeenrollment option
+// WithCertificateLifetime(time.Duration) to specify the lifetime of the
+// generated cert(s).
 func RotateRoots(ctx context.Context, workerAuthRepo *WorkerAuthRepositoryStorage, opt ...nodeenrollment.Option) (*types.RootCertificates, error) {
 	const op = "server.RotateRoots"
 	if workerAuthRepo == nil {
@@ -28,12 +32,50 @@ func RotateRoots(ctx context.Context, workerAuthRepo *WorkerAuthRepositoryStorag
 	if err != nil && err != nodeenrollment.ErrNotFound {
 		return nil, errors.Wrap(ctx, err, op)
 	}
-	opt = append(opt, nodeenrollment.WithState(rootCerts.GetState()))
+	opt = append(opt,
+		nodeenrollment.WithState(rootCerts.GetState()),
+		nodeenrollment.WithNotAfterClockSkew(0),
+		nodeenrollment.WithNotBeforeClockSkew(0),
+	)
+
+	var current, next string
+	if rootCerts.Current != nil {
+		current = rootCerts.Current.GetId()
+	}
+	if rootCerts.Next != nil {
+		next = rootCerts.Next.GetId()
+	}
 
 	roots, err := rotation.RotateRootCertificates(ctx, workerAuthRepo, opt...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
+	if roots != nil && (roots.Current.GetId() != current || roots.Next.GetId() != next) {
+		var args []any
+		currentKeyId, err := nodeenrollment.KeyIdFromPkix(roots.Current.PublicKeyPkix)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error deriving pkix string from current ca certificate public key after rotation: %w", err))
+		} else {
+			args = append(args,
+				"current_ca_cert_id", currentKeyId,
+				"current_ca_cert_not_before", roots.Current.GetNotBefore().AsTime().Format(time.RFC3339),
+				"current_ca_cert_not_after", roots.Current.GetNotAfter().AsTime().Format(time.RFC3339),
+			)
+		}
+		nextKeyId, err := nodeenrollment.KeyIdFromPkix(roots.Next.PublicKeyPkix)
+		if err != nil {
+			event.WriteError(ctx, op, fmt.Errorf("error deriving pkix string from next ca certificate public key after rotation: %w", err))
+		} else {
+			args = append(args,
+				"next_ca_cert_id", nextKeyId,
+				"next_ca_cert_not_before", roots.Next.GetNotBefore().AsTime().Format(time.RFC3339),
+				"next_ca_cert_not_after", roots.Next.GetNotAfter().AsTime().Format(time.RFC3339),
+			)
+		}
+		event.WriteSysEvent(ctx, op, "worker auth root certificates were rotated", args...)
+
+	}
+
 	return roots, nil
 }