Enhance worker connection security (#1641)
This stores worker connection nonces within the database to ensure uniqueness of connections. Prior the connection required any attacker to be able to decrypt the incoming encrypted authentication message in order to replay (and if the attacker has access to the KMS you have bigger problems), but with this change even if an attacker does have access to the KMS key, they cannot reuse the same message. (If they're able to encrypt against the KMS key, they can still act as a worker with a new message, so this only really helps if they only have decryption capabilities.)
This commit is contained in:
1 parent
a679300b50
commit
1782f6a4e4
23 files changed
+426
-63
No files matched your search
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/hashicorp/boundary/internal/observability/event"
|
||||
"github.com/hashicorp/boundary/internal/perms"
|
||||
"github.com/hashicorp/boundary/internal/requests"
|
||||
"github.com/hashicorp/boundary/internal/servers"
|
||||
"github.com/hashicorp/boundary/internal/servers/controller/common"
|
||||
"github.com/hashicorp/boundary/internal/servers/controller/handlers"
|
||||
"github.com/hashicorp/boundary/internal/types/action"
|
||||
@@ -391,7 +392,7 @@ func (v *verifier) decryptToken(ctx context.Context) {
|
||||
v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
|
||||
return
|
||||
}
|
||||
if err := repo.AddRecoveryNonce(v.ctx, info.Nonce); err != nil {
|
||||
if err := repo.AddNonce(v.ctx, info.Nonce, servers.NoncePurposeRecovery); err != nil {
|
||||
event.WriteError(ctx, op, err, event.WithInfoMsg("decrypt recovery token: error adding nonce to database (possible replay attack)"))
|
||||
v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
|
||||
return
|
||||
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
"github.com/hashicorp/boundary/internal/target"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-secure-stdlib/mlock"
|
||||
"github.com/patrickmn/go-cache"
|
||||
ua "go.uber.org/atomic"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
@@ -39,10 +38,10 @@ type Controller struct {
|
||||
baseCancel context.CancelFunc
|
||||
started *ua.Bool
|
||||
|
||||
tickerWg sync.WaitGroup
|
||||
schedulerWg sync.WaitGroup
|
||||
tickerWg *sync.WaitGroup
|
||||
schedulerWg *sync.WaitGroup
|
||||
|
||||
workerAuthCache *cache.Cache
|
||||
workerAuthCache *sync.Map
|
||||
|
||||
// Used for testing and tracking worker health
|
||||
workerStatusUpdateTimes *sync.Map
|
||||
@@ -74,6 +73,9 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
|
||||
conf: conf,
|
||||
logger: conf.Logger.Named("controller"),
|
||||
started: ua.NewBool(false),
|
||||
tickerWg: new(sync.WaitGroup),
|
||||
schedulerWg: new(sync.WaitGroup),
|
||||
workerAuthCache: new(sync.Map),
|
||||
workerStatusUpdateTimes: new(sync.Map),
|
||||
}
|
||||
|
||||
@@ -163,7 +165,6 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
|
||||
c.SessionRepoFn = func() (*session.Repository, error) {
|
||||
return session.NewRepository(dbase, dbase, c.kms)
|
||||
}
|
||||
c.workerAuthCache = cache.New(0, 0)
|
||||
|
||||
return c, nil
|
||||
}
|
||||
@@ -178,7 +179,7 @@ func (c *Controller) Start() error {
|
||||
if err := c.registerJobs(); err != nil {
|
||||
return fmt.Errorf("error registering jobs: %w", err)
|
||||
}
|
||||
if err := c.scheduler.Start(c.baseContext, &c.schedulerWg); err != nil {
|
||||
if err := c.scheduler.Start(c.baseContext, c.schedulerWg); err != nil {
|
||||
return fmt.Errorf("error starting scheduler: %w", err)
|
||||
}
|
||||
if err := c.startListeners(c.baseContext); err != nil {
|
||||
@@ -192,7 +193,7 @@ func (c *Controller) Start() error {
|
||||
}()
|
||||
go func() {
|
||||
defer c.tickerWg.Done()
|
||||
c.startRecoveryNonceCleanupTicking(c.baseContext)
|
||||
c.startNonceCleanupTicking(c.baseContext)
|
||||
}()
|
||||
go func() {
|
||||
defer c.tickerWg.Done()
|
||||
|
||||
@@ -195,9 +195,5 @@ func ValidNameDescription(in string) bool {
|
||||
return !unicode.IsPrint(c)
|
||||
})
|
||||
|
||||
if idx != -1 {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
return idx == -1
|
||||
}
|
||||
@@ -70,7 +70,7 @@ func (m *interceptingListener) Accept() (net.Conn, error) {
|
||||
}
|
||||
return nil, fmt.Errorf("error reading nonce from worker, expected %d bytes, got %d", 20, read)
|
||||
}
|
||||
workerInfoRaw, found := m.c.workerAuthCache.Get(string(nonce))
|
||||
workerInfoRaw, found := m.c.workerAuthCache.Load(string(nonce))
|
||||
if !found {
|
||||
if err := conn.Close(); err != nil {
|
||||
event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
|
||||
|
||||
@@ -17,7 +17,7 @@ const (
|
||||
)
|
||||
|
||||
// This is exported so it can be tweaked in tests
|
||||
var RecoveryNonceCleanupInterval = 2 * time.Minute
|
||||
var NonceCleanupInterval = 2 * time.Minute
|
||||
|
||||
func (c *Controller) startStatusTicking(cancelCtx context.Context) {
|
||||
const op = "controller.(Controller).startStatusTicking"
|
||||
@@ -51,8 +51,8 @@ func (c *Controller) startStatusTicking(cancelCtx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) startRecoveryNonceCleanupTicking(cancelCtx context.Context) {
|
||||
const op = "controller.(Controller).startRecoveryNonceCleanupTicking"
|
||||
func (c *Controller) startNonceCleanupTicking(cancelCtx context.Context) {
|
||||
const op = "controller.(Controller).startNonceCleanupTicking"
|
||||
timer := time.NewTimer(0)
|
||||
for {
|
||||
select {
|
||||
@@ -72,7 +72,7 @@ func (c *Controller) startRecoveryNonceCleanupTicking(cancelCtx context.Context)
|
||||
event.WriteSysEvent(cancelCtx, op, "recovery nonce cleanup successful", "nonces_cleaned", nonceCount)
|
||||
}
|
||||
}
|
||||
timer.Reset(RecoveryNonceCleanupInterval)
|
||||
timer.Reset(NonceCleanupInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,12 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/boundary/internal/cmd/base"
|
||||
"github.com/hashicorp/boundary/internal/servers"
|
||||
wrapping "github.com/hashicorp/go-kms-wrapping"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
@@ -20,16 +22,30 @@ type workerAuthEntry struct {
|
||||
conn net.Conn
|
||||
}
|
||||
|
||||
// validateWorkerTls is called by the Go TLS stack with client info. It calls
|
||||
// v1WorkerAuthConfig to validate the encryption and ensure the nonce has not
|
||||
// been replayed, then stores connection information into the auth cache, which
|
||||
// is used by the intercepting listener to log information.
|
||||
func (c Controller) validateWorkerTls(hello *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
for _, p := range hello.SupportedProtos {
|
||||
switch {
|
||||
case strings.HasPrefix(p, "v1workerauth-"):
|
||||
tlsConf, workerInfo, err := c.v1WorkerAuthConfig(hello.SupportedProtos)
|
||||
if err == nil {
|
||||
// Set the info we need to prevent replays
|
||||
c.workerAuthCache.Set(workerInfo.ConnectionNonce, &workerAuthEntry{
|
||||
// Store info that can be retrieved in the intercepting
|
||||
// listener.
|
||||
//
|
||||
// NOTE: This is not a load-or-store because we need to ensure
|
||||
// replays can't happen across controllers, hence storing in the
|
||||
// database in v1WorkerAuthConfig. Although this could serve as
|
||||
// a secondary check, a load-or-store here keeps the test from
|
||||
// ensuring that it is rejected at the database level via the
|
||||
// nonce unique constraint, which is by far the more important
|
||||
// thing to validate (although there are DB-level tests for that
|
||||
// too).
|
||||
c.workerAuthCache.Store(workerInfo.ConnectionNonce, &workerAuthEntry{
|
||||
WorkerAuthInfo: workerInfo,
|
||||
}, 0)
|
||||
})
|
||||
}
|
||||
return tlsConf, err
|
||||
}
|
||||
@@ -37,6 +53,13 @@ func (c Controller) validateWorkerTls(hello *tls.ClientHelloInfo) (*tls.Config,
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// v1WorkerAuthConfig:
|
||||
//
|
||||
// * Reads the information from ALPN, combining across protos
|
||||
// * Combines them to form the encrypted auth information
|
||||
// * Validates it by decrypting against the configured KMS
|
||||
// * Ensures that the nonce is unique to prevent replay attacks
|
||||
// * Returns the shared TLS configuration that is used to establish the connection
|
||||
func (c Controller) v1WorkerAuthConfig(protos []string) (*tls.Config, *base.WorkerAuthInfo, error) {
|
||||
var firstMatchProto string
|
||||
var encString string
|
||||
@@ -69,6 +92,15 @@ func (c Controller) v1WorkerAuthConfig(protos []string) (*tls.Config, *base.Work
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Check for replays
|
||||
serversRepo, err := c.ServersRepoFn()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to fetch servers repo: %w", err)
|
||||
}
|
||||
if err := serversRepo.AddNonce(c.baseContext, info.ConnectionNonce, servers.NoncePurposeWorkerAuth); err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to add connection nonce to database: %w", err)
|
||||
}
|
||||
|
||||
rootCAs := x509.NewCertPool()
|
||||
if ok := rootCAs.AppendCertsFromPEM(info.CertPEM); !ok {
|
||||
return nil, info, errors.New("unable to add ca cert to cert pool")
|
||||
|
||||
Reference in new issue
Block a user