Enhance worker connection security (#1641)

This stores worker connection nonces within the database to ensure
uniqueness of connections. Prior the connection required any attacker to
be able to decrypt the incoming encrypted authentication message in
order to replay (and if the attacker has access to the KMS you have
bigger problems), but with this change even if an attacker does have
access to the KMS key, they cannot reuse the same message. (If they're
able to encrypt against the KMS key, they can still act as a worker with
a new message, so this only really helps if they only have decryption
capabilities.)
This commit is contained in:
Jeff Mitchell authored and GitHub committed 2021-10-27 20:32:07 -04:00
1 parent a679300b50
commit 1782f6a4e4
23 files changed
+426 -63

No files matched your search

+2 -1
View File
@@ -18,6 +18,7 @@ import (
"github.com/hashicorp/boundary/internal/observability/event"
"github.com/hashicorp/boundary/internal/perms"
"github.com/hashicorp/boundary/internal/requests"
"github.com/hashicorp/boundary/internal/servers"
"github.com/hashicorp/boundary/internal/servers/controller/common"
"github.com/hashicorp/boundary/internal/servers/controller/handlers"
"github.com/hashicorp/boundary/internal/types/action"
@@ -391,7 +392,7 @@ func (v *verifier) decryptToken(ctx context.Context) {
v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
return
}
if err := repo.AddRecoveryNonce(v.ctx, info.Nonce); err != nil {
if err := repo.AddNonce(v.ctx, info.Nonce, servers.NoncePurposeRecovery); err != nil {
event.WriteError(ctx, op, err, event.WithInfoMsg("decrypt recovery token: error adding nonce to database (possible replay attack)"))
v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
return
+8 -7
View File
@@ -26,7 +26,6 @@ import (
"github.com/hashicorp/boundary/internal/target"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-secure-stdlib/mlock"
"github.com/patrickmn/go-cache"
ua "go.uber.org/atomic"
"google.golang.org/grpc"
)
@@ -39,10 +38,10 @@ type Controller struct {
baseCancel context.CancelFunc
started *ua.Bool
tickerWg sync.WaitGroup
schedulerWg sync.WaitGroup
tickerWg *sync.WaitGroup
schedulerWg *sync.WaitGroup
workerAuthCache *cache.Cache
workerAuthCache *sync.Map
// Used for testing and tracking worker health
workerStatusUpdateTimes *sync.Map
@@ -74,6 +73,9 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
conf: conf,
logger: conf.Logger.Named("controller"),
started: ua.NewBool(false),
tickerWg: new(sync.WaitGroup),
schedulerWg: new(sync.WaitGroup),
workerAuthCache: new(sync.Map),
workerStatusUpdateTimes: new(sync.Map),
}
@@ -163,7 +165,6 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
c.SessionRepoFn = func() (*session.Repository, error) {
return session.NewRepository(dbase, dbase, c.kms)
}
c.workerAuthCache = cache.New(0, 0)
return c, nil
}
@@ -178,7 +179,7 @@ func (c *Controller) Start() error {
if err := c.registerJobs(); err != nil {
return fmt.Errorf("error registering jobs: %w", err)
}
if err := c.scheduler.Start(c.baseContext, &c.schedulerWg); err != nil {
if err := c.scheduler.Start(c.baseContext, c.schedulerWg); err != nil {
return fmt.Errorf("error starting scheduler: %w", err)
}
if err := c.startListeners(c.baseContext); err != nil {
@@ -192,7 +193,7 @@ func (c *Controller) Start() error {
}()
go func() {
defer c.tickerWg.Done()
c.startRecoveryNonceCleanupTicking(c.baseContext)
c.startNonceCleanupTicking(c.baseContext)
}()
go func() {
defer c.tickerWg.Done()
@@ -195,9 +195,5 @@ func ValidNameDescription(in string) bool {
return !unicode.IsPrint(c)
})
if idx != -1 {
return false
}
return true
return idx == -1
}
@@ -70,7 +70,7 @@ func (m *interceptingListener) Accept() (net.Conn, error) {
}
return nil, fmt.Errorf("error reading nonce from worker, expected %d bytes, got %d", 20, read)
}
workerInfoRaw, found := m.c.workerAuthCache.Get(string(nonce))
workerInfoRaw, found := m.c.workerAuthCache.Load(string(nonce))
if !found {
if err := conn.Close(); err != nil {
event.WriteError(ctx, op, err, event.WithInfoMsg("error closing worker connection"))
+4 -4
View File
@@ -17,7 +17,7 @@ const (
)
// This is exported so it can be tweaked in tests
var RecoveryNonceCleanupInterval = 2 * time.Minute
var NonceCleanupInterval = 2 * time.Minute
func (c *Controller) startStatusTicking(cancelCtx context.Context) {
const op = "controller.(Controller).startStatusTicking"
@@ -51,8 +51,8 @@ func (c *Controller) startStatusTicking(cancelCtx context.Context) {
}
}
func (c *Controller) startRecoveryNonceCleanupTicking(cancelCtx context.Context) {
const op = "controller.(Controller).startRecoveryNonceCleanupTicking"
func (c *Controller) startNonceCleanupTicking(cancelCtx context.Context) {
const op = "controller.(Controller).startNonceCleanupTicking"
timer := time.NewTimer(0)
for {
select {
@@ -72,7 +72,7 @@ func (c *Controller) startRecoveryNonceCleanupTicking(cancelCtx context.Context)
event.WriteSysEvent(cancelCtx, op, "recovery nonce cleanup successful", "nonces_cleaned", nonceCount)
}
}
timer.Reset(RecoveryNonceCleanupInterval)
timer.Reset(NonceCleanupInterval)
}
}
}
@@ -7,10 +7,12 @@ import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net"
"strings"
"github.com/hashicorp/boundary/internal/cmd/base"
"github.com/hashicorp/boundary/internal/servers"
wrapping "github.com/hashicorp/go-kms-wrapping"
"google.golang.org/protobuf/proto"
)
@@ -20,16 +22,30 @@ type workerAuthEntry struct {
conn net.Conn
}
// validateWorkerTls is called by the Go TLS stack with client info. It calls
// v1WorkerAuthConfig to validate the encryption and ensure the nonce has not
// been replayed, then stores connection information into the auth cache, which
// is used by the intercepting listener to log information.
func (c Controller) validateWorkerTls(hello *tls.ClientHelloInfo) (*tls.Config, error) {
for _, p := range hello.SupportedProtos {
switch {
case strings.HasPrefix(p, "v1workerauth-"):
tlsConf, workerInfo, err := c.v1WorkerAuthConfig(hello.SupportedProtos)
if err == nil {
// Set the info we need to prevent replays
c.workerAuthCache.Set(workerInfo.ConnectionNonce, &workerAuthEntry{
// Store info that can be retrieved in the intercepting
// listener.
//
// NOTE: This is not a load-or-store because we need to ensure
// replays can't happen across controllers, hence storing in the
// database in v1WorkerAuthConfig. Although this could serve as
// a secondary check, a load-or-store here keeps the test from
// ensuring that it is rejected at the database level via the
// nonce unique constraint, which is by far the more important
// thing to validate (although there are DB-level tests for that
// too).
c.workerAuthCache.Store(workerInfo.ConnectionNonce, &workerAuthEntry{
WorkerAuthInfo: workerInfo,
}, 0)
})
}
return tlsConf, err
}
@@ -37,6 +53,13 @@ func (c Controller) validateWorkerTls(hello *tls.ClientHelloInfo) (*tls.Config,
return nil, nil
}
// v1WorkerAuthConfig:
//
// * Reads the information from ALPN, combining across protos
// * Combines them to form the encrypted auth information
// * Validates it by decrypting against the configured KMS
// * Ensures that the nonce is unique to prevent replay attacks
// * Returns the shared TLS configuration that is used to establish the connection
func (c Controller) v1WorkerAuthConfig(protos []string) (*tls.Config, *base.WorkerAuthInfo, error) {
var firstMatchProto string
var encString string
@@ -69,6 +92,15 @@ func (c Controller) v1WorkerAuthConfig(protos []string) (*tls.Config, *base.Work
return nil, nil, err
}
// Check for replays
serversRepo, err := c.ServersRepoFn()
if err != nil {
return nil, nil, fmt.Errorf("unable to fetch servers repo: %w", err)
}
if err := serversRepo.AddNonce(c.baseContext, info.ConnectionNonce, servers.NoncePurposeWorkerAuth); err != nil {
return nil, nil, fmt.Errorf("unable to add connection nonce to database: %w", err)
}
rootCAs := x509.NewCertPool()
if ok := rootCAs.AppendCertsFromPEM(info.CertPEM); !ok {
return nil, info, errors.New("unable to add ca cert to cert pool")