Several session.Repository methods need to encrypt or decrypt fields
when persisting or retrieving a Session. These encrypt/decrypt
operations where being performed while within a database transaction.
The first step in the encrypt/decrypt operation is to retrieve the
kms.Wrapper. While the wrappers get cached in memory, the retrieval
still results in a database query to check that the kms collection
version has not changed. This query happens with a separate db.Reader,
and therefore uses a separate database connection from the transaction
being used by the session.Repository.
If the controllers are configured with a relatively low
`max_open_connections`, and be under sufficient load interactive with
sessions, the controller can consume all of the available database
connections, and get stuck with open transactions while waiting for a
database connection to execute the kms related queries. This can be see
by checking pg_stat_activity and seeing transactions stuck in `idle in
transaction` with a wait_event of ClientRead.
This commit changes all of the Repository methods the perform any
encryption or decryption operations, including the retrieval of the
kms.Wrapper, outside of a database transaction. An alternative approach
would be to unsure that the kms.Wrapper retrieval happens within the
same database transaction. However, it seems preferable to avoid doing
these relatively expensive operations within a transaction when it is
not necessary.
As part of this fix, it also eliminates some logic that is no longer
needed. During previous would with KMS wrapping, the method of how
sessions were encrypted was changed, and some logic to lazily re-encrypt
sessions was introduced. However, given that terminated sessions are
deleted, and the KMS changes was introduced several versions ago
(0.13.0), this lazy re-encrypt logic should no longer be necessary.
Refs: edd323b73a
See:
https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-ACTIVITY-VIEW
https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-IDLE-IN-TRANSACTION-SESSION-TIMEOUT
https://developer.hashicorp.com/boundary/docs/configuration/controller#max_open_connections
128 lines
5.0 KiB
Go
128 lines
5.0 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package session
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/hashicorp/boundary/internal/db"
|
|
"github.com/hashicorp/boundary/internal/errors"
|
|
"github.com/hashicorp/boundary/internal/kms"
|
|
"github.com/hashicorp/boundary/internal/util"
|
|
)
|
|
|
|
func init() {
|
|
kms.RegisterTableRewrapFn(defaultSessionTableName, sessionRewrapFn)
|
|
kms.RegisterTableRewrapFn("session_credential", sessionCredentialRewrapFn)
|
|
}
|
|
|
|
func rewrapParameterChecks(ctx context.Context, dataKeyVersionId string, scopeId string, reader db.Reader, writer db.Writer, kmsRepo kms.GetWrapperer) string {
|
|
if dataKeyVersionId == "" {
|
|
return "missing data key version id"
|
|
}
|
|
if scopeId == "" {
|
|
return "missing scope id"
|
|
}
|
|
if util.IsNil(reader) {
|
|
return "missing database reader"
|
|
}
|
|
if util.IsNil(writer) {
|
|
return "missing database writer"
|
|
}
|
|
if kmsRepo == nil {
|
|
return "missing kms repository"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func sessionCredentialRewrapFn(ctx context.Context, dataKeyVersionId, scopeId string, reader db.Reader, writer db.Writer, kmsRepo kms.GetWrapperer) error {
|
|
const op = "session.sessionCredentialRewrapFn"
|
|
if errStr := rewrapParameterChecks(ctx, dataKeyVersionId, scopeId, reader, writer, kmsRepo); errStr != "" {
|
|
return errors.New(ctx, errors.InvalidParameter, op, errStr)
|
|
}
|
|
var creds []*credential
|
|
// An index exists on (session_id, credential_sha256), so we can query workers via scope and refine with key id.
|
|
// This is the fastest query we can use without creating a new index on key_id.
|
|
rows, err := reader.Query(ctx, sessionCredentialRewrapQuery, []interface{}{scopeId, dataKeyVersionId})
|
|
if err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to query sql for rows that need rewrapping"))
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
cred := &credential{}
|
|
if err := rows.Scan(
|
|
&cred.SessionId,
|
|
&cred.KeyId,
|
|
&cred.CtCredential,
|
|
&cred.CredentialSha256,
|
|
); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to failed to scan row"))
|
|
}
|
|
creds = append(creds, cred)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to iterate over retrieved rows"))
|
|
}
|
|
wrapper, err := kmsRepo.GetWrapper(ctx, scopeId, kms.KeyPurposeDatabase)
|
|
if err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to fetch kms wrapper for rewrapping"))
|
|
}
|
|
for _, cred := range creds {
|
|
if err := cred.decrypt(ctx, wrapper); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to decrypt session credential"))
|
|
}
|
|
if err := cred.encrypt(ctx, wrapper); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to re-encrypt session credential"))
|
|
}
|
|
if _, err := writer.Exec(ctx, sessionCredentialRewrapUpdate, []interface{}{
|
|
cred.CtCredential,
|
|
cred.KeyId,
|
|
cred.SessionId,
|
|
cred.CredentialSha256,
|
|
}); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to update session credential row with rewrapped fields"))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sessionRewrapFn(ctx context.Context, dataKeyVersionId string, scopeId string, reader db.Reader, writer db.Writer, kmsRepo kms.GetWrapperer) error {
|
|
const op = "session.sessionRewrapFn"
|
|
if errStr := rewrapParameterChecks(ctx, dataKeyVersionId, scopeId, reader, writer, kmsRepo); errStr != "" {
|
|
return errors.New(ctx, errors.InvalidParameter, op, errStr)
|
|
}
|
|
var sessions []*Session
|
|
// An index exists on (project_id, user_id, termination_reason), so we can query sessions via scope and refine with key id.
|
|
// This is the fastest query we can use without creating a new index on key_id.
|
|
if err := reader.SearchWhere(ctx, &sessions, "project_id=? and key_id=?", []interface{}{scopeId, dataKeyVersionId}, db.WithLimit(-1)); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to query sql for rows that need rewrapping"))
|
|
}
|
|
for _, session := range sessions {
|
|
if session.ProjectId == "" || session.UserId == "" {
|
|
// Skip decryption if Project ID or UserId is missing,
|
|
// since it will just lead to errors, and the session
|
|
// is already canceled. Unset KeyId to allow the key to
|
|
// be destroyed.
|
|
if _, err := writer.Update(ctx, session, nil, []string{"KeyId"}); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to unset key ID in canceled session"))
|
|
}
|
|
continue
|
|
}
|
|
if err := decrypt(ctx, kmsRepo, session); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to decrypt session"))
|
|
}
|
|
wrapper, err := kmsRepo.GetWrapper(ctx, session.GetProjectId(), kms.KeyPurposeSessions)
|
|
if err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to fetch kms wrapper for rewrapping"))
|
|
}
|
|
if err := session.encrypt(ctx, wrapper); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to re-encrypt session"))
|
|
}
|
|
if _, err := writer.Update(ctx, session, []string{"CtTofuToken", "CtCertificatePrivateKey", "KeyId"}, nil); err != nil {
|
|
return errors.Wrap(ctx, err, op, errors.WithMsg("failed to update session row with rewrapped fields"))
|
|
}
|
|
}
|
|
return nil
|
|
}
|