Files
threshold/internal/servers/repository.go
T
Chris Marchesi 5a70875726 internal/servers/controller: Worker failure connection cleanup (#1340)
This commit adds the support to do the following:

* Mark connections for non-reporting workers as closed. This is the
controller counterpart to the worker functionality (see #1330). This is
written as a scheduled job that does the work DB-side in a single atomic
query.

* Works to reconcile states if such a broken controller-worker
connection resumes and a worker reports a connection as connected that
should be disconnected. In this case, the controller will send an update
request, and the worker will honor it and terminate the connection.

* Further refinement of the grace period setting has been added here.
We have converged on the current server "liveness" setting as our
default here, which is half of the previous 30s (15 seconds, in other
words). Additionally, this is now configurable on the controller and
worker side, with the caveat that it's currently impossible to do so in
config as the setting has been untagged in HCL. This is exposed so that
we can run some sophisticated testing scenarios where we skew the grace
period to either the controller or worker to ensure the aforementioned
reconciliation works.

* Some repository functions have been added to support the new
functionality, in addition to some test code to the worker to allow
querying of session state while testing.

* Finally, we've added some add timestamp subtraction functions
as well, basically serving as the opposite of the addition functions.
2021-07-14 09:14:24 -07:00

250 lines
7.4 KiB
Go

package servers
import (
"context"
"fmt"
"time"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/kms"
"github.com/hashicorp/boundary/internal/types/resource"
)
const (
// DefaultLiveness is the setting that controls the server "liveness" time,
// or the maximum allowable time that a worker can't send a status update to
// the controller for. After this, the server is considered dead, and it will
// be taken out of the rotation for allowable workers for connections, and
// connections will possibly start to be terminated and marked as closed
// depending on the grace period setting (see
// base.Server.StatusGracePeriodDuration). This value serves as the default
// and minimum allowable setting for the grace period.
DefaultLiveness = 15 * time.Second
)
type ServerType string
const (
ServerTypeController ServerType = "controller"
ServerTypeWorker ServerType = "worker"
)
func (s ServerType) String() string {
return string(s)
}
// Repository is the servers database repository
type Repository struct {
reader db.Reader
writer db.Writer
kms *kms.Kms
}
// NewRepository creates a new servers Repository. Supports the options: WithLimit
// which sets a default limit on results returned by repo operations.
func NewRepository(r db.Reader, w db.Writer, kms *kms.Kms) (*Repository, error) {
const op = "servers.NewRepository"
if r == nil {
return nil, errors.New(errors.InvalidParameter, op, "nil reader")
}
if w == nil {
return nil, errors.New(errors.InvalidParameter, op, "nil writer")
}
if kms == nil {
return nil, errors.New(errors.InvalidParameter, op, "nil kms")
}
return &Repository{
reader: r,
writer: w,
kms: kms,
}, nil
}
// ListServers is a passthrough to listServersWithReader that uses the repo's
// normal reader.
func (r *Repository) ListServers(ctx context.Context, serverType ServerType, opt ...Option) ([]*Server, error) {
return r.listServersWithReader(ctx, r.reader, serverType, opt...)
}
// listServersWithReader will return a listing of resources and honor the
// WithLimit option or the repo defaultLimit. It accepts a reader, allowing it
// to be used within a transaction or without.
func (r *Repository) listServersWithReader(ctx context.Context, reader db.Reader, serverType ServerType, opt ...Option) ([]*Server, error) {
opts := getOpts(opt...)
liveness := opts.withLiveness
if liveness == 0 {
liveness = DefaultLiveness
}
var where string
if liveness > 0 {
where = fmt.Sprintf("type = $1 and update_time > now() - interval '%d seconds'", uint32(liveness.Seconds()))
} else {
where = "type = $1"
}
var servers []*Server
if err := reader.SearchWhere(
ctx,
&servers,
where,
[]interface{}{serverType},
db.WithLimit(-1),
); err != nil {
return nil, errors.Wrap(err, "servers.listServersWithReader")
}
return servers, nil
}
// ServerTag holds the information for the server_tag table for Gorm.
type ServerTag struct {
ServerId string
Key string
Value string
}
// TableName overrides the table name used by ServerTag to `server_tag`
func (ServerTag) TableName() string {
return "server_tag"
}
// ListTagsForServers pulls out tag tuples into ServerTag structs for the
// given server ID values.
func (r *Repository) ListTagsForServers(ctx context.Context, serverIds []string, opt ...Option) ([]*ServerTag, error) {
var serverTags []*ServerTag
if err := r.reader.SearchWhere(
ctx,
&serverTags,
"server_id in (?)",
[]interface{}{serverIds},
db.WithLimit(-1),
); err != nil {
return nil, errors.Wrap(err, "servers.ListTagsForServers", errors.WithMsg(fmt.Sprintf("server IDs %v", serverIds)))
}
return serverTags, nil
}
// UpsertServer adds or updates a server in the DB
func (r *Repository) UpsertServer(ctx context.Context, server *Server, opt ...Option) ([]*Server, int, error) {
const op = "servers.UpsertServer"
if server == nil {
return nil, db.NoRowsAffected, errors.New(errors.InvalidParameter, op, "server is nil")
}
opts := getOpts(opt...)
var rowsUpdated int
var controllers []*Server
_, err := r.writer.DoTx(
ctx,
db.StdRetryCnt,
db.ExpBackoff{},
func(read db.Reader, w db.Writer) error {
var err error
rowsUpdated, err = w.Exec(ctx,
serverUpsertQuery,
[]interface{}{
server.PrivateId,
server.Type,
server.Description,
server.Address,
})
if err != nil {
return errors.Wrap(err, op+":Upsert")
}
// If it's a worker, fetch the current controllers to feed to them
if server.Type == resource.Worker.String() {
// Fetch current controllers to feed to the workers
controllers, err = r.listServersWithReader(ctx, read, ServerTypeController)
if err != nil {
return errors.Wrap(err, op+":ListServer")
}
}
// If we've been told to update tags, we need to clean out old
// ones and add new ones. Within the current transaction, simply
// delete all tags for the given worker, then add the new ones
// we've been sent.
if opts.withUpdateTags {
_, err = w.Delete(ctx, &ServerTag{}, db.WithWhere(deleteTagsSql, server.PrivateId))
if err != nil {
return errors.Wrap(err, op+":DeleteTags", errors.WithMsg(server.PrivateId))
}
// If tags were cleared out entirely, then we'll have nothing
// to do here, e.g., it will result in deletion of all tags.
// Otherwise, go through and stage each tuple for insertion
// below.
if len(server.Tags) > 0 {
tags := make([]interface{}, 0, len(server.Tags))
for k, v := range server.Tags {
if v == nil {
return errors.New(errors.InvalidParameter, op+":RangeTags", fmt.Sprintf("found nil tag value for worker %s and key %s", server.PrivateId, k))
}
for _, val := range v.Values {
tags = append(tags, ServerTag{
ServerId: server.PrivateId,
Key: k,
Value: val,
})
}
}
if err = w.CreateItems(ctx, tags); err != nil {
return errors.Wrap(err, op+":CreateTags", errors.WithMsg(server.PrivateId))
}
}
}
return nil
},
)
if err != nil {
return nil, db.NoRowsAffected, err
}
return controllers, rowsUpdated, nil
}
type RecoveryNonce struct {
Nonce string
}
// AddRecoveryNonce adds a nonce
func (r *Repository) AddRecoveryNonce(ctx context.Context, nonce string, opt ...Option) error {
const op = "servers.AddRecoveryNonce"
if nonce == "" {
return errors.New(errors.InvalidParameter, op, "empty nonce")
}
rn := &RecoveryNonce{Nonce: nonce}
if err := r.writer.Create(ctx, rn); err != nil {
return errors.Wrap(err, op+":Insertion")
}
return nil
}
// CleanupNonces removes nonces that no longer need to be stored
func (r *Repository) CleanupNonces(ctx context.Context, opt ...Option) (int, error) {
// If something was inserted before 3x the actual validity period, clean it out
endTime := time.Now().Add(-3 * globals.RecoveryTokenValidityPeriod)
rows, err := r.writer.Delete(ctx, &RecoveryNonce{}, db.WithWhere(deleteWhereCreateTimeSql, endTime))
if err != nil {
return db.NoRowsAffected, errors.Wrap(err, "servers.CleanupNonces")
}
return rows, nil
}
// ListNonces lists nonces. Used only for tests at the moment.
func (r *Repository) ListNonces(ctx context.Context, opt ...Option) ([]*RecoveryNonce, error) {
var nonces []*RecoveryNonce
if err := r.reader.SearchWhere(ctx, &nonces, "", nil, db.WithLimit(-1)); err != nil {
return nil, errors.Wrap(err, "servers.ListNonces")
}
return nonces, nil
}