Files
threshold/internal/session/job_session_cleanup.go
T
hashicorp-copywrite[bot]andJohan Brandhorst-Satzkorn 29da0bcb92 [COMPLIANCE] License changes (#3567)
* Adding explicit MPL license for sub-package.

This directory and its subdirectories (packages) contain files licensed with the MPLv2 `LICENSE` file in this directory and are intentionally licensed separately from the BSL `LICENSE` file at the root of this repository.

* Adding explicit MPL license for sub-package

This directory and its subdirectories (packages) contain files licensed with the MPLv2 `LICENSE` file in this directory and are intentionally licensed separately from the BSL `LICENSE` file at the root of this repository.

* Updating the license from MPL to Business Source License

Going forward, this project will be licensed under the Business Source License v1.1. Please see our blog post for more details at https://hashi.co/bsl-blog, FAQ at www.hashicorp.com/licensing-faq, and details of the license at www.hashicorp.com/bsl.

* Update copyright file headers to BUS-1.1

* Rerun make gen

This will pick up the last of the license changes

* Revert "Rerun make gen"

This reverts commit dbe0968bffd92745d8663c6940debae098395a6b.

* Remove copywrite from generation script

The current version of copywrite is not sophisticated enough
to only apply the copywrite changes to certain subdirectories,
so to avoid accidentally relicensing our api and sdk modules,
we will stop running the copywrite script until such time
that it can do the right thing.

* License protobuf API as MPL

The protobuf API files are used to generate the SDK,
since we want the SDK to be MPL, the protobuf files
must also be MPL. This extends to all protobuf files
whose sources end up in the SDK directory, including:

- internal/proto/controller/api
- internal/proto/controller/custom_options
- internal/proto/plugin

It also includes the generated service directory in
internal/gen/controller/api/services, since they are
generated from the protobuf sources used for the SDK.

* Change website license file name

This makes the naming scheme consistent with the rest
of the repository.

* Add note on licensing to README

* Set copywrite license to BUSL

* Revert "Add note on licensing to README"

This reverts commit 4d865a91a7a4f11605f0dc4609355d7e3c3cfffe.

---------

Co-authored-by: hashicorp-copywrite[bot] <110428419+hashicorp-copywrite[bot]@users.noreply.github.com>
Co-authored-by: Johan Brandhorst-Satzkorn <johan.brandhorst@gmail.com>
2023-08-11 10:31:53 -07:00

202 lines
6.1 KiB
Go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package session
import (
"context"
"database/sql"
stderrors "errors"
"sync/atomic"
"time"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/observability/event"
"github.com/hashicorp/boundary/internal/scheduler"
)
type closeConnectionsForDeadWorkersResult struct {
WorkerId string
LastUpdateTime time.Time
NumberConnectionsClosed int
}
// sessionConnectionCleanupJob defines a periodic job that monitors workers for
// loss of connection and terminates connections on workers that have
// not sent a heartbeat in a significant period of time.
//
// Relevant connections are simply marked as disconnected in the
// database. Connections will be independently terminated by the
// worker, or the event of a synchronization issue between the two,
// the controller will win out and order that the connections be
// closed on the worker.
type sessionConnectionCleanupJob struct {
writer db.Writer
// The amount of time to give disconnected workers before marking their
// connections as closed. This should be larger than the liveness setting
// for the worker.
gracePeriod *atomic.Int64
// The total number of connections closed in the last run.
totalClosed int
}
// newSessionConnectionCleanupJob instantiates the session cleanup job.
func newSessionConnectionCleanupJob(
ctx context.Context,
writer db.Writer,
gracePeriod *atomic.Int64,
) (*sessionConnectionCleanupJob, error) {
const op = "session.newNewSessionConnectionCleanupJob"
switch {
case writer == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing db writer")
case gracePeriod == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grace period")
case gracePeriod.Load() == 0:
return nil, errors.New(ctx, errors.InvalidParameter, op, "grace period is zero")
}
return &sessionConnectionCleanupJob{
writer: writer,
gracePeriod: gracePeriod,
}, nil
}
// Name returns a short, unique name for the job.
func (j *sessionConnectionCleanupJob) Name() string { return "session_cleanup" }
// Description returns the description for the job.
func (j *sessionConnectionCleanupJob) Description() string {
return "Clean up session connections from disconnected workers"
}
// NextRunIn returns the next run time after a job is completed.
//
// The next run time is defined for sessionConnectionCleanupJob as one second.
// This is because the job should run continuously to terminate
// connections as soon as a worker has not reported in for a long
// enough time. Only one job will ever run at once, so there is no
// reason why it cannot run again immediately.
func (j *sessionConnectionCleanupJob) NextRunIn(_ context.Context) (time.Duration, error) {
return time.Second, nil
}
// Status returns the status of the running job.
func (j *sessionConnectionCleanupJob) Status() scheduler.JobStatus {
return scheduler.JobStatus{
Completed: j.totalClosed,
Total: j.totalClosed,
}
}
// Run executes the job.
func (j *sessionConnectionCleanupJob) Run(ctx context.Context) error {
const op = "session.(sessionConnectionCleanupJob).Run"
j.totalClosed = 0
// Run the atomic dead worker cleanup job.
gracePeriod := time.Duration(j.gracePeriod.Load())
results, err := j.closeConnectionsForDeadWorkers(ctx, gracePeriod)
if err != nil {
return errors.Wrap(ctx, err, op)
}
for _, result := range results {
event.WriteError(ctx, op, stderrors.New("worker has not reported status within acceptable grace period, all connections closed"),
event.WithInfo(
"public_id", result.WorkerId,
"update_time", result.LastUpdateTime,
"grace_period_seconds", gracePeriod.Seconds(),
"number_connections_closed", result.NumberConnectionsClosed,
))
j.totalClosed += result.NumberConnectionsClosed
}
{
count, err := j.closeWorkerlessConnections(ctx)
if err != nil {
return errors.Wrap(ctx, err, op)
}
if count > 0 {
event.WriteError(ctx, op, stderrors.New("connections with no worker closed"),
event.WithInfo(
"number_connections_closed", count,
))
j.totalClosed += count
}
}
return nil
}
// closeWorkerlessConnections will close all connections which do not have a
// worker id associated with them.
func (j *sessionConnectionCleanupJob) closeWorkerlessConnections(ctx context.Context) (int, error) {
const op = "session.(sessionConnectionCleanupJob).closeWorkerlessConnections"
count := 0
_, err := j.writer.DoTx(
ctx,
db.StdRetryCnt,
db.ExpBackoff{},
func(reader db.Reader, w db.Writer) error {
var err error
count, err = w.Exec(ctx, closeWorkerlessConnections, []any{})
if err != nil {
return errors.Wrap(ctx, err, op)
}
return nil
},
)
if err != nil {
return count, errors.Wrap(ctx, err, op)
}
return count, nil
}
// closeConnectionsForDeadWorkers will run
// closeConnectionsForDeadServersCte to look for connections that
// should be marked because they are on a server that is no longer
// sending status updates to the controller(s).
//
// The only input to the method is the grace period, in seconds.
func (j *sessionConnectionCleanupJob) closeConnectionsForDeadWorkers(ctx context.Context, gracePeriod time.Duration) ([]closeConnectionsForDeadWorkersResult, error) {
const op = "session.(sessionConnectionCleanupJob).closeConnectionsForDeadWorkers"
args := []any{
sql.Named("grace_period_seconds", gracePeriod.Seconds()),
}
results := make([]closeConnectionsForDeadWorkersResult, 0)
_, err := j.writer.DoTx(
ctx,
db.StdRetryCnt,
db.ExpBackoff{},
func(reader db.Reader, w db.Writer) error {
rows, err := w.Query(ctx, closeConnectionsForDeadServersCte, args)
if err != nil {
return errors.Wrap(ctx, err, op)
}
defer rows.Close()
for rows.Next() {
var result closeConnectionsForDeadWorkersResult
if err := w.ScanRows(ctx, rows, &result); err != nil {
return errors.Wrap(ctx, err, op)
}
results = append(results, result)
}
return nil
},
)
if err != nil {
return nil, errors.Wrap(ctx, err, op)
}
return results, nil
}