Merge remote-tracking branch 'origin/main' into llb-byow

This commit is contained in:
Jeff Mitchell committed 2022-06-10 19:05:27 -04:00
commit 5d3facf561
56 files changed
+1455 -320

No files matched your search

@@ -0,0 +1,75 @@
package session
import (
"context"
"time"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/scheduler"
)
type deleteTerminatedJob struct {
repo *Repository
// the amount of time that a session must be in the terminated
// state for it to be deleted.
threshold time.Duration
// the number of sessions deleted in the most recent run
deletedInRun int
}
func newDeleteTerminatedJob(ctx context.Context, repo *Repository, threshold time.Duration) (*deleteTerminatedJob, error) {
const op = "session.newDeleteTerminatedJob"
switch {
case repo == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing repository")
}
return &deleteTerminatedJob{
repo: repo,
threshold: threshold,
}, nil
}
// Status reports the job’s current status. The status is periodically persisted by
// the scheduler when a job is running, and will be used to verify a job is making progress.
func (d *deleteTerminatedJob) Status() scheduler.JobStatus {
return scheduler.JobStatus{
Completed: d.deletedInRun,
Total: d.deletedInRun,
}
}
// Run performs the required work depending on the implementation.
// The context is used to notify the job that it should exit early.
func (d *deleteTerminatedJob) Run(ctx context.Context) error {
const op = "session.(deleteTerminatedJob).Run"
d.deletedInRun = 0
var err error
d.deletedInRun, err = d.repo.deleteSessionsTerminatedBefore(ctx, d.threshold)
if err != nil {
return errors.Wrap(ctx, err, op)
}
return nil
}
// NextRunIn returns the duration until the next job run should be scheduled. This
// method is invoked after a run has successfully completed and the next run time
// is being persisted by the scheduler. If an error is returned, the error will be logged
// but the duration returned will still be used in scheduling. If a zero duration is returned
// the job will be scheduled to run again immediately.
func (d *deleteTerminatedJob) NextRunIn(_ context.Context) (time.Duration, error) {
return time.Minute * 30, nil
}
// Name is the unique name of the job.
func (d *deleteTerminatedJob) Name() string {
return "delete_terminated_sessions"
}
// Description is the human readable description of the job.
func (d *deleteTerminatedJob) Description() string {
return "Delete terminated sessions that were terminated "
}
@@ -0,0 +1,106 @@
package session
import (
"context"
"fmt"
"testing"
"time"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/iam"
"github.com/hashicorp/boundary/internal/kms"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDeleteTermiantedSessionsJob(t *testing.T) {
ctx := context.Background()
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
iamRepo := iam.TestRepo(t, conn, wrapper)
kms := kms.TestKms(t, conn, wrapper)
repo, err := NewRepository(rw, rw, kms)
require.NoError(t, err)
composedOf := TestSessionParams(t, conn, wrapper, iamRepo)
require.NoError(t, err)
cases := []struct {
sessionCount int
terminateCount int
threshold time.Duration
expected int
}{
{
0,
0,
time.Nanosecond,
0,
},
{
1,
1,
time.Nanosecond,
1,
},
{
1,
1,
time.Hour,
0,
},
{
10,
10,
time.Nanosecond,
10,
},
{
10,
4,
time.Nanosecond,
4,
},
{
10,
0,
time.Nanosecond,
0,
},
{
10,
10,
time.Hour,
0,
},
}
for _, tc := range cases {
t.Run(fmt.Sprintf("%d_%d_%s", tc.sessionCount, tc.terminateCount, tc.threshold), func(t *testing.T) {
t.Cleanup(func() {
sdb, err := conn.SqlDB(ctx)
require.NoError(t, err)
_, err = sdb.Exec(`delete from session;`)
require.NoError(t, err)
})
for i := 0; i < tc.sessionCount; i++ {
s := TestSession(t, conn, wrapper, composedOf)
if i < tc.terminateCount {
_, err = repo.CancelSession(ctx, s.PublicId, s.Version)
require.NoError(t, err)
}
}
c, err := repo.TerminateCompletedSessions(ctx)
require.NoError(t, err)
assert.Equal(t, tc.terminateCount, c)
job, err := newDeleteTerminatedJob(ctx, repo, tc.threshold)
require.NoError(t, err)
err = job.Run(ctx)
require.NoError(t, err)
assert.Equal(t, tc.expected, job.deletedInRun)
})
}
}
+18 -2
View File
@@ -6,12 +6,16 @@ import (
"time"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/kms"
"github.com/hashicorp/boundary/internal/scheduler"
)
// RegisterJobs registers plugin host related jobs with the provided scheduler.
func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writer, gracePeriod time.Duration) error {
const deleteTerminatedThreshold = time.Hour
// RegisterJobs registers session related jobs with the provided scheduler.
func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writer, r db.Reader, k *kms.Kms, gracePeriod time.Duration) error {
const op = "session.RegisterJobs"
sessionConnectionCleanupJob, err := newSessionConnectionCleanupJob(w, gracePeriod)
if err != nil {
return fmt.Errorf("error creating session cleanup job: %w", err)
@@ -20,5 +24,17 @@ func RegisterJobs(ctx context.Context, scheduler *scheduler.Scheduler, w db.Writ
return fmt.Errorf("error registering session cleanup job: %w", err)
}
repo, err := NewRepository(r, w, k)
if err != nil {
return fmt.Errorf("error creating repository: %w", err)
}
deleteTerminatedJob, err := newDeleteTerminatedJob(ctx, repo, deleteTerminatedThreshold)
if err != nil {
return fmt.Errorf("error creating delete terminated session job: %w", err)
}
if err = scheduler.RegisterJob(ctx, deleteTerminatedJob); err != nil {
return fmt.Errorf("error registering delete terminated session job: %w", err)
}
return nil
}
+8
View File
@@ -31,6 +31,7 @@ type options struct {
withSessionIds []string
withDbOpts []db.Option
withWorkerStateDelay time.Duration
withTerminated bool
}
func getDefaultOptions() options {
@@ -112,3 +113,10 @@ func WithWorkerStateDelay(d time.Duration) Option {
o.withWorkerStateDelay = d
}
}
// WithTerminated is used to include terminated sessions in a list request.
func WithTerminated(withTerminated bool) Option {
return func(o *options) {
o.withTerminated = withTerminated
}
}
+21 -1
View File
@@ -117,11 +117,20 @@ session_connection_limit(expiration_time, connection_limit) as (
select expiration_time, connection_limit, current_connection_count
from
session_connection_limit, session_connection_count;
`
nonTerminatedSessionPublicIdList = `
select public_id, scope_id, user_id from session
where
session.termination_reason is null
and
scope_id = any(@scope_ids)
;
`
sessionPublicIdList = `
select public_id, scope_id, user_id from session
%s
where
scope_id = any(@scope_ids)
;
`
@@ -402,6 +411,17 @@ where
and ss.end_time is null
%s
;
`
deleteTerminated = `
delete from session
using session_state
where
session.public_id = session_state.session_id
and
session_state.state = 'terminated'
and
session_state.start_time < wt_sub_seconds_from_now(@threshold_seconds)
;
`
)
-25
View File
@@ -83,8 +83,6 @@ func (r *Repository) convertToSessions(ctx context.Context, sessionList []*sessi
return nil, nil
}
sessions := []*Session{}
// deduplication map of connections by connection_id
connections := map[string]*Connection{}
// deduplication map of states by end_time
states := map[*timestamp.Timestamp]*State{}
var prevSessionId string
@@ -92,10 +90,6 @@ func (r *Repository) convertToSessions(ctx context.Context, sessionList []*sessi
for _, sv := range sessionList {
if sv.PublicId != prevSessionId {
if prevSessionId != "" {
for _, c := range connections {
workingSession.Connections = append(workingSession.Connections, c)
}
connections = map[string]*Connection{}
for _, s := range states {
workingSession.States = append(workingSession.States, s)
}
@@ -155,22 +149,6 @@ func (r *Repository) convertToSessions(ctx context.Context, sessionList []*sessi
}
}
if sv.ConnectionId != "" {
if _, ok := connections[sv.ConnectionId]; !ok {
connections[sv.ConnectionId] = &Connection{
PublicId: sv.ConnectionId,
SessionId: sv.PublicId,
ClientTcpAddress: sv.ClientTcpAddress,
ClientTcpPort: sv.ClientTcpPort,
EndpointTcpAddress: sv.EndpointTcpAddress,
EndpointTcpPort: sv.EndpointTcpPort,
BytesUp: sv.BytesUp,
BytesDown: sv.BytesDown,
ClosedReason: sv.ClosedReason,
}
}
}
}
for _, s := range states {
workingSession.States = append(workingSession.States, s)
@@ -178,9 +156,6 @@ func (r *Repository) convertToSessions(ctx context.Context, sessionList []*sessi
sort.Slice(workingSession.States, func(i, j int) bool {
return workingSession.States[i].StartTime.GetTimestamp().AsTime().After(workingSession.States[j].StartTime.GetTimestamp().AsTime())
})
for _, c := range connections {
workingSession.Connections = append(workingSession.Connections, c)
}
sessions = append(sessions, workingSession)
return sessions, nil
}
+35 -21
View File
@@ -7,6 +7,7 @@ import (
"database/sql"
"fmt"
"strings"
"time"
"github.com/hashicorp/boundary/internal/boundary"
"github.com/hashicorp/boundary/internal/db"
@@ -200,32 +201,31 @@ func (r *Repository) LookupSession(ctx context.Context, sessionId string, _ ...O
return &session, authzSummary, nil
}
// FetchAuthzProtectedEntitiesByScope implements boundary.AuthzProtectedEntityProvider
func (r *Repository) FetchAuthzProtectedEntitiesByScope(ctx context.Context, scopeIds []string) (map[string][]boundary.AuthzProtectedEntity, error) {
const op = "session.(Repository).FetchAuthzProtectedEntityInfo"
// fetchAuthzProtectedSessionsByScope fetches sessions for the given scopes.
// Note that the sessions are not fully populated, and only contain the
// necessary information to implement the boundary.AuthzProtectedEntity
// interface. Supports the WithTerminated option.
func (r *Repository) fetchAuthzProtectedSessionsByScope(
ctx context.Context, scopeIds []string, opt ...Option,
) (map[string][]boundary.AuthzProtectedEntity, error) {
const op = "session.(Repository).fetchAuthzProtectedSessionsByScope"
var where string
var args []interface{}
opts := getOpts(opt...)
inClauseCnt := 0
switch len(scopeIds) {
case 0:
if len(scopeIds) == 0 {
return nil, errors.New(ctx, errors.InvalidParameter, op, "no scopes given")
case 1:
inClauseCnt += 1
where, args = fmt.Sprintf("where scope_id = @%d", inClauseCnt), append(args, sql.Named(fmt.Sprintf("%d", inClauseCnt), scopeIds[0]))
default:
idsInClause := make([]string, 0, len(scopeIds))
for _, id := range scopeIds {
inClauseCnt += 1
idsInClause, args = append(idsInClause, fmt.Sprintf("@%d", inClauseCnt)), append(args, sql.Named(fmt.Sprintf("%d", inClauseCnt), id))
}
where = fmt.Sprintf("where scope_id in (%s)", strings.Join(idsInClause, ","))
}
q := sessionPublicIdList
query := fmt.Sprintf(q, where)
args := []interface{}{
sql.Named("scope_ids", "{"+strings.Join(scopeIds, ",")+"}"),
}
var query string
if opts.withTerminated {
query = sessionPublicIdList
} else {
query = nonTerminatedSessionPublicIdList
}
rows, err := r.reader.Query(ctx, query, args)
if err != nil {
@@ -681,6 +681,20 @@ func (r *Repository) checkIfNoLongerActive(ctx context.Context, reportedSessions
return notActive, nil
}
func (r *Repository) deleteSessionsTerminatedBefore(ctx context.Context, threshold time.Duration) (int, error) {
const op = "session.(Repository).deleteTerminated"
args := []any{
sql.Named("threshold_seconds", threshold.Seconds()),
}
c, err := r.writer.Exec(ctx, deleteTerminated, args)
if err != nil {
return 0, errors.Wrap(ctx, err, op, errors.WithMsg("error deleting terminated sessions"))
}
return c, nil
}
func fetchStates(ctx context.Context, r db.Reader, sessionId string, opt ...db.Option) ([]*State, error) {
const op = "session.fetchStates"
var states []*State
+319 -1
View File
@@ -125,7 +125,8 @@ func TestRepository_ListSession(t *testing.T) {
require.NoError(err)
assert.Equal(tt.wantCnt, len(got))
for i := 0; i < len(got); i++ {
assert.Equal(tt.withConnections, len(got[i].Connections))
// connections should not be returned for list requests
assert.Equal(0, len(got[i].Connections))
for _, c := range got[i].Connections {
assert.Equal("127.0.0.1", c.ClientTcpAddress)
assert.Equal(uint32(22), c.ClientTcpPort)
@@ -1473,3 +1474,320 @@ func TestRepository_deleteTargetFKey(t *testing.T) {
})
}
}
func TestRepository_deleteTerminated(t *testing.T) {
ctx := context.Background()
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
iamRepo := iam.TestRepo(t, conn, wrapper)
kms := kms.TestKms(t, conn, wrapper)
repo, err := NewRepository(rw, rw, kms)
composedOf := TestSessionParams(t, conn, wrapper, iamRepo)
cases := []struct {
sessionCount int
terminateCount int
threshold time.Duration
expected int
}{
{
0,
0,
time.Nanosecond,
0,
},
{
1,
1,
time.Nanosecond,
1,
},
{
1,
1,
time.Hour,
0,
},
{
10,
10,
time.Nanosecond,
10,
},
{
10,
4,
time.Nanosecond,
4,
},
{
10,
0,
time.Nanosecond,
0,
},
{
10,
10,
time.Hour,
0,
},
}
for _, tc := range cases {
t.Run(fmt.Sprintf("%d_%d_%s", tc.sessionCount, tc.terminateCount, tc.threshold), func(t *testing.T) {
t.Cleanup(func() {
sdb, err := conn.SqlDB(ctx)
require.NoError(t, err)
_, err = sdb.Exec(`delete from session;`)
require.NoError(t, err)
})
for i := 0; i < tc.sessionCount; i++ {
s := TestSession(t, conn, wrapper, composedOf)
if i < tc.terminateCount {
_, err = repo.CancelSession(ctx, s.PublicId, s.Version)
require.NoError(t, err)
}
}
c, err := repo.TerminateCompletedSessions(ctx)
require.NoError(t, err)
assert.Equal(t, tc.terminateCount, c)
c, err = repo.deleteSessionsTerminatedBefore(ctx, tc.threshold)
require.NoError(t, err)
assert.Equal(t, tc.expected, c)
})
}
}
func TestFetchAuthzProtectedSessionsByScopes(t *testing.T) {
conn, _ := db.TestSetup(t, "postgres")
ctx := context.Background()
wrapper := db.TestWrapper(t)
iamRepo := iam.TestRepo(t, conn, wrapper)
rw := db.New(conn)
kms := kms.TestKms(t, conn, wrapper)
repo, err := NewRepository(rw, rw, kms)
require.NoError(t, err)
composedOf := TestSessionParams(t, conn, wrapper, iamRepo)
_, pWithOtherSessions := iam.TestScopes(t, iamRepo)
hcOther := static.TestCatalogs(t, conn, pWithOtherSessions.GetPublicId(), 1)[0]
hsOther := static.TestSets(t, conn, hcOther.GetPublicId(), 1)[0]
hOther := static.TestHosts(t, conn, hcOther.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hsOther.GetPublicId(), []*static.Host{hOther})
tarOther := tcp.TestTarget(ctx, t, conn, pWithOtherSessions.GetPublicId(), "test", target.WithHostSources([]string{hsOther.GetPublicId()}))
composedOfOther := ComposedOf{
UserId: composedOf.UserId,
HostId: hOther.GetPublicId(),
TargetId: tarOther.GetPublicId(),
HostSetId: hsOther.GetPublicId(),
AuthTokenId: composedOf.AuthTokenId,
ScopeId: pWithOtherSessions.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
}
type testCase struct {
name string
createCnt int
terminateCnt int
otherCnt int
otherTerminateCnt int
opts []Option
reqScopes []string
wantCnt int
wantOtherCnt int
wantErr bool
}
cases := []testCase{
{
name: "NonTerminated/none",
createCnt: 0,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 0,
wantErr: false,
},
{
name: "NonTerminated/one",
createCnt: 1,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 1,
wantErr: false,
},
{
name: "NonTerminated/many",
createCnt: 5,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 5,
wantErr: false,
},
{
name: "NonTerminated/many one terminated",
createCnt: 5,
terminateCnt: 1,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 4,
wantErr: false,
},
{
name: "NonTerminated/many terminated",
createCnt: 5,
terminateCnt: 3,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 2,
wantErr: false,
},
{
name: "NonTerminated/many multiple scopes",
createCnt: 5,
terminateCnt: 3,
reqScopes: []string{composedOf.ScopeId, composedOfOther.ScopeId},
wantCnt: 2,
wantErr: false,
},
{
name: "NonTerminated/many multiple scopes",
createCnt: 5,
terminateCnt: 3,
otherCnt: 3,
otherTerminateCnt: 1,
reqScopes: []string{composedOf.ScopeId, composedOfOther.ScopeId},
wantCnt: 2,
wantOtherCnt: 2,
wantErr: false,
},
{
name: "NonTerminated/no scopes",
createCnt: 2,
terminateCnt: 1,
otherCnt: 2,
otherTerminateCnt: 1,
reqScopes: []string{},
wantErr: true,
},
{
name: "none",
opts: []Option{WithTerminated(true)},
createCnt: 0,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 0,
wantErr: false,
},
{
name: "one",
opts: []Option{WithTerminated(true)},
createCnt: 1,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 1,
wantErr: false,
},
{
name: "many",
opts: []Option{WithTerminated(true)},
createCnt: 5,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 5,
wantErr: false,
},
{
name: "many one terminated",
opts: []Option{WithTerminated(true)},
createCnt: 5,
terminateCnt: 1,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 5,
wantErr: false,
},
{
name: "many terminated",
opts: []Option{WithTerminated(true)},
createCnt: 5,
terminateCnt: 3,
reqScopes: []string{composedOf.ScopeId},
wantCnt: 5,
wantErr: false,
},
{
name: "many multiple scopes",
opts: []Option{WithTerminated(true)},
createCnt: 5,
terminateCnt: 3,
reqScopes: []string{composedOf.ScopeId, composedOfOther.ScopeId},
wantCnt: 5,
wantErr: false,
},
{
name: "many multiple scopes",
opts: []Option{WithTerminated(true)},
createCnt: 5,
terminateCnt: 3,
otherCnt: 3,
otherTerminateCnt: 1,
reqScopes: []string{composedOf.ScopeId, composedOfOther.ScopeId},
wantCnt: 5,
wantOtherCnt: 3,
wantErr: false,
},
{
name: "no scopes",
opts: []Option{WithTerminated(true)},
createCnt: 2,
terminateCnt: 1,
otherCnt: 2,
otherTerminateCnt: 1,
reqScopes: []string{},
wantErr: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
assert, require := assert.New(t), require.New(t)
db.TestDeleteWhere(t, conn, func() interface{} { i := AllocSession(); return &i }(), "1=1")
testSessions := []*Session{}
for i := 0; i < tt.createCnt; i++ {
s := TestSession(t, conn, wrapper, composedOf)
_ = TestState(t, conn, s.PublicId, StatusActive)
testSessions = append(testSessions, s)
}
for i := 0; i < tt.terminateCnt; i++ {
_, err := repo.CancelSession(ctx, testSessions[i].PublicId, testSessions[i].Version)
require.NoError(err)
}
terminated, err := repo.TerminateCompletedSessions(ctx)
require.NoError(err)
require.Equal(tt.terminateCnt, terminated)
otherTestSessions := []*Session{}
for i := 0; i < tt.otherCnt; i++ {
s := TestSession(t, conn, wrapper, composedOfOther)
_ = TestState(t, conn, s.PublicId, StatusActive)
otherTestSessions = append(otherTestSessions, s)
}
for i := 0; i < tt.otherTerminateCnt; i++ {
_, err := repo.CancelSession(ctx, otherTestSessions[i].PublicId, otherTestSessions[i].Version)
require.NoError(err)
}
terminated, err = repo.TerminateCompletedSessions(ctx)
require.NoError(err)
require.Equal(tt.otherTerminateCnt, terminated)
assert.Equal(tt.otherCnt, len(otherTestSessions))
got, err := repo.fetchAuthzProtectedSessionsByScope(ctx, tt.reqScopes, tt.opts...)
if tt.wantErr {
require.Error(err)
return
}
require.NoError(err)
assert.Equal(tt.wantCnt, len(got[composedOf.ScopeId]))
assert.Equal(tt.wantOtherCnt, len(got[composedOfOther.ScopeId]))
})
}
}
@@ -0,0 +1,23 @@
package session
import (
"context"
"github.com/hashicorp/boundary/internal/boundary"
)
type listForAuthzCheckFunc func(ctx context.Context, scopeIds []string) (map[string][]boundary.AuthzProtectedEntity, error)
func (a listForAuthzCheckFunc) FetchAuthzProtectedEntitiesByScope(ctx context.Context, scopeIds []string) (map[string][]boundary.AuthzProtectedEntity, error) {
return a(ctx, scopeIds)
}
// ListForAuthzCheck returns a functions that fetches sessions for the given
// scopes. Note that the sessions are not fully populated, and only contain the
// necessary information to implement the boundary.AuthzProtectedEntity
// interface. Supports the WithTerminated option.
func ListForAuthzCheck(repo *Repository, opt ...Option) listForAuthzCheckFunc {
return func(ctx context.Context, scopeIds []string) (map[string][]boundary.AuthzProtectedEntity, error) {
return repo.fetchAuthzProtectedSessionsByScope(ctx, scopeIds, opt...)
}
}
-10
View File
@@ -458,16 +458,6 @@ type sessionListView struct {
PreviousEndTime *timestamp.Timestamp `json:"previous_end_time,omitempty" gorm:"default:current_timestamp"`
StartTime *timestamp.Timestamp `json:"start_time,omitempty" gorm:"default:current_timestamp;primary_key"`
EndTime *timestamp.Timestamp `json:"end_time,omitempty" gorm:"default:current_timestamp"`
// Connection fields
ConnectionId string `json:"connection_id,omitempty" gorm:"default:null"`
ClientTcpAddress string `json:"client_tcp_address,omitempty" gorm:"default:null"`
ClientTcpPort uint32 `json:"client_tcp_port,omitempty" gorm:"default:null"`
EndpointTcpAddress string `json:"endpoint_tcp_address,omitempty" gorm:"default:null"`
EndpointTcpPort uint32 `json:"endpoint_tcp_port,omitempty" gorm:"default:null"`
BytesUp uint64 `json:"bytes_up,omitempty" gorm:"default:null"`
BytesDown uint64 `json:"bytes_down,omitempty" gorm:"default:null"`
ClosedReason string `json:"closed_reason,omitempty" gorm:"default:null"`
}
// TableName returns the tablename to override the default gorm table name