CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ec0a3f964bc7b58654221acb4d07cc7bdd8aa44a
threshold/internal/session/service_statistics_test.go
T
Johan Brandhorst-SatzkornandDamian Debkowski ec0a3f964b Worker status refactor (#5417) 
* oss: feat(proto): statistics rpc added to server coordination service

* oss: feat(session): add update statistics status service

* oss: feat(handlers): add Statistics method to worker service server

* oss: feat(worker): add statistics ticking

* oss: internal/proto: define the new RoutingInfo RPC

* oss: internal/server: repository changes for RoutingInfo RPC

* oss: internal/daemon: add RoutingInfo handler to worker service

The new RoutingInfo RPC is used to get information from the worker
that is used to make routing decisions on the controller. It also
handles the initial request from the worker.

* oss: feat(db): server_worker_session_info table

* oss: feat(proto): session info rpc for server coordination service

* oss: feat(server): worker session info request data type

* oss: feat(worker): session info ticker

* oss: feat(handler): server coordination handler for session info

* oss: feat(session): utilize session info request time for session cleanup

* oss: proto: fix some gotags comments

Some of these fields were missing comments, while others had
incorrectly formatted comments.

* oss: worker: send connection status to server

This is required for the correct tracking of orphaned connections

* oss: internal/daemon/worker: add function to wait for Statistics RPC

* oss: internal/daemon/worker: add routing info RPC

The new RoutingInfo RPC replaces part of the Status RPC,
namely the part which is used by the controller and worker
to perform decisions around the establishment of new sessions
and new worker connections.

* oss: internal/daemon/worker: remove unnecessary error check

* oss: internal/daemon/worker: lock access to config and upstreams

The new confLock protects access to the conf value and to the
addressReceivers slice.

* oss: worker: replace Status RPC with RoutingInfo

* oss: worker: deprecate the Status RPC and related types

The Status RPC and related messages have been replaced
by the RoutingInfo/SessionInfo/Statistics RPCs. The controller
will continue to support the Status RPC until two new minor
versions have been released.

* oss: internal/event: add default deny for new worker RPCs

* oss: internal/daemon/worker: allow RPC interval to be set

The new SessionInfo/RoutingInfo/Statistics RPCs run at
long intervals, which make some of our cluster tests take
a long time to complete. This allows their interval to be
configured in tests.

* oss: all: use TestWorkerRPCInterval in slow tests

This should speed up these tests significantly.

* oss: internal/tests/cluster: reset sys eventer in test

---------

Co-authored-by: Damian Debkowski <damian.debkowski@hashicorp.com>
2025-01-09 10:26:37 -08:00

535 lines
20 KiB
Go
Raw Blame History

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package session
import (
"context"
"testing"
"github.com/hashicorp/boundary/internal/authtoken"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/host/static"
"github.com/hashicorp/boundary/internal/iam"
"github.com/hashicorp/boundary/internal/kms"
"github.com/hashicorp/boundary/internal/server"
"github.com/hashicorp/boundary/internal/server/store"
"github.com/hashicorp/boundary/internal/target"
"github.com/hashicorp/boundary/internal/target/tcp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCloseOrphanedConnections(t *testing.T) {
t.Parallel()
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
testKms := kms.TestKms(t, conn, wrapper)
connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms)
require.NoError(t, err)
sessRepo, err := NewRepository(context.Background(), rw, rw, testKms)
require.NoError(t, err)
serverRepo, _ := server.NewRepository(context.Background(), rw, rw, testKms)
_, err = serverRepo.UpsertController(context.Background(), &store.Controller{
PrivateId: "test_controller1",
Address: "127.0.0.1",
})
require.NoError(t, err)
cases := []struct {
name string
connRepo *ConnectionRepository
setup func() (workerId string, conns []*Connection, expectedClosedConnections []string)
wantErrMsg string
}{
{
name: "missing connection repository",
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
return
},
wantErrMsg: "session.CloseOrphanedConnections: missing connection repository: parameter violation: error #100",
},
{
name: "empty worker id",
connRepo: &ConnectionRepository{},
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
return
},
wantErrMsg: "session.CloseOrphanedConnections: missing worker id: parameter violation: error #100",
},
{
name: "nil sessions",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
workerId = w.GetPublicId()
return
},
},
{
name: "empty connections",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
workerId = w.GetPublicId()
conns = make([]*Connection, 0)
return
},
},
{
name: "ignore unknown connection",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
workerId = w.GetPublicId()
conns = []*Connection{
{
PublicId: "c_dne",
},
}
return
},
},
{
name: "session already canceled",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(t, err)
require.NoError(t, err)
_, err = connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
require.NoError(t, err)
_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
require.NoError(t, err)
workerId = w.GetPublicId()
conns = []*Connection{}
return
},
},
{
name: "active session",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(t, err)
c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId())
require.NoError(t, err)
workerId = w.GetPublicId()
conns = []*Connection{
{
PublicId: c1.GetPublicId(),
},
}
return
},
},
{
name: "closed session",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(t, err)
c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
require.NoError(t, err)
_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
require.NoError(t, err)
workerId = w.GetPublicId()
conns = []*Connection{
{
PublicId: c1.GetPublicId(),
},
}
return
},
},
{
name: "multiple closed sessions",
connRepo: connRepo,
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(t, err)
c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
require.NoError(t, err)
_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
require.NoError(t, err)
s2 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t))
require.NoError(t, err)
c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
require.NoError(t, err)
_, err = sessRepo.CancelSession(context.Background(), s2.PublicId, s2.Version)
require.NoError(t, err)
workerId = w.GetPublicId()
conns = []*Connection{
{
PublicId: c1.GetPublicId(),
},
{
PublicId: c2.GetPublicId(),
},
}
return
},
},
{
name: "close orphaned connections",
connRepo: func() *ConnectionRepository {
connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms, WithWorkerStateDelay(0))
require.NoError(t, err)
return connRepo
}(),
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(t, err)
c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
require.NoError(t, err)
s2 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t))
require.NoError(t, err)
c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
require.NoError(t, err)
require.NotEqual(t, c1.PublicId, c2.PublicId)
workerId = w.GetPublicId()
conns = []*Connection{
{
PublicId: c2.GetPublicId(),
},
}
expectedClosedConnections = []string{c1.PublicId}
return
},
},
{
name: "multiple sessions and orphaned connections",
connRepo: func() *ConnectionRepository {
connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms, WithWorkerStateDelay(0))
require.NoError(t, err)
return connRepo
}(),
setup: func() (workerId string, conns []*Connection, expectedClosedConnections []string) {
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(t, err)
c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.PublicId)
require.NoError(t, err)
_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
require.NoError(t, err)
s2 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s2, _, err = sessRepo.ActivateSession(context.Background(), s2.PublicId, s2.Version, TestTofu(t))
require.NoError(t, err)
c2, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
require.NoError(t, err)
c3, err := connRepo.AuthorizeConnection(context.Background(), s2.PublicId, w.PublicId)
require.NoError(t, err)
_, err = sessRepo.CancelSession(context.Background(), s2.PublicId, s2.Version)
require.NoError(t, err)
workerId = w.GetPublicId()
conns = []*Connection{
{
PublicId: c1.GetPublicId(),
},
{
PublicId: c2.GetPublicId(),
},
}
expectedClosedConnections = []string{c3.PublicId}
return
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
assert, require := assert.New(t), require.New(t)
require.NotNil(tc.setup)
workerId, stats, expectedClosedConnections := tc.setup()
actualClosedConnections, err := CloseOrphanedConnections(context.Background(), tc.connRepo, workerId, stats)
if tc.wantErrMsg != "" {
require.Error(err)
assert.ErrorContains(err, tc.wantErrMsg)
assert.Empty(actualClosedConnections)
return
}
require.NoError(err)
assert.Equal(expectedClosedConnections, actualClosedConnections)
for _, connectionId := range expectedClosedConnections {
conn, err := connRepo.LookupConnection(context.Background(), connectionId)
require.NoError(err)
assert.Equal(ConnectionSystemError, ClosedReason(conn.ClosedReason))
assert.Equal(StatusClosed, ConnectionStatusFromString(conn.Status))
}
})
}
}
func TestUpdateConnectionBytesUpDown(t *testing.T) {
t.Parallel()
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
wrapper := db.TestWrapper(t)
testKms := kms.TestKms(t, conn, wrapper)
connRepo, err := NewConnectionRepository(context.Background(), rw, rw, testKms)
require.NoError(t, err)
sessRepo, err := NewRepository(context.Background(), rw, rw, testKms)
require.NoError(t, err)
serverRepo, _ := server.NewRepository(context.Background(), rw, rw, testKms)
_, err = serverRepo.UpsertController(context.Background(), &store.Controller{
PrivateId: "test_controller1",
Address: "127.0.0.1",
})
require.NoError(t, err)
t.Run("nil connection repo", func(t *testing.T) {
assert, require := assert.New(t), require.New(t)
err := UpdateConnectionBytesUpDown(context.Background(), nil, nil)
require.Error(err)
assert.ErrorContains(err, "missing connection repository")
})
t.Run("bytes up and bytes down", func(t *testing.T) {
assert, require := assert.New(t), require.New(t)
w := server.TestKmsWorker(t, conn, wrapper)
org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
authToken := authtoken.TestAuthToken(t, conn, testKms, org.GetPublicId())
userId := authToken.GetIamUserId()
hostCatalog := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
hostSet := static.TestSets(t, conn, hostCatalog.GetPublicId(), 1)[0]
host := static.TestHosts(t, conn, hostCatalog.GetPublicId(), 1)[0]
static.TestSetMembers(t, conn, hostSet.GetPublicId(), []*static.Host{host})
tar := tcp.TestTarget(
context.Background(),
t, conn, prj.GetPublicId(), "test",
target.WithHostSources([]string{hostSet.GetPublicId()}),
target.WithSessionConnectionLimit(-1),
)
s1 := TestSession(t, conn, wrapper, ComposedOf{
UserId: userId,
HostId: host.GetPublicId(),
TargetId: tar.GetPublicId(),
HostSetId: hostSet.GetPublicId(),
AuthTokenId: authToken.GetPublicId(),
ProjectId: prj.GetPublicId(),
Endpoint: "tcp://127.0.0.1:22",
ConnectionLimit: 10,
})
s1, _, err = sessRepo.ActivateSession(context.Background(), s1.PublicId, s1.Version, TestTofu(t))
require.NoError(err)
c1, err := connRepo.AuthorizeConnection(context.Background(), s1.PublicId, w.GetPublicId())
require.NoError(err)
// write to active session
var expectedBytesUp int64 = 1024
var expectedBytesDown int64 = 2048
conns := []*Connection{
{
PublicId: c1.GetPublicId(),
BytesUp: expectedBytesUp,
BytesDown: expectedBytesDown,
},
}
err = UpdateConnectionBytesUpDown(context.Background(), connRepo, conns)
require.NoError(err)
conn, err := connRepo.LookupConnection(context.Background(), c1.PublicId)
require.NoError(err)
assert.Equal(StatusAuthorized, ConnectionStatusFromString(conn.Status))
assert.Equal(expectedBytesUp, conn.BytesUp)
assert.Equal(expectedBytesDown, conn.BytesDown)
// write to canceled session
_, err = sessRepo.CancelSession(context.Background(), s1.PublicId, s1.Version)
require.NoError(err)
expectedBytesUp = 4096
expectedBytesDown = 8192
conns = []*Connection{
{
PublicId: c1.GetPublicId(),
BytesUp: expectedBytesUp,
BytesDown: expectedBytesDown,
},
}
err = UpdateConnectionBytesUpDown(context.Background(), connRepo, conns)
require.NoError(err)
conn, err = connRepo.LookupConnection(context.Background(), c1.PublicId)
require.NoError(err)
assert.Equal(expectedBytesUp, conn.BytesUp)
assert.Equal(expectedBytesDown, conn.BytesDown)
})
}
Reference in new issue View Git Blame Copy Permalink