This backports some test work that is being added in #1340 to assert that the worker is actually marking connections closed in its *local* state on a failure to contact the controller. This was missed in #1357 during some late-stage refactoring. Summary of the changes: * If we can't actually contact the controller during connection closure to mark the connections as closed, we actually construct a "fake" response that just simply includes a closed state for all connections requested. This is then used to update the local connection state. * Added a 3 second timeout to the CloseConnection request to the controller to ensure the call to close connections completed in a timely manner and did not hang indefinitely. Testing enhancements: * Added a new helper in TestHelper called LookupSession that allows the lookup of a session from the worker's internal session state. * This is used in the new ExpectConnectionStateOnWorker check in the E2E test, which is part of a larger amount of enhancements to the E2E test that are coming in #1340. * Added a ConnectionStatusFromProtoVal helper function to the session package to allow the conversion of protobuf connection status values to their higher-level counterparts; essentially it is a reverse of ConnectionStatus.ProtoVal. This is used in some of our tests (with more coming in #1340) to just help with keeping things more idiomatic. Finally, there is some minor error refactoring, and we also change the timeout on internal status and connection close requests to the controller to 5 seconds.
286 lines
7.5 KiB
Go
286 lines
7.5 KiB
Go
package worker
|
|
|
|
import (
|
|
"errors"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
|
|
"github.com/hashicorp/boundary/internal/session"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestWorkerMakeCloseConnectionRequest(t *testing.T) {
|
|
require := require.New(t)
|
|
in := map[string]string{"foo": "one", "bar": "two"}
|
|
expected := &pbs.CloseConnectionRequest{
|
|
CloseRequestData: []*pbs.CloseConnectionRequestData{
|
|
{ConnectionId: "foo", Reason: session.UnknownReason.String()},
|
|
{ConnectionId: "bar", Reason: session.UnknownReason.String()},
|
|
},
|
|
}
|
|
actual := new(Worker).makeCloseConnectionRequest(in)
|
|
require.ElementsMatch(expected.GetCloseRequestData(), actual.GetCloseRequestData())
|
|
}
|
|
|
|
func TestMakeSessionCloseInfo(t *testing.T) {
|
|
require := require.New(t)
|
|
closeInfo := map[string]string{"foo": "one", "bar": "two"}
|
|
response := &pbs.CloseConnectionResponse{
|
|
CloseResponseData: []*pbs.CloseConnectionResponseData{
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
}
|
|
expected := map[string][]*pbs.CloseConnectionResponseData{
|
|
"one": {
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
"two": {
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
}
|
|
actual, err := new(Worker).makeSessionCloseInfo(closeInfo, response)
|
|
require.NoError(err)
|
|
require.Equal(expected, actual)
|
|
}
|
|
|
|
func TestMakeSessionCloseInfoErrorIfCloseInfoNil(t *testing.T) {
|
|
require := require.New(t)
|
|
actual, err := new(Worker).makeSessionCloseInfo(nil, nil)
|
|
require.Nil(actual)
|
|
require.ErrorIs(err, errMakeSessionCloseInfoNilCloseInfo)
|
|
}
|
|
|
|
func TestMakeSessionCloseInfoEmpty(t *testing.T) {
|
|
require := require.New(t)
|
|
actual, err := new(Worker).makeSessionCloseInfo(make(map[string]string), nil)
|
|
require.NoError(err)
|
|
require.Equal(
|
|
make(map[string][]*pbs.CloseConnectionResponseData),
|
|
actual,
|
|
)
|
|
}
|
|
|
|
func TestMakeFakeSessionCloseInfo(t *testing.T) {
|
|
require := require.New(t)
|
|
closeInfo := map[string]string{"foo": "one", "bar": "two"}
|
|
expected := map[string][]*pbs.CloseConnectionResponseData{
|
|
"one": {
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
"two": {
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
}
|
|
actual, err := new(Worker).makeFakeSessionCloseInfo(closeInfo)
|
|
require.NoError(err)
|
|
require.Equal(expected, actual)
|
|
}
|
|
|
|
func TestMakeFakeSessionCloseInfoPanicIfCloseInfoNil(t *testing.T) {
|
|
require := require.New(t)
|
|
actual, err := new(Worker).makeFakeSessionCloseInfo(nil)
|
|
require.Nil(actual)
|
|
require.ErrorIs(err, errMakeSessionCloseInfoNilCloseInfo)
|
|
}
|
|
|
|
func TestMakeFakeSessionCloseInfoEmpty(t *testing.T) {
|
|
require := require.New(t)
|
|
actual, err := new(Worker).makeFakeSessionCloseInfo(make(map[string]string))
|
|
require.NoError(err)
|
|
require.Equal(
|
|
make(map[string][]*pbs.CloseConnectionResponseData),
|
|
actual,
|
|
)
|
|
}
|
|
|
|
func TestWorkerSetCloseTimeForResponse(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
sessionCloseInfo map[string][]*pbs.CloseConnectionResponseData
|
|
sessionInfoMap func() *sync.Map
|
|
expected []string
|
|
expectedClosed map[string]struct{}
|
|
expectedErr []error
|
|
}{
|
|
{
|
|
name: "basic",
|
|
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
|
|
"one": {
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
"two": {
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
},
|
|
sessionInfoMap: func() *sync.Map {
|
|
m := new(sync.Map)
|
|
m.Store("one", &sessionInfo{
|
|
id: "one",
|
|
connInfoMap: map[string]*connInfo{
|
|
"foo": {id: "foo"},
|
|
},
|
|
})
|
|
m.Store("two", &sessionInfo{
|
|
id: "two",
|
|
connInfoMap: map[string]*connInfo{
|
|
"bar": {id: "bar"},
|
|
},
|
|
})
|
|
m.Store("three", &sessionInfo{
|
|
id: "three",
|
|
connInfoMap: map[string]*connInfo{
|
|
"baz": {id: "baz"},
|
|
},
|
|
})
|
|
|
|
return m
|
|
},
|
|
expected: []string{"foo", "bar"},
|
|
expectedClosed: map[string]struct{}{
|
|
"foo": {},
|
|
"bar": {},
|
|
},
|
|
},
|
|
{
|
|
name: "not closed",
|
|
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
|
|
"one": {
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
"two": {
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED},
|
|
},
|
|
},
|
|
sessionInfoMap: func() *sync.Map {
|
|
m := new(sync.Map)
|
|
m.Store("one", &sessionInfo{
|
|
id: "one",
|
|
connInfoMap: map[string]*connInfo{
|
|
"foo": {id: "foo"},
|
|
},
|
|
})
|
|
m.Store("two", &sessionInfo{
|
|
id: "two",
|
|
connInfoMap: map[string]*connInfo{
|
|
"bar": {id: "bar"},
|
|
},
|
|
})
|
|
|
|
return m
|
|
},
|
|
expected: []string{"foo"},
|
|
expectedClosed: map[string]struct{}{
|
|
"foo": {},
|
|
},
|
|
},
|
|
{
|
|
name: "missing session",
|
|
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
|
|
"one": {
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
"two": {
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
},
|
|
sessionInfoMap: func() *sync.Map {
|
|
m := new(sync.Map)
|
|
m.Store("one", &sessionInfo{
|
|
id: "one",
|
|
connInfoMap: map[string]*connInfo{
|
|
"foo": {id: "foo"},
|
|
},
|
|
})
|
|
|
|
return m
|
|
},
|
|
expected: []string{"foo"},
|
|
expectedClosed: map[string]struct{}{
|
|
"foo": {},
|
|
},
|
|
expectedErr: []error{
|
|
errors.New(`could not find session ID "two" in local state after closing connections`),
|
|
},
|
|
},
|
|
{
|
|
name: "missing connection",
|
|
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
|
|
"one": {
|
|
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
"two": {
|
|
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
|
|
},
|
|
},
|
|
sessionInfoMap: func() *sync.Map {
|
|
m := new(sync.Map)
|
|
m.Store("one", &sessionInfo{
|
|
id: "one",
|
|
connInfoMap: map[string]*connInfo{
|
|
"foo": {id: "foo"},
|
|
},
|
|
})
|
|
m.Store("two", &sessionInfo{id: "two"})
|
|
|
|
return m
|
|
},
|
|
expected: []string{"foo"},
|
|
expectedClosed: map[string]struct{}{
|
|
"foo": {},
|
|
},
|
|
expectedErr: []error{
|
|
errors.New(`could not find connection ID "bar" for session ID "two" in local state after closing connections`),
|
|
},
|
|
},
|
|
{
|
|
name: "empty",
|
|
sessionCloseInfo: make(map[string][]*pbs.CloseConnectionResponseData),
|
|
sessionInfoMap: func() *sync.Map {
|
|
m := new(sync.Map)
|
|
m.Store("one", &sessionInfo{
|
|
id: "one",
|
|
connInfoMap: map[string]*connInfo{
|
|
"foo": {id: "foo"},
|
|
},
|
|
})
|
|
|
|
return m
|
|
},
|
|
expected: []string{},
|
|
expectedClosed: map[string]struct{}{},
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
require := require.New(t)
|
|
w := &Worker{
|
|
sessionInfoMap: tc.sessionInfoMap(),
|
|
}
|
|
actual, actualErr := w.setCloseTimeForResponse(tc.sessionCloseInfo)
|
|
|
|
// Assert all close times were set
|
|
w.sessionInfoMap.Range(func(key, value interface{}) bool {
|
|
t.Helper()
|
|
for _, ci := range value.(*sessionInfo).connInfoMap {
|
|
if _, ok := tc.expectedClosed[ci.id]; ok {
|
|
require.NotEqual(time.Time{}, ci.closeTime)
|
|
} else {
|
|
require.Equal(time.Time{}, ci.closeTime)
|
|
}
|
|
}
|
|
|
|
return true
|
|
})
|
|
|
|
// Assert return values
|
|
require.ElementsMatch(tc.expected, actual)
|
|
require.ElementsMatch(tc.expectedErr, actualErr)
|
|
})
|
|
}
|
|
}
|