Files
threshold/internal/servers/worker/session_test.go
T
Chris Marchesi 9f9acce972 [Merge to main] internal/servers/worker: Controller failure connection cleanup (#1330) (#1357)
This commit adds some rudimentary functionality for cleaning up
connections on controller failure, or more specifically, a failure to
communicate with the controllers.

In this scenario, when a status cannot be sent for any reason for longer
than 30 seconds, the worker will close all connections that are normally
not cleaned up (so for sessions that currently are *not* in the progress
of being canceled or terminated). Status requests now time out after 10
seconds if they are not made successfully.

Session status is not touched during this process, but natural cleanup
or connectionless sessions will still occur. This should be okay, as the
session should be added on re-connect if need be.

A few other changes are made to cancelConnections: first, error logging
has been moved to the function so that we can handle errors requesting
cancellation directly within the function; this entails simply just
locally marking these connections as closed with a warning. Second,
there was a minor logging bug where closed connections were not properly
being noted in the last trace log, this has been fixed.

Additional changes:

* Circle: Set acceptance test no_output_timeout to 30m
2021-06-28 17:32:13 -07:00

250 lines
6.8 KiB
Go

package worker
import (
"errors"
"sync"
"testing"
"time"
pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
"github.com/hashicorp/boundary/internal/session"
"github.com/stretchr/testify/require"
)
func TestWorkerMakeCloseConnectionRequest(t *testing.T) {
require := require.New(t)
in := map[string]string{"foo": "one", "bar": "two"}
expected := &pbs.CloseConnectionRequest{
CloseRequestData: []*pbs.CloseConnectionRequestData{
{ConnectionId: "foo", Reason: session.UnknownReason.String()},
{ConnectionId: "bar", Reason: session.UnknownReason.String()},
},
}
actual := new(Worker).makeCloseConnectionRequest(in)
require.ElementsMatch(expected.GetCloseRequestData(), actual.GetCloseRequestData())
}
func TestMakeSessionCloseInfo(t *testing.T) {
require := require.New(t)
closeInfo := map[string]string{"foo": "one", "bar": "two"}
response := &pbs.CloseConnectionResponse{
CloseResponseData: []*pbs.CloseConnectionResponseData{
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
}
expected := map[string][]*pbs.CloseConnectionResponseData{
"one": []*pbs.CloseConnectionResponseData{
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
"two": []*pbs.CloseConnectionResponseData{
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
}
actual := new(Worker).makeSessionCloseInfo(closeInfo, response)
require.Equal(expected, actual)
}
func TestMakeSessionCloseInfoPanicIfCloseInfoNil(t *testing.T) {
require := require.New(t)
require.Panics(func() {
new(Worker).makeSessionCloseInfo(nil, nil)
})
}
func TestMakeSessionCloseInfoEmpty(t *testing.T) {
require := require.New(t)
require.Equal(
make(map[string][]*pbs.CloseConnectionResponseData),
new(Worker).makeSessionCloseInfo(make(map[string]string), nil),
)
}
func TestWorkerSetCloseTimeForResponse(t *testing.T) {
cases := []struct {
name string
sessionCloseInfo map[string][]*pbs.CloseConnectionResponseData
sessionInfoMap func() *sync.Map
expected []string
expectedClosed map[string]struct{}
expectedErr []error
}{
{
name: "basic",
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
"one": []*pbs.CloseConnectionResponseData{
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
"two": []*pbs.CloseConnectionResponseData{
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
},
sessionInfoMap: func() *sync.Map {
m := new(sync.Map)
m.Store("one", &sessionInfo{
id: "one",
connInfoMap: map[string]*connInfo{
"foo": &connInfo{id: "foo"},
},
})
m.Store("two", &sessionInfo{
id: "two",
connInfoMap: map[string]*connInfo{
"bar": &connInfo{id: "bar"},
},
})
m.Store("three", &sessionInfo{
id: "three",
connInfoMap: map[string]*connInfo{
"baz": &connInfo{id: "baz"},
},
})
return m
},
expected: []string{"foo", "bar"},
expectedClosed: map[string]struct{}{
"foo": struct{}{},
"bar": struct{}{},
},
},
{
name: "not closed",
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
"one": []*pbs.CloseConnectionResponseData{
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
"two": []*pbs.CloseConnectionResponseData{
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CONNECTED},
},
},
sessionInfoMap: func() *sync.Map {
m := new(sync.Map)
m.Store("one", &sessionInfo{
id: "one",
connInfoMap: map[string]*connInfo{
"foo": &connInfo{id: "foo"},
},
})
m.Store("two", &sessionInfo{
id: "two",
connInfoMap: map[string]*connInfo{
"bar": &connInfo{id: "bar"},
},
})
return m
},
expected: []string{"foo"},
expectedClosed: map[string]struct{}{
"foo": struct{}{},
},
},
{
name: "missing session",
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
"one": []*pbs.CloseConnectionResponseData{
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
"two": []*pbs.CloseConnectionResponseData{
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
},
sessionInfoMap: func() *sync.Map {
m := new(sync.Map)
m.Store("one", &sessionInfo{
id: "one",
connInfoMap: map[string]*connInfo{
"foo": &connInfo{id: "foo"},
},
})
return m
},
expected: []string{"foo"},
expectedClosed: map[string]struct{}{
"foo": struct{}{},
},
expectedErr: []error{
errors.New(`could not find session ID "two" in local state after closing connections`),
},
},
{
name: "missing connection",
sessionCloseInfo: map[string][]*pbs.CloseConnectionResponseData{
"one": []*pbs.CloseConnectionResponseData{
{ConnectionId: "foo", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
"two": []*pbs.CloseConnectionResponseData{
{ConnectionId: "bar", Status: pbs.CONNECTIONSTATUS_CONNECTIONSTATUS_CLOSED},
},
},
sessionInfoMap: func() *sync.Map {
m := new(sync.Map)
m.Store("one", &sessionInfo{
id: "one",
connInfoMap: map[string]*connInfo{
"foo": &connInfo{id: "foo"},
},
})
m.Store("two", &sessionInfo{id: "two"})
return m
},
expected: []string{"foo"},
expectedClosed: map[string]struct{}{
"foo": struct{}{},
},
expectedErr: []error{
errors.New(`could not find connection ID "bar" for session ID "two" in local state after closing connections`),
},
},
{
name: "empty",
sessionCloseInfo: make(map[string][]*pbs.CloseConnectionResponseData),
sessionInfoMap: func() *sync.Map {
m := new(sync.Map)
m.Store("one", &sessionInfo{
id: "one",
connInfoMap: map[string]*connInfo{
"foo": &connInfo{id: "foo"},
},
})
return m
},
expected: []string{},
expectedClosed: map[string]struct{}{},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)
w := &Worker{
sessionInfoMap: tc.sessionInfoMap(),
}
actual, actualErr := w.setCloseTimeForResponse(tc.sessionCloseInfo)
// Assert all close times were set
w.sessionInfoMap.Range(func(key, value interface{}) bool {
t.Helper()
for _, ci := range value.(*sessionInfo).connInfoMap {
if _, ok := tc.expectedClosed[ci.id]; ok {
require.NotEqual(time.Time{}, ci.closeTime)
} else {
require.Equal(time.Time{}, ci.closeTime)
}
}
return true
})
// Assert return values
require.ElementsMatch(tc.expected, actual)
require.ElementsMatch(tc.expectedErr, actualErr)
})
}
}