This commit adds some rudimentary functionality for cleaning up connections on controller failure, or more specifically, a failure to communicate with the controllers. In this scenario, when a status cannot be sent for any reason for longer than 30 seconds, the worker will close all connections that are normally not cleaned up (so for sessions that currently are *not* in the progress of being canceled or terminated). Status requests now time out after 10 seconds if they are not made successfully. Session status is not touched during this process, but natural cleanup or connectionless sessions will still occur. This should be okay, as the session should be added on re-connect if need be. A few other changes are made to cancelConnections: first, error logging has been moved to the function so that we can handle errors requesting cancellation directly within the function; this entails simply just locally marking these connections as closed with a warning. Second, there was a minor logging bug where closed connections were not properly being noted in the last trace log, this has been fixed. Additional changes: * Circle: Set acceptance test no_output_timeout to 30m
This commit is contained in:
1 parent
2b4155d01e
commit
9f9acce972
10 files changed
+1008
-45
No files matched your search
@@ -367,6 +367,10 @@ type TestControllerOpts struct {
|
||||
|
||||
// The logger to use, or one will be created
|
||||
Logger hclog.Logger
|
||||
|
||||
// A cluster address for overriding the advertised controller listener
|
||||
// (overrides address provided in config, if any)
|
||||
PublicClusterAddr string
|
||||
}
|
||||
|
||||
func NewTestController(t *testing.T, opts *TestControllerOpts) *TestController {
|
||||
@@ -503,6 +507,11 @@ func NewTestController(t *testing.T, opts *TestControllerOpts) *TestController {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set cluster address if we supplied one (overrides one in config)
|
||||
if opts.PublicClusterAddr != "" {
|
||||
opts.Config.Controller.PublicClusterAddr = opts.PublicClusterAddr
|
||||
}
|
||||
|
||||
if opts.DatabaseUrl != "" {
|
||||
tc.b.DatabaseUrl = opts.DatabaseUrl
|
||||
if _, err := schema.MigrateStore(ctx, "postgres", tc.b.DatabaseUrl); err != nil {
|
||||
@@ -600,6 +609,7 @@ func (tc *TestController) AddClusterControllerMember(t *testing.T, opts *TestCon
|
||||
DefaultPassword: tc.b.DevPassword,
|
||||
DisableKmsKeyCreation: true,
|
||||
DisableAuthMethodCreation: true,
|
||||
PublicClusterAddr: opts.PublicClusterAddr,
|
||||
}
|
||||
if opts.Logger != nil {
|
||||
nextOpts.Logger = opts.Logger
|
||||
|
||||
Reference in new issue
Block a user