feat(config): Add database config options for idle connection management (#2176)

This adds two new config options for the `database` stanza:

- `max_idle_connections`
- `max_idle_time`

These correspond to settings for the underlying `sql.DB` instance. They
can be used to have finer control over the controller's database
connection pool. For example this can be used to mitigate the high
connection overhead when connecting to postgres by setting a high
`max_idle_connections` and either setting `max_idle_time` to zero or to
a high duration.

Alternatively, if using a connection pool like pgpool-II or pgbouncer,
`max_idle_connections` can be set to 0.

See:
    https://pkg.go.dev/database/sql#DB.SetMaxIdleConns
    https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime
    https://www.pgpool.net/mediawiki/index.php/Main_Page
    https://www.pgbouncer.org/

internal/cmd/base/servers.go

@@ -135,9 +135,12 @@ type Server struct {
 
 	DevOidcSetup oidcSetup
 
-	DatabaseUrl                string
-	DatabaseMaxOpenConnections int
-	DevDatabaseCleanupFunc     func() error
+	DatabaseUrl                     string
+	DatabaseMaxOpenConnections      int
+	DatabaseMaxIdleConnections      *int
+	DatabaseConnMaxIdleTimeDuration *time.Duration
+
+	DevDatabaseCleanupFunc func() error
 
 	Database *db.DB
 
@@ -675,11 +678,13 @@ func (b *Server) ConnectToDatabase(ctx context.Context, dialect string) error {
 	}
 	opts := []db.Option{
 		db.WithMaxOpenConnections(b.DatabaseMaxOpenConnections),
+		db.WithMaxIdleConnections(b.DatabaseMaxIdleConnections),
+		db.WithConnMaxIdleTimeDuration(b.DatabaseConnMaxIdleTimeDuration),
 	}
 	if os.Getenv("BOUNDARY_DISABLE_GORM_FORMATTER") == "" {
 		opts = append(opts, db.WithGormFormatter(b.Logger))
 	}
-	dbase, err := db.Open(dbType, b.DatabaseUrl, opts...)
+	dbase, err := db.Open(ctx, dbType, b.DatabaseUrl, opts...)
 	if err != nil {
 		return fmt.Errorf("unable to create db object with dialect %s: %w", dialect, err)
 	}

internal/cmd/commands/server/server.go

@@ -410,6 +410,8 @@ func (c *Command) Run(args []string) int {
 			return base.CommandUserError
 		}
 		c.DatabaseMaxOpenConnections = c.Config.Controller.Database.MaxOpenConnections
+		c.DatabaseMaxIdleConnections = c.Config.Controller.Database.MaxIdleConnections
+		c.DatabaseConnMaxIdleTimeDuration = c.Config.Controller.Database.ConnMaxIdleTimeDuration
 
 		if err := c.ConnectToDatabase(c.Context, "postgres"); err != nil {
 			c.UI.Error(fmt.Errorf("Error connecting to database: %w", err).Error())

internal/cmd/config/config.go

@@ -215,10 +215,14 @@ type Worker struct {
 }
 
 type Database struct {
-	Url                   string      `hcl:"url"`
-	MigrationUrl          string      `hcl:"migration_url"`
-	MaxOpenConnections    int         `hcl:"-"`
-	MaxOpenConnectionsRaw interface{} `hcl:"max_open_connections"`
+	Url                     string         `hcl:"url"`
+	MigrationUrl            string         `hcl:"migration_url"`
+	MaxOpenConnections      int            `hcl:"-"`
+	MaxOpenConnectionsRaw   interface{}    `hcl:"max_open_connections"`
+	MaxIdleConnections      *int           `hcl:"-"`
+	MaxIdleConnectionsRaw   interface{}    `hcl:"max_idle_connections"`
+	ConnMaxIdleTime         interface{}    `hcl:"max_idle_time"`
+	ConnMaxIdleTimeDuration *time.Duration `hcl:"-"`
 }
 
 type Plugins struct {
@@ -375,7 +379,7 @@ func Parse(d string) (*Config, error) {
 				switch t := result.Controller.Database.MaxOpenConnectionsRaw.(type) {
 				case string:
 					maxOpenConnectionsString, err := parseutil.ParsePath(t)
-					if err != nil {
+					if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
 						return nil, fmt.Errorf("Error parsing database max open connections: %w", err)
 					}
 					result.Controller.Database.MaxOpenConnections, err = strconv.Atoi(maxOpenConnectionsString)
@@ -389,6 +393,43 @@ func Parse(d string) (*Config, error) {
 						reflect.TypeOf(t).String())
 				}
 			}
+			if result.Controller.Database.MaxIdleConnectionsRaw != nil {
+				switch t := result.Controller.Database.MaxIdleConnectionsRaw.(type) {
+				case string:
+					maxIdleConnectionsString, err := parseutil.ParsePath(t)
+					if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
+						return nil, fmt.Errorf("Error parsing database max idle connections: %w", err)
+					}
+					idleConns, err := strconv.Atoi(maxIdleConnectionsString)
+					if err != nil {
+						return nil, fmt.Errorf("Database max idle connections value is not a uint: %w", err)
+					}
+					result.Controller.Database.MaxIdleConnections = &idleConns
+				case int:
+					result.Controller.Database.MaxIdleConnections = &t
+				default:
+					return nil, fmt.Errorf("Database max idle connections: unsupported type %q",
+						reflect.TypeOf(t).String())
+				}
+			}
+			if result.Controller.Database.ConnMaxIdleTime != nil {
+				switch t := result.Controller.Database.ConnMaxIdleTime.(type) {
+				case string:
+					durationString, err := parseutil.ParsePath(t)
+					if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
+						return nil, fmt.Errorf("Error parsing connection max idle time: %w", err)
+					}
+					connMaxIdleTime, err := parseutil.ParseDurationSecond(durationString)
+					if err != nil {
+						return nil, fmt.Errorf("Connection max idle time is not a duration: %w", err)
+					}
+					result.Controller.Database.ConnMaxIdleTimeDuration = &connMaxIdleTime
+				default:
+					return nil, fmt.Errorf("Database connection max idle time: unsupported type %q",
+						reflect.TypeOf(t).String())
+				}
+			}
+
 		}
 	}
 

internal/cmd/config/config_test.go

@@ -1389,6 +1389,18 @@ func TestDatabaseMaxConnections(t *testing.T) {
 			expMaxOpenConnections: 5,
 			expErr:                false,
 		},
+		{
+			name: "Valid string value",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_open_connections = "5"
+			  	}
+			}`,
+			expMaxOpenConnections: 5,
+			expErr:                false,
+		},
 		{
 			name: "Invalid value string",
 			in: `
@@ -1460,3 +1472,175 @@ func TestDatabaseMaxConnections(t *testing.T) {
 		})
 	}
 }
+
+func TestDatabaseMaxIdleConnections(t *testing.T) {
+	tests := []struct {
+		name                  string
+		in                    string
+		envMaxIdleConnections string
+		expMaxIdleConnections int
+		expErr                bool
+		expErrStr             string
+	}{
+		{
+			name: "Valid integer value",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_connections = 5
+			  	}
+			}`,
+			expMaxIdleConnections: 5,
+			expErr:                false,
+		},
+		{
+			name: "Valid integer string",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_connections = "5"
+			  	}
+			}`,
+			expMaxIdleConnections: 5,
+			expErr:                false,
+		},
+		{
+			name: "Invalid value string",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_connections = "string bad"
+				}
+			}`,
+			expErr: true,
+			expErrStr: "Database max idle connections value is not a uint: " +
+				"strconv.Atoi: parsing \"string bad\": invalid syntax",
+		},
+		{
+			name: "Invalid value type",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_connections = false
+				}
+			}`,
+			expErr:    true,
+			expErrStr: "Database max idle connections: unsupported type \"bool\"",
+		},
+		{
+			name: "Valid env var",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_connections = "env://ENV_MAX_IDLE_CONN"
+			  	}
+			}`,
+			expMaxIdleConnections: 8,
+			envMaxIdleConnections: "8",
+			expErr:                false,
+		},
+		{
+			name: "Invalid env var",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_connections = "env://ENV_MAX_IDLE_CONN"
+			  	}
+			}`,
+			envMaxIdleConnections: "bogus value",
+			expErr:                true,
+			expErrStr: "Database max idle connections value is not a uint: " +
+				"strconv.Atoi: parsing \"bogus value\": invalid syntax",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv("ENV_MAX_IDLE_CONN", tt.envMaxIdleConnections)
+			c, err := Parse(tt.in)
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrStr)
+				require.Nil(t, c)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, c)
+			require.NotNil(t, c.Controller)
+			require.NotNil(t, c.Controller.Database)
+			require.Equal(t, tt.expMaxIdleConnections, *c.Controller.Database.MaxIdleConnections)
+		})
+	}
+}
+
+func TestDatabaseConnMaxIdleTimeDuration(t *testing.T) {
+	tests := []struct {
+		name                       string
+		in                         string
+		envConnMaxIdleTimeDuration string
+		expConnMaxIdleTimeDuration time.Duration
+		expErr                     bool
+		expErrStr                  string
+	}{
+		{
+			name: "Valid duration value",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_time = "5m"
+			  	}
+			}`,
+			expConnMaxIdleTimeDuration: time.Minute * 5,
+			expErr:                     false,
+		},
+		{
+			name:                       "Valid env var value",
+			envConnMaxIdleTimeDuration: "5m",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_time = "env://ENV_CONN_MAX_IDLE_TIME"
+			  	}
+			}`,
+			expConnMaxIdleTimeDuration: time.Minute * 5,
+			expErr:                     false,
+		},
+		{
+			name: "Invalid value string",
+			in: `
+			controller {
+				name = "example-controller"
+				database {
+					max_idle_time = "string bad"
+				}
+			}`,
+			expErr: true,
+			expErrStr: "Connection max idle time is not a duration: " +
+				"strconv.ParseInt: parsing \"string ba\": invalid syntax",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv("ENV_CONN_MAX_IDLE_TIME", tt.envConnMaxIdleTimeDuration)
+			c, err := Parse(tt.in)
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrStr)
+				require.Nil(t, c)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, c)
+			require.NotNil(t, c.Controller)
+			require.NotNil(t, c.Controller.Database)
+			require.Equal(t, tt.expConnMaxIdleTimeDuration, *c.Controller.Database.ConnMaxIdleTimeDuration)
+		})
+	}
+}

internal/db/db.go

@@ -77,7 +77,7 @@ func (d *DB) Close(ctx context.Context) error {
 // Note: Consider if you need to call Close() on the returned DB.  Typically the
 // answer is no, but there are occasions when it's necessary.  See the sql.DB
 // docs for more information.
-func Open(dbType DbType, connectionUrl string, opt ...Option) (*DB, error) {
+func Open(ctx context.Context, dbType DbType, connectionUrl string, opt ...Option) (*DB, error) {
 	const op = "db.Open"
 	var dialect dbw.Dialector
 	switch dbType {
@@ -105,5 +105,19 @@ func Open(dbType DbType, connectionUrl string, opt ...Option) (*DB, error) {
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", op, err)
 	}
+
+	sdb, err := wrapped.SqlDB(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if opts.withMaxIdleConnections != nil {
+		sdb.SetMaxIdleConns(*opts.withMaxIdleConnections)
+	}
+
+	if opts.withConnMaxIdleTimeDuration != nil {
+		sdb.SetConnMaxIdleTime(*opts.withConnMaxIdleTimeDuration)
+	}
+
 	return &DB{wrapped: wrapped}, nil
 }

internal/db/db_test.go

@@ -57,7 +57,7 @@ func TestOpen(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := Open(tt.args.dbType, tt.args.connectionUrl)
+			got, err := Open(ctx, tt.args.dbType, tt.args.connectionUrl)
 			defer func() {
 				if err == nil {
 					sqlDB, err := got.SqlDB(ctx)

internal/db/option.go

@@ -3,6 +3,7 @@ package db
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/oplog"
@@ -166,8 +167,10 @@ type Options struct {
 	// withPrngValues is used to switch the ID generation to a pseudo-random mode
 	withPrngValues []string
 
-	withGormFormatter      hclog.Logger
-	withMaxOpenConnections int
+	withGormFormatter           hclog.Logger
+	withMaxOpenConnections      int
+	withMaxIdleConnections      *int
+	withConnMaxIdleTimeDuration *time.Duration
 
 	// withDebug indicates that the given operation should invoke Gorm's debug
 	// mode
@@ -189,11 +192,13 @@ func getDefaultOptions() Options {
 			wrapper:  nil,
 			metadata: oplog.Metadata{},
 		},
-		withLookup:         false,
-		WithFieldMaskPaths: []string{},
-		WithNullPaths:      []string{},
-		WithLimit:          0,
-		WithVersion:        nil,
+		withLookup:                  false,
+		WithFieldMaskPaths:          []string{},
+		WithNullPaths:               []string{},
+		WithLimit:                   0,
+		WithVersion:                 nil,
+		withMaxIdleConnections:      nil,
+		withConnMaxIdleTimeDuration: nil,
 	}
 }
 
@@ -306,7 +311,7 @@ func WithGormFormatter(l hclog.Logger) Option {
 	}
 }
 
-// WithMaxOpenConnections specifices and optional max open connections for the
+// WithMaxOpenConnections specifies an optional max open connections for the
 // database
 func WithMaxOpenConnections(max int) Option {
 	return func(o *Options) {
@@ -314,6 +319,24 @@ func WithMaxOpenConnections(max int) Option {
 	}
 }
 
+// WithMaxIdleConnections specifies an optional max idle connections for the
+// database.
+// This corresponds with: https://pkg.go.dev/database/sql#DB.SetMaxIdleConns
+func WithMaxIdleConnections(max *int) Option {
+	return func(o *Options) {
+		o.withMaxIdleConnections = max
+	}
+}
+
+// WithConnMaxIdleTimeDuration specifies an optional connection max idle time
+// for the database.
+// This corresponds with: https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime
+func WithConnMaxIdleTimeDuration(max *time.Duration) Option {
+	return func(o *Options) {
+		o.withConnMaxIdleTimeDuration = max
+	}
+}
+
 // WithDebug specifies the given operation should invoke debug mode in Gorm
 func WithDebug(with bool) Option {
 	return func(o *Options) {

internal/db/option_test.go

@@ -2,6 +2,7 @@ package db
 
 import (
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/internal/oplog"
 	"github.com/hashicorp/go-hclog"
@@ -192,6 +193,26 @@ func Test_getOpts(t *testing.T) {
 		testOpts.withMaxOpenConnections = 22
 		assert.Equal(opts, testOpts)
 	})
+	t.Run("WithMaxIdleConnections", func(t *testing.T) {
+		assert := assert.New(t)
+		opts := GetOpts()
+		testOpts := getDefaultOptions()
+		assert.Equal(opts, testOpts)
+		ic := 22
+		opts = GetOpts(WithMaxIdleConnections(&ic))
+		testOpts.withMaxIdleConnections = &ic
+		assert.Equal(opts, testOpts)
+	})
+	t.Run("WithConnMaxIdleTimeDuration", func(t *testing.T) {
+		assert := assert.New(t)
+		opts := GetOpts()
+		testOpts := getDefaultOptions()
+		assert.Equal(opts, testOpts)
+		d := time.Minute * 5
+		opts = GetOpts(WithConnMaxIdleTimeDuration(&d))
+		testOpts.withConnMaxIdleTimeDuration = &d
+		assert.Equal(opts, testOpts)
+	})
 	t.Run("WithDebug", func(t *testing.T) {
 		assert := assert.New(t)
 		// test default of false

internal/db/schema/migrations/oss/postgres_26_01_test.go

@@ -55,7 +55,7 @@ func TestMigrations_WareHouse_HostAddresses(t *testing.T) {
 	// get a connection
 	dbType, err := db.StringToDbType(dialect)
 	require.NoError(t, err)
-	conn, err := db.Open(dbType, u)
+	conn, err := db.Open(ctx, dbType, u)
 	require.NoError(t, err)
 	rw := db.New(conn)
 

internal/db/schema/migrations/oss/postgres_30_01_test.go

@@ -58,7 +58,7 @@ func TestMigrations_KMS_Refactor(t *testing.T) {
 	// get a connection
 	dbType, err := db.StringToDbType(dialect)
 	require.NoError(t, err)
-	conn, err := db.Open(dbType, u)
+	conn, err := db.Open(ctx, dbType, u)
 	require.NoError(t, err)
 	rw := db.New(conn)
 
@@ -97,7 +97,7 @@ func TestMigrations_KMS_Refactor(t *testing.T) {
 		// get a new connection
 		dbType, err := db.StringToDbType(dialect)
 		require.NoError(t, err)
-		conn, err := db.Open(dbType, u)
+		conn, err := db.Open(ctx, dbType, u)
 		require.NoError(t, err)
 		rw := db.New(conn)
 

internal/db/schema/migrations/oss/postgres_32_01_test.go

@@ -57,7 +57,7 @@ func TestMigrations_user_password_Migration(t *testing.T) {
 	// get a connection
 	dbType, err := db.StringToDbType(dialect)
 	require.NoError(t, err)
-	conn, err := db.Open(dbType, u)
+	conn, err := db.Open(ctx, dbType, u)
 	require.NoError(t, err)
 	rw := db.New(conn)
 

internal/db/testing.go

@@ -58,7 +58,7 @@ func TestSetup(t testing.TB, dialect string, opt ...TestOption) (*DB, string) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	db, err := Open(dbType, url)
+	db, err := Open(ctx, dbType, url)
 	if err != nil {
 		t.Fatal(err)
 	}

website/content/docs/configuration/controller.mdx

@@ -35,18 +35,41 @@ description will be read.
     Boundary will not be able to connect by default. To run Boundary without a TLS connection
     to Postgres (not recommended for production usage), add the `sslmode=disable` parameter to
     your connection string, such as `url = "postgresql://postgres:boundary@192.168.1.1:5432/boundary?sslmode=disable"`
+    This value can refer to a file on disk (file://) from which a URL will be read; an env
+    var (env://) from which the URL will be read; or a direct database URL (postgres://).
   - `migration_url` - Can be used to specify a different URL for migrations, as that
     usually requires higher privileges.
+    This value can refer to a file on disk (file://) from which a URL will be read; an env
+    var (env://) from which the URL will be read; or a direct database URL (postgres://).
   - `max_open_connections` - Can be used to control the maximum number of
     connections that can be opened by the controller.
     The minimum number of connections required is 5.
-    Setting this value to 0 will allow the controller to open as many
-    connections as needed. This value can be a string representing the max number of connections, can refer to a file
-    on disk (file://) from which the number of connections will be read; or an env var (env://) from which the
-    number of connections will be read.
-
-  Either URL can refer to a file on disk (file://) from which a URL will be read; an env
-  var (env://) from which the URL will be read; or a direct database URL (postgres://).
+    Setting this value to 0 will allow the controller to open as many connections as needed.
+    This value can be a string or an integer representing the max number of connections,
+    or a string that can refer to a file
+    on disk (file://) from which the number of connections will be read,
+    or an env var (env://) from which the number of connections will be read.
+  - `max_idle_connections` - Can be used to control the maximum number of
+    idle connections in the idle connection pool.
+    If `max_open_connections` is greater than 0 but less than
+    `max_idle_connections`, then `max_idle_connections` will be reduced to match the `max_open_connections` limit. 
+    Setting this value to 0 will mean that no idle connections are retained.
+    If not set or set to less than 0, the default
+    [sql.DB](https://pkg.go.dev/database/sql#DB.SetMaxIdleConns) setting will be used.
+    This value can be a string or an integer representing the max number of connections,
+    or a string that can refer to a file on disk (file://) from which the number of connections will be read,
+    or an env var (env://) from which the number of connections will be read.
+  - `max_idle_time` - Can be used to control the maximum amount of time
+    a connection may be idle.
+    Setting this value to 0 will mean that connections are not closed due to a
+    connections idle time.
+    If not set or set to less than 0, the default
+    [sql.DB](https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime) setting will be used.
+    This value can be a string representing the duration,
+    or a string that can refer to a file on disk (file://) from which the duration will be read,
+    or an env var (env://) from which the duration will be read.
+    Valid time units are anything specified by Golang's
+    [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method.
 
 - `public_cluster_addr` - Specifies the public host or IP address (and
   optionally port) at which the controller can be reached _by workers_. This will