From fd5d15af274f690e48d2c59760421194348c9c74 Mon Sep 17 00:00:00 2001 From: Timothy Messier Date: Tue, 5 Jul 2022 15:25:55 -0400 Subject: [PATCH] feat(config): Add database config options for idle connection management (#2176) This adds two new config options for the `database` stanza: - `max_idle_connections` - `max_idle_time` These correspond to settings for the underlying `sql.DB` instance. They can be used to have finer control over the controller's database connection pool. For example this can be used to mitigate the high connection overhead when connecting to postgres by setting a high `max_idle_connections` and either setting `max_idle_time` to zero or to a high duration. Alternatively, if using a connection pool like pgpool-II or pgbouncer, `max_idle_connections` can be set to 0. See: https://pkg.go.dev/database/sql#DB.SetMaxIdleConns https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime https://www.pgpool.net/mediawiki/index.php/Main_Page https://www.pgbouncer.org/ --- internal/cmd/base/servers.go | 13 +- internal/cmd/commands/server/server.go | 2 + internal/cmd/config/config.go | 51 ++++- internal/cmd/config/config_test.go | 184 ++++++++++++++++++ internal/db/db.go | 16 +- internal/db/db_test.go | 2 +- internal/db/option.go | 39 +++- internal/db/option_test.go | 21 ++ .../migrations/oss/postgres_26_01_test.go | 2 +- .../migrations/oss/postgres_30_01_test.go | 4 +- .../migrations/oss/postgres_32_01_test.go | 2 +- internal/db/testing.go | 2 +- .../content/docs/configuration/controller.mdx | 37 +++- 13 files changed, 344 insertions(+), 31 deletions(-) diff --git a/internal/cmd/base/servers.go b/internal/cmd/base/servers.go index 4ad69d092..32257fcc9 100644 --- a/internal/cmd/base/servers.go +++ b/internal/cmd/base/servers.go @@ -135,9 +135,12 @@ type Server struct { DevOidcSetup oidcSetup - DatabaseUrl string - DatabaseMaxOpenConnections int - DevDatabaseCleanupFunc func() error + DatabaseUrl string + DatabaseMaxOpenConnections int + DatabaseMaxIdleConnections *int + DatabaseConnMaxIdleTimeDuration *time.Duration + + DevDatabaseCleanupFunc func() error Database *db.DB @@ -675,11 +678,13 @@ func (b *Server) ConnectToDatabase(ctx context.Context, dialect string) error { } opts := []db.Option{ db.WithMaxOpenConnections(b.DatabaseMaxOpenConnections), + db.WithMaxIdleConnections(b.DatabaseMaxIdleConnections), + db.WithConnMaxIdleTimeDuration(b.DatabaseConnMaxIdleTimeDuration), } if os.Getenv("BOUNDARY_DISABLE_GORM_FORMATTER") == "" { opts = append(opts, db.WithGormFormatter(b.Logger)) } - dbase, err := db.Open(dbType, b.DatabaseUrl, opts...) + dbase, err := db.Open(ctx, dbType, b.DatabaseUrl, opts...) if err != nil { return fmt.Errorf("unable to create db object with dialect %s: %w", dialect, err) } diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go index d6bb25c49..243772635 100644 --- a/internal/cmd/commands/server/server.go +++ b/internal/cmd/commands/server/server.go @@ -410,6 +410,8 @@ func (c *Command) Run(args []string) int { return base.CommandUserError } c.DatabaseMaxOpenConnections = c.Config.Controller.Database.MaxOpenConnections + c.DatabaseMaxIdleConnections = c.Config.Controller.Database.MaxIdleConnections + c.DatabaseConnMaxIdleTimeDuration = c.Config.Controller.Database.ConnMaxIdleTimeDuration if err := c.ConnectToDatabase(c.Context, "postgres"); err != nil { c.UI.Error(fmt.Errorf("Error connecting to database: %w", err).Error()) diff --git a/internal/cmd/config/config.go b/internal/cmd/config/config.go index 1a59811f7..d2b297d6e 100644 --- a/internal/cmd/config/config.go +++ b/internal/cmd/config/config.go @@ -215,10 +215,14 @@ type Worker struct { } type Database struct { - Url string `hcl:"url"` - MigrationUrl string `hcl:"migration_url"` - MaxOpenConnections int `hcl:"-"` - MaxOpenConnectionsRaw interface{} `hcl:"max_open_connections"` + Url string `hcl:"url"` + MigrationUrl string `hcl:"migration_url"` + MaxOpenConnections int `hcl:"-"` + MaxOpenConnectionsRaw interface{} `hcl:"max_open_connections"` + MaxIdleConnections *int `hcl:"-"` + MaxIdleConnectionsRaw interface{} `hcl:"max_idle_connections"` + ConnMaxIdleTime interface{} `hcl:"max_idle_time"` + ConnMaxIdleTimeDuration *time.Duration `hcl:"-"` } type Plugins struct { @@ -375,7 +379,7 @@ func Parse(d string) (*Config, error) { switch t := result.Controller.Database.MaxOpenConnectionsRaw.(type) { case string: maxOpenConnectionsString, err := parseutil.ParsePath(t) - if err != nil { + if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { return nil, fmt.Errorf("Error parsing database max open connections: %w", err) } result.Controller.Database.MaxOpenConnections, err = strconv.Atoi(maxOpenConnectionsString) @@ -389,6 +393,43 @@ func Parse(d string) (*Config, error) { reflect.TypeOf(t).String()) } } + if result.Controller.Database.MaxIdleConnectionsRaw != nil { + switch t := result.Controller.Database.MaxIdleConnectionsRaw.(type) { + case string: + maxIdleConnectionsString, err := parseutil.ParsePath(t) + if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { + return nil, fmt.Errorf("Error parsing database max idle connections: %w", err) + } + idleConns, err := strconv.Atoi(maxIdleConnectionsString) + if err != nil { + return nil, fmt.Errorf("Database max idle connections value is not a uint: %w", err) + } + result.Controller.Database.MaxIdleConnections = &idleConns + case int: + result.Controller.Database.MaxIdleConnections = &t + default: + return nil, fmt.Errorf("Database max idle connections: unsupported type %q", + reflect.TypeOf(t).String()) + } + } + if result.Controller.Database.ConnMaxIdleTime != nil { + switch t := result.Controller.Database.ConnMaxIdleTime.(type) { + case string: + durationString, err := parseutil.ParsePath(t) + if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) { + return nil, fmt.Errorf("Error parsing connection max idle time: %w", err) + } + connMaxIdleTime, err := parseutil.ParseDurationSecond(durationString) + if err != nil { + return nil, fmt.Errorf("Connection max idle time is not a duration: %w", err) + } + result.Controller.Database.ConnMaxIdleTimeDuration = &connMaxIdleTime + default: + return nil, fmt.Errorf("Database connection max idle time: unsupported type %q", + reflect.TypeOf(t).String()) + } + } + } } diff --git a/internal/cmd/config/config_test.go b/internal/cmd/config/config_test.go index 6afefae26..2e0dc1382 100644 --- a/internal/cmd/config/config_test.go +++ b/internal/cmd/config/config_test.go @@ -1389,6 +1389,18 @@ func TestDatabaseMaxConnections(t *testing.T) { expMaxOpenConnections: 5, expErr: false, }, + { + name: "Valid string value", + in: ` + controller { + name = "example-controller" + database { + max_open_connections = "5" + } + }`, + expMaxOpenConnections: 5, + expErr: false, + }, { name: "Invalid value string", in: ` @@ -1460,3 +1472,175 @@ func TestDatabaseMaxConnections(t *testing.T) { }) } } + +func TestDatabaseMaxIdleConnections(t *testing.T) { + tests := []struct { + name string + in string + envMaxIdleConnections string + expMaxIdleConnections int + expErr bool + expErrStr string + }{ + { + name: "Valid integer value", + in: ` + controller { + name = "example-controller" + database { + max_idle_connections = 5 + } + }`, + expMaxIdleConnections: 5, + expErr: false, + }, + { + name: "Valid integer string", + in: ` + controller { + name = "example-controller" + database { + max_idle_connections = "5" + } + }`, + expMaxIdleConnections: 5, + expErr: false, + }, + { + name: "Invalid value string", + in: ` + controller { + name = "example-controller" + database { + max_idle_connections = "string bad" + } + }`, + expErr: true, + expErrStr: "Database max idle connections value is not a uint: " + + "strconv.Atoi: parsing \"string bad\": invalid syntax", + }, + { + name: "Invalid value type", + in: ` + controller { + name = "example-controller" + database { + max_idle_connections = false + } + }`, + expErr: true, + expErrStr: "Database max idle connections: unsupported type \"bool\"", + }, + { + name: "Valid env var", + in: ` + controller { + name = "example-controller" + database { + max_idle_connections = "env://ENV_MAX_IDLE_CONN" + } + }`, + expMaxIdleConnections: 8, + envMaxIdleConnections: "8", + expErr: false, + }, + { + name: "Invalid env var", + in: ` + controller { + name = "example-controller" + database { + max_idle_connections = "env://ENV_MAX_IDLE_CONN" + } + }`, + envMaxIdleConnections: "bogus value", + expErr: true, + expErrStr: "Database max idle connections value is not a uint: " + + "strconv.Atoi: parsing \"bogus value\": invalid syntax", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("ENV_MAX_IDLE_CONN", tt.envMaxIdleConnections) + c, err := Parse(tt.in) + if tt.expErr { + require.EqualError(t, err, tt.expErrStr) + require.Nil(t, c) + return + } + + require.NoError(t, err) + require.NotNil(t, c) + require.NotNil(t, c.Controller) + require.NotNil(t, c.Controller.Database) + require.Equal(t, tt.expMaxIdleConnections, *c.Controller.Database.MaxIdleConnections) + }) + } +} + +func TestDatabaseConnMaxIdleTimeDuration(t *testing.T) { + tests := []struct { + name string + in string + envConnMaxIdleTimeDuration string + expConnMaxIdleTimeDuration time.Duration + expErr bool + expErrStr string + }{ + { + name: "Valid duration value", + in: ` + controller { + name = "example-controller" + database { + max_idle_time = "5m" + } + }`, + expConnMaxIdleTimeDuration: time.Minute * 5, + expErr: false, + }, + { + name: "Valid env var value", + envConnMaxIdleTimeDuration: "5m", + in: ` + controller { + name = "example-controller" + database { + max_idle_time = "env://ENV_CONN_MAX_IDLE_TIME" + } + }`, + expConnMaxIdleTimeDuration: time.Minute * 5, + expErr: false, + }, + { + name: "Invalid value string", + in: ` + controller { + name = "example-controller" + database { + max_idle_time = "string bad" + } + }`, + expErr: true, + expErrStr: "Connection max idle time is not a duration: " + + "strconv.ParseInt: parsing \"string ba\": invalid syntax", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("ENV_CONN_MAX_IDLE_TIME", tt.envConnMaxIdleTimeDuration) + c, err := Parse(tt.in) + if tt.expErr { + require.EqualError(t, err, tt.expErrStr) + require.Nil(t, c) + return + } + + require.NoError(t, err) + require.NotNil(t, c) + require.NotNil(t, c.Controller) + require.NotNil(t, c.Controller.Database) + require.Equal(t, tt.expConnMaxIdleTimeDuration, *c.Controller.Database.ConnMaxIdleTimeDuration) + }) + } +} diff --git a/internal/db/db.go b/internal/db/db.go index bb387807f..279e488fd 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -77,7 +77,7 @@ func (d *DB) Close(ctx context.Context) error { // Note: Consider if you need to call Close() on the returned DB. Typically the // answer is no, but there are occasions when it's necessary. See the sql.DB // docs for more information. -func Open(dbType DbType, connectionUrl string, opt ...Option) (*DB, error) { +func Open(ctx context.Context, dbType DbType, connectionUrl string, opt ...Option) (*DB, error) { const op = "db.Open" var dialect dbw.Dialector switch dbType { @@ -105,5 +105,19 @@ func Open(dbType DbType, connectionUrl string, opt ...Option) (*DB, error) { if err != nil { return nil, fmt.Errorf("%s: %w", op, err) } + + sdb, err := wrapped.SqlDB(ctx) + if err != nil { + return nil, err + } + + if opts.withMaxIdleConnections != nil { + sdb.SetMaxIdleConns(*opts.withMaxIdleConnections) + } + + if opts.withConnMaxIdleTimeDuration != nil { + sdb.SetConnMaxIdleTime(*opts.withConnMaxIdleTimeDuration) + } + return &DB{wrapped: wrapped}, nil } diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 8af72e482..4db1bf0fc 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -57,7 +57,7 @@ func TestOpen(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := Open(tt.args.dbType, tt.args.connectionUrl) + got, err := Open(ctx, tt.args.dbType, tt.args.connectionUrl) defer func() { if err == nil { sqlDB, err := got.SqlDB(ctx) diff --git a/internal/db/option.go b/internal/db/option.go index 6ef166b8b..399aec53a 100644 --- a/internal/db/option.go +++ b/internal/db/option.go @@ -3,6 +3,7 @@ package db import ( "context" "fmt" + "time" "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/internal/oplog" @@ -166,8 +167,10 @@ type Options struct { // withPrngValues is used to switch the ID generation to a pseudo-random mode withPrngValues []string - withGormFormatter hclog.Logger - withMaxOpenConnections int + withGormFormatter hclog.Logger + withMaxOpenConnections int + withMaxIdleConnections *int + withConnMaxIdleTimeDuration *time.Duration // withDebug indicates that the given operation should invoke Gorm's debug // mode @@ -189,11 +192,13 @@ func getDefaultOptions() Options { wrapper: nil, metadata: oplog.Metadata{}, }, - withLookup: false, - WithFieldMaskPaths: []string{}, - WithNullPaths: []string{}, - WithLimit: 0, - WithVersion: nil, + withLookup: false, + WithFieldMaskPaths: []string{}, + WithNullPaths: []string{}, + WithLimit: 0, + WithVersion: nil, + withMaxIdleConnections: nil, + withConnMaxIdleTimeDuration: nil, } } @@ -306,7 +311,7 @@ func WithGormFormatter(l hclog.Logger) Option { } } -// WithMaxOpenConnections specifices and optional max open connections for the +// WithMaxOpenConnections specifies an optional max open connections for the // database func WithMaxOpenConnections(max int) Option { return func(o *Options) { @@ -314,6 +319,24 @@ func WithMaxOpenConnections(max int) Option { } } +// WithMaxIdleConnections specifies an optional max idle connections for the +// database. +// This corresponds with: https://pkg.go.dev/database/sql#DB.SetMaxIdleConns +func WithMaxIdleConnections(max *int) Option { + return func(o *Options) { + o.withMaxIdleConnections = max + } +} + +// WithConnMaxIdleTimeDuration specifies an optional connection max idle time +// for the database. +// This corresponds with: https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime +func WithConnMaxIdleTimeDuration(max *time.Duration) Option { + return func(o *Options) { + o.withConnMaxIdleTimeDuration = max + } +} + // WithDebug specifies the given operation should invoke debug mode in Gorm func WithDebug(with bool) Option { return func(o *Options) { diff --git a/internal/db/option_test.go b/internal/db/option_test.go index 60b9c62ab..6f2546108 100644 --- a/internal/db/option_test.go +++ b/internal/db/option_test.go @@ -2,6 +2,7 @@ package db import ( "testing" + "time" "github.com/hashicorp/boundary/internal/oplog" "github.com/hashicorp/go-hclog" @@ -192,6 +193,26 @@ func Test_getOpts(t *testing.T) { testOpts.withMaxOpenConnections = 22 assert.Equal(opts, testOpts) }) + t.Run("WithMaxIdleConnections", func(t *testing.T) { + assert := assert.New(t) + opts := GetOpts() + testOpts := getDefaultOptions() + assert.Equal(opts, testOpts) + ic := 22 + opts = GetOpts(WithMaxIdleConnections(&ic)) + testOpts.withMaxIdleConnections = &ic + assert.Equal(opts, testOpts) + }) + t.Run("WithConnMaxIdleTimeDuration", func(t *testing.T) { + assert := assert.New(t) + opts := GetOpts() + testOpts := getDefaultOptions() + assert.Equal(opts, testOpts) + d := time.Minute * 5 + opts = GetOpts(WithConnMaxIdleTimeDuration(&d)) + testOpts.withConnMaxIdleTimeDuration = &d + assert.Equal(opts, testOpts) + }) t.Run("WithDebug", func(t *testing.T) { assert := assert.New(t) // test default of false diff --git a/internal/db/schema/migrations/oss/postgres_26_01_test.go b/internal/db/schema/migrations/oss/postgres_26_01_test.go index c6f1b9f8f..8d6e3b1cf 100644 --- a/internal/db/schema/migrations/oss/postgres_26_01_test.go +++ b/internal/db/schema/migrations/oss/postgres_26_01_test.go @@ -55,7 +55,7 @@ func TestMigrations_WareHouse_HostAddresses(t *testing.T) { // get a connection dbType, err := db.StringToDbType(dialect) require.NoError(t, err) - conn, err := db.Open(dbType, u) + conn, err := db.Open(ctx, dbType, u) require.NoError(t, err) rw := db.New(conn) diff --git a/internal/db/schema/migrations/oss/postgres_30_01_test.go b/internal/db/schema/migrations/oss/postgres_30_01_test.go index 539ffd32e..51bb052b7 100644 --- a/internal/db/schema/migrations/oss/postgres_30_01_test.go +++ b/internal/db/schema/migrations/oss/postgres_30_01_test.go @@ -58,7 +58,7 @@ func TestMigrations_KMS_Refactor(t *testing.T) { // get a connection dbType, err := db.StringToDbType(dialect) require.NoError(t, err) - conn, err := db.Open(dbType, u) + conn, err := db.Open(ctx, dbType, u) require.NoError(t, err) rw := db.New(conn) @@ -97,7 +97,7 @@ func TestMigrations_KMS_Refactor(t *testing.T) { // get a new connection dbType, err := db.StringToDbType(dialect) require.NoError(t, err) - conn, err := db.Open(dbType, u) + conn, err := db.Open(ctx, dbType, u) require.NoError(t, err) rw := db.New(conn) diff --git a/internal/db/schema/migrations/oss/postgres_32_01_test.go b/internal/db/schema/migrations/oss/postgres_32_01_test.go index 45163b102..0cbcc48e0 100644 --- a/internal/db/schema/migrations/oss/postgres_32_01_test.go +++ b/internal/db/schema/migrations/oss/postgres_32_01_test.go @@ -57,7 +57,7 @@ func TestMigrations_user_password_Migration(t *testing.T) { // get a connection dbType, err := db.StringToDbType(dialect) require.NoError(t, err) - conn, err := db.Open(dbType, u) + conn, err := db.Open(ctx, dbType, u) require.NoError(t, err) rw := db.New(conn) diff --git a/internal/db/testing.go b/internal/db/testing.go index 4b058a10d..4ce84583c 100644 --- a/internal/db/testing.go +++ b/internal/db/testing.go @@ -58,7 +58,7 @@ func TestSetup(t testing.TB, dialect string, opt ...TestOption) (*DB, string) { if err != nil { t.Fatal(err) } - db, err := Open(dbType, url) + db, err := Open(ctx, dbType, url) if err != nil { t.Fatal(err) } diff --git a/website/content/docs/configuration/controller.mdx b/website/content/docs/configuration/controller.mdx index fb26fea57..b395a8edf 100644 --- a/website/content/docs/configuration/controller.mdx +++ b/website/content/docs/configuration/controller.mdx @@ -35,18 +35,41 @@ description will be read. Boundary will not be able to connect by default. To run Boundary without a TLS connection to Postgres (not recommended for production usage), add the `sslmode=disable` parameter to your connection string, such as `url = "postgresql://postgres:boundary@192.168.1.1:5432/boundary?sslmode=disable"` + This value can refer to a file on disk (file://) from which a URL will be read; an env + var (env://) from which the URL will be read; or a direct database URL (postgres://). - `migration_url` - Can be used to specify a different URL for migrations, as that usually requires higher privileges. + This value can refer to a file on disk (file://) from which a URL will be read; an env + var (env://) from which the URL will be read; or a direct database URL (postgres://). - `max_open_connections` - Can be used to control the maximum number of connections that can be opened by the controller. The minimum number of connections required is 5. - Setting this value to 0 will allow the controller to open as many - connections as needed. This value can be a string representing the max number of connections, can refer to a file - on disk (file://) from which the number of connections will be read; or an env var (env://) from which the - number of connections will be read. - - Either URL can refer to a file on disk (file://) from which a URL will be read; an env - var (env://) from which the URL will be read; or a direct database URL (postgres://). + Setting this value to 0 will allow the controller to open as many connections as needed. + This value can be a string or an integer representing the max number of connections, + or a string that can refer to a file + on disk (file://) from which the number of connections will be read, + or an env var (env://) from which the number of connections will be read. + - `max_idle_connections` - Can be used to control the maximum number of + idle connections in the idle connection pool. + If `max_open_connections` is greater than 0 but less than + `max_idle_connections`, then `max_idle_connections` will be reduced to match the `max_open_connections` limit. + Setting this value to 0 will mean that no idle connections are retained. + If not set or set to less than 0, the default + [sql.DB](https://pkg.go.dev/database/sql#DB.SetMaxIdleConns) setting will be used. + This value can be a string or an integer representing the max number of connections, + or a string that can refer to a file on disk (file://) from which the number of connections will be read, + or an env var (env://) from which the number of connections will be read. + - `max_idle_time` - Can be used to control the maximum amount of time + a connection may be idle. + Setting this value to 0 will mean that connections are not closed due to a + connections idle time. + If not set or set to less than 0, the default + [sql.DB](https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime) setting will be used. + This value can be a string representing the duration, + or a string that can refer to a file on disk (file://) from which the duration will be read, + or an env var (env://) from which the duration will be read. + Valid time units are anything specified by Golang's + [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. - `public_cluster_addr` - Specifies the public host or IP address (and optionally port) at which the controller can be reached _by workers_. This will