Restrict concurrent password hashes (#5437)

* internal/auth/password: add resizable pool

The resizable pool wraps the permit pool with a mutex
to allow it to be resized at runtime.

* internal/auth/password: add concurrency limit

The default concurrency limit is 1, and it can be
set using the SetHashingPermits function.

* internal/cmd/config: allow configuring hashing limits

The new concurrent_password_hash_workers configuration
value can be used to loosen the constraints on the
userpass auth method hashing operation.

* internal/cmd: set password hashing constraints

Sets the password hashing concurrency constraints on
startup and config reload.

* CHANGELOG: add notice about password concurrency limit

internal/cmd/commands/server/server.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/globals"
+	"github.com/hashicorp/boundary/internal/auth/password"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/cmd/ops"
@@ -870,6 +871,10 @@ func (c *Command) Reload(newConf *config.Config) error {
 		}
 	}
 
+	if newConf != nil && newConf.Controller != nil && newConf.Controller.ConcurrentPasswordHashWorkers > 0 {
+		reloadErrors = stderrors.Join(reloadErrors, password.SetHashingPermits(int(newConf.Controller.ConcurrentPasswordHashWorkers)))
+	}
+
 	// Send a message that we reloaded. This prevents "guessing" sleep times
 	// in tests.
 	if c.reloadedCh != nil {

internal/cmd/config/config.go

@@ -317,6 +317,14 @@ type Controller struct {
 	// it is rejected by the controller.
 	MaxPageSizeRaw any  `hcl:"max_page_size"`
 	MaxPageSize    uint `hcl:"-"`
+
+	// ConcurrentPasswordHashWorkers controls the number of concurrent password
+	// hashing workers is allowed. The default value is 1. Increasing this number
+	// will increase the authentication throughput of all userpass auth methods,
+	// but at the cost of bursty memory and CPU use. Can also be controlled via
+	// the environment variable BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS.
+	ConcurrentPasswordHashWorkersRaw any  `hcl:"concurrent_password_hash_workers"`
+	ConcurrentPasswordHashWorkers    uint `hcl:"-"`
 }
 
 func (c *Controller) InitNameIfEmpty(ctx context.Context) error {
@@ -871,6 +879,37 @@ func Parse(d string) (*Config, error) {
 		if result.Controller.ApiRateLimiterMaxQuotas <= 0 {
 			result.Controller.ApiRateLimiterMaxQuotas = ratelimit.DefaultLimiterMaxQuotas()
 		}
+
+		switch t := result.Controller.ConcurrentPasswordHashWorkersRaw.(type) {
+		case string:
+			concurrentPasswordWorkersString, err := parseutil.ParsePath(t)
+			if err != nil && !errors.Is(err, parseutil.ErrNotAUrl) {
+				return nil, fmt.Errorf("Error parsing concurrent password hash workers: %w", err)
+			}
+			concurrentWorkers, err := strconv.Atoi(concurrentPasswordWorkersString)
+			if err != nil {
+				return nil, fmt.Errorf("Concurrent password hash workers value is not an int: %w", err)
+			}
+			if concurrentWorkers <= 0 {
+				return nil, fmt.Errorf("Concurrent password hash workers value must be at least 1, was %d", concurrentWorkers)
+			}
+			result.Controller.ConcurrentPasswordHashWorkers = uint(concurrentWorkers)
+		case int:
+			if t <= 0 {
+				return nil, fmt.Errorf("Concurrent password hash workers value must be at least 1, was %d", t)
+			}
+			result.Controller.ConcurrentPasswordHashWorkers = uint(t)
+		case nil:
+			if envVal := os.Getenv("BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS"); envVal != "" {
+				concurrentPasswordWorkers, err := strconv.Atoi(envVal)
+				if err != nil {
+					return nil, fmt.Errorf("BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS value is not an int: %w", err)
+				}
+				result.Controller.ConcurrentPasswordHashWorkers = uint(concurrentPasswordWorkers)
+			}
+		default:
+			return nil, fmt.Errorf("Concurrent password hash workers: unsupported type %q", reflect.TypeOf(t).String())
+		}
 	}
 
 	// Parse worker tags

internal/cmd/config/config_test.go

@@ -3348,7 +3348,7 @@ func TestMaxPageSize(t *testing.T) {
 			in: `
 			controller {
 				name = "example-controller"
-				max_page_size = "env://ENV_MAX_CONN"
+				max_page_size = "env://ENV_MAX_PAGE_SIZE"
 			}`,
 			expMaxPageSize: 8,
 			envMaxPageSize: "8",
@@ -3359,7 +3359,7 @@ func TestMaxPageSize(t *testing.T) {
 			in: `
 			controller {
 				name = "example-controller"
-				max_page_size = "env://ENV_MAX_CONN"
+				max_page_size = "env://ENV_MAX_PAGE_SIZE"
 			}`,
 			envMaxPageSize: "bogus value",
 			expErr:         true,
@@ -3369,7 +3369,7 @@ func TestMaxPageSize(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			t.Setenv("ENV_MAX_CONN", tt.envMaxPageSize)
+			t.Setenv("ENV_MAX_PAGE_SIZE", tt.envMaxPageSize)
 			c, err := Parse(tt.in)
 			if tt.expErr {
 				require.EqualError(t, err, tt.expErrStr)
@@ -3384,3 +3384,154 @@ func TestMaxPageSize(t *testing.T) {
 		})
 	}
 }
+
+func TestConcurrentPasswordHashWorkers(t *testing.T) {
+	tests := []struct {
+		name                             string
+		in                               string
+		envConcurrentPasswordHashWorkers string
+		expConcurrentPasswordHashWorkers uint
+		expErr                           bool
+		expErrStr                        string
+	}{
+		{
+			name: "Valid integer value",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = 5
+			}`,
+			expConcurrentPasswordHashWorkers: 5,
+			expErr:                           false,
+		},
+		{
+			name: "Valid string value",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = "5"
+			}`,
+			expConcurrentPasswordHashWorkers: 5,
+			expErr:                           false,
+		},
+		{
+			name: "Invalid value integer",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = 0
+			}`,
+			expErr:    true,
+			expErrStr: "Concurrent password hash workers value must be at least 1, was 0",
+		},
+		{
+			name: "Invalid value string",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = "string bad"
+			}`,
+			expErr: true,
+			expErrStr: "Concurrent password hash workers value is not an int: " +
+				"strconv.Atoi: parsing \"string bad\": invalid syntax",
+		},
+		{
+			name: "Invalid value string integer",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = "-1"
+			}`,
+			expErr:    true,
+			expErrStr: "Concurrent password hash workers value must be at least 1, was -1",
+		},
+		{
+			name: "Invalid value type",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = false
+			}`,
+			expErr:    true,
+			expErrStr: "Concurrent password hash workers: unsupported type \"bool\"",
+		},
+		{
+			name: "Valid env var",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = "env://ENV_MAX_PW_WORKERS"
+			}`,
+			expConcurrentPasswordHashWorkers: 8,
+			envConcurrentPasswordHashWorkers: "8",
+			expErr:                           false,
+		},
+		{
+			name: "Invalid env var",
+			in: `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = "env://ENV_MAX_PW_WORKERS"
+			}`,
+			envConcurrentPasswordHashWorkers: "bogus value",
+			expErr:                           true,
+			expErrStr: "Concurrent password hash workers value is not an int: " +
+				"strconv.Atoi: parsing \"bogus value\": invalid syntax",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv("ENV_MAX_PW_WORKERS", tt.envConcurrentPasswordHashWorkers)
+			c, err := Parse(tt.in)
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrStr)
+				require.Nil(t, c)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, c)
+			require.NotNil(t, c.Controller)
+			require.Equal(t, tt.expConcurrentPasswordHashWorkers, c.Controller.ConcurrentPasswordHashWorkers)
+		})
+	}
+
+	t.Run("using environment variable", func(t *testing.T) {
+		t.Setenv("BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS", "2")
+		in := `
+			controller {
+				name = "example-controller"
+			}`
+		c, err := Parse(in)
+		require.NoError(t, err)
+		require.NotNil(t, c)
+		require.NotNil(t, c.Controller)
+		require.EqualValues(t, 2, c.Controller.ConcurrentPasswordHashWorkers)
+	})
+
+	t.Run("using environment variable with invalid value", func(t *testing.T) {
+		t.Setenv("BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS", "invalid")
+		in := `
+			controller {
+				name = "example-controller"
+			}`
+		c, err := Parse(in)
+		require.Error(t, err)
+		require.Nil(t, c)
+		require.Contains(t, err.Error(), "BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS value is not an int")
+	})
+
+	t.Run("using environment variable and config value uses config value", func(t *testing.T) {
+		t.Setenv("BOUNDARY_CONTROLLER_CONCURRENT_PASSWORD_HASH_WORKERS", "2")
+		in := `
+			controller {
+				name = "example-controller"
+				concurrent_password_hash_workers = 3
+			}`
+		c, err := Parse(in)
+		require.NoError(t, err)
+		require.NotNil(t, c)
+		require.NotNil(t, c.Controller)
+		require.EqualValues(t, 3, c.Controller.ConcurrentPasswordHashWorkers)
+	})
+}