threshold/internal/cmd/commands/server/controller_ratelimit_reload_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package server

import (
	"fmt"
	"net/http"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/hashicorp/boundary/internal/cmd/config"
	"github.com/mitchellh/cli"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	ratelimitConfig = `
disable_mlock = true

telemetry {
	prometheus_retention_time = "24h"
	disable_hostname = true
}

controller {
	name = "test-controller"
	description = "A default controller created for tests"
	database {
		url = "%s"
	}

	api_rate_limit {
		resources = ["*"]
		actions   = ["*"]
		per       = "total"
		limit     = 2
		period    = "1m"
	}
}

kms "aead" {
	purpose = "root"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_root"
}

kms "aead" {
	purpose = "worker-auth"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_worker-auth"
}

kms "aead" {
	purpose = "recovery"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_recovery"
}

listener "tcp" {
	purpose = "api"
	address = "127.0.0.1:9500"
	tls_disable = true
}

listener "tcp" {
	address = "127.0.0.1:9501"
	purpose = "cluster"
}
`

	ratelimitConfigReload = `
disable_mlock = true

telemetry {
	prometheus_retention_time = "24h"
	disable_hostname = true
}

controller {
	name = "test-controller"
	description = "A default controller created for tests"
	database {
		url = "%s"
	}

	api_rate_limit {
		resources = ["*"]
		actions   = ["*"]
		per       = "total"
		limit     = 5
		period    = "1m"
	}
}

kms "aead" {
	purpose = "root"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_root"
}

kms "aead" {
	purpose = "worker-auth"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_worker-auth"
}

kms "aead" {
	purpose = "recovery"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_recovery"
}

listener "tcp" {
	purpose = "api"
	address = "127.0.0.1:9500"
	tls_disable = true
}

listener "tcp" {
	address = "127.0.0.1:9501"
	purpose = "cluster"
}
`

	ratelimitConfigDisabledReload = `
disable_mlock = true

telemetry {
	prometheus_retention_time = "24h"
	disable_hostname = true
}

controller {
	name = "test-controller"
	description = "A default controller created for tests"
	database {
		url = "%s"
	}

	api_rate_limit_disable = true
}

kms "aead" {
	purpose = "root"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_root"
}

kms "aead" {
	purpose = "worker-auth"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_worker-auth"
}

kms "aead" {
	purpose = "recovery"
	aead_type = "aes-gcm"
	key = "%s"
	key_id = "global_recovery"
}

listener "tcp" {
	purpose = "api"
	address = "127.0.0.1:9500"
	tls_disable = true
}

listener "tcp" {
	address = "127.0.0.1:9501"
	purpose = "cluster"
}
`
)

func TestReloadControllerRateLimits(t *testing.T) {
	td := t.TempDir()

	controllerKey := config.DevKeyGeneration()

	closeDB, url, _, err := getInitDatabase(t, controllerKey)
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, closeDB()) })

	cmd := testServerCommand(t, testServerCommandOpts{})

	workerAuthKey := config.DevKeyGeneration()
	recoveryKey := config.DevKeyGeneration()
	cfgHcl := fmt.Sprintf(ratelimitConfig, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()

		args := []string{"-config", td + "/config.hcl"}
		exitCode := cmd.Run(args)
		if exitCode != 0 {
			output := cmd.UI.(*cli.MockUi).ErrorWriter.String() + cmd.UI.(*cli.MockUi).OutputWriter.String()
			fmt.Printf("%s: got a non-zero exit status: %s", t.Name(), output)
		}
	}()

	// Wait until things are up and running (or timeout).
	select {
	case <-cmd.startedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Change config so it is ready for reloading
	cfgHcl = fmt.Sprintf(ratelimitConfigReload, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	c := http.Client{}
	r, err := c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=1, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// out of quota, so we expect a 429
	assert.Equal(t, http.StatusTooManyRequests, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	cmd.SighupCh <- struct{}{}
	select {
	case <-cmd.reloadedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Make another request, the limit should have reset and the new limit
	// should get reported via the headers.
	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=5, remaining=4, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `5;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	cmd.ShutdownCh <- struct{}{}
	wg.Wait()
}

func TestReloadControllerRateLimitsSameConfig(t *testing.T) {
	td := t.TempDir()

	// Create and migrate database A and B.
	controllerKey := config.DevKeyGeneration()

	closeDB, url, _, err := getInitDatabase(t, controllerKey)
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, closeDB()) })

	cmd := testServerCommand(t, testServerCommandOpts{})

	workerAuthKey := config.DevKeyGeneration()
	recoveryKey := config.DevKeyGeneration()
	cfgHcl := fmt.Sprintf(ratelimitConfig, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()

		args := []string{"-config", td + "/config.hcl"}
		exitCode := cmd.Run(args)
		if exitCode != 0 {
			output := cmd.UI.(*cli.MockUi).ErrorWriter.String() + cmd.UI.(*cli.MockUi).OutputWriter.String()
			fmt.Printf("%s: got a non-zero exit status: %s", t.Name(), output)
		}
	}()

	// Wait until things are up and running (or timeout).
	select {
	case <-cmd.startedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	c := http.Client{}
	r, err := c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=1, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// out of quota, so we expect a 429
	assert.Equal(t, http.StatusTooManyRequests, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	cmd.SighupCh <- struct{}{}
	select {
	case <-cmd.reloadedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Make another request, the limit should not have reset, and the current
	// quota should still apply.
	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// should still be rate limited, so 429
	assert.Equal(t, http.StatusTooManyRequests, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	cmd.ShutdownCh <- struct{}{}
	wg.Wait()
}

func TestReloadControllerRateLimitsDisable(t *testing.T) {
	td := t.TempDir()

	controllerKey := config.DevKeyGeneration()

	closeDB, url, _, err := getInitDatabase(t, controllerKey)
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, closeDB()) })

	cmd := testServerCommand(t, testServerCommandOpts{})

	workerAuthKey := config.DevKeyGeneration()
	recoveryKey := config.DevKeyGeneration()
	cfgHcl := fmt.Sprintf(ratelimitConfig, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()

		args := []string{"-config", td + "/config.hcl"}
		exitCode := cmd.Run(args)
		if exitCode != 0 {
			output := cmd.UI.(*cli.MockUi).ErrorWriter.String() + cmd.UI.(*cli.MockUi).OutputWriter.String()
			fmt.Printf("%s: got a non-zero exit status: %s", t.Name(), output)
		}
	}()

	// Wait until things are up and running (or timeout).
	select {
	case <-cmd.startedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Change config so it is ready for reloading
	cfgHcl = fmt.Sprintf(ratelimitConfigDisabledReload, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	c := http.Client{}
	r, err := c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=1, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// out of quota, so we expect a 429
	assert.Equal(t, http.StatusTooManyRequests, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	cmd.SighupCh <- struct{}{}
	select {
	case <-cmd.reloadedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Make another request, the limit should be disabled
	// should get reported via the headers.
	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, ``, r.Header.Get("Ratelimit"))
	assert.Equal(t, ``, r.Header.Get("Ratelimit-Policy"))

	cmd.ShutdownCh <- struct{}{}
	wg.Wait()
}

func TestReloadControllerRateLimitsEnable(t *testing.T) {
	td := t.TempDir()

	controllerKey := config.DevKeyGeneration()

	closeDB, url, _, err := getInitDatabase(t, controllerKey)
	require.NoError(t, err)
	t.Cleanup(func() { require.NoError(t, closeDB()) })

	cmd := testServerCommand(t, testServerCommandOpts{})

	workerAuthKey := config.DevKeyGeneration()
	recoveryKey := config.DevKeyGeneration()
	// Start with rate limiting diasabled
	cfgHcl := fmt.Sprintf(ratelimitConfigDisabledReload, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()

		args := []string{"-config", td + "/config.hcl"}
		exitCode := cmd.Run(args)
		if exitCode != 0 {
			output := cmd.UI.(*cli.MockUi).ErrorWriter.String() + cmd.UI.(*cli.MockUi).OutputWriter.String()
			fmt.Printf("%s: got a non-zero exit status: %s", t.Name(), output)
		}
	}()

	// Wait until things are up and running (or timeout).
	select {
	case <-cmd.startedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Change config so it is ready for reloading
	cfgHcl = fmt.Sprintf(ratelimitConfig, url, controllerKey, workerAuthKey, recoveryKey)
	require.NoError(t, os.WriteFile(td+"/config.hcl", []byte(cfgHcl), 0o644))

	c := http.Client{}
	r, err := c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, ``, r.Header.Get("Ratelimit"))
	assert.Equal(t, ``, r.Header.Get("Ratelimit-Policy"))

	cmd.SighupCh <- struct{}{}
	select {
	case <-cmd.reloadedCh:
	case <-time.After(15 * time.Second):
		t.Fatal("timeout")
	}

	// Make another request, the limit should be enabled
	// should get reported via the headers.
	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=1, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// unauthed request, so we expect a 400
	assert.Equal(t, http.StatusBadRequest, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	r, err = c.Do(func() *http.Request {
		r, err := http.NewRequest(http.MethodGet, `http://127.0.0.1:9500/v1/targets`, nil)
		require.NoError(t, err)
		return r
	}())
	require.NoError(t, err)
	// out of quota, so we expect a 429
	assert.Equal(t, http.StatusTooManyRequests, r.StatusCode)
	assert.Equal(t, `limit=2, remaining=0, reset=60`, r.Header.Get("Ratelimit"))
	assert.Equal(t, `2;w=60;comment="total", 1500;w=30;comment="ip-address", 150;w=30;comment="auth-token"`, r.Header.Get("Ratelimit-Policy"))

	cmd.ShutdownCh <- struct{}{}
	wg.Wait()
}