feat(controller): Add health endpoint (#1882)

* feat(controller): New health service and server

This change also implements a new listener purpose, `ops`.

`ops` (short for operations) is a new listener purpose that is
responsible for serving Boundary's operations endpoints, like health
and possibly metrics.

At Startup: `ops` listeners get configured as late as possible to catch
any errors with the startup process;

At Shutdown: If the health endpoint has been set-up, we introduce a
configurable delay in the process. This is to enable the health endpoint
to report unhealthy for that amount of time. `ops` listeners are also
the last listeners to be shutdown.

CHANGELOG.md

@@ -3,10 +3,15 @@
 Canonical reference for changes, improvements, and bugfixes for Boundary.
 
 ## Next
+
+### New and Improved
+* controller: new health endpoint ([PR](https://github.com/hashicorp/boundary/pull/1882)).
+
 ### Bug Fixes
 * worker: create new error to prevent `event.newError: missing error: invalid parameter` and handle session cancel 
-with no TOFU token ([Issue](https://github.com/hashicorp/boundary/issues/1902),
+  with no TOFU token ([Issue](https://github.com/hashicorp/boundary/issues/1902),
   [PR](https://github.com/hashicorp/boundary/pull/1929))
+  
 ## 0.7.6 (2022/03/15)
 
 ### Bug Fixes

internal/cmd/base/listener.go

@@ -128,6 +128,8 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U
 			l.Address = "127.0.0.1:9201"
 		case "proxy":
 			l.Address = "127.0.0.1:9202"
+		case "ops":
+			l.Address = "127.0.0.1:9203"
 		default:
 			l.Address = "127.0.0.1:9200"
 		}
@@ -141,6 +143,8 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U
 				port = "9201"
 			case "proxy":
 				port = "9202"
+			case "ops":
+				port = "9203"
 			default:
 				port = "9200"
 			}

internal/cmd/commands/dev/dev.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/boundary/internal/auth/password"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
+	"github.com/hashicorp/boundary/internal/cmd/ops"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/intglobals"
@@ -39,6 +40,7 @@ var (
 
 type Command struct {
 	*base.Server
+	opsServer *ops.Server
 
 	SighupCh      chan struct{}
 	childSighupCh []chan struct{}
@@ -68,6 +70,7 @@ type Command struct {
 	flagWorkerAuthKey                string
 	flagWorkerProxyListenAddr        string
 	flagWorkerPublicAddr             string
+	flagOpsListenAddr                string
 	flagUiPassthroughDir             string
 	flagRecoveryKey                  string
 	flagDatabaseUrl                  string
@@ -213,6 +216,12 @@ func (c *Command) Flags() *base.FlagSets {
 		EnvVar: "BOUNDARY_DEV_CONTROLLER_PUBLIC_CLUSTER_ADDRESS",
 		Usage:  "Public address at which the controller is reachable for cluster tasks (like worker connections).",
 	})
+	f.StringVar(&base.StringVar{
+		Name:   "ops-listen-address",
+		Target: &c.flagOpsListenAddr,
+		EnvVar: "BOUNDARY_DEV_OPS_LISTEN_ADDRESS",
+		Usage:  "Address to bind to for \"ops\" purpose. If this begins with a forward slash, it will be assumed to be a Unix domain socket path.",
+	})
 
 	f.BoolVar(&base.BoolVar{
 		Name:   "controller-only",
@@ -449,6 +458,14 @@ func (c *Command) Run(args []string) int {
 				l.Type = "unix"
 			}
 
+		case "ops":
+			if c.flagOpsListenAddr != "" {
+				l.Address = c.flagOpsListenAddr
+			}
+			if strings.HasPrefix(l.Address, "/") {
+				l.Type = "unix"
+			}
+
 		case "proxy":
 			if c.flagWorkerProxyListenAddr != "" {
 				l.Address = c.flagWorkerProxyListenAddr
@@ -543,7 +560,7 @@ func (c *Command) Run(args []string) int {
 	c.Info["[Recovery] AEAD Key Bytes"] = c.Config.DevRecoveryKey
 
 	// Initialize the listeners
-	if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy"}); err != nil {
+	if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy", "ops"}); err != nil {
 		c.UI.Error(err.Error())
 		return base.CommandUserError
 	}
@@ -636,7 +653,7 @@ func (c *Command) Run(args []string) int {
 		var err error
 		c.worker, err = worker.New(conf)
 		if err != nil {
-			c.UI.Error(fmt.Errorf("Error initializing controller: %w", err).Error())
+			c.UI.Error(fmt.Errorf("Error initializing worker: %w", err).Error())
 			return base.CommandCliError
 		}
 
@@ -654,6 +671,15 @@ func (c *Command) Run(args []string) int {
 		}
 	}
 
+	opsServer, err := ops.NewServer(c.Logger, c.controller, c.Listeners...)
+	if err != nil {
+		c.UI.Error(fmt.Errorf("Failed to start ops listeners: %w", err).Error())
+		return base.CommandCliError
+
+	}
+	c.opsServer = opsServer
+	c.opsServer.Start()
+
 	// Wait for shutdown
 	shutdownTriggered := false
 
@@ -678,6 +704,10 @@ func (c *Command) Run(args []string) int {
 				}
 			}()
 
+			if c.Config.Controller != nil {
+				c.opsServer.WaitIfHealthExists(c.Config.Controller.GracefulShutdownWaitDuration, c.UI)
+			}
+
 			if !c.flagControllerOnly {
 				if err := c.worker.Shutdown(false); err != nil {
 					c.UI.Error(fmt.Errorf("Error shutting down worker: %w", err).Error())
@@ -688,6 +718,11 @@ func (c *Command) Run(args []string) int {
 				c.UI.Error(fmt.Errorf("Error shutting down controller: %w", err).Error())
 			}
 
+			err := c.opsServer.Shutdown()
+			if err != nil {
+				c.UI.Error(fmt.Errorf("Failed to shutdown ops listeners: %w", err).Error())
+			}
+
 			shutdownTriggered = true
 
 		case <-c.SigUSR2Ch:

internal/cmd/commands/server/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/cmd/config"
+	"github.com/hashicorp/boundary/internal/cmd/ops"
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/db/schema"
 	"github.com/hashicorp/boundary/internal/errors"
@@ -44,6 +45,7 @@ var (
 
 type Command struct {
 	*base.Server
+	opsServer *ops.Server
 
 	SighupCh  chan struct{}
 	SigUSR2Ch chan struct{}
@@ -233,6 +235,7 @@ func (c *Command) Run(args []string) int {
 				if lnConfig.Address == "" {
 					lnConfig.Address = "127.0.0.1:9202"
 				}
+			case "ops":
 			default:
 				c.UI.Error(fmt.Sprintf("Unknown listener purpose %q", lnConfig.Purpose[0]))
 				return base.CommandUserError
@@ -333,7 +336,7 @@ func (c *Command) Run(args []string) int {
 			}
 		}
 	}
-	if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy"}); err != nil {
+	if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy", "ops"}); err != nil {
 		c.UI.Error(err.Error())
 		return base.CommandUserError
 	}
@@ -473,6 +476,14 @@ func (c *Command) Run(args []string) int {
 		}
 	}
 
+	opsServer, err := ops.NewServer(c.Logger, c.controller, c.Listeners...)
+	if err != nil {
+		c.UI.Error(err.Error())
+		return base.CommandCliError
+	}
+	c.opsServer = opsServer
+	c.opsServer.Start()
+
 	// Inform any tests that the server is ready
 	if c.startedCh != nil {
 		close(c.startedCh)
@@ -660,6 +671,10 @@ func (c *Command) WaitForInterrupt() int {
 				}
 			}()
 
+			if c.Config.Controller != nil {
+				c.opsServer.WaitIfHealthExists(c.Config.Controller.GracefulShutdownWaitDuration, c.UI)
+			}
+
 			// Do worker shutdown
 			if c.Config.Worker != nil {
 				if err := c.worker.Shutdown(false); err != nil {
@@ -674,6 +689,11 @@ func (c *Command) WaitForInterrupt() int {
 				}
 			}
 
+			err := c.opsServer.Shutdown()
+			if err != nil {
+				c.UI.Error(fmt.Errorf("Error shutting down ops listeners: %w", err).Error())
+			}
+
 			shutdownTriggered = true
 
 		case <-c.SighupCh:

internal/cmd/config/config.go

@@ -74,6 +74,11 @@ listener "tcp" {
 listener "tcp" {
 	purpose = "cluster"
 }
+
+listener "tcp" {
+	purpose = "ops"
+	tls_disable = true
+}
 `
 
 	devWorkerExtraConfig = `
@@ -128,6 +133,12 @@ type Controller struct {
 	AuthTokenTimeToStale         interface{} `hcl:"auth_token_time_to_stale"`
 	AuthTokenTimeToStaleDuration time.Duration
 
+	// GracefulShutdownWait is the amount of time that we'll wait before actually
+	// starting the Controller shutdown. This allows the health endpoint to
+	// return a status code to indicate that the instance is shutting down.
+	GracefulShutdownWait         interface{} `hcl:"graceful_shutdown_wait_duration"`
+	GracefulShutdownWaitDuration time.Duration
+
 	// StatusGracePeriod represents the period of time (as a duration) that the
 	// controller will wait before marking connections from a disconnected worker
 	// as invalid.
@@ -341,6 +352,14 @@ func Parse(d string) (*Config, error) {
 			result.Controller.AuthTokenTimeToStaleDuration = t
 		}
 
+		if result.Controller.GracefulShutdownWait != "" {
+			t, err := parseutil.ParseDurationSecond(result.Controller.GracefulShutdownWait)
+			if err != nil {
+				return result, err
+			}
+			result.Controller.GracefulShutdownWaitDuration = t
+		}
+
 		if result.Controller.Database != nil {
 			if result.Controller.Database.MaxOpenConnectionsRaw != nil {
 				switch t := result.Controller.Database.MaxOpenConnectionsRaw.(type) {

internal/cmd/config/config_test.go

@@ -38,6 +38,11 @@ func TestDevController(t *testing.T) {
 					Type:    "tcp",
 					Purpose: []string{"cluster"},
 				},
+				{
+					Type:       "tcp",
+					Purpose:    []string{"ops"},
+					TLSDisable: true,
+				},
 			},
 			Seals: []*configutil.KMS{
 				{
@@ -75,6 +80,7 @@ func TestDevController(t *testing.T) {
 
 	exp.Listeners[0].RawConfig = actual.Listeners[0].RawConfig
 	exp.Listeners[1].RawConfig = actual.Listeners[1].RawConfig
+	exp.Listeners[2].RawConfig = actual.Listeners[2].RawConfig
 	exp.Seals[0].Config["key"] = actual.Seals[0].Config["key"]
 	exp.Seals[1].Config["key"] = actual.Seals[1].Config["key"]
 	exp.Seals[2].Config["key"] = actual.Seals[2].Config["key"]

internal/cmd/ops/server.go

@@ -0,0 +1,214 @@
+// Package ops encapsulates the lifecycle of Boundary's ops-purpose listeners
+// and servers: Creating, setting them up, starting and shutdown.
+package ops
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/cmd/base"
+	"github.com/hashicorp/boundary/internal/libs/alpnmux"
+	"github.com/hashicorp/boundary/internal/servers/controller"
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-secure-stdlib/listenerutil"
+	"github.com/mitchellh/cli"
+	"github.com/pkg/errors"
+)
+
+// Server is a collection of all state required to serve
+// multiple ops endpoints through a single object.
+type Server struct {
+	bundles    []*opsBundle
+	controller *controller.Controller
+}
+
+type opsBundle struct {
+	ln      *base.ServerListener
+	h       http.Handler
+	startFn []func()
+}
+
+// NewServer iterates through all the listeners and sets up HTTP Servers for each, along with individual handlers.
+// If Controller is set-up, NewServer will set-up a health endpoint for it.
+func NewServer(l hclog.Logger, c *controller.Controller, listeners ...*base.ServerListener) (*Server, error) {
+	const op = "ops.NewServer()"
+	if l == nil {
+		return nil, fmt.Errorf("%s: missing logger", op)
+	}
+
+	bundles := make([]*opsBundle, 0, len(listeners))
+	for _, ln := range listeners {
+		if ln == nil || ln.Config == nil {
+			continue
+		}
+		if ln.Config.Purpose[0] != "ops" {
+			continue
+		}
+
+		h, err := createOpsHandler(ln.Config, c)
+		if err != nil {
+			return nil, err
+		}
+
+		b := &opsBundle{ln: ln, h: h}
+		b.ln.HTTPServer = createHttpServer(l, b.h, b.ln.Config)
+
+		funcs, err := getStartFn(b.ln)
+		if err != nil {
+			return nil, err
+		}
+		b.startFn = funcs
+
+		bundles = append(bundles, b)
+	}
+
+	return &Server{bundles, c}, nil
+}
+
+// Starts all goroutines that were set-up in NewServer.
+// These goroutines start the HTTP Servers on the appropriate
+// listeners (as defined by the bundle).
+func (s *Server) Start() {
+	for _, b := range s.bundles {
+		for _, f := range b.startFn {
+			f()
+		}
+	}
+}
+
+// Shutdown attempts to cleanly shutdown all running ops listeners and HTTP servers.
+func (s *Server) Shutdown() error {
+	const op = "ops.(Server).Shutdown"
+
+	var closeErrors *multierror.Error
+	for _, b := range s.bundles {
+		if b == nil || b.ln == nil || b.ln.Config == nil || b.ln.Mux == nil || b.ln.HTTPServer == nil {
+			return fmt.Errorf("%s: missing bundle, listener or its fields", op)
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), b.ln.Config.MaxRequestDuration)
+		defer cancel()
+
+		err := b.ln.HTTPServer.Shutdown(ctx)
+		if err != nil {
+			multierror.Append(closeErrors, fmt.Errorf("%s: failed to shutdown http server: %w", op, err))
+		}
+
+		err = b.ln.Mux.Close()
+		err = listenerCloseErrorCheck(b.ln.Config.Type, err)
+		if err != nil {
+			multierror.Append(closeErrors, fmt.Errorf("%s: failed to close listener mux: %w", op, err))
+		}
+	}
+
+	return closeErrors.ErrorOrNil()
+}
+
+// WaitIfHealthExists waits for a configurable period of time `d` if the health endpoint has been
+// configured (i.e the Controller exists and ops listeners have been set-up)
+func (s *Server) WaitIfHealthExists(d time.Duration, ui cli.Ui) {
+	if s.controller == nil || s.controller.HealthService == nil {
+		return
+	}
+	if len(s.bundles) == 0 {
+		return
+	}
+
+	// If we have ops listeners and the health endpoint is up,
+	// we wait for a configurable amount of time before shutting down.
+	// This is to give time for health to report unhealthy and for external
+	// systems to pick up on that.
+	ui.Output(fmt.Sprintf("==> Health is enabled, waiting %s before shutdown", d.String()))
+	s.controller.HealthService.StartServiceUnavailableReplies()
+	<-time.After(d)
+}
+
+func createOpsHandler(lncfg *listenerutil.ListenerConfig, c *controller.Controller) (http.Handler, error) {
+	mux := http.NewServeMux()
+	if c != nil && c.HealthService != nil {
+		h, err := c.GetHealthHandler(lncfg)
+		if err != nil {
+			return nil, err
+		}
+		mux.Handle("/health", h)
+	}
+
+	return cleanhttp.PrintablePathCheckHandler(mux, nil), nil
+}
+
+func createHttpServer(l hclog.Logger, h http.Handler, lncfg *listenerutil.ListenerConfig) *http.Server {
+	s := &http.Server{
+		Handler:           h,
+		ReadHeaderTimeout: 10 * time.Second,
+		ReadTimeout:       30 * time.Second,
+		IdleTimeout:       5 * time.Minute,
+		ErrorLog:          l.StandardLogger(nil),
+	}
+
+	if lncfg.HTTPReadHeaderTimeout > 0 {
+		s.ReadHeaderTimeout = lncfg.HTTPReadHeaderTimeout
+	}
+	if lncfg.HTTPReadTimeout > 0 {
+		s.ReadTimeout = lncfg.HTTPReadTimeout
+	}
+	if lncfg.HTTPWriteTimeout > 0 {
+		s.WriteTimeout = lncfg.HTTPWriteTimeout
+	}
+	if lncfg.HTTPIdleTimeout > 0 {
+		s.IdleTimeout = lncfg.HTTPIdleTimeout
+	}
+
+	return s
+}
+
+func getStartFn(ln *base.ServerListener) ([]func(), error) {
+	const op = "getStartFn()"
+
+	funcs := make([]func(), 0)
+	switch ln.Config.TLSDisable {
+	case true:
+		l, err := ln.Mux.RegisterProto(alpnmux.NoProto, nil)
+		if err != nil {
+			return nil, fmt.Errorf("%s: error getting non-tls listener: %w", op, err)
+		}
+		if l == nil {
+			return nil, fmt.Errorf("%s: could not get non-tls listener", op)
+		}
+		funcs = append(funcs, func() { go ln.HTTPServer.Serve(l) })
+
+	default:
+		for _, v := range []string{"", "http/1.1", "h2"} {
+			l := ln.Mux.GetListener(v)
+			if l == nil {
+				return nil, fmt.Errorf("%s: could not get tls proto %q listener", op, v)
+			}
+			funcs = append(funcs, func() { go ln.HTTPServer.Serve(l) })
+		}
+	}
+
+	return funcs, nil
+}
+
+func listenerCloseErrorCheck(lnType string, err error) error {
+	if errors.Is(err, net.ErrClosed) {
+		// Ignore net.ErrClosed - The listener was already closed,
+		// so there's nothing else to do.
+		return nil
+	}
+	if _, ok := err.(*os.PathError); ok && lnType == "unix" {
+		// The underlying rmListener probably tried to remove
+		// the file but it didn't exist, ignore the error;
+		// this is a conflict between rmListener and the
+		// default Go behavior of removing auto-vivified
+		// Unix domain sockets.
+		return nil
+	}
+
+	return err
+}

internal/cmd/ops/server_test.go

@@ -0,0 +1,867 @@
+package ops
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"net"
+	"net/http"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/cmd/base"
+	"github.com/hashicorp/boundary/internal/libs/alpnmux"
+	"github.com/hashicorp/boundary/internal/servers/controller"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/health"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-secure-stdlib/base62"
+	"github.com/hashicorp/go-secure-stdlib/configutil/v2"
+	"github.com/hashicorp/go-secure-stdlib/listenerutil"
+	"github.com/hashicorp/go-secure-stdlib/reloadutil"
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewServer(t *testing.T) {
+	tests := []struct {
+		name       string
+		logger     hclog.Logger
+		c          *controller.Controller
+		listeners  []*base.ServerListener
+		assertions func(t *testing.T, s *Server)
+		expErr     bool
+		expErrMsg  string
+	}{
+		{
+			name:      "nil logger",
+			logger:    nil,
+			listeners: []*base.ServerListener{{}},
+			expErr:    true,
+			expErrMsg: "ops.NewServer(): missing logger",
+		},
+		{
+			name:      "nil ServerListeners",
+			logger:    hclog.Default(),
+			listeners: nil,
+			expErr:    false,
+			assertions: func(t *testing.T, s *Server) {
+				require.Len(t, s.bundles, 0)
+			},
+		},
+		{
+			name:      "empty ServerListeners",
+			logger:    hclog.Default(),
+			listeners: []*base.ServerListener{},
+			expErr:    false,
+			assertions: func(t *testing.T, s *Server) {
+				require.Len(t, s.bundles, 0)
+			},
+		},
+		{
+			name:      "nil listeners",
+			logger:    hclog.Default(),
+			listeners: []*base.ServerListener{nil, nil},
+			expErr:    false,
+			assertions: func(t *testing.T, s *Server) {
+				require.Len(t, s.bundles, 0)
+			},
+		},
+		{
+			name:   "listeners with nil config",
+			logger: hclog.Default(),
+			listeners: []*base.ServerListener{
+				{
+					Config: nil,
+				},
+				{
+					Config: nil,
+				},
+			},
+			expErr: false,
+			assertions: func(t *testing.T, s *Server) {
+				require.Len(t, s.bundles, 0)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, err := NewServer(tt.logger, tt.c, tt.listeners...)
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrMsg)
+				require.Nil(t, s)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, s)
+
+			if tt.assertions != nil {
+				tt.assertions(t, s)
+			}
+		})
+	}
+}
+
+func TestNewServerIntegration(t *testing.T) {
+	tests := []struct {
+		name       string
+		setup      func(t *testing.T)
+		listeners  []*listenerutil.ListenerConfig
+		assertions func(t *testing.T, addrs []string)
+		expErr     bool
+		expErrMsg  string
+	}{
+		{
+			name: "one tcp ops listener",
+			listeners: []*listenerutil.ListenerConfig{
+				{
+					Type:       "tcp",
+					Purpose:    []string{"ops"},
+					Address:    "127.0.0.1:0",
+					TLSDisable: true,
+				},
+			},
+			assertions: func(t *testing.T, addrs []string) {
+				_, err := http.Get("http://" + addrs[0])
+				require.NoError(t, err)
+			},
+		},
+		{
+			name: "multiple tcp ops listeners",
+			listeners: []*listenerutil.ListenerConfig{
+				{
+					Type:       "tcp",
+					Purpose:    []string{"ops"},
+					Address:    "127.0.0.1:0",
+					TLSDisable: true,
+				},
+				{
+					Type:       "tcp",
+					Purpose:    []string{"ops"},
+					Address:    "127.0.0.1:0",
+					TLSDisable: true,
+				},
+			},
+			assertions: func(t *testing.T, addrs []string) {
+				_, err := http.Get("http://" + addrs[0])
+				require.NoError(t, err)
+
+				_, err = http.Get("http://" + addrs[1])
+				require.NoError(t, err)
+			},
+		},
+		{
+			name: "one unix socket ops listener",
+			listeners: []*listenerutil.ListenerConfig{
+				{
+					Type:       "unix",
+					Purpose:    []string{"ops"},
+					Address:    fmt.Sprintf("/tmp/boundary-opslistener-test-%s.sock", base62.MustRandom(5)),
+					TLSDisable: true,
+				},
+			},
+			expErr: false,
+			assertions: func(t *testing.T, addrs []string) {
+				conn, err := net.Dial("unix", addrs[0])
+				require.NoError(t, err)
+				t.Cleanup(func() { conn.Close() })
+
+				cl := http.Client{
+					Transport: &http.Transport{
+						Dial: func(network, addr string) (net.Conn, error) { return conn, nil },
+					},
+				}
+				_, err = cl.Get("http://random.domain")
+				require.NoError(t, err)
+			},
+		},
+		{
+			name: "multiple unix socket ops listeners",
+			listeners: []*listenerutil.ListenerConfig{
+				{
+					Type:       "unix",
+					Purpose:    []string{"ops"},
+					Address:    fmt.Sprintf("/tmp/boundary-opslistener-test-%s.sock", base62.MustRandom(5)),
+					TLSDisable: true,
+				},
+				{
+					Type:       "unix",
+					Purpose:    []string{"ops"},
+					Address:    fmt.Sprintf("/tmp/boundary-opslistener-test-%s.sock", base62.MustRandom(5)),
+					TLSDisable: true,
+				},
+			},
+			expErr: false,
+			assertions: func(t *testing.T, addrs []string) {
+				for _, addr := range addrs {
+					conn, err := net.Dial("unix", addr)
+					require.NoError(t, err)
+					t.Cleanup(func() { conn.Close() })
+
+					cl := http.Client{
+						Transport: &http.Transport{
+							Dial: func(network, addr string) (net.Conn, error) { return conn, nil },
+						},
+					}
+					_, err = cl.Get("http://random.domain")
+					require.NoError(t, err)
+				}
+			},
+		},
+		{
+			name: "multiple tcp ops listeners (tls)",
+			setup: func(t *testing.T) {
+				testTlsSetup(t, "./boundary-opslistener-cert1.cert", "./boundary-opslistener-pk1.key")
+				testTlsSetup(t, "./boundary-opslistener-cert2.cert", "./boundary-opslistener-pk2.key")
+			},
+			listeners: []*listenerutil.ListenerConfig{
+				{
+					Type:        "tcp",
+					Purpose:     []string{"ops"},
+					Address:     "127.0.0.1:0",
+					TLSKeyFile:  "./boundary-opslistener-pk1.key",
+					TLSCertFile: "./boundary-opslistener-cert1.cert",
+				},
+				{
+					Type:        "tcp",
+					Purpose:     []string{"ops"},
+					Address:     "127.0.0.1:0",
+					TLSKeyFile:  "./boundary-opslistener-pk2.key",
+					TLSCertFile: "./boundary-opslistener-cert2.cert",
+				},
+			},
+			assertions: func(t *testing.T, addrs []string) {
+				cl := testTlsHttpClient(t, "./boundary-opslistener-cert1.cert")
+				_, err := cl.Get("https://" + addrs[0])
+				require.NoError(t, err)
+
+				cl = testTlsHttpClient(t, "./boundary-opslistener-cert2.cert")
+				_, err = cl.Get("https://" + addrs[1])
+				require.NoError(t, err)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.setup != nil {
+				tt.setup(t)
+			}
+
+			bs := &base.Server{ // These fields are only needed to successfully call `SetupListeners`
+				Info:            make(map[string]string),
+				ReloadFuncs:     make(map[string][]reloadutil.ReloadFunc),
+				ReloadFuncsLock: &sync.RWMutex{},
+				Logger:          hclog.Default(),
+			}
+			err := bs.SetupListeners(nil, &configutil.SharedConfig{Listeners: tt.listeners}, []string{"ops"})
+			require.NoError(t, err)
+
+			s, err := NewServer(hclog.Default(), nil, bs.Listeners...)
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrMsg)
+				require.Nil(t, s)
+				return
+			}
+
+			require.NoError(t, err)
+			require.NotNil(t, s)
+			s.Start()
+			t.Cleanup(func() { s.Shutdown() })
+
+			addrs := make([]string, 0, len(s.bundles))
+			for _, b := range s.bundles {
+				addrs = append(addrs, b.ln.Mux.Addr().String())
+			}
+			if tt.assertions != nil {
+				tt.assertions(t, addrs)
+			}
+		})
+	}
+}
+
+func TestShutdown(t *testing.T) {
+	tests := []struct {
+		name        string
+		opsServerFn func(t *testing.T) *Server
+		assertions  func(t *testing.T, o *Server)
+		expErr      bool
+		expErrStr   string
+	}{
+		{
+			name: "nil bundle slice",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{bundles: nil}
+			},
+			expErr: false, // No-op
+		},
+		{
+			name: "empty bundle slice",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{bundles: []*opsBundle{}}
+			},
+			expErr: false, // No-op
+		},
+		{
+			name: "nil bundle",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{bundles: []*opsBundle{nil}}
+			},
+			expErr:    true,
+			expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields",
+		},
+		{
+			name: "empty bundle",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{bundles: []*opsBundle{{}}}
+			},
+			expErr:    true,
+			expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields",
+		},
+		{
+			name: "missing listener config",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{
+					bundles: []*opsBundle{
+						{
+							ln: &base.ServerListener{
+								Mux:        &alpnmux.ALPNMux{},
+								HTTPServer: &http.Server{},
+							},
+						},
+					},
+				}
+			},
+			expErr:    true,
+			expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields",
+		},
+		{
+			name: "missing listener mux",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{
+					bundles: []*opsBundle{
+						{
+							ln: &base.ServerListener{
+								Config:     &listenerutil.ListenerConfig{},
+								HTTPServer: &http.Server{},
+							},
+						},
+					},
+				}
+			},
+			expErr:    true,
+			expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields",
+		},
+		{
+			name: "missing http server",
+			opsServerFn: func(t *testing.T) *Server {
+				return &Server{
+					bundles: []*opsBundle{
+						{
+							ln: &base.ServerListener{
+								Mux:    &alpnmux.ALPNMux{},
+								Config: &listenerutil.ListenerConfig{},
+							},
+						},
+					},
+				}
+			},
+			expErr:    true,
+			expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields",
+		},
+		{
+			name: "listener already closed",
+			opsServerFn: func(t *testing.T) *Server {
+				l1, err := net.Listen("tcp", "127.0.0.1:0")
+				require.NoError(t, err)
+				require.NoError(t, l1.Close())
+
+				s1 := &http.Server{}
+				return &Server{
+					bundles: []*opsBundle{
+						{
+							ln: &base.ServerListener{
+								ALPNListener: l1,
+								HTTPServer:   s1,
+								Mux:          alpnmux.New(l1),
+								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+							},
+						},
+					},
+				}
+			},
+			assertions: func(t *testing.T, s *Server) {
+				// The HTTP Server must be closed.
+				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed)
+
+				// The underlying listener must be closed.
+				require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed)
+			},
+			expErr: false,
+		},
+		{
+			name: "http server already closed",
+			opsServerFn: func(t *testing.T) *Server {
+				l1, err := net.Listen("tcp", "127.0.0.1:0")
+				require.NoError(t, err)
+
+				s1 := &http.Server{}
+				go func() {
+					require.ErrorIs(t, s1.Serve(l1), http.ErrServerClosed)
+				}()
+
+				err = s1.Shutdown(context.Background())
+				require.NoError(t, err)
+
+				return &Server{
+					bundles: []*opsBundle{
+						{
+							ln: &base.ServerListener{
+								ALPNListener: l1,
+								HTTPServer:   s1,
+								Mux:          alpnmux.New(l1),
+								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+							},
+						},
+					},
+				}
+			},
+			assertions: func(t *testing.T, s *Server) {
+				// The HTTP Server must be closed.
+				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed)
+
+				// The underlying listener must be closed.
+				require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed)
+			},
+		},
+		{
+			name: "multiple bundles to close",
+			opsServerFn: func(t *testing.T) *Server {
+				l1, err := net.Listen("tcp", "127.0.0.1:0")
+				require.NoError(t, err)
+
+				s1 := &http.Server{}
+				go func() {
+					require.ErrorIs(t, s1.Serve(l1), http.ErrServerClosed)
+				}()
+
+				l2, err := net.Listen("tcp", "127.0.0.1:0")
+				require.NoError(t, err)
+
+				s2 := &http.Server{}
+				go func() {
+					require.ErrorIs(t, s2.Serve(l2), http.ErrServerClosed)
+				}()
+
+				return &Server{
+					bundles: []*opsBundle{
+						{
+							ln: &base.ServerListener{
+								ALPNListener: l1,
+								HTTPServer:   s1,
+								Mux:          alpnmux.New(l1),
+								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+							},
+						},
+						{
+							ln: &base.ServerListener{
+								ALPNListener: l2,
+								HTTPServer:   s2,
+								Mux:          alpnmux.New(l2),
+								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+							},
+						},
+					},
+				}
+			},
+			assertions: func(t *testing.T, s *Server) {
+				// The HTTP Server must be closed.
+				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed)
+				require.ErrorIs(t, s.bundles[1].ln.HTTPServer.Serve(s.bundles[1].ln.ALPNListener), http.ErrServerClosed)
+
+				// The underlying listener must be closed.
+				require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed)
+				require.ErrorIs(t, s.bundles[1].ln.Mux.Close(), net.ErrClosed)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			o := tt.opsServerFn(t)
+			err := o.Shutdown()
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrStr)
+				return
+			}
+
+			require.NoError(t, err)
+			if tt.assertions != nil {
+				tt.assertions(t, o)
+			}
+		})
+	}
+}
+
+func TestHealthEndpointLifecycle(t *testing.T) {
+	listeners := []*listenerutil.ListenerConfig{
+		{
+			Type:       "tcp",
+			Address:    "127.0.0.1:0",
+			Purpose:    []string{"ops"},
+			TLSDisable: true,
+		},
+
+		// We only care about the ops listener, but the following
+		// listeners are required to start a Controller.
+		{
+			Type:       "tcp",
+			Address:    "127.0.0.1:0",
+			Purpose:    []string{"api"},
+			TLSDisable: true,
+		},
+		{
+			Type:    "tcp",
+			Address: "127.0.0.1:0",
+			Purpose: []string{"cluster"},
+		},
+	}
+
+	// Get test a controller so we can actually start a controller.
+	tc := controller.NewTestController(t, &controller.TestControllerOpts{DisableAutoStart: true})
+	tc.Config().RawConfig.SharedConfig.Listeners = listeners // Override with our own listeners.
+
+	// Setup listeners that we've set up above to turn them into correct `ServerListener` objects
+	// that the controller can handle. This will set `Server.Listeners`.
+	err := tc.Server().SetupListeners(nil, tc.Config().RawConfig.SharedConfig, []string{"api", "cluster", "ops"})
+	require.NoError(t, err)
+
+	// Start controller. This sets `cmd.controller`.
+	err = tc.Controller().Start()
+	require.NoError(t, err)
+	t.Cleanup(tc.Shutdown)
+
+	// Controller has started and is set onto our Command object, start ops.
+	opsServer, err := NewServer(hclog.Default(), tc.Controller(), tc.Config().Listeners...)
+	require.NoError(t, err)
+	opsServer.Start()
+
+	// Assert the ops endpoint is up and returning 200 OK.
+	rsp, err := http.Get("http://" + tc.Config().Listeners[0].Mux.Addr().String() + "/health")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rsp.StatusCode)
+
+	// Start replying with 503 Service Unavailable from the health endpoint.
+	tc.Controller().HealthService.StartServiceUnavailableReplies()
+
+	// Assert we're receiving 503 Service Unavailable now instead of 200 OK.
+	rsp, err = http.Get("http://" + tc.Config().Listeners[0].Mux.Addr().String() + "/health")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode)
+}
+
+func TestWaitIfHealthExists(t *testing.T) {
+	tests := []struct {
+		name        string
+		bundles     []*opsBundle
+		gracePeriod time.Duration
+		controller  *controller.Controller
+		expMinWait  time.Duration
+		expMaxWait  time.Duration
+	}{
+		{
+			name:        "nil bundles",
+			bundles:     nil,
+			controller:  &controller.Controller{HealthService: &health.Service{}},
+			gracePeriod: 1 * time.Minute,
+			expMinWait:  0 * time.Second,
+			expMaxWait:  500 * time.Microsecond,
+		},
+		{
+			name:        "no bundles",
+			bundles:     []*opsBundle{},
+			controller:  &controller.Controller{HealthService: &health.Service{}},
+			gracePeriod: 1 * time.Minute,
+			expMinWait:  0 * time.Second,
+			expMaxWait:  500 * time.Microsecond,
+		},
+		{
+			name:        "nil controller",
+			controller:  nil,
+			gracePeriod: 1 * time.Minute,
+			expMinWait:  0 * time.Second,
+			expMaxWait:  500 * time.Microsecond,
+		},
+		{
+			name:        "controller with no health service",
+			controller:  &controller.Controller{HealthService: nil},
+			gracePeriod: 1 * time.Minute,
+			expMinWait:  0 * time.Second,
+			expMaxWait:  500 * time.Microsecond,
+		},
+		{
+			name:       "no grace period set",
+			controller: &controller.Controller{HealthService: &health.Service{}},
+			expMinWait: 0 * time.Second,
+			expMaxWait: 500 * time.Microsecond,
+		},
+		{
+			name:        "wait 500ms",
+			bundles:     []*opsBundle{{}},
+			controller:  &controller.Controller{HealthService: &health.Service{}},
+			gracePeriod: 500 * time.Millisecond,
+			expMinWait:  500 * time.Millisecond,
+			expMaxWait:  600 * time.Millisecond,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Server{
+				bundles:    tt.bundles,
+				controller: tt.controller,
+			}
+
+			start := time.Now()
+			s.WaitIfHealthExists(tt.gracePeriod, cli.NewMockUi())
+			actualWait := time.Since(start)
+
+			require.GreaterOrEqual(t, actualWait, tt.expMinWait)
+			require.LessOrEqual(t, actualWait, tt.expMaxWait)
+		})
+	}
+}
+
+func TestCreateOpsHandler(t *testing.T) {
+	tests := []struct {
+		name            string
+		setupController bool
+		lncfg           *listenerutil.ListenerConfig
+		expErr          bool
+		expErrMsg       string
+		assertions      func(t *testing.T, addr string)
+	}{
+		{
+			name:            "no controller set",
+			setupController: false,
+			lncfg:           &listenerutil.ListenerConfig{},
+			assertions: func(t *testing.T, addr string) {
+				rsp, err := http.Get("http://" + addr + "/health")
+				require.NoError(t, err)                               // We can do the GET request
+				require.Equal(t, http.StatusNotFound, rsp.StatusCode) // But the endpoint doesn't exist
+			},
+		},
+		{
+			name:            "controller set",
+			setupController: true,
+			lncfg:           &listenerutil.ListenerConfig{},
+			assertions: func(t *testing.T, addr string) {
+				rsp, err := http.Get("http://" + addr + "/health")
+				require.NoError(t, err)                         // We can do the GET request
+				require.Equal(t, http.StatusOK, rsp.StatusCode) // And the endpoint exists
+			},
+		},
+		{
+			name:            "controller set, but nil listener config",
+			setupController: true,
+			lncfg:           nil,
+			expErr:          true,
+			expErrMsg:       "controller.(Controller).GetHealthHandler: received nil listener config",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var c *controller.Controller
+			if tt.setupController {
+				tc := controller.NewTestController(t, &controller.TestControllerOpts{})
+				t.Cleanup(tc.Shutdown)
+
+				c = tc.Controller()
+			}
+
+			h, err := createOpsHandler(tt.lncfg, c)
+			if tt.expErr {
+				require.EqualError(t, err, tt.expErrMsg)
+				require.Nil(t, h)
+				return
+			}
+
+			require.NoError(t, err)
+
+			s := http.Server{Handler: h}
+			l, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(t, err)
+			go s.Serve(l)
+
+			t.Cleanup(func() {
+				require.NoError(t, s.Shutdown(context.Background()))
+			})
+
+			tt.assertions(t, l.Addr().String())
+		})
+	}
+}
+
+func TestCreateHttpServer(t *testing.T) {
+	tests := []struct {
+		name       string
+		lncfg      *listenerutil.ListenerConfig
+		assertions func(t *testing.T, s *http.Server)
+	}{
+		{
+			name:  "default values are set if none are set in listener config",
+			lncfg: &listenerutil.ListenerConfig{},
+			assertions: func(t *testing.T, s *http.Server) {
+				require.Equal(t, 10*time.Second, s.ReadHeaderTimeout)
+				require.Equal(t, 30*time.Second, s.ReadTimeout)
+				require.Equal(t, 5*time.Minute, s.IdleTimeout)
+			},
+		},
+		{
+			name: "configured values are respected",
+			lncfg: &listenerutil.ListenerConfig{
+				HTTPReadHeaderTimeout: time.Minute,
+				HTTPReadTimeout:       2 * time.Minute,
+				HTTPWriteTimeout:      3 * time.Minute,
+				HTTPIdleTimeout:       4 * time.Minute,
+			},
+			assertions: func(t *testing.T, s *http.Server) {
+				require.Equal(t, time.Minute, s.ReadHeaderTimeout)
+				require.Equal(t, 2*time.Minute, s.ReadTimeout)
+				require.Equal(t, 3*time.Minute, s.WriteTimeout)
+				require.Equal(t, 4*time.Minute, s.IdleTimeout)
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			l := hclog.Default()
+
+			s := createHttpServer(l, http.NotFoundHandler(), tt.lncfg)
+			require.NotNil(t, s)
+			require.NotNil(t, s.Handler)
+			require.NotNil(t, s.ErrorLog)
+
+			tt.assertions(t, s)
+		})
+	}
+}
+
+func TestListenerCloseErrorCheck(t *testing.T) {
+	tests := []struct {
+		name   string
+		lnType string
+		err    error
+		expErr error
+	}{
+		{
+			name:   "nil err",
+			lnType: "tcp",
+			err:    nil,
+			expErr: nil,
+		},
+		{
+			name:   "net.Closed",
+			lnType: "tcp",
+			err:    net.ErrClosed,
+			expErr: nil,
+		},
+		{
+			name:   "path err not unix type",
+			lnType: "tcp",
+			err:    &os.PathError{Op: "test"},
+			expErr: &os.PathError{Op: "test"},
+		},
+		{
+			name:   "path err unix type",
+			lnType: "unix",
+			err:    &os.PathError{Op: "test"},
+			expErr: nil,
+		},
+		{
+			name:   "literally anything else",
+			lnType: "tcp",
+			err:    fmt.Errorf("oops I errored"),
+			expErr: fmt.Errorf("oops I errored"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := listenerCloseErrorCheck(tt.lnType, tt.err)
+			require.Equal(t, tt.expErr, err)
+		})
+	}
+}
+
+func testTlsSetup(t *testing.T, certPath, keyPath string) {
+	t.Cleanup(func() {
+		require.NoError(t, os.Remove(certPath))
+		require.NoError(t, os.Remove(keyPath))
+	})
+
+	certBytes, _, priv := createTestCert(t)
+
+	certFile, err := os.Create(certPath)
+	require.NoError(t, err)
+	require.NoError(t, pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes}))
+	require.NoError(t, certFile.Close())
+
+	keyFile, err := os.Create(keyPath)
+	require.NoError(t, err)
+
+	marshaledKey, err := x509.MarshalPKCS8PrivateKey(priv)
+	require.NoError(t, err)
+
+	require.NoError(t, pem.Encode(keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: marshaledKey}))
+	require.NoError(t, keyFile.Close())
+}
+
+func testTlsHttpClient(t *testing.T, certPath string) *http.Client {
+	f, err := os.Open(certPath)
+	require.NoError(t, err)
+
+	certBytes, err := ioutil.ReadAll(f)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	certPool := x509.NewCertPool()
+	certPool.AppendCertsFromPEM(certBytes)
+
+	return &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{RootCAs: certPool},
+		},
+	}
+}
+
+func createTestCert(t *testing.T) ([]byte, ed25519.PublicKey, ed25519.PrivateKey) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	require.NoError(t, err)
+
+	template := &x509.Certificate{
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement | x509.KeyUsageCertSign,
+		SerialNumber:          big.NewInt(0),
+		NotBefore:             time.Now().Add(-30 * time.Second),
+		NotAfter:              time.Now().Add(5 * time.Minute),
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+		DNSNames:              []string{"/tmp/boundary-opslistener-test0.sock", "/tmp/boundary-opslistener-test1.sock"},
+	}
+
+	certBytes, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv)
+	require.NoError(t, err)
+
+	return certBytes, pub, priv
+}

internal/gen/controller/ops/services/health_service.pb.go

@@ -0,0 +1,209 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.27.1
+// 	protoc        v3.17.3
+// source: controller/ops/services/v1/health_service.proto
+
+package services
+
+import (
+	_ "github.com/hashicorp/boundary/sdk/pbs/controller/ops/resources/health"
+	_ "google.golang.org/genproto/googleapis/api/annotations"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type GetHealthRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *GetHealthRequest) Reset() {
+	*x = GetHealthRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GetHealthRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetHealthRequest) ProtoMessage() {}
+
+func (x *GetHealthRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetHealthRequest.ProtoReflect.Descriptor instead.
+func (*GetHealthRequest) Descriptor() ([]byte, []int) {
+	return file_controller_ops_services_v1_health_service_proto_rawDescGZIP(), []int{0}
+}
+
+type GetHealthResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *GetHealthResponse) Reset() {
+	*x = GetHealthResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GetHealthResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetHealthResponse) ProtoMessage() {}
+
+func (x *GetHealthResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetHealthResponse.ProtoReflect.Descriptor instead.
+func (*GetHealthResponse) Descriptor() ([]byte, []int) {
+	return file_controller_ops_services_v1_health_service_proto_rawDescGZIP(), []int{1}
+}
+
+var File_controller_ops_services_v1_health_service_proto protoreflect.FileDescriptor
+
+var file_controller_ops_services_v1_health_service_proto_rawDesc = []byte{
+	0x0a, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73,
+	0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x68, 0x65, 0x61,
+	0x6c, 0x74, 0x68, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x1a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70,
+	0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2f, 0x63, 0x6f, 0x6e,
+	0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, 0x2f, 0x72, 0x65, 0x73, 0x6f,
+	0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x2f,
+	0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x12, 0x0a, 0x10,
+	0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x8a, 0x01, 0x0a, 0x0d, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68,
+	0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x79, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x48, 0x65,
+	0x61, 0x6c, 0x74, 0x68, 0x12, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65,
+	0x72, 0x2e, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76,
+	0x31, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e,
+	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e,
+	0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x22, 0x0f, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x09, 0x12, 0x07, 0x2f, 0x68, 0x65, 0x61, 0x6c,
+	0x74, 0x68, 0x42, 0x4d, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+	0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64,
+	0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e,
+	0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, 0x2f,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_controller_ops_services_v1_health_service_proto_rawDescOnce sync.Once
+	file_controller_ops_services_v1_health_service_proto_rawDescData = file_controller_ops_services_v1_health_service_proto_rawDesc
+)
+
+func file_controller_ops_services_v1_health_service_proto_rawDescGZIP() []byte {
+	file_controller_ops_services_v1_health_service_proto_rawDescOnce.Do(func() {
+		file_controller_ops_services_v1_health_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_controller_ops_services_v1_health_service_proto_rawDescData)
+	})
+	return file_controller_ops_services_v1_health_service_proto_rawDescData
+}
+
+var file_controller_ops_services_v1_health_service_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_controller_ops_services_v1_health_service_proto_goTypes = []interface{}{
+	(*GetHealthRequest)(nil),  // 0: controller.ops.services.v1.GetHealthRequest
+	(*GetHealthResponse)(nil), // 1: controller.ops.services.v1.GetHealthResponse
+}
+var file_controller_ops_services_v1_health_service_proto_depIdxs = []int32{
+	0, // 0: controller.ops.services.v1.HealthService.GetHealth:input_type -> controller.ops.services.v1.GetHealthRequest
+	1, // 1: controller.ops.services.v1.HealthService.GetHealth:output_type -> controller.ops.services.v1.GetHealthResponse
+	1, // [1:2] is the sub-list for method output_type
+	0, // [0:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_controller_ops_services_v1_health_service_proto_init() }
+func file_controller_ops_services_v1_health_service_proto_init() {
+	if File_controller_ops_services_v1_health_service_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_controller_ops_services_v1_health_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetHealthRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_controller_ops_services_v1_health_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*GetHealthResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_controller_ops_services_v1_health_service_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_controller_ops_services_v1_health_service_proto_goTypes,
+		DependencyIndexes: file_controller_ops_services_v1_health_service_proto_depIdxs,
+		MessageInfos:      file_controller_ops_services_v1_health_service_proto_msgTypes,
+	}.Build()
+	File_controller_ops_services_v1_health_service_proto = out.File
+	file_controller_ops_services_v1_health_service_proto_rawDesc = nil
+	file_controller_ops_services_v1_health_service_proto_goTypes = nil
+	file_controller_ops_services_v1_health_service_proto_depIdxs = nil
+}

internal/gen/controller/ops/services/health_service.pb.gw.go

@@ -0,0 +1,151 @@
+// Code generated by protoc-gen-grpc-gateway. DO NOT EDIT.
+// source: controller/ops/services/v1/health_service.proto
+
+/*
+Package services is a reverse proxy.
+
+It translates gRPC into RESTful JSON APIs.
+*/
+package services
+
+import (
+	"context"
+	"io"
+	"net/http"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/utilities"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+// Suppress "imported and not used" errors
+var _ codes.Code
+var _ io.Reader
+var _ status.Status
+var _ = runtime.String
+var _ = utilities.NewDoubleArray
+var _ = metadata.Join
+
+func request_HealthService_GetHealth_0(ctx context.Context, marshaler runtime.Marshaler, client HealthServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq GetHealthRequest
+	var metadata runtime.ServerMetadata
+
+	msg, err := client.GetHealth(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
+	return msg, metadata, err
+
+}
+
+func local_request_HealthService_GetHealth_0(ctx context.Context, marshaler runtime.Marshaler, server HealthServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
+	var protoReq GetHealthRequest
+	var metadata runtime.ServerMetadata
+
+	msg, err := server.GetHealth(ctx, &protoReq)
+	return msg, metadata, err
+
+}
+
+// RegisterHealthServiceHandlerServer registers the http handlers for service HealthService to "mux".
+// UnaryRPC     :call HealthServiceServer directly.
+// StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906.
+// Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterHealthServiceHandlerFromEndpoint instead.
+func RegisterHealthServiceHandlerServer(ctx context.Context, mux *runtime.ServeMux, server HealthServiceServer) error {
+
+	mux.Handle("GET", pattern_HealthService_GetHealth_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		var stream runtime.ServerTransportStream
+		ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		rctx, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/controller.ops.services.v1.HealthService/GetHealth", runtime.WithHTTPPathPattern("/health"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := local_request_HealthService_GetHealth_0(rctx, inboundMarshaler, server, req, pathParams)
+		md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_HealthService_GetHealth_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	return nil
+}
+
+// RegisterHealthServiceHandlerFromEndpoint is same as RegisterHealthServiceHandler but
+// automatically dials to "endpoint" and closes the connection when "ctx" gets done.
+func RegisterHealthServiceHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) {
+	conn, err := grpc.Dial(endpoint, opts...)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			if cerr := conn.Close(); cerr != nil {
+				grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr)
+			}
+			return
+		}
+		go func() {
+			<-ctx.Done()
+			if cerr := conn.Close(); cerr != nil {
+				grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr)
+			}
+		}()
+	}()
+
+	return RegisterHealthServiceHandler(ctx, mux, conn)
+}
+
+// RegisterHealthServiceHandler registers the http handlers for service HealthService to "mux".
+// The handlers forward requests to the grpc endpoint over "conn".
+func RegisterHealthServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error {
+	return RegisterHealthServiceHandlerClient(ctx, mux, NewHealthServiceClient(conn))
+}
+
+// RegisterHealthServiceHandlerClient registers the http handlers for service HealthService
+// to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "HealthServiceClient".
+// Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "HealthServiceClient"
+// doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in
+// "HealthServiceClient" to call the correct interceptors.
+func RegisterHealthServiceHandlerClient(ctx context.Context, mux *runtime.ServeMux, client HealthServiceClient) error {
+
+	mux.Handle("GET", pattern_HealthService_GetHealth_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+		inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
+		rctx, err := runtime.AnnotateContext(ctx, mux, req, "/controller.ops.services.v1.HealthService/GetHealth", runtime.WithHTTPPathPattern("/health"))
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+		resp, md, err := request_HealthService_GetHealth_0(rctx, inboundMarshaler, client, req, pathParams)
+		ctx = runtime.NewServerMetadataContext(ctx, md)
+		if err != nil {
+			runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
+			return
+		}
+
+		forward_HealthService_GetHealth_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
+
+	})
+
+	return nil
+}
+
+var (
+	pattern_HealthService_GetHealth_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"health"}, ""))
+)
+
+var (
+	forward_HealthService_GetHealth_0 = runtime.ForwardResponseMessage
+)

internal/gen/controller/ops/services/health_service_grpc.pb.go

@@ -0,0 +1,101 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+
+package services
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+// HealthServiceClient is the client API for HealthService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type HealthServiceClient interface {
+	GetHealth(ctx context.Context, in *GetHealthRequest, opts ...grpc.CallOption) (*GetHealthResponse, error)
+}
+
+type healthServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewHealthServiceClient(cc grpc.ClientConnInterface) HealthServiceClient {
+	return &healthServiceClient{cc}
+}
+
+func (c *healthServiceClient) GetHealth(ctx context.Context, in *GetHealthRequest, opts ...grpc.CallOption) (*GetHealthResponse, error) {
+	out := new(GetHealthResponse)
+	err := c.cc.Invoke(ctx, "/controller.ops.services.v1.HealthService/GetHealth", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// HealthServiceServer is the server API for HealthService service.
+// All implementations must embed UnimplementedHealthServiceServer
+// for forward compatibility
+type HealthServiceServer interface {
+	GetHealth(context.Context, *GetHealthRequest) (*GetHealthResponse, error)
+	mustEmbedUnimplementedHealthServiceServer()
+}
+
+// UnimplementedHealthServiceServer must be embedded to have forward compatible implementations.
+type UnimplementedHealthServiceServer struct {
+}
+
+func (UnimplementedHealthServiceServer) GetHealth(context.Context, *GetHealthRequest) (*GetHealthResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetHealth not implemented")
+}
+func (UnimplementedHealthServiceServer) mustEmbedUnimplementedHealthServiceServer() {}
+
+// UnsafeHealthServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to HealthServiceServer will
+// result in compilation errors.
+type UnsafeHealthServiceServer interface {
+	mustEmbedUnimplementedHealthServiceServer()
+}
+
+func RegisterHealthServiceServer(s grpc.ServiceRegistrar, srv HealthServiceServer) {
+	s.RegisterService(&HealthService_ServiceDesc, srv)
+}
+
+func _HealthService_GetHealth_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetHealthRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(HealthServiceServer).GetHealth(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/controller.ops.services.v1.HealthService/GetHealth",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(HealthServiceServer).GetHealth(ctx, req.(*GetHealthRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// HealthService_ServiceDesc is the grpc.ServiceDesc for HealthService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var HealthService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "controller.ops.services.v1.HealthService",
+	HandlerType: (*HealthServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetHealth",
+			Handler:    _HealthService_GetHealth_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "controller/ops/services/v1/health_service.proto",
+}

internal/proto/local/controller/ops/resources/health/v1/health.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package controller.ops.resources.health.v1;
+
+option go_package = "github.com/hashicorp/boundary/sdk/pbs/controller/ops/resources/health;health";
+
+// Health is the underlying resource used by the Health Service.
+// Empty for now as we don't want to commit to an API yet.
+message Health {}

internal/proto/local/controller/ops/services/v1/health_service.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package controller.ops.services.v1;
+
+option go_package = "github.com/hashicorp/boundary/internal/gen/controller/ops/services;services";
+
+import "google/api/annotations.proto";
+import "controller/ops/resources/health/v1/health.proto";
+
+service HealthService {
+  rpc GetHealth(GetHealthRequest) returns (GetHealthResponse) {
+    option (google.api.http) = {
+      get : "/health"
+    };
+  }
+}
+
+message GetHealthRequest {}
+
+message GetHealthResponse {}

internal/servers/controller/controller.go

@@ -25,6 +25,7 @@ import (
 	"github.com/hashicorp/boundary/internal/scheduler/job"
 	"github.com/hashicorp/boundary/internal/servers"
 	"github.com/hashicorp/boundary/internal/servers/controller/common"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/health"
 	"github.com/hashicorp/boundary/internal/session"
 	"github.com/hashicorp/boundary/internal/target"
 	"github.com/hashicorp/boundary/internal/types/scope"
@@ -80,6 +81,10 @@ type Controller struct {
 	kms *kms.Kms
 
 	enabledPlugins []base.EnabledPlugin
+
+	// Used to signal the Health Service to start
+	// replying to queries with "503 Service Unavailable".
+	HealthService *health.Service
 }
 
 func New(ctx context.Context, conf *Config) (*Controller, error) {
@@ -92,6 +97,7 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 		workerAuthCache:         new(sync.Map),
 		workerStatusUpdateTimes: new(sync.Map),
 		enabledPlugins:          conf.Server.EnabledPlugins,
+		apiListeners:            make([]*base.ServerListener, 0),
 	}
 
 	c.started.Store(false)

internal/servers/controller/handler.go

@@ -18,6 +18,7 @@ import (
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
+	opsservices "github.com/hashicorp/boundary/internal/gen/controller/ops/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/boundary/internal/servers/common"
 	"github.com/hashicorp/boundary/internal/servers/controller/auth"
@@ -27,6 +28,7 @@ import (
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/credentiallibraries"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/credentialstores"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/groups"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/health"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_catalogs"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_sets"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/hosts"
@@ -74,6 +76,28 @@ func (c *Controller) apiHandler(props HandlerProperties) (http.Handler, error) {
 	return eventsHandler, err
 }
 
+// GetHealthHandler returns a gRPC Gateway mux that is registered against the
+// controller's gRPC health service to make it accessible from an HTTP API.
+func (c *Controller) GetHealthHandler(lcfg *listenerutil.ListenerConfig) (http.Handler, error) {
+	const op = "controller.(Controller).GetHealthHandler"
+	if lcfg == nil {
+		return nil, fmt.Errorf("%s: received nil listener config", op)
+	}
+
+	healthGrpcGwMux := newGrpcGatewayMux()
+	err := registerHealthGrpcGatewayEndpoint(c.baseContext, healthGrpcGwMux, gatewayDialOptions(c.apiGrpcServerListener)...)
+	if err != nil {
+		return nil, fmt.Errorf("%s: failed to register health service handler: %w", op, err)
+	}
+
+	wrapped := wrapHandlerWithCommonFuncs(healthGrpcGwMux, c, HandlerProperties{lcfg, c.baseContext})
+	return common.WrapWithEventsHandler(wrapped, c.conf.Eventer, c.kms, lcfg)
+}
+
+func registerHealthGrpcGatewayEndpoint(ctx context.Context, gwMux *runtime.ServeMux, dialOptions ...grpc.DialOption) error {
+	return opsservices.RegisterHealthServiceHandlerFromEndpoint(ctx, gwMux, gatewayTarget, dialOptions)
+}
+
 func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 	// We have to check against the current services because the gRPC lib treats a duplicate
 	// register call as an error and os.Exits.
@@ -193,6 +217,11 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 		}
 		services.RegisterCredentialLibraryServiceServer(s, cl)
 	}
+	if _, ok := s.GetServiceInfo()[opsservices.HealthService_ServiceDesc.ServiceName]; !ok {
+		hs := health.NewService()
+		opsservices.RegisterHealthServiceServer(s, hs)
+		c.HealthService = hs
+	}
 
 	return nil
 }

internal/servers/controller/handlers/health/health.go

@@ -0,0 +1,38 @@
+package health
+
+import (
+	"context"
+	"net/http"
+
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/ops/services"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
+)
+
+type Service struct {
+	pbs.UnimplementedHealthServiceServer
+
+	replyWithServiceUnavailable bool
+}
+
+func NewService() *Service {
+	s := Service{}
+	return &s
+}
+
+func (s *Service) GetHealth(ctx context.Context, req *pbs.GetHealthRequest) (*pbs.GetHealthResponse, error) {
+	if s.replyWithServiceUnavailable {
+		err := handlers.SetStatusCode(ctx, http.StatusServiceUnavailable)
+		if err != nil {
+			return nil, err
+		}
+		return &pbs.GetHealthResponse{}, nil
+	}
+
+	return &pbs.GetHealthResponse{}, nil
+}
+
+// StartServiceUnavailableReplies gets returned to the caller of NewService.
+// When invoked, we start responding to any health queries with a 503.
+func (s *Service) StartServiceUnavailableReplies() {
+	s.replyWithServiceUnavailable = true
+}

internal/servers/controller/handlers/health/health_test.go

@@ -0,0 +1,86 @@
+package health
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/gen/controller/ops/services"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestGetHealth(t *testing.T) {
+	tests := []struct {
+		name                 string
+		ctx                  context.Context
+		serviceUnavailable   bool
+		expGetHealthResponse *services.GetHealthResponse
+		expErr               bool
+		expErrMsg            string
+	}{
+		{
+			name:                 "healthy reply",
+			ctx:                  context.Background(),
+			serviceUnavailable:   false,
+			expGetHealthResponse: &services.GetHealthResponse{},
+			expErr:               false,
+		},
+		{
+			name:                 "service unavailable reply",
+			ctx:                  grpc.NewContextWithServerTransportStream(context.Background(), &testServerTransportStream{expHttpCode: "503"}),
+			serviceUnavailable:   true,
+			expGetHealthResponse: &services.GetHealthResponse{},
+			expErr:               false,
+		},
+		{
+			name:               "get health error",
+			ctx:                context.Background(),
+			serviceUnavailable: true,
+			expErr:             true,
+			expErrMsg:          "handlers.SetHttpStatusCode: internal error, unknown: error #500: rpc error: code = Internal desc = grpc: failed to fetch the stream from the context context.Background",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hs := NewService()
+
+			if tt.serviceUnavailable {
+				hs.StartServiceUnavailableReplies()
+			}
+
+			rsp, err := hs.GetHealth(tt.ctx, &services.GetHealthRequest{})
+			if tt.expErr {
+				require.Error(t, err)
+				require.EqualError(t, err, tt.expErrMsg)
+				return
+			}
+
+			require.NoError(t, err)
+			require.EqualValues(t, tt.expGetHealthResponse, rsp)
+		})
+	}
+}
+
+type testServerTransportStream struct {
+	expHttpCode string
+}
+
+func (s *testServerTransportStream) Method() string                  { return "" }
+func (s *testServerTransportStream) SendHeader(md metadata.MD) error { return nil }
+func (s *testServerTransportStream) SetTrailer(md metadata.MD) error { return nil }
+func (s *testServerTransportStream) SetHeader(md metadata.MD) error {
+	codes, ok := md["x-http-code"]
+	if !ok {
+		return fmt.Errorf("x-http-code header not found")
+	}
+	if len(codes) != 1 {
+		return fmt.Errorf("expected only one element in http codes, got %d", len(codes))
+	}
+	if codes[0] != s.expHttpCode {
+		return fmt.Errorf("expected http code %q, got %q", s.expHttpCode, codes[0])
+	}
+	return nil
+}

internal/servers/controller/testing.go

@@ -576,7 +576,7 @@ func TestControllerConfig(t *testing.T, ctx context.Context, tc *TestController,
 	for _, listener := range opts.Config.Listeners {
 		listener.RandomPort = true
 	}
-	if err := tc.b.SetupListeners(nil, opts.Config.SharedConfig, []string{"api", "cluster"}); err != nil {
+	if err := tc.b.SetupListeners(nil, opts.Config.SharedConfig, []string{"api", "cluster", "ops"}); err != nil {
 		t.Fatal(err)
 	}
 	if err := tc.b.SetupControllerPublicClusterAddress(opts.Config, ""); err != nil {

internal/servers/worker/listeners.go

@@ -30,7 +30,7 @@ func (w *Worker) startListeners() error {
 	for _, ln := range w.conf.Listeners {
 		for _, purpose := range ln.Config.Purpose {
 			switch purpose {
-			case "api", "cluster":
+			case "api", "cluster", "ops":
 				// We may have this in dev mode; ignore
 				continue
 

sdk/pbs/controller/ops/resources/health/health.pb.go

@@ -0,0 +1,141 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.27.1
+// 	protoc        v3.17.3
+// source: controller/ops/resources/health/v1/health.proto
+
+package health
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// Health is the underlying resource used by the Health Service.
+// Empty for now as we don't want to commit to an API yet.
+type Health struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *Health) Reset() {
+	*x = Health{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_controller_ops_resources_health_v1_health_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Health) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Health) ProtoMessage() {}
+
+func (x *Health) ProtoReflect() protoreflect.Message {
+	mi := &file_controller_ops_resources_health_v1_health_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Health.ProtoReflect.Descriptor instead.
+func (*Health) Descriptor() ([]byte, []int) {
+	return file_controller_ops_resources_health_v1_health_proto_rawDescGZIP(), []int{0}
+}
+
+var File_controller_ops_resources_health_v1_health_proto protoreflect.FileDescriptor
+
+var file_controller_ops_resources_health_v1_health_proto_rawDesc = []byte{
+	0x0a, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73,
+	0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74,
+	0x68, 0x2f, 0x76, 0x31, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x12, 0x22, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70,
+	0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x68, 0x65, 0x61, 0x6c,
+	0x74, 0x68, 0x2e, 0x76, 0x31, 0x22, 0x08, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x42,
+	0x4e, 0x5a, 0x4c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61,
+	0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79,
+	0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x70, 0x62, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
+	0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x73, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x3b, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_controller_ops_resources_health_v1_health_proto_rawDescOnce sync.Once
+	file_controller_ops_resources_health_v1_health_proto_rawDescData = file_controller_ops_resources_health_v1_health_proto_rawDesc
+)
+
+func file_controller_ops_resources_health_v1_health_proto_rawDescGZIP() []byte {
+	file_controller_ops_resources_health_v1_health_proto_rawDescOnce.Do(func() {
+		file_controller_ops_resources_health_v1_health_proto_rawDescData = protoimpl.X.CompressGZIP(file_controller_ops_resources_health_v1_health_proto_rawDescData)
+	})
+	return file_controller_ops_resources_health_v1_health_proto_rawDescData
+}
+
+var file_controller_ops_resources_health_v1_health_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_controller_ops_resources_health_v1_health_proto_goTypes = []interface{}{
+	(*Health)(nil), // 0: controller.ops.resources.health.v1.Health
+}
+var file_controller_ops_resources_health_v1_health_proto_depIdxs = []int32{
+	0, // [0:0] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_controller_ops_resources_health_v1_health_proto_init() }
+func file_controller_ops_resources_health_v1_health_proto_init() {
+	if File_controller_ops_resources_health_v1_health_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_controller_ops_resources_health_v1_health_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Health); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_controller_ops_resources_health_v1_health_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_controller_ops_resources_health_v1_health_proto_goTypes,
+		DependencyIndexes: file_controller_ops_resources_health_v1_health_proto_depIdxs,
+		MessageInfos:      file_controller_ops_resources_health_v1_health_proto_msgTypes,
+	}.Build()
+	File_controller_ops_resources_health_v1_health_proto = out.File
+	file_controller_ops_resources_health_v1_health_proto_rawDesc = nil
+	file_controller_ops_resources_health_v1_health_proto_goTypes = nil
+	file_controller_ops_resources_health_v1_health_proto_depIdxs = nil
+}

website/content/docs/configuration/controller.mdx

@@ -61,15 +61,23 @@ description will be read.
 - `auth_token_time_to_live` - Maximum time to live (TTL) for all auth tokens globally (pertains
   to all tokens from all auth methods). Valid time units are anything specified by Golang's
   [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. Default is 7 days.
+
 - `auth_token_time_to_stale` - Maximum time of inactivity for all auth tokens globally (pertains
   to all tokens from all auth methods). Valid time units are anything specified by Golang's
   [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. Default is 1 day.
 
+- `graceful_shutdown_wait_duration` - Amount of time Boundary will wait before initiating the shutdown procedure,
+  after receiving a shutdown signal. In this state, Boundary still processes requests as normal but replies 
+  with `503 Service Unavailable` to any health requests. This is designed to allow an operator to configure
+  load-balancers to preemptively stop new traffic to a Boundary instance that is going away. Valid time units
+  are anything specified by Go's [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. Only
+  used when an `ops` listener is set and the Controller is present. Default is 0 seconds.
+
 ## KMS Configuration
 
 The controller requires two KMS stanzas for `root` and `worker-auth` purposes:
 
-```kms
+```hcl
 # Root KMS configuration block: this is the root key for Boundary
 # Use a production KMS such as AWS KMS in production installs
 kms "aead" {
@@ -130,6 +138,9 @@ controller {
   name = "demo-controller-1"
   description = "A controller for a demo!"
 
+  # After receiving a shutdown signal, Boundary will wait 10s before initiating the shutdown process.
+  graceful_shutdown_wait_duration = "10s"
+
   # Database URL for postgres. This can be a direct "postgres://"
   # URL, or it can be "file://" to read the contents of a file to
   # supply the url, or "env://" to name an environment variable
@@ -162,6 +173,16 @@ listener "tcp" {
   purpose = "cluster"
 }
 
+listener "tcp" {
+  # Should be the address of the NIC where your external systems'
+  # (eg: Load-Balancer) will connect on.
+  address = "10.0.0.1"
+  # The purpose of this listener block
+  purpose = "ops"
+
+  tls_disable = false
+}
+
 # Root KMS configuration block: this is the root key for Boundary
 # Use a production KMS such as AWS KMS in production installs
 kms "aead" {

website/content/docs/configuration/index.mdx

@@ -30,13 +30,15 @@ After the configuration is written, use the `-config` flag to specify a local pa
 - [`listener`](/docs/configuration/listener): Configures the listeners on which
   Boundary serves traffic (API, cluster, and proxy).
 
-  Controllers will have two listener blocks: one marked for `api` purpose and
-  the other marked for `cluster` purpose. By default, the API listener runs on
-  :9200 and the cluster listener (currently used for worker communication) uses
-  :9201.
+  Controllers must have at least two listener blocks defined: one marked for
+  `api` purpose and the other marked for `cluster` purpose. 
 
   Workers will have only one listener, marked for `proxy` purpose.
 
+  Optionally, the `ops` purpose listener block serves as a higher-level listener
+  where Boundary's operational endpoints live (eg: /health).
+
+
 - [`kms`](/docs/configuration/kms): Configures KMS blocks [for various
   purposes](/docs/concepts/security/data-encryption).
 

website/content/docs/configuration/listener/index.mdx

@@ -9,7 +9,35 @@ description: |-
 # `listener` Stanza
 
 The `listener` stanza configures the addresses and ports on which Boundary will
-respond to requests. At this time, there are two listeners - [TCP][tcp] and [Unix][unix].
+respond to requests. At this time, there are two listener types:
+[TCP](/docs/configuration/listener/tcp) and [Unix](/docs/configuration/listener/unix).
+
+
+## `listener` Purpose
+
+The `purpose` field is used to determine what part of Boundary's functionality is
+started and exposed for a given listener block. Each listener must have a `purpose` set.
+
+Currently, Boundary recognizes the following listener purposes:
+
+- `api`: Starts up and exposes Boundary's controller API.
+  At least one `api` listener must be present if a Boundary
+  instance is set-up as a controller. 
+  By default, it runs on `:9200`.
+
+- `cluster`: Starts up and exposes Boundary's worker and
+  controller communication layer. Only one `cluster` listener
+  is allowed per Boundary instance. Must be present if a Boundary
+  instance is set-up as a controller. 
+  By default, it runs on `:9201`.
+
+- `proxy`: Starts up a Boundary Worker. By default, it runs
+on `:9202`.
+
+- `ops`: Starts up and exposes Boundary's operational
+endpoints (eg: /health). By default, it runs on `:9203`.
+This listener's exposed functionality depends on what Boundary
+components you're running on any given Boundary instance (eg: Health does not
+run unless your Boundary instance has a controller running)
+
 
-[tcp]: /docs/configuration/listener/tcp
-[unix]: /docs/configuration/listener/unix

website/content/docs/configuration/listener/tcp.mdx

@@ -25,8 +25,8 @@ allowed.
 
 ### General
 
-- `purpose` `(string: "")` - Specifies the purpose. Can be `api`, `cluster`, or
-  `proxy`.
+- `purpose` `(string: "")` - Specifies the purpose. Can be `api`, `cluster`,
+  `proxy`, or `ops`.
 
 - `address` `(string: "127.0.0.1:9200")` – Specifies the address to bind to for
   listening.
@@ -86,9 +86,9 @@ allowed.
 
 ### TLS
 
-~> All `tls` parameters are valid only for the `api` listener. `cluster` and
-`proxy` connections use their own ephemeral TLS stacks. For more information,
-see [the connections security concepts
+~> `tls` parameters are valid for `api` and `ops` listeners. `cluster`
+and `proxy` connections use their own ephemeral TLS stacks. For more 
+information, see [the connections security concepts
 page](https://www.boundaryproject.io/docs/concepts/security/connections-tls).
 
 - `tls_disable` `(string: "false")` – Specifies if TLS will be disabled. Boundary

website/content/docs/configuration/listener/unix.mdx

@@ -27,7 +27,7 @@ allowed.
 
 ### General
 
-- `purpose` `(string: "")` - Specifies the purpose. Can be `api` or `cluster`.
+- `purpose` `(string: "")` - Specifies the purpose. Can be `api`, `ops` or `cluster`.
 
 - `address` `(string: "")` – Specifies the address (filesystem path) to bind to
   for listening.
@@ -87,9 +87,9 @@ allowed.
 
 ### TLS
 
-~> All `tls` parameters are valid only for the `api` listener. `cluster` and
-`proxy` connections use their own ephemeral TLS stacks. For more information,
-see [the connections security concepts
+~> `tls` parameters are valid for `api` and `ops` listeners. `cluster`
+and `proxy` connections use their own ephemeral TLS stacks. For more 
+information, see [the connections security concepts
 page](https://www.boundaryproject.io/docs/concepts/security/connections-tls).
 
 - `tls_disable` `(string: "false")` – Specifies if TLS will be disabled. Boundary