diff --git a/CHANGELOG.md b/CHANGELOG.md index 54e4bf82b..94afe9a94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,15 @@ Canonical reference for changes, improvements, and bugfixes for Boundary. ## Next + +### New and Improved +* controller: new health endpoint ([PR](https://github.com/hashicorp/boundary/pull/1882)). + ### Bug Fixes * worker: create new error to prevent `event.newError: missing error: invalid parameter` and handle session cancel -with no TOFU token ([Issue](https://github.com/hashicorp/boundary/issues/1902), + with no TOFU token ([Issue](https://github.com/hashicorp/boundary/issues/1902), [PR](https://github.com/hashicorp/boundary/pull/1929)) + ## 0.7.6 (2022/03/15) ### Bug Fixes diff --git a/internal/cmd/base/listener.go b/internal/cmd/base/listener.go index 97bfbec0a..fef18788a 100644 --- a/internal/cmd/base/listener.go +++ b/internal/cmd/base/listener.go @@ -128,6 +128,8 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U l.Address = "127.0.0.1:9201" case "proxy": l.Address = "127.0.0.1:9202" + case "ops": + l.Address = "127.0.0.1:9203" default: l.Address = "127.0.0.1:9200" } @@ -141,6 +143,8 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U port = "9201" case "proxy": port = "9202" + case "ops": + port = "9203" default: port = "9200" } diff --git a/internal/cmd/commands/dev/dev.go b/internal/cmd/commands/dev/dev.go index 086586824..aa625fa94 100644 --- a/internal/cmd/commands/dev/dev.go +++ b/internal/cmd/commands/dev/dev.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/boundary/internal/auth/password" "github.com/hashicorp/boundary/internal/cmd/base" "github.com/hashicorp/boundary/internal/cmd/config" + "github.com/hashicorp/boundary/internal/cmd/ops" "github.com/hashicorp/boundary/internal/host/static" "github.com/hashicorp/boundary/internal/iam" "github.com/hashicorp/boundary/internal/intglobals" @@ -39,6 +40,7 @@ var ( type Command struct { *base.Server + opsServer *ops.Server SighupCh chan struct{} childSighupCh []chan struct{} @@ -68,6 +70,7 @@ type Command struct { flagWorkerAuthKey string flagWorkerProxyListenAddr string flagWorkerPublicAddr string + flagOpsListenAddr string flagUiPassthroughDir string flagRecoveryKey string flagDatabaseUrl string @@ -213,6 +216,12 @@ func (c *Command) Flags() *base.FlagSets { EnvVar: "BOUNDARY_DEV_CONTROLLER_PUBLIC_CLUSTER_ADDRESS", Usage: "Public address at which the controller is reachable for cluster tasks (like worker connections).", }) + f.StringVar(&base.StringVar{ + Name: "ops-listen-address", + Target: &c.flagOpsListenAddr, + EnvVar: "BOUNDARY_DEV_OPS_LISTEN_ADDRESS", + Usage: "Address to bind to for \"ops\" purpose. If this begins with a forward slash, it will be assumed to be a Unix domain socket path.", + }) f.BoolVar(&base.BoolVar{ Name: "controller-only", @@ -449,6 +458,14 @@ func (c *Command) Run(args []string) int { l.Type = "unix" } + case "ops": + if c.flagOpsListenAddr != "" { + l.Address = c.flagOpsListenAddr + } + if strings.HasPrefix(l.Address, "/") { + l.Type = "unix" + } + case "proxy": if c.flagWorkerProxyListenAddr != "" { l.Address = c.flagWorkerProxyListenAddr @@ -543,7 +560,7 @@ func (c *Command) Run(args []string) int { c.Info["[Recovery] AEAD Key Bytes"] = c.Config.DevRecoveryKey // Initialize the listeners - if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy"}); err != nil { + if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy", "ops"}); err != nil { c.UI.Error(err.Error()) return base.CommandUserError } @@ -636,7 +653,7 @@ func (c *Command) Run(args []string) int { var err error c.worker, err = worker.New(conf) if err != nil { - c.UI.Error(fmt.Errorf("Error initializing controller: %w", err).Error()) + c.UI.Error(fmt.Errorf("Error initializing worker: %w", err).Error()) return base.CommandCliError } @@ -654,6 +671,15 @@ func (c *Command) Run(args []string) int { } } + opsServer, err := ops.NewServer(c.Logger, c.controller, c.Listeners...) + if err != nil { + c.UI.Error(fmt.Errorf("Failed to start ops listeners: %w", err).Error()) + return base.CommandCliError + + } + c.opsServer = opsServer + c.opsServer.Start() + // Wait for shutdown shutdownTriggered := false @@ -678,6 +704,10 @@ func (c *Command) Run(args []string) int { } }() + if c.Config.Controller != nil { + c.opsServer.WaitIfHealthExists(c.Config.Controller.GracefulShutdownWaitDuration, c.UI) + } + if !c.flagControllerOnly { if err := c.worker.Shutdown(false); err != nil { c.UI.Error(fmt.Errorf("Error shutting down worker: %w", err).Error()) @@ -688,6 +718,11 @@ func (c *Command) Run(args []string) int { c.UI.Error(fmt.Errorf("Error shutting down controller: %w", err).Error()) } + err := c.opsServer.Shutdown() + if err != nil { + c.UI.Error(fmt.Errorf("Failed to shutdown ops listeners: %w", err).Error()) + } + shutdownTriggered = true case <-c.SigUSR2Ch: diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go index 2d5763ad0..129ce2909 100644 --- a/internal/cmd/commands/server/server.go +++ b/internal/cmd/commands/server/server.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/boundary/globals" "github.com/hashicorp/boundary/internal/cmd/base" "github.com/hashicorp/boundary/internal/cmd/config" + "github.com/hashicorp/boundary/internal/cmd/ops" "github.com/hashicorp/boundary/internal/db" "github.com/hashicorp/boundary/internal/db/schema" "github.com/hashicorp/boundary/internal/errors" @@ -44,6 +45,7 @@ var ( type Command struct { *base.Server + opsServer *ops.Server SighupCh chan struct{} SigUSR2Ch chan struct{} @@ -233,6 +235,7 @@ func (c *Command) Run(args []string) int { if lnConfig.Address == "" { lnConfig.Address = "127.0.0.1:9202" } + case "ops": default: c.UI.Error(fmt.Sprintf("Unknown listener purpose %q", lnConfig.Purpose[0])) return base.CommandUserError @@ -333,7 +336,7 @@ func (c *Command) Run(args []string) int { } } } - if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy"}); err != nil { + if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy", "ops"}); err != nil { c.UI.Error(err.Error()) return base.CommandUserError } @@ -473,6 +476,14 @@ func (c *Command) Run(args []string) int { } } + opsServer, err := ops.NewServer(c.Logger, c.controller, c.Listeners...) + if err != nil { + c.UI.Error(err.Error()) + return base.CommandCliError + } + c.opsServer = opsServer + c.opsServer.Start() + // Inform any tests that the server is ready if c.startedCh != nil { close(c.startedCh) @@ -660,6 +671,10 @@ func (c *Command) WaitForInterrupt() int { } }() + if c.Config.Controller != nil { + c.opsServer.WaitIfHealthExists(c.Config.Controller.GracefulShutdownWaitDuration, c.UI) + } + // Do worker shutdown if c.Config.Worker != nil { if err := c.worker.Shutdown(false); err != nil { @@ -674,6 +689,11 @@ func (c *Command) WaitForInterrupt() int { } } + err := c.opsServer.Shutdown() + if err != nil { + c.UI.Error(fmt.Errorf("Error shutting down ops listeners: %w", err).Error()) + } + shutdownTriggered = true case <-c.SighupCh: diff --git a/internal/cmd/config/config.go b/internal/cmd/config/config.go index 9feaa4b68..5b862cd94 100644 --- a/internal/cmd/config/config.go +++ b/internal/cmd/config/config.go @@ -74,6 +74,11 @@ listener "tcp" { listener "tcp" { purpose = "cluster" } + +listener "tcp" { + purpose = "ops" + tls_disable = true +} ` devWorkerExtraConfig = ` @@ -128,6 +133,12 @@ type Controller struct { AuthTokenTimeToStale interface{} `hcl:"auth_token_time_to_stale"` AuthTokenTimeToStaleDuration time.Duration + // GracefulShutdownWait is the amount of time that we'll wait before actually + // starting the Controller shutdown. This allows the health endpoint to + // return a status code to indicate that the instance is shutting down. + GracefulShutdownWait interface{} `hcl:"graceful_shutdown_wait_duration"` + GracefulShutdownWaitDuration time.Duration + // StatusGracePeriod represents the period of time (as a duration) that the // controller will wait before marking connections from a disconnected worker // as invalid. @@ -341,6 +352,14 @@ func Parse(d string) (*Config, error) { result.Controller.AuthTokenTimeToStaleDuration = t } + if result.Controller.GracefulShutdownWait != "" { + t, err := parseutil.ParseDurationSecond(result.Controller.GracefulShutdownWait) + if err != nil { + return result, err + } + result.Controller.GracefulShutdownWaitDuration = t + } + if result.Controller.Database != nil { if result.Controller.Database.MaxOpenConnectionsRaw != nil { switch t := result.Controller.Database.MaxOpenConnectionsRaw.(type) { diff --git a/internal/cmd/config/config_test.go b/internal/cmd/config/config_test.go index 7cebfc82e..73bfca194 100644 --- a/internal/cmd/config/config_test.go +++ b/internal/cmd/config/config_test.go @@ -38,6 +38,11 @@ func TestDevController(t *testing.T) { Type: "tcp", Purpose: []string{"cluster"}, }, + { + Type: "tcp", + Purpose: []string{"ops"}, + TLSDisable: true, + }, }, Seals: []*configutil.KMS{ { @@ -75,6 +80,7 @@ func TestDevController(t *testing.T) { exp.Listeners[0].RawConfig = actual.Listeners[0].RawConfig exp.Listeners[1].RawConfig = actual.Listeners[1].RawConfig + exp.Listeners[2].RawConfig = actual.Listeners[2].RawConfig exp.Seals[0].Config["key"] = actual.Seals[0].Config["key"] exp.Seals[1].Config["key"] = actual.Seals[1].Config["key"] exp.Seals[2].Config["key"] = actual.Seals[2].Config["key"] diff --git a/internal/cmd/ops/server.go b/internal/cmd/ops/server.go new file mode 100644 index 000000000..8ecfd8d8e --- /dev/null +++ b/internal/cmd/ops/server.go @@ -0,0 +1,214 @@ +// Package ops encapsulates the lifecycle of Boundary's ops-purpose listeners +// and servers: Creating, setting them up, starting and shutdown. +package ops + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "time" + + "github.com/hashicorp/boundary/internal/cmd/base" + "github.com/hashicorp/boundary/internal/libs/alpnmux" + "github.com/hashicorp/boundary/internal/servers/controller" + "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-secure-stdlib/listenerutil" + "github.com/mitchellh/cli" + "github.com/pkg/errors" +) + +// Server is a collection of all state required to serve +// multiple ops endpoints through a single object. +type Server struct { + bundles []*opsBundle + controller *controller.Controller +} + +type opsBundle struct { + ln *base.ServerListener + h http.Handler + startFn []func() +} + +// NewServer iterates through all the listeners and sets up HTTP Servers for each, along with individual handlers. +// If Controller is set-up, NewServer will set-up a health endpoint for it. +func NewServer(l hclog.Logger, c *controller.Controller, listeners ...*base.ServerListener) (*Server, error) { + const op = "ops.NewServer()" + if l == nil { + return nil, fmt.Errorf("%s: missing logger", op) + } + + bundles := make([]*opsBundle, 0, len(listeners)) + for _, ln := range listeners { + if ln == nil || ln.Config == nil { + continue + } + if ln.Config.Purpose[0] != "ops" { + continue + } + + h, err := createOpsHandler(ln.Config, c) + if err != nil { + return nil, err + } + + b := &opsBundle{ln: ln, h: h} + b.ln.HTTPServer = createHttpServer(l, b.h, b.ln.Config) + + funcs, err := getStartFn(b.ln) + if err != nil { + return nil, err + } + b.startFn = funcs + + bundles = append(bundles, b) + } + + return &Server{bundles, c}, nil +} + +// Starts all goroutines that were set-up in NewServer. +// These goroutines start the HTTP Servers on the appropriate +// listeners (as defined by the bundle). +func (s *Server) Start() { + for _, b := range s.bundles { + for _, f := range b.startFn { + f() + } + } +} + +// Shutdown attempts to cleanly shutdown all running ops listeners and HTTP servers. +func (s *Server) Shutdown() error { + const op = "ops.(Server).Shutdown" + + var closeErrors *multierror.Error + for _, b := range s.bundles { + if b == nil || b.ln == nil || b.ln.Config == nil || b.ln.Mux == nil || b.ln.HTTPServer == nil { + return fmt.Errorf("%s: missing bundle, listener or its fields", op) + } + + ctx, cancel := context.WithTimeout(context.Background(), b.ln.Config.MaxRequestDuration) + defer cancel() + + err := b.ln.HTTPServer.Shutdown(ctx) + if err != nil { + multierror.Append(closeErrors, fmt.Errorf("%s: failed to shutdown http server: %w", op, err)) + } + + err = b.ln.Mux.Close() + err = listenerCloseErrorCheck(b.ln.Config.Type, err) + if err != nil { + multierror.Append(closeErrors, fmt.Errorf("%s: failed to close listener mux: %w", op, err)) + } + } + + return closeErrors.ErrorOrNil() +} + +// WaitIfHealthExists waits for a configurable period of time `d` if the health endpoint has been +// configured (i.e the Controller exists and ops listeners have been set-up) +func (s *Server) WaitIfHealthExists(d time.Duration, ui cli.Ui) { + if s.controller == nil || s.controller.HealthService == nil { + return + } + if len(s.bundles) == 0 { + return + } + + // If we have ops listeners and the health endpoint is up, + // we wait for a configurable amount of time before shutting down. + // This is to give time for health to report unhealthy and for external + // systems to pick up on that. + ui.Output(fmt.Sprintf("==> Health is enabled, waiting %s before shutdown", d.String())) + s.controller.HealthService.StartServiceUnavailableReplies() + <-time.After(d) +} + +func createOpsHandler(lncfg *listenerutil.ListenerConfig, c *controller.Controller) (http.Handler, error) { + mux := http.NewServeMux() + if c != nil && c.HealthService != nil { + h, err := c.GetHealthHandler(lncfg) + if err != nil { + return nil, err + } + mux.Handle("/health", h) + } + + return cleanhttp.PrintablePathCheckHandler(mux, nil), nil +} + +func createHttpServer(l hclog.Logger, h http.Handler, lncfg *listenerutil.ListenerConfig) *http.Server { + s := &http.Server{ + Handler: h, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + IdleTimeout: 5 * time.Minute, + ErrorLog: l.StandardLogger(nil), + } + + if lncfg.HTTPReadHeaderTimeout > 0 { + s.ReadHeaderTimeout = lncfg.HTTPReadHeaderTimeout + } + if lncfg.HTTPReadTimeout > 0 { + s.ReadTimeout = lncfg.HTTPReadTimeout + } + if lncfg.HTTPWriteTimeout > 0 { + s.WriteTimeout = lncfg.HTTPWriteTimeout + } + if lncfg.HTTPIdleTimeout > 0 { + s.IdleTimeout = lncfg.HTTPIdleTimeout + } + + return s +} + +func getStartFn(ln *base.ServerListener) ([]func(), error) { + const op = "getStartFn()" + + funcs := make([]func(), 0) + switch ln.Config.TLSDisable { + case true: + l, err := ln.Mux.RegisterProto(alpnmux.NoProto, nil) + if err != nil { + return nil, fmt.Errorf("%s: error getting non-tls listener: %w", op, err) + } + if l == nil { + return nil, fmt.Errorf("%s: could not get non-tls listener", op) + } + funcs = append(funcs, func() { go ln.HTTPServer.Serve(l) }) + + default: + for _, v := range []string{"", "http/1.1", "h2"} { + l := ln.Mux.GetListener(v) + if l == nil { + return nil, fmt.Errorf("%s: could not get tls proto %q listener", op, v) + } + funcs = append(funcs, func() { go ln.HTTPServer.Serve(l) }) + } + } + + return funcs, nil +} + +func listenerCloseErrorCheck(lnType string, err error) error { + if errors.Is(err, net.ErrClosed) { + // Ignore net.ErrClosed - The listener was already closed, + // so there's nothing else to do. + return nil + } + if _, ok := err.(*os.PathError); ok && lnType == "unix" { + // The underlying rmListener probably tried to remove + // the file but it didn't exist, ignore the error; + // this is a conflict between rmListener and the + // default Go behavior of removing auto-vivified + // Unix domain sockets. + return nil + } + + return err +} diff --git a/internal/cmd/ops/server_test.go b/internal/cmd/ops/server_test.go new file mode 100644 index 000000000..459117214 --- /dev/null +++ b/internal/cmd/ops/server_test.go @@ -0,0 +1,867 @@ +package ops + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "net" + "net/http" + "os" + "sync" + "testing" + "time" + + "github.com/hashicorp/boundary/internal/cmd/base" + "github.com/hashicorp/boundary/internal/libs/alpnmux" + "github.com/hashicorp/boundary/internal/servers/controller" + "github.com/hashicorp/boundary/internal/servers/controller/handlers/health" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-secure-stdlib/base62" + "github.com/hashicorp/go-secure-stdlib/configutil/v2" + "github.com/hashicorp/go-secure-stdlib/listenerutil" + "github.com/hashicorp/go-secure-stdlib/reloadutil" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestNewServer(t *testing.T) { + tests := []struct { + name string + logger hclog.Logger + c *controller.Controller + listeners []*base.ServerListener + assertions func(t *testing.T, s *Server) + expErr bool + expErrMsg string + }{ + { + name: "nil logger", + logger: nil, + listeners: []*base.ServerListener{{}}, + expErr: true, + expErrMsg: "ops.NewServer(): missing logger", + }, + { + name: "nil ServerListeners", + logger: hclog.Default(), + listeners: nil, + expErr: false, + assertions: func(t *testing.T, s *Server) { + require.Len(t, s.bundles, 0) + }, + }, + { + name: "empty ServerListeners", + logger: hclog.Default(), + listeners: []*base.ServerListener{}, + expErr: false, + assertions: func(t *testing.T, s *Server) { + require.Len(t, s.bundles, 0) + }, + }, + { + name: "nil listeners", + logger: hclog.Default(), + listeners: []*base.ServerListener{nil, nil}, + expErr: false, + assertions: func(t *testing.T, s *Server) { + require.Len(t, s.bundles, 0) + }, + }, + { + name: "listeners with nil config", + logger: hclog.Default(), + listeners: []*base.ServerListener{ + { + Config: nil, + }, + { + Config: nil, + }, + }, + expErr: false, + assertions: func(t *testing.T, s *Server) { + require.Len(t, s.bundles, 0) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, err := NewServer(tt.logger, tt.c, tt.listeners...) + if tt.expErr { + require.EqualError(t, err, tt.expErrMsg) + require.Nil(t, s) + return + } + + require.NoError(t, err) + require.NotNil(t, s) + + if tt.assertions != nil { + tt.assertions(t, s) + } + }) + } +} + +func TestNewServerIntegration(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T) + listeners []*listenerutil.ListenerConfig + assertions func(t *testing.T, addrs []string) + expErr bool + expErrMsg string + }{ + { + name: "one tcp ops listener", + listeners: []*listenerutil.ListenerConfig{ + { + Type: "tcp", + Purpose: []string{"ops"}, + Address: "127.0.0.1:0", + TLSDisable: true, + }, + }, + assertions: func(t *testing.T, addrs []string) { + _, err := http.Get("http://" + addrs[0]) + require.NoError(t, err) + }, + }, + { + name: "multiple tcp ops listeners", + listeners: []*listenerutil.ListenerConfig{ + { + Type: "tcp", + Purpose: []string{"ops"}, + Address: "127.0.0.1:0", + TLSDisable: true, + }, + { + Type: "tcp", + Purpose: []string{"ops"}, + Address: "127.0.0.1:0", + TLSDisable: true, + }, + }, + assertions: func(t *testing.T, addrs []string) { + _, err := http.Get("http://" + addrs[0]) + require.NoError(t, err) + + _, err = http.Get("http://" + addrs[1]) + require.NoError(t, err) + }, + }, + { + name: "one unix socket ops listener", + listeners: []*listenerutil.ListenerConfig{ + { + Type: "unix", + Purpose: []string{"ops"}, + Address: fmt.Sprintf("/tmp/boundary-opslistener-test-%s.sock", base62.MustRandom(5)), + TLSDisable: true, + }, + }, + expErr: false, + assertions: func(t *testing.T, addrs []string) { + conn, err := net.Dial("unix", addrs[0]) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + + cl := http.Client{ + Transport: &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { return conn, nil }, + }, + } + _, err = cl.Get("http://random.domain") + require.NoError(t, err) + }, + }, + { + name: "multiple unix socket ops listeners", + listeners: []*listenerutil.ListenerConfig{ + { + Type: "unix", + Purpose: []string{"ops"}, + Address: fmt.Sprintf("/tmp/boundary-opslistener-test-%s.sock", base62.MustRandom(5)), + TLSDisable: true, + }, + { + Type: "unix", + Purpose: []string{"ops"}, + Address: fmt.Sprintf("/tmp/boundary-opslistener-test-%s.sock", base62.MustRandom(5)), + TLSDisable: true, + }, + }, + expErr: false, + assertions: func(t *testing.T, addrs []string) { + for _, addr := range addrs { + conn, err := net.Dial("unix", addr) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + + cl := http.Client{ + Transport: &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { return conn, nil }, + }, + } + _, err = cl.Get("http://random.domain") + require.NoError(t, err) + } + }, + }, + { + name: "multiple tcp ops listeners (tls)", + setup: func(t *testing.T) { + testTlsSetup(t, "./boundary-opslistener-cert1.cert", "./boundary-opslistener-pk1.key") + testTlsSetup(t, "./boundary-opslistener-cert2.cert", "./boundary-opslistener-pk2.key") + }, + listeners: []*listenerutil.ListenerConfig{ + { + Type: "tcp", + Purpose: []string{"ops"}, + Address: "127.0.0.1:0", + TLSKeyFile: "./boundary-opslistener-pk1.key", + TLSCertFile: "./boundary-opslistener-cert1.cert", + }, + { + Type: "tcp", + Purpose: []string{"ops"}, + Address: "127.0.0.1:0", + TLSKeyFile: "./boundary-opslistener-pk2.key", + TLSCertFile: "./boundary-opslistener-cert2.cert", + }, + }, + assertions: func(t *testing.T, addrs []string) { + cl := testTlsHttpClient(t, "./boundary-opslistener-cert1.cert") + _, err := cl.Get("https://" + addrs[0]) + require.NoError(t, err) + + cl = testTlsHttpClient(t, "./boundary-opslistener-cert2.cert") + _, err = cl.Get("https://" + addrs[1]) + require.NoError(t, err) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup(t) + } + + bs := &base.Server{ // These fields are only needed to successfully call `SetupListeners` + Info: make(map[string]string), + ReloadFuncs: make(map[string][]reloadutil.ReloadFunc), + ReloadFuncsLock: &sync.RWMutex{}, + Logger: hclog.Default(), + } + err := bs.SetupListeners(nil, &configutil.SharedConfig{Listeners: tt.listeners}, []string{"ops"}) + require.NoError(t, err) + + s, err := NewServer(hclog.Default(), nil, bs.Listeners...) + if tt.expErr { + require.EqualError(t, err, tt.expErrMsg) + require.Nil(t, s) + return + } + + require.NoError(t, err) + require.NotNil(t, s) + s.Start() + t.Cleanup(func() { s.Shutdown() }) + + addrs := make([]string, 0, len(s.bundles)) + for _, b := range s.bundles { + addrs = append(addrs, b.ln.Mux.Addr().String()) + } + if tt.assertions != nil { + tt.assertions(t, addrs) + } + }) + } +} + +func TestShutdown(t *testing.T) { + tests := []struct { + name string + opsServerFn func(t *testing.T) *Server + assertions func(t *testing.T, o *Server) + expErr bool + expErrStr string + }{ + { + name: "nil bundle slice", + opsServerFn: func(t *testing.T) *Server { + return &Server{bundles: nil} + }, + expErr: false, // No-op + }, + { + name: "empty bundle slice", + opsServerFn: func(t *testing.T) *Server { + return &Server{bundles: []*opsBundle{}} + }, + expErr: false, // No-op + }, + { + name: "nil bundle", + opsServerFn: func(t *testing.T) *Server { + return &Server{bundles: []*opsBundle{nil}} + }, + expErr: true, + expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields", + }, + { + name: "empty bundle", + opsServerFn: func(t *testing.T) *Server { + return &Server{bundles: []*opsBundle{{}}} + }, + expErr: true, + expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields", + }, + { + name: "missing listener config", + opsServerFn: func(t *testing.T) *Server { + return &Server{ + bundles: []*opsBundle{ + { + ln: &base.ServerListener{ + Mux: &alpnmux.ALPNMux{}, + HTTPServer: &http.Server{}, + }, + }, + }, + } + }, + expErr: true, + expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields", + }, + { + name: "missing listener mux", + opsServerFn: func(t *testing.T) *Server { + return &Server{ + bundles: []*opsBundle{ + { + ln: &base.ServerListener{ + Config: &listenerutil.ListenerConfig{}, + HTTPServer: &http.Server{}, + }, + }, + }, + } + }, + expErr: true, + expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields", + }, + { + name: "missing http server", + opsServerFn: func(t *testing.T) *Server { + return &Server{ + bundles: []*opsBundle{ + { + ln: &base.ServerListener{ + Mux: &alpnmux.ALPNMux{}, + Config: &listenerutil.ListenerConfig{}, + }, + }, + }, + } + }, + expErr: true, + expErrStr: "ops.(Server).Shutdown: missing bundle, listener or its fields", + }, + { + name: "listener already closed", + opsServerFn: func(t *testing.T) *Server { + l1, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + require.NoError(t, l1.Close()) + + s1 := &http.Server{} + return &Server{ + bundles: []*opsBundle{ + { + ln: &base.ServerListener{ + ALPNListener: l1, + HTTPServer: s1, + Mux: alpnmux.New(l1), + Config: &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}}, + }, + }, + }, + } + }, + assertions: func(t *testing.T, s *Server) { + // The HTTP Server must be closed. + require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed) + + // The underlying listener must be closed. + require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed) + }, + expErr: false, + }, + { + name: "http server already closed", + opsServerFn: func(t *testing.T) *Server { + l1, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + s1 := &http.Server{} + go func() { + require.ErrorIs(t, s1.Serve(l1), http.ErrServerClosed) + }() + + err = s1.Shutdown(context.Background()) + require.NoError(t, err) + + return &Server{ + bundles: []*opsBundle{ + { + ln: &base.ServerListener{ + ALPNListener: l1, + HTTPServer: s1, + Mux: alpnmux.New(l1), + Config: &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}}, + }, + }, + }, + } + }, + assertions: func(t *testing.T, s *Server) { + // The HTTP Server must be closed. + require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed) + + // The underlying listener must be closed. + require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed) + }, + }, + { + name: "multiple bundles to close", + opsServerFn: func(t *testing.T) *Server { + l1, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + s1 := &http.Server{} + go func() { + require.ErrorIs(t, s1.Serve(l1), http.ErrServerClosed) + }() + + l2, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + s2 := &http.Server{} + go func() { + require.ErrorIs(t, s2.Serve(l2), http.ErrServerClosed) + }() + + return &Server{ + bundles: []*opsBundle{ + { + ln: &base.ServerListener{ + ALPNListener: l1, + HTTPServer: s1, + Mux: alpnmux.New(l1), + Config: &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}}, + }, + }, + { + ln: &base.ServerListener{ + ALPNListener: l2, + HTTPServer: s2, + Mux: alpnmux.New(l2), + Config: &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}}, + }, + }, + }, + } + }, + assertions: func(t *testing.T, s *Server) { + // The HTTP Server must be closed. + require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed) + require.ErrorIs(t, s.bundles[1].ln.HTTPServer.Serve(s.bundles[1].ln.ALPNListener), http.ErrServerClosed) + + // The underlying listener must be closed. + require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed) + require.ErrorIs(t, s.bundles[1].ln.Mux.Close(), net.ErrClosed) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := tt.opsServerFn(t) + err := o.Shutdown() + if tt.expErr { + require.EqualError(t, err, tt.expErrStr) + return + } + + require.NoError(t, err) + if tt.assertions != nil { + tt.assertions(t, o) + } + }) + } +} + +func TestHealthEndpointLifecycle(t *testing.T) { + listeners := []*listenerutil.ListenerConfig{ + { + Type: "tcp", + Address: "127.0.0.1:0", + Purpose: []string{"ops"}, + TLSDisable: true, + }, + + // We only care about the ops listener, but the following + // listeners are required to start a Controller. + { + Type: "tcp", + Address: "127.0.0.1:0", + Purpose: []string{"api"}, + TLSDisable: true, + }, + { + Type: "tcp", + Address: "127.0.0.1:0", + Purpose: []string{"cluster"}, + }, + } + + // Get test a controller so we can actually start a controller. + tc := controller.NewTestController(t, &controller.TestControllerOpts{DisableAutoStart: true}) + tc.Config().RawConfig.SharedConfig.Listeners = listeners // Override with our own listeners. + + // Setup listeners that we've set up above to turn them into correct `ServerListener` objects + // that the controller can handle. This will set `Server.Listeners`. + err := tc.Server().SetupListeners(nil, tc.Config().RawConfig.SharedConfig, []string{"api", "cluster", "ops"}) + require.NoError(t, err) + + // Start controller. This sets `cmd.controller`. + err = tc.Controller().Start() + require.NoError(t, err) + t.Cleanup(tc.Shutdown) + + // Controller has started and is set onto our Command object, start ops. + opsServer, err := NewServer(hclog.Default(), tc.Controller(), tc.Config().Listeners...) + require.NoError(t, err) + opsServer.Start() + + // Assert the ops endpoint is up and returning 200 OK. + rsp, err := http.Get("http://" + tc.Config().Listeners[0].Mux.Addr().String() + "/health") + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) + + // Start replying with 503 Service Unavailable from the health endpoint. + tc.Controller().HealthService.StartServiceUnavailableReplies() + + // Assert we're receiving 503 Service Unavailable now instead of 200 OK. + rsp, err = http.Get("http://" + tc.Config().Listeners[0].Mux.Addr().String() + "/health") + require.NoError(t, err) + require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode) +} + +func TestWaitIfHealthExists(t *testing.T) { + tests := []struct { + name string + bundles []*opsBundle + gracePeriod time.Duration + controller *controller.Controller + expMinWait time.Duration + expMaxWait time.Duration + }{ + { + name: "nil bundles", + bundles: nil, + controller: &controller.Controller{HealthService: &health.Service{}}, + gracePeriod: 1 * time.Minute, + expMinWait: 0 * time.Second, + expMaxWait: 500 * time.Microsecond, + }, + { + name: "no bundles", + bundles: []*opsBundle{}, + controller: &controller.Controller{HealthService: &health.Service{}}, + gracePeriod: 1 * time.Minute, + expMinWait: 0 * time.Second, + expMaxWait: 500 * time.Microsecond, + }, + { + name: "nil controller", + controller: nil, + gracePeriod: 1 * time.Minute, + expMinWait: 0 * time.Second, + expMaxWait: 500 * time.Microsecond, + }, + { + name: "controller with no health service", + controller: &controller.Controller{HealthService: nil}, + gracePeriod: 1 * time.Minute, + expMinWait: 0 * time.Second, + expMaxWait: 500 * time.Microsecond, + }, + { + name: "no grace period set", + controller: &controller.Controller{HealthService: &health.Service{}}, + expMinWait: 0 * time.Second, + expMaxWait: 500 * time.Microsecond, + }, + { + name: "wait 500ms", + bundles: []*opsBundle{{}}, + controller: &controller.Controller{HealthService: &health.Service{}}, + gracePeriod: 500 * time.Millisecond, + expMinWait: 500 * time.Millisecond, + expMaxWait: 600 * time.Millisecond, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Server{ + bundles: tt.bundles, + controller: tt.controller, + } + + start := time.Now() + s.WaitIfHealthExists(tt.gracePeriod, cli.NewMockUi()) + actualWait := time.Since(start) + + require.GreaterOrEqual(t, actualWait, tt.expMinWait) + require.LessOrEqual(t, actualWait, tt.expMaxWait) + }) + } +} + +func TestCreateOpsHandler(t *testing.T) { + tests := []struct { + name string + setupController bool + lncfg *listenerutil.ListenerConfig + expErr bool + expErrMsg string + assertions func(t *testing.T, addr string) + }{ + { + name: "no controller set", + setupController: false, + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, addr string) { + rsp, err := http.Get("http://" + addr + "/health") + require.NoError(t, err) // We can do the GET request + require.Equal(t, http.StatusNotFound, rsp.StatusCode) // But the endpoint doesn't exist + }, + }, + { + name: "controller set", + setupController: true, + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, addr string) { + rsp, err := http.Get("http://" + addr + "/health") + require.NoError(t, err) // We can do the GET request + require.Equal(t, http.StatusOK, rsp.StatusCode) // And the endpoint exists + }, + }, + { + name: "controller set, but nil listener config", + setupController: true, + lncfg: nil, + expErr: true, + expErrMsg: "controller.(Controller).GetHealthHandler: received nil listener config", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var c *controller.Controller + if tt.setupController { + tc := controller.NewTestController(t, &controller.TestControllerOpts{}) + t.Cleanup(tc.Shutdown) + + c = tc.Controller() + } + + h, err := createOpsHandler(tt.lncfg, c) + if tt.expErr { + require.EqualError(t, err, tt.expErrMsg) + require.Nil(t, h) + return + } + + require.NoError(t, err) + + s := http.Server{Handler: h} + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + go s.Serve(l) + + t.Cleanup(func() { + require.NoError(t, s.Shutdown(context.Background())) + }) + + tt.assertions(t, l.Addr().String()) + }) + } +} + +func TestCreateHttpServer(t *testing.T) { + tests := []struct { + name string + lncfg *listenerutil.ListenerConfig + assertions func(t *testing.T, s *http.Server) + }{ + { + name: "default values are set if none are set in listener config", + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, s *http.Server) { + require.Equal(t, 10*time.Second, s.ReadHeaderTimeout) + require.Equal(t, 30*time.Second, s.ReadTimeout) + require.Equal(t, 5*time.Minute, s.IdleTimeout) + }, + }, + { + name: "configured values are respected", + lncfg: &listenerutil.ListenerConfig{ + HTTPReadHeaderTimeout: time.Minute, + HTTPReadTimeout: 2 * time.Minute, + HTTPWriteTimeout: 3 * time.Minute, + HTTPIdleTimeout: 4 * time.Minute, + }, + assertions: func(t *testing.T, s *http.Server) { + require.Equal(t, time.Minute, s.ReadHeaderTimeout) + require.Equal(t, 2*time.Minute, s.ReadTimeout) + require.Equal(t, 3*time.Minute, s.WriteTimeout) + require.Equal(t, 4*time.Minute, s.IdleTimeout) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := hclog.Default() + + s := createHttpServer(l, http.NotFoundHandler(), tt.lncfg) + require.NotNil(t, s) + require.NotNil(t, s.Handler) + require.NotNil(t, s.ErrorLog) + + tt.assertions(t, s) + }) + } +} + +func TestListenerCloseErrorCheck(t *testing.T) { + tests := []struct { + name string + lnType string + err error + expErr error + }{ + { + name: "nil err", + lnType: "tcp", + err: nil, + expErr: nil, + }, + { + name: "net.Closed", + lnType: "tcp", + err: net.ErrClosed, + expErr: nil, + }, + { + name: "path err not unix type", + lnType: "tcp", + err: &os.PathError{Op: "test"}, + expErr: &os.PathError{Op: "test"}, + }, + { + name: "path err unix type", + lnType: "unix", + err: &os.PathError{Op: "test"}, + expErr: nil, + }, + { + name: "literally anything else", + lnType: "tcp", + err: fmt.Errorf("oops I errored"), + expErr: fmt.Errorf("oops I errored"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := listenerCloseErrorCheck(tt.lnType, tt.err) + require.Equal(t, tt.expErr, err) + }) + } +} + +func testTlsSetup(t *testing.T, certPath, keyPath string) { + t.Cleanup(func() { + require.NoError(t, os.Remove(certPath)) + require.NoError(t, os.Remove(keyPath)) + }) + + certBytes, _, priv := createTestCert(t) + + certFile, err := os.Create(certPath) + require.NoError(t, err) + require.NoError(t, pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes})) + require.NoError(t, certFile.Close()) + + keyFile, err := os.Create(keyPath) + require.NoError(t, err) + + marshaledKey, err := x509.MarshalPKCS8PrivateKey(priv) + require.NoError(t, err) + + require.NoError(t, pem.Encode(keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: marshaledKey})) + require.NoError(t, keyFile.Close()) +} + +func testTlsHttpClient(t *testing.T, certPath string) *http.Client { + f, err := os.Open(certPath) + require.NoError(t, err) + + certBytes, err := ioutil.ReadAll(f) + require.NoError(t, err) + require.NoError(t, f.Close()) + + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(certBytes) + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: certPool}, + }, + } +} + +func createTestCert(t *testing.T) ([]byte, ed25519.PublicKey, ed25519.PrivateKey) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement | x509.KeyUsageCertSign, + SerialNumber: big.NewInt(0), + NotBefore: time.Now().Add(-30 * time.Second), + NotAfter: time.Now().Add(5 * time.Minute), + BasicConstraintsValid: true, + IsCA: true, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"/tmp/boundary-opslistener-test0.sock", "/tmp/boundary-opslistener-test1.sock"}, + } + + certBytes, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) + require.NoError(t, err) + + return certBytes, pub, priv +} diff --git a/internal/gen/controller/ops/services/health_service.pb.go b/internal/gen/controller/ops/services/health_service.pb.go new file mode 100644 index 000000000..2f425caa5 --- /dev/null +++ b/internal/gen/controller/ops/services/health_service.pb.go @@ -0,0 +1,209 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.17.3 +// source: controller/ops/services/v1/health_service.proto + +package services + +import ( + _ "github.com/hashicorp/boundary/sdk/pbs/controller/ops/resources/health" + _ "google.golang.org/genproto/googleapis/api/annotations" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GetHealthRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetHealthRequest) Reset() { + *x = GetHealthRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetHealthRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetHealthRequest) ProtoMessage() {} + +func (x *GetHealthRequest) ProtoReflect() protoreflect.Message { + mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetHealthRequest.ProtoReflect.Descriptor instead. +func (*GetHealthRequest) Descriptor() ([]byte, []int) { + return file_controller_ops_services_v1_health_service_proto_rawDescGZIP(), []int{0} +} + +type GetHealthResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetHealthResponse) Reset() { + *x = GetHealthResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetHealthResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetHealthResponse) ProtoMessage() {} + +func (x *GetHealthResponse) ProtoReflect() protoreflect.Message { + mi := &file_controller_ops_services_v1_health_service_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetHealthResponse.ProtoReflect.Descriptor instead. +func (*GetHealthResponse) Descriptor() ([]byte, []int) { + return file_controller_ops_services_v1_health_service_proto_rawDescGZIP(), []int{1} +} + +var File_controller_ops_services_v1_health_service_proto protoreflect.FileDescriptor + +var file_controller_ops_services_v1_health_service_proto_rawDesc = []byte{ + 0x0a, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x68, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x1a, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, + 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, 0x2f, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, 0x2f, + 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x12, 0x0a, 0x10, + 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x8a, 0x01, 0x0a, 0x0d, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x79, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x48, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x12, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, + 0x72, 0x2e, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, + 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x0f, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x09, 0x12, 0x07, 0x2f, 0x68, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x42, 0x4d, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x61, 0x72, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, 0x2f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x3b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_controller_ops_services_v1_health_service_proto_rawDescOnce sync.Once + file_controller_ops_services_v1_health_service_proto_rawDescData = file_controller_ops_services_v1_health_service_proto_rawDesc +) + +func file_controller_ops_services_v1_health_service_proto_rawDescGZIP() []byte { + file_controller_ops_services_v1_health_service_proto_rawDescOnce.Do(func() { + file_controller_ops_services_v1_health_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_controller_ops_services_v1_health_service_proto_rawDescData) + }) + return file_controller_ops_services_v1_health_service_proto_rawDescData +} + +var file_controller_ops_services_v1_health_service_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_controller_ops_services_v1_health_service_proto_goTypes = []interface{}{ + (*GetHealthRequest)(nil), // 0: controller.ops.services.v1.GetHealthRequest + (*GetHealthResponse)(nil), // 1: controller.ops.services.v1.GetHealthResponse +} +var file_controller_ops_services_v1_health_service_proto_depIdxs = []int32{ + 0, // 0: controller.ops.services.v1.HealthService.GetHealth:input_type -> controller.ops.services.v1.GetHealthRequest + 1, // 1: controller.ops.services.v1.HealthService.GetHealth:output_type -> controller.ops.services.v1.GetHealthResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_controller_ops_services_v1_health_service_proto_init() } +func file_controller_ops_services_v1_health_service_proto_init() { + if File_controller_ops_services_v1_health_service_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_controller_ops_services_v1_health_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetHealthRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_controller_ops_services_v1_health_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetHealthResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_controller_ops_services_v1_health_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_controller_ops_services_v1_health_service_proto_goTypes, + DependencyIndexes: file_controller_ops_services_v1_health_service_proto_depIdxs, + MessageInfos: file_controller_ops_services_v1_health_service_proto_msgTypes, + }.Build() + File_controller_ops_services_v1_health_service_proto = out.File + file_controller_ops_services_v1_health_service_proto_rawDesc = nil + file_controller_ops_services_v1_health_service_proto_goTypes = nil + file_controller_ops_services_v1_health_service_proto_depIdxs = nil +} diff --git a/internal/gen/controller/ops/services/health_service.pb.gw.go b/internal/gen/controller/ops/services/health_service.pb.gw.go new file mode 100644 index 000000000..b9cb84cca --- /dev/null +++ b/internal/gen/controller/ops/services/health_service.pb.gw.go @@ -0,0 +1,151 @@ +// Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. +// source: controller/ops/services/v1/health_service.proto + +/* +Package services is a reverse proxy. + +It translates gRPC into RESTful JSON APIs. +*/ +package services + +import ( + "context" + "io" + "net/http" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/grpc-ecosystem/grpc-gateway/v2/utilities" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// Suppress "imported and not used" errors +var _ codes.Code +var _ io.Reader +var _ status.Status +var _ = runtime.String +var _ = utilities.NewDoubleArray +var _ = metadata.Join + +func request_HealthService_GetHealth_0(ctx context.Context, marshaler runtime.Marshaler, client HealthServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq GetHealthRequest + var metadata runtime.ServerMetadata + + msg, err := client.GetHealth(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + +func local_request_HealthService_GetHealth_0(ctx context.Context, marshaler runtime.Marshaler, server HealthServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq GetHealthRequest + var metadata runtime.ServerMetadata + + msg, err := server.GetHealth(ctx, &protoReq) + return msg, metadata, err + +} + +// RegisterHealthServiceHandlerServer registers the http handlers for service HealthService to "mux". +// UnaryRPC :call HealthServiceServer directly. +// StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. +// Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterHealthServiceHandlerFromEndpoint instead. +func RegisterHealthServiceHandlerServer(ctx context.Context, mux *runtime.ServeMux, server HealthServiceServer) error { + + mux.Handle("GET", pattern_HealthService_GetHealth_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/controller.ops.services.v1.HealthService/GetHealth", runtime.WithHTTPPathPattern("/health")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_HealthService_GetHealth_0(rctx, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_HealthService_GetHealth_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + return nil +} + +// RegisterHealthServiceHandlerFromEndpoint is same as RegisterHealthServiceHandler but +// automatically dials to "endpoint" and closes the connection when "ctx" gets done. +func RegisterHealthServiceHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { + conn, err := grpc.Dial(endpoint, opts...) + if err != nil { + return err + } + defer func() { + if err != nil { + if cerr := conn.Close(); cerr != nil { + grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr) + } + return + } + go func() { + <-ctx.Done() + if cerr := conn.Close(); cerr != nil { + grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr) + } + }() + }() + + return RegisterHealthServiceHandler(ctx, mux, conn) +} + +// RegisterHealthServiceHandler registers the http handlers for service HealthService to "mux". +// The handlers forward requests to the grpc endpoint over "conn". +func RegisterHealthServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { + return RegisterHealthServiceHandlerClient(ctx, mux, NewHealthServiceClient(conn)) +} + +// RegisterHealthServiceHandlerClient registers the http handlers for service HealthService +// to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "HealthServiceClient". +// Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "HealthServiceClient" +// doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in +// "HealthServiceClient" to call the correct interceptors. +func RegisterHealthServiceHandlerClient(ctx context.Context, mux *runtime.ServeMux, client HealthServiceClient) error { + + mux.Handle("GET", pattern_HealthService_GetHealth_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req, "/controller.ops.services.v1.HealthService/GetHealth", runtime.WithHTTPPathPattern("/health")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_HealthService_GetHealth_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_HealthService_GetHealth_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + return nil +} + +var ( + pattern_HealthService_GetHealth_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"health"}, "")) +) + +var ( + forward_HealthService_GetHealth_0 = runtime.ForwardResponseMessage +) diff --git a/internal/gen/controller/ops/services/health_service_grpc.pb.go b/internal/gen/controller/ops/services/health_service_grpc.pb.go new file mode 100644 index 000000000..1abc10cdc --- /dev/null +++ b/internal/gen/controller/ops/services/health_service_grpc.pb.go @@ -0,0 +1,101 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package services + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// HealthServiceClient is the client API for HealthService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type HealthServiceClient interface { + GetHealth(ctx context.Context, in *GetHealthRequest, opts ...grpc.CallOption) (*GetHealthResponse, error) +} + +type healthServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewHealthServiceClient(cc grpc.ClientConnInterface) HealthServiceClient { + return &healthServiceClient{cc} +} + +func (c *healthServiceClient) GetHealth(ctx context.Context, in *GetHealthRequest, opts ...grpc.CallOption) (*GetHealthResponse, error) { + out := new(GetHealthResponse) + err := c.cc.Invoke(ctx, "/controller.ops.services.v1.HealthService/GetHealth", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// HealthServiceServer is the server API for HealthService service. +// All implementations must embed UnimplementedHealthServiceServer +// for forward compatibility +type HealthServiceServer interface { + GetHealth(context.Context, *GetHealthRequest) (*GetHealthResponse, error) + mustEmbedUnimplementedHealthServiceServer() +} + +// UnimplementedHealthServiceServer must be embedded to have forward compatible implementations. +type UnimplementedHealthServiceServer struct { +} + +func (UnimplementedHealthServiceServer) GetHealth(context.Context, *GetHealthRequest) (*GetHealthResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetHealth not implemented") +} +func (UnimplementedHealthServiceServer) mustEmbedUnimplementedHealthServiceServer() {} + +// UnsafeHealthServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HealthServiceServer will +// result in compilation errors. +type UnsafeHealthServiceServer interface { + mustEmbedUnimplementedHealthServiceServer() +} + +func RegisterHealthServiceServer(s grpc.ServiceRegistrar, srv HealthServiceServer) { + s.RegisterService(&HealthService_ServiceDesc, srv) +} + +func _HealthService_GetHealth_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetHealthRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServiceServer).GetHealth(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/controller.ops.services.v1.HealthService/GetHealth", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServiceServer).GetHealth(ctx, req.(*GetHealthRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// HealthService_ServiceDesc is the grpc.ServiceDesc for HealthService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var HealthService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "controller.ops.services.v1.HealthService", + HandlerType: (*HealthServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetHealth", + Handler: _HealthService_GetHealth_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "controller/ops/services/v1/health_service.proto", +} diff --git a/internal/proto/local/controller/ops/resources/health/v1/health.proto b/internal/proto/local/controller/ops/resources/health/v1/health.proto new file mode 100644 index 000000000..1cfd4833d --- /dev/null +++ b/internal/proto/local/controller/ops/resources/health/v1/health.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package controller.ops.resources.health.v1; + +option go_package = "github.com/hashicorp/boundary/sdk/pbs/controller/ops/resources/health;health"; + +// Health is the underlying resource used by the Health Service. +// Empty for now as we don't want to commit to an API yet. +message Health {} diff --git a/internal/proto/local/controller/ops/services/v1/health_service.proto b/internal/proto/local/controller/ops/services/v1/health_service.proto new file mode 100644 index 000000000..94bdfd9e7 --- /dev/null +++ b/internal/proto/local/controller/ops/services/v1/health_service.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package controller.ops.services.v1; + +option go_package = "github.com/hashicorp/boundary/internal/gen/controller/ops/services;services"; + +import "google/api/annotations.proto"; +import "controller/ops/resources/health/v1/health.proto"; + +service HealthService { + rpc GetHealth(GetHealthRequest) returns (GetHealthResponse) { + option (google.api.http) = { + get : "/health" + }; + } +} + +message GetHealthRequest {} + +message GetHealthResponse {} diff --git a/internal/servers/controller/controller.go b/internal/servers/controller/controller.go index 214d6e40d..6e1e60ddd 100644 --- a/internal/servers/controller/controller.go +++ b/internal/servers/controller/controller.go @@ -25,6 +25,7 @@ import ( "github.com/hashicorp/boundary/internal/scheduler/job" "github.com/hashicorp/boundary/internal/servers" "github.com/hashicorp/boundary/internal/servers/controller/common" + "github.com/hashicorp/boundary/internal/servers/controller/handlers/health" "github.com/hashicorp/boundary/internal/session" "github.com/hashicorp/boundary/internal/target" "github.com/hashicorp/boundary/internal/types/scope" @@ -80,6 +81,10 @@ type Controller struct { kms *kms.Kms enabledPlugins []base.EnabledPlugin + + // Used to signal the Health Service to start + // replying to queries with "503 Service Unavailable". + HealthService *health.Service } func New(ctx context.Context, conf *Config) (*Controller, error) { @@ -92,6 +97,7 @@ func New(ctx context.Context, conf *Config) (*Controller, error) { workerAuthCache: new(sync.Map), workerStatusUpdateTimes: new(sync.Map), enabledPlugins: conf.Server.EnabledPlugins, + apiListeners: make([]*base.ServerListener, 0), } c.started.Store(false) diff --git a/internal/servers/controller/handler.go b/internal/servers/controller/handler.go index eab9ce957..f0df293b3 100644 --- a/internal/servers/controller/handler.go +++ b/internal/servers/controller/handler.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/boundary/internal/auth/oidc" "github.com/hashicorp/boundary/internal/gen/controller/api/services" authpb "github.com/hashicorp/boundary/internal/gen/controller/auth" + opsservices "github.com/hashicorp/boundary/internal/gen/controller/ops/services" "github.com/hashicorp/boundary/internal/observability/event" "github.com/hashicorp/boundary/internal/servers/common" "github.com/hashicorp/boundary/internal/servers/controller/auth" @@ -27,6 +28,7 @@ import ( "github.com/hashicorp/boundary/internal/servers/controller/handlers/credentiallibraries" "github.com/hashicorp/boundary/internal/servers/controller/handlers/credentialstores" "github.com/hashicorp/boundary/internal/servers/controller/handlers/groups" + "github.com/hashicorp/boundary/internal/servers/controller/handlers/health" "github.com/hashicorp/boundary/internal/servers/controller/handlers/host_catalogs" "github.com/hashicorp/boundary/internal/servers/controller/handlers/host_sets" "github.com/hashicorp/boundary/internal/servers/controller/handlers/hosts" @@ -74,6 +76,28 @@ func (c *Controller) apiHandler(props HandlerProperties) (http.Handler, error) { return eventsHandler, err } +// GetHealthHandler returns a gRPC Gateway mux that is registered against the +// controller's gRPC health service to make it accessible from an HTTP API. +func (c *Controller) GetHealthHandler(lcfg *listenerutil.ListenerConfig) (http.Handler, error) { + const op = "controller.(Controller).GetHealthHandler" + if lcfg == nil { + return nil, fmt.Errorf("%s: received nil listener config", op) + } + + healthGrpcGwMux := newGrpcGatewayMux() + err := registerHealthGrpcGatewayEndpoint(c.baseContext, healthGrpcGwMux, gatewayDialOptions(c.apiGrpcServerListener)...) + if err != nil { + return nil, fmt.Errorf("%s: failed to register health service handler: %w", op, err) + } + + wrapped := wrapHandlerWithCommonFuncs(healthGrpcGwMux, c, HandlerProperties{lcfg, c.baseContext}) + return common.WrapWithEventsHandler(wrapped, c.conf.Eventer, c.kms, lcfg) +} + +func registerHealthGrpcGatewayEndpoint(ctx context.Context, gwMux *runtime.ServeMux, dialOptions ...grpc.DialOption) error { + return opsservices.RegisterHealthServiceHandlerFromEndpoint(ctx, gwMux, gatewayTarget, dialOptions) +} + func (c *Controller) registerGrpcServices(s *grpc.Server) error { // We have to check against the current services because the gRPC lib treats a duplicate // register call as an error and os.Exits. @@ -193,6 +217,11 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error { } services.RegisterCredentialLibraryServiceServer(s, cl) } + if _, ok := s.GetServiceInfo()[opsservices.HealthService_ServiceDesc.ServiceName]; !ok { + hs := health.NewService() + opsservices.RegisterHealthServiceServer(s, hs) + c.HealthService = hs + } return nil } diff --git a/internal/servers/controller/handlers/health/health.go b/internal/servers/controller/handlers/health/health.go new file mode 100644 index 000000000..f73257339 --- /dev/null +++ b/internal/servers/controller/handlers/health/health.go @@ -0,0 +1,38 @@ +package health + +import ( + "context" + "net/http" + + pbs "github.com/hashicorp/boundary/internal/gen/controller/ops/services" + "github.com/hashicorp/boundary/internal/servers/controller/handlers" +) + +type Service struct { + pbs.UnimplementedHealthServiceServer + + replyWithServiceUnavailable bool +} + +func NewService() *Service { + s := Service{} + return &s +} + +func (s *Service) GetHealth(ctx context.Context, req *pbs.GetHealthRequest) (*pbs.GetHealthResponse, error) { + if s.replyWithServiceUnavailable { + err := handlers.SetStatusCode(ctx, http.StatusServiceUnavailable) + if err != nil { + return nil, err + } + return &pbs.GetHealthResponse{}, nil + } + + return &pbs.GetHealthResponse{}, nil +} + +// StartServiceUnavailableReplies gets returned to the caller of NewService. +// When invoked, we start responding to any health queries with a 503. +func (s *Service) StartServiceUnavailableReplies() { + s.replyWithServiceUnavailable = true +} diff --git a/internal/servers/controller/handlers/health/health_test.go b/internal/servers/controller/handlers/health/health_test.go new file mode 100644 index 000000000..612c3cb7c --- /dev/null +++ b/internal/servers/controller/handlers/health/health_test.go @@ -0,0 +1,86 @@ +package health + +import ( + "context" + "fmt" + "testing" + + "github.com/hashicorp/boundary/internal/gen/controller/ops/services" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +func TestGetHealth(t *testing.T) { + tests := []struct { + name string + ctx context.Context + serviceUnavailable bool + expGetHealthResponse *services.GetHealthResponse + expErr bool + expErrMsg string + }{ + { + name: "healthy reply", + ctx: context.Background(), + serviceUnavailable: false, + expGetHealthResponse: &services.GetHealthResponse{}, + expErr: false, + }, + { + name: "service unavailable reply", + ctx: grpc.NewContextWithServerTransportStream(context.Background(), &testServerTransportStream{expHttpCode: "503"}), + serviceUnavailable: true, + expGetHealthResponse: &services.GetHealthResponse{}, + expErr: false, + }, + { + name: "get health error", + ctx: context.Background(), + serviceUnavailable: true, + expErr: true, + expErrMsg: "handlers.SetHttpStatusCode: internal error, unknown: error #500: rpc error: code = Internal desc = grpc: failed to fetch the stream from the context context.Background", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hs := NewService() + + if tt.serviceUnavailable { + hs.StartServiceUnavailableReplies() + } + + rsp, err := hs.GetHealth(tt.ctx, &services.GetHealthRequest{}) + if tt.expErr { + require.Error(t, err) + require.EqualError(t, err, tt.expErrMsg) + return + } + + require.NoError(t, err) + require.EqualValues(t, tt.expGetHealthResponse, rsp) + }) + } +} + +type testServerTransportStream struct { + expHttpCode string +} + +func (s *testServerTransportStream) Method() string { return "" } +func (s *testServerTransportStream) SendHeader(md metadata.MD) error { return nil } +func (s *testServerTransportStream) SetTrailer(md metadata.MD) error { return nil } +func (s *testServerTransportStream) SetHeader(md metadata.MD) error { + codes, ok := md["x-http-code"] + if !ok { + return fmt.Errorf("x-http-code header not found") + } + if len(codes) != 1 { + return fmt.Errorf("expected only one element in http codes, got %d", len(codes)) + } + if codes[0] != s.expHttpCode { + return fmt.Errorf("expected http code %q, got %q", s.expHttpCode, codes[0]) + } + return nil +} diff --git a/internal/servers/controller/testing.go b/internal/servers/controller/testing.go index d6e34af98..b433e6972 100644 --- a/internal/servers/controller/testing.go +++ b/internal/servers/controller/testing.go @@ -576,7 +576,7 @@ func TestControllerConfig(t *testing.T, ctx context.Context, tc *TestController, for _, listener := range opts.Config.Listeners { listener.RandomPort = true } - if err := tc.b.SetupListeners(nil, opts.Config.SharedConfig, []string{"api", "cluster"}); err != nil { + if err := tc.b.SetupListeners(nil, opts.Config.SharedConfig, []string{"api", "cluster", "ops"}); err != nil { t.Fatal(err) } if err := tc.b.SetupControllerPublicClusterAddress(opts.Config, ""); err != nil { diff --git a/internal/servers/worker/listeners.go b/internal/servers/worker/listeners.go index 615672743..0342ae3d3 100644 --- a/internal/servers/worker/listeners.go +++ b/internal/servers/worker/listeners.go @@ -30,7 +30,7 @@ func (w *Worker) startListeners() error { for _, ln := range w.conf.Listeners { for _, purpose := range ln.Config.Purpose { switch purpose { - case "api", "cluster": + case "api", "cluster", "ops": // We may have this in dev mode; ignore continue diff --git a/sdk/pbs/controller/ops/resources/health/health.pb.go b/sdk/pbs/controller/ops/resources/health/health.pb.go new file mode 100644 index 000000000..343fa909a --- /dev/null +++ b/sdk/pbs/controller/ops/resources/health/health.pb.go @@ -0,0 +1,141 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.17.3 +// source: controller/ops/resources/health/v1/health.proto + +package health + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Health is the underlying resource used by the Health Service. +// Empty for now as we don't want to commit to an API yet. +type Health struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Health) Reset() { + *x = Health{} + if protoimpl.UnsafeEnabled { + mi := &file_controller_ops_resources_health_v1_health_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Health) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Health) ProtoMessage() {} + +func (x *Health) ProtoReflect() protoreflect.Message { + mi := &file_controller_ops_resources_health_v1_health_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Health.ProtoReflect.Descriptor instead. +func (*Health) Descriptor() ([]byte, []int) { + return file_controller_ops_resources_health_v1_health_proto_rawDescGZIP(), []int{0} +} + +var File_controller_ops_resources_health_v1_health_proto protoreflect.FileDescriptor + +var file_controller_ops_resources_health_v1_health_proto_rawDesc = []byte{ + 0x0a, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, + 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x2f, 0x76, 0x31, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x22, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x70, + 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x2e, 0x68, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x2e, 0x76, 0x31, 0x22, 0x08, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x42, + 0x4e, 0x5a, 0x4c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, + 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x70, 0x62, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x73, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x3b, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_controller_ops_resources_health_v1_health_proto_rawDescOnce sync.Once + file_controller_ops_resources_health_v1_health_proto_rawDescData = file_controller_ops_resources_health_v1_health_proto_rawDesc +) + +func file_controller_ops_resources_health_v1_health_proto_rawDescGZIP() []byte { + file_controller_ops_resources_health_v1_health_proto_rawDescOnce.Do(func() { + file_controller_ops_resources_health_v1_health_proto_rawDescData = protoimpl.X.CompressGZIP(file_controller_ops_resources_health_v1_health_proto_rawDescData) + }) + return file_controller_ops_resources_health_v1_health_proto_rawDescData +} + +var file_controller_ops_resources_health_v1_health_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_controller_ops_resources_health_v1_health_proto_goTypes = []interface{}{ + (*Health)(nil), // 0: controller.ops.resources.health.v1.Health +} +var file_controller_ops_resources_health_v1_health_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_controller_ops_resources_health_v1_health_proto_init() } +func file_controller_ops_resources_health_v1_health_proto_init() { + if File_controller_ops_resources_health_v1_health_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_controller_ops_resources_health_v1_health_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Health); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_controller_ops_resources_health_v1_health_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_controller_ops_resources_health_v1_health_proto_goTypes, + DependencyIndexes: file_controller_ops_resources_health_v1_health_proto_depIdxs, + MessageInfos: file_controller_ops_resources_health_v1_health_proto_msgTypes, + }.Build() + File_controller_ops_resources_health_v1_health_proto = out.File + file_controller_ops_resources_health_v1_health_proto_rawDesc = nil + file_controller_ops_resources_health_v1_health_proto_goTypes = nil + file_controller_ops_resources_health_v1_health_proto_depIdxs = nil +} diff --git a/website/content/docs/configuration/controller.mdx b/website/content/docs/configuration/controller.mdx index 8b6678108..fb26fea57 100644 --- a/website/content/docs/configuration/controller.mdx +++ b/website/content/docs/configuration/controller.mdx @@ -61,15 +61,23 @@ description will be read. - `auth_token_time_to_live` - Maximum time to live (TTL) for all auth tokens globally (pertains to all tokens from all auth methods). Valid time units are anything specified by Golang's [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. Default is 7 days. + - `auth_token_time_to_stale` - Maximum time of inactivity for all auth tokens globally (pertains to all tokens from all auth methods). Valid time units are anything specified by Golang's [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. Default is 1 day. +- `graceful_shutdown_wait_duration` - Amount of time Boundary will wait before initiating the shutdown procedure, + after receiving a shutdown signal. In this state, Boundary still processes requests as normal but replies + with `503 Service Unavailable` to any health requests. This is designed to allow an operator to configure + load-balancers to preemptively stop new traffic to a Boundary instance that is going away. Valid time units + are anything specified by Go's [ParseDuration()](https://golang.org/pkg/time/#ParseDuration) method. Only + used when an `ops` listener is set and the Controller is present. Default is 0 seconds. + ## KMS Configuration The controller requires two KMS stanzas for `root` and `worker-auth` purposes: -```kms +```hcl # Root KMS configuration block: this is the root key for Boundary # Use a production KMS such as AWS KMS in production installs kms "aead" { @@ -130,6 +138,9 @@ controller { name = "demo-controller-1" description = "A controller for a demo!" + # After receiving a shutdown signal, Boundary will wait 10s before initiating the shutdown process. + graceful_shutdown_wait_duration = "10s" + # Database URL for postgres. This can be a direct "postgres://" # URL, or it can be "file://" to read the contents of a file to # supply the url, or "env://" to name an environment variable @@ -162,6 +173,16 @@ listener "tcp" { purpose = "cluster" } +listener "tcp" { + # Should be the address of the NIC where your external systems' + # (eg: Load-Balancer) will connect on. + address = "10.0.0.1" + # The purpose of this listener block + purpose = "ops" + + tls_disable = false +} + # Root KMS configuration block: this is the root key for Boundary # Use a production KMS such as AWS KMS in production installs kms "aead" { diff --git a/website/content/docs/configuration/index.mdx b/website/content/docs/configuration/index.mdx index f19ef7b3d..f55c9916a 100644 --- a/website/content/docs/configuration/index.mdx +++ b/website/content/docs/configuration/index.mdx @@ -30,13 +30,15 @@ After the configuration is written, use the `-config` flag to specify a local pa - [`listener`](/docs/configuration/listener): Configures the listeners on which Boundary serves traffic (API, cluster, and proxy). - Controllers will have two listener blocks: one marked for `api` purpose and - the other marked for `cluster` purpose. By default, the API listener runs on - :9200 and the cluster listener (currently used for worker communication) uses - :9201. + Controllers must have at least two listener blocks defined: one marked for + `api` purpose and the other marked for `cluster` purpose. Workers will have only one listener, marked for `proxy` purpose. + Optionally, the `ops` purpose listener block serves as a higher-level listener + where Boundary's operational endpoints live (eg: /health). + + - [`kms`](/docs/configuration/kms): Configures KMS blocks [for various purposes](/docs/concepts/security/data-encryption). diff --git a/website/content/docs/configuration/listener/index.mdx b/website/content/docs/configuration/listener/index.mdx index 05bc781c2..0de81e63d 100644 --- a/website/content/docs/configuration/listener/index.mdx +++ b/website/content/docs/configuration/listener/index.mdx @@ -9,7 +9,35 @@ description: |- # `listener` Stanza The `listener` stanza configures the addresses and ports on which Boundary will -respond to requests. At this time, there are two listeners - [TCP][tcp] and [Unix][unix]. +respond to requests. At this time, there are two listener types: +[TCP](/docs/configuration/listener/tcp) and [Unix](/docs/configuration/listener/unix). + + +## `listener` Purpose + +The `purpose` field is used to determine what part of Boundary's functionality is +started and exposed for a given listener block. Each listener must have a `purpose` set. + +Currently, Boundary recognizes the following listener purposes: + +- `api`: Starts up and exposes Boundary's controller API. + At least one `api` listener must be present if a Boundary + instance is set-up as a controller. + By default, it runs on `:9200`. + +- `cluster`: Starts up and exposes Boundary's worker and + controller communication layer. Only one `cluster` listener + is allowed per Boundary instance. Must be present if a Boundary + instance is set-up as a controller. + By default, it runs on `:9201`. + +- `proxy`: Starts up a Boundary Worker. By default, it runs +on `:9202`. + +- `ops`: Starts up and exposes Boundary's operational +endpoints (eg: /health). By default, it runs on `:9203`. +This listener's exposed functionality depends on what Boundary +components you're running on any given Boundary instance (eg: Health does not +run unless your Boundary instance has a controller running) + -[tcp]: /docs/configuration/listener/tcp -[unix]: /docs/configuration/listener/unix diff --git a/website/content/docs/configuration/listener/tcp.mdx b/website/content/docs/configuration/listener/tcp.mdx index 804091b4a..cf1c2b175 100644 --- a/website/content/docs/configuration/listener/tcp.mdx +++ b/website/content/docs/configuration/listener/tcp.mdx @@ -25,8 +25,8 @@ allowed. ### General -- `purpose` `(string: "")` - Specifies the purpose. Can be `api`, `cluster`, or - `proxy`. +- `purpose` `(string: "")` - Specifies the purpose. Can be `api`, `cluster`, + `proxy`, or `ops`. - `address` `(string: "127.0.0.1:9200")` – Specifies the address to bind to for listening. @@ -86,9 +86,9 @@ allowed. ### TLS -~> All `tls` parameters are valid only for the `api` listener. `cluster` and -`proxy` connections use their own ephemeral TLS stacks. For more information, -see [the connections security concepts +~> `tls` parameters are valid for `api` and `ops` listeners. `cluster` +and `proxy` connections use their own ephemeral TLS stacks. For more +information, see [the connections security concepts page](https://www.boundaryproject.io/docs/concepts/security/connections-tls). - `tls_disable` `(string: "false")` – Specifies if TLS will be disabled. Boundary diff --git a/website/content/docs/configuration/listener/unix.mdx b/website/content/docs/configuration/listener/unix.mdx index 44d1f8c49..968f41d9a 100644 --- a/website/content/docs/configuration/listener/unix.mdx +++ b/website/content/docs/configuration/listener/unix.mdx @@ -27,7 +27,7 @@ allowed. ### General -- `purpose` `(string: "")` - Specifies the purpose. Can be `api` or `cluster`. +- `purpose` `(string: "")` - Specifies the purpose. Can be `api`, `ops` or `cluster`. - `address` `(string: "")` – Specifies the address (filesystem path) to bind to for listening. @@ -87,9 +87,9 @@ allowed. ### TLS -~> All `tls` parameters are valid only for the `api` listener. `cluster` and -`proxy` connections use their own ephemeral TLS stacks. For more information, -see [the connections security concepts +~> `tls` parameters are valid for `api` and `ops` listeners. `cluster` +and `proxy` connections use their own ephemeral TLS stacks. For more +information, see [the connections security concepts page](https://www.boundaryproject.io/docs/concepts/security/connections-tls). - `tls_disable` `(string: "false")` – Specifies if TLS will be disabled. Boundary