feat(controller): Add health endpoint (#1882)

* feat(controller): New health service and server

This change also implements a new listener purpose, `ops`.

`ops` (short for operations) is a new listener purpose that is
responsible for serving Boundary's operations endpoints, like health
and possibly metrics.

At Startup: `ops` listeners get configured as late as possible to catch
any errors with the startup process;

At Shutdown: If the health endpoint has been set-up, we introduce a
configurable delay in the process. This is to enable the health endpoint
to report unhealthy for that amount of time. `ops` listeners are also
the last listeners to be shutdown.

internal/servers/controller/handler.go

@@ -18,6 +18,7 @@ import (
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
+	opsservices "github.com/hashicorp/boundary/internal/gen/controller/ops/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/boundary/internal/servers/common"
 	"github.com/hashicorp/boundary/internal/servers/controller/auth"
@@ -27,6 +28,7 @@ import (
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/credentiallibraries"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/credentialstores"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/groups"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/health"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_catalogs"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_sets"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/hosts"
@@ -74,6 +76,28 @@ func (c *Controller) apiHandler(props HandlerProperties) (http.Handler, error) {
 	return eventsHandler, err
 }
 
+// GetHealthHandler returns a gRPC Gateway mux that is registered against the
+// controller's gRPC health service to make it accessible from an HTTP API.
+func (c *Controller) GetHealthHandler(lcfg *listenerutil.ListenerConfig) (http.Handler, error) {
+	const op = "controller.(Controller).GetHealthHandler"
+	if lcfg == nil {
+		return nil, fmt.Errorf("%s: received nil listener config", op)
+	}
+
+	healthGrpcGwMux := newGrpcGatewayMux()
+	err := registerHealthGrpcGatewayEndpoint(c.baseContext, healthGrpcGwMux, gatewayDialOptions(c.apiGrpcServerListener)...)
+	if err != nil {
+		return nil, fmt.Errorf("%s: failed to register health service handler: %w", op, err)
+	}
+
+	wrapped := wrapHandlerWithCommonFuncs(healthGrpcGwMux, c, HandlerProperties{lcfg, c.baseContext})
+	return common.WrapWithEventsHandler(wrapped, c.conf.Eventer, c.kms, lcfg)
+}
+
+func registerHealthGrpcGatewayEndpoint(ctx context.Context, gwMux *runtime.ServeMux, dialOptions ...grpc.DialOption) error {
+	return opsservices.RegisterHealthServiceHandlerFromEndpoint(ctx, gwMux, gatewayTarget, dialOptions)
+}
+
 func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 	// We have to check against the current services because the gRPC lib treats a duplicate
 	// register call as an error and os.Exits.
@@ -193,6 +217,11 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 		}
 		services.RegisterCredentialLibraryServiceServer(s, cl)
 	}
+	if _, ok := s.GetServiceInfo()[opsservices.HealthService_ServiceDesc.ServiceName]; !ok {
+		hs := health.NewService()
+		opsservices.RegisterHealthServiceServer(s, hs)
+		c.HealthService = hs
+	}
 
 	return nil
 }