feat(pprof): Enable pprof endpoints behind flag (#6644)

Makefile

@@ -71,13 +71,6 @@ install: build
 install-no-plugins: export SKIP_PLUGIN_BUILD=1
 install-no-plugins: install
 
-.PHONY: build-pprof
-build-pprof: BUILD_TAGS+=pprof
-build-pprof: BUILD_TAGS+=ui
-build-pprof:
-	@echo "==> Building Boundary with memory pprof enabled"
-	@CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/build.sh'"
-
 .PHONY: build-memprof
 build-memprof: BUILD_TAGS+=memprofiler
 build-memprof:

internal/cmd/base/pprof_off.go

@@ -1,14 +0,0 @@
-// Copyright IBM Corp. 2020, 2026
-// SPDX-License-Identifier: BUSL-1.1
-
-//go:build !pprof
-// +build !pprof
-
-package base
-
-import (
-	"context"
-)
-
-func StartPprof(_ context.Context) {
-}

internal/cmd/base/pprof_on.go

@@ -1,39 +0,0 @@
-// Copyright IBM Corp. 2020, 2026
-// SPDX-License-Identifier: BUSL-1.1
-
-//go:build pprof
-// +build pprof
-
-package base
-
-import (
-	"context"
-	"errors"
-	"net/http"
-	"net/http/pprof"
-
-	"github.com/hashicorp/boundary/internal/event"
-)
-
-func StartPprof(ctx context.Context) {
-	const op = "base.StartPprof"
-	go func() {
-		const addr = "localhost:6060"
-		mux := http.NewServeMux()
-		mux.HandleFunc("/debug/pprof/", pprof.Index)
-		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		mux.Handle("/debug/pprof/block", pprof.Handler("block"))
-		mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
-		mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
-		mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
-		mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
-		mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
-		event.WriteSysEvent(ctx, op, "starting pprof HTTP server", "addr", addr)
-		if err := http.ListenAndServe(addr, mux); err != nil && !errors.Is(err, http.ErrServerClosed) {
-			event.WriteSysEvent(ctx, op, "failed to serve pprof HTTP server", "error", err.Error())
-		}
-	}()
-}

internal/cmd/commands/dev/dev.go

@@ -87,6 +87,7 @@ type Command struct {
 	flagWorkerProxyListenAddr                          string
 	flagWorkerPublicAddr                               string
 	flagOpsListenAddr                                  string
+	flagDebug                                          bool
 	flagUiPassthroughDir                               string
 	flagRecoveryKey                                    string
 	flagDatabaseUrl                                    string
@@ -263,6 +264,13 @@ func (c *Command) Flags() *base.FlagSets {
 		Usage:  "Address to bind to for \"ops\" purpose. If this begins with a forward slash, it will be assumed to be a Unix domain socket path.",
 	})
 
+	f.BoolVar(&base.BoolVar{
+		Name:   "debug",
+		Target: &c.flagDebug,
+		Usage:  "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.",
+		Hidden: true,
+	})
+
 	f.BoolVar(&base.BoolVar{
 		Name:   "controller-only",
 		Target: &c.flagControllerOnly,
@@ -731,8 +739,6 @@ func (c *Command) Run(args []string) int {
 		return base.CommandCliError
 	}
 
-	base.StartPprof(c.Context)
-
 	if c.flagRecoveryKey != "" {
 		c.Config.DevRecoveryKey = c.flagRecoveryKey
 	}
@@ -1002,7 +1008,7 @@ func (c *Command) Run(args []string) int {
 		return base.CommandCliError
 	}
 
-	opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...)
+	opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...)
 	if err != nil {
 		c.UI.Error(fmt.Errorf("Failed to start ops listeners: %w", err).Error())
 		return base.CommandCliError

internal/cmd/commands/server/server.go

@@ -62,6 +62,7 @@ type Command struct {
 	flagLogLevel                string
 	flagLogFormat               string
 	flagCombineLogs             bool
+	flagDebug                   bool
 	flagSkipPlugins             bool
 	flagSkipAliasTargetCreation bool
 	flagWorkerDnsServer         string
@@ -148,7 +149,12 @@ func (c *Command) Flags() *base.FlagSets {
 		Target: &c.flagWorkerAuthCaReinitialize,
 		Hidden: true,
 	})
-
+	f.BoolVar(&base.BoolVar{
+		Name:   "debug",
+		Target: &c.flagDebug,
+		Usage:  "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.",
+		Hidden: true,
+	})
 	f.BoolVar(&base.BoolVar{
 		Name:   "skip-plugins",
 		Target: &c.flagSkipPlugins,
@@ -217,7 +223,6 @@ func (c *Command) Run(args []string) int {
 	c.WorkerAuthDebuggingEnabled.Store(c.Config.EnableWorkerAuthDebugging)
 
 	base.StartMemProfiler(c.Context)
-	base.StartPprof(c.Context)
 
 	// Note: the checks directly after this must remain where they are because
 	// they rely on the state of configured KMSes.
@@ -548,7 +553,7 @@ func (c *Command) Run(args []string) int {
 		return base.CommandCliError
 	}
 
-	opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...)
+	opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...)
 	if err != nil {
 		c.UI.Error(err.Error())
 		return base.CommandCliError

internal/cmd/ops/server.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"time"
 
@@ -39,7 +40,7 @@ type opsBundle struct {
 
 // NewServer iterates through all the listeners and sets up HTTP Servers for each, along with individual handlers.
 // If Controller is set-up, NewServer will set-up a health endpoint for it.
-func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, listeners ...*base.ServerListener) (*Server, error) {
+func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, enableDebug bool, listeners ...*base.ServerListener) (*Server, error) {
 	const op = "ops.NewServer()"
 	if l == nil {
 		return nil, fmt.Errorf("%s: missing logger", op)
@@ -57,7 +58,7 @@ func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w
 			return nil, fmt.Errorf("%s: missing ops listener", op)
 		}
 
-		h, err := createOpsHandler(ctx, ln.Config, c, w)
+		h, err := createOpsHandler(ctx, ln.Config, c, w, enableDebug)
 		if err != nil {
 			return nil, err
 		}
@@ -131,7 +132,7 @@ func (s *Server) WaitIfHealthExists(d time.Duration, ui cli.Ui) {
 	<-time.After(d)
 }
 
-func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker) (http.Handler, error) {
+func createOpsHandler(_ context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker, enableDebug bool) (http.Handler, error) {
 	mux := http.NewServeMux()
 	var h http.Handler
 	var err error
@@ -156,6 +157,20 @@ func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c
 		mux.Handle("/health", h)
 	}
 	mux.Handle("/metrics", promhttp.Handler())
+	if enableDebug {
+		// Turn on pprof endpoints if debug is enabled.
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+		mux.Handle("/debug/pprof/block", pprof.Handler("block"))
+		mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		mux.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+		mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+		mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
+		mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
+	}
 	return cleanhttp.PrintablePathCheckHandler(mux, nil), nil
 }
 

internal/cmd/ops/server_test.go

@@ -106,7 +106,7 @@ func TestNewServer(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, tt.listeners...)
+			s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, false, tt.listeners...)
 			if tt.expErr {
 				require.EqualError(t, err, tt.expErrMsg)
 				require.Nil(t, s)
@@ -297,7 +297,7 @@ func TestNewServerIntegration(t *testing.T) {
 			err := bs.SetupListeners(nil, &configutil.SharedConfig{Listeners: tt.listeners}, []string{"ops"})
 			require.NoError(t, err)
 
-			s, err := NewServer(context.Background(), hclog.Default(), nil, nil, bs.Listeners...)
+			s, err := NewServer(context.Background(), hclog.Default(), nil, nil, false, bs.Listeners...)
 			if tt.expErr {
 				require.EqualError(t, err, tt.expErrMsg)
 				require.Nil(t, s)
@@ -597,7 +597,7 @@ func TestHealthEndpointLifecycle(t *testing.T) {
 	require.NoError(t, err)
 
 	// Controller has started and is set onto our Command object, start ops.
-	opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, tc.Config().Listeners...)
+	opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, false, tc.Config().Listeners...)
 	require.NoError(t, err)
 	opsServer.Start()
 
@@ -692,6 +692,7 @@ func TestCreateOpsHandler(t *testing.T) {
 		name            string
 		setupController bool
 		setupWorker     bool
+		enableDebug     bool
 		lncfg           *listenerutil.ListenerConfig
 		expErr          bool
 		expErrMsg       string
@@ -800,6 +801,28 @@ func TestCreateOpsHandler(t *testing.T) {
 			expErr:          true,
 			expErrMsg:       "controller.(Controller).GetHealthHandler: received nil listener config",
 		},
+		{
+			name:        "pprof disabled by debug flag",
+			enableDebug: false,
+			lncfg:       &listenerutil.ListenerConfig{},
+			assertions: func(t *testing.T, addr string) {
+				rsp, err := http.Get("http://" + addr + "/debug/pprof/")
+				require.NoError(t, err)
+				require.Equal(t, http.StatusNotFound, rsp.StatusCode)
+				require.NoError(t, rsp.Body.Close())
+			},
+		},
+		{
+			name:        "pprof enabled by debug flag",
+			enableDebug: true,
+			lncfg:       &listenerutil.ListenerConfig{},
+			assertions: func(t *testing.T, addr string) {
+				rsp, err := http.Get("http://" + addr + "/debug/pprof/")
+				require.NoError(t, err)
+				require.Equal(t, http.StatusOK, rsp.StatusCode)
+				require.NoError(t, rsp.Body.Close())
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -816,7 +839,7 @@ func TestCreateOpsHandler(t *testing.T) {
 				w = tc.Worker()
 			}
 
-			h, err := createOpsHandler(ctx, tt.lncfg, c, w)
+			h, err := createOpsHandler(ctx, tt.lncfg, c, w, tt.enableDebug)
 			if tt.expErr {
 				require.EqualError(t, err, tt.expErrMsg)
 				require.Nil(t, h)

testing/TRACING.md

@@ -1,19 +1,19 @@
 # Tracing in Boundary
 
 Boundary includes a small number of runtime tracing user regions, which can be used to see where Boundary spends its time during execution.
-To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint is as simple as building with the `pprof` build tag or running
+To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint requires enabling the runtime `-debug` flag on a process with an `ops` listener.
 
 ```
-make build-pprof 
+make build
+boundary dev -debug -ops-listen-address=127.0.0.1:9203
 ```
 
-This will create a new HTTP endpoint on `localhost:6060` of the running binary. As such, it's only accessible to the users on the same machine.
+This will expose the pprof endpoints on the configured ops listener. With the example above, that means `127.0.0.1:9203`, so it's only accessible to users on the same machine.
-Remember to remove this code again once you're done testing.
 
 To create a trace, one can use any tool that allows creating HTTP requests, e.g. `curl`. To create a 3 second trace:
 
 ```
-$ curl -o trace.out http://localhost:6060/debug/pprof/trace?seconds=3
+$ curl -o trace.out http://127.0.0.1:9203/debug/pprof/trace?seconds=3
 ```
 
 Traces are most interesting if they contain some request handling, so it is recommended to prepare some HTTP requests that trigger the behavior you want to understand that you can run while the trace is being collected.