From bb806026b396e13a2494a16455b3c43fd3775b84 Mon Sep 17 00:00:00 2001 From: Louis Ruch Date: Mon, 27 Apr 2026 09:35:40 -0700 Subject: [PATCH] feat(pprof): Enable pprof endpoints behind flag (#6644) --- Makefile | 7 ----- internal/cmd/base/pprof_off.go | 14 --------- internal/cmd/base/pprof_on.go | 39 -------------------------- internal/cmd/commands/dev/dev.go | 12 ++++++-- internal/cmd/commands/server/server.go | 11 ++++++-- internal/cmd/ops/server.go | 21 ++++++++++++-- internal/cmd/ops/server_test.go | 31 +++++++++++++++++--- testing/TRACING.md | 10 +++---- 8 files changed, 67 insertions(+), 78 deletions(-) delete mode 100644 internal/cmd/base/pprof_off.go delete mode 100644 internal/cmd/base/pprof_on.go diff --git a/Makefile b/Makefile index 819d692e7..6db2a701e 100644 --- a/Makefile +++ b/Makefile @@ -71,13 +71,6 @@ install: build install-no-plugins: export SKIP_PLUGIN_BUILD=1 install-no-plugins: install -.PHONY: build-pprof -build-pprof: BUILD_TAGS+=pprof -build-pprof: BUILD_TAGS+=ui -build-pprof: - @echo "==> Building Boundary with memory pprof enabled" - @CGO_ENABLED=$(CGO_ENABLED) BUILD_TAGS='$(BUILD_TAGS)' sh -c "'$(CURDIR)/scripts/build.sh'" - .PHONY: build-memprof build-memprof: BUILD_TAGS+=memprofiler build-memprof: diff --git a/internal/cmd/base/pprof_off.go b/internal/cmd/base/pprof_off.go deleted file mode 100644 index cd680b3e8..000000000 --- a/internal/cmd/base/pprof_off.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright IBM Corp. 2020, 2026 -// SPDX-License-Identifier: BUSL-1.1 - -//go:build !pprof -// +build !pprof - -package base - -import ( - "context" -) - -func StartPprof(_ context.Context) { -} diff --git a/internal/cmd/base/pprof_on.go b/internal/cmd/base/pprof_on.go deleted file mode 100644 index afc08b354..000000000 --- a/internal/cmd/base/pprof_on.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright IBM Corp. 2020, 2026 -// SPDX-License-Identifier: BUSL-1.1 - -//go:build pprof -// +build pprof - -package base - -import ( - "context" - "errors" - "net/http" - "net/http/pprof" - - "github.com/hashicorp/boundary/internal/event" -) - -func StartPprof(ctx context.Context) { - const op = "base.StartPprof" - go func() { - const addr = "localhost:6060" - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - mux.Handle("/debug/pprof/block", pprof.Handler("block")) - mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) - mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) - mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) - mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) - event.WriteSysEvent(ctx, op, "starting pprof HTTP server", "addr", addr) - if err := http.ListenAndServe(addr, mux); err != nil && !errors.Is(err, http.ErrServerClosed) { - event.WriteSysEvent(ctx, op, "failed to serve pprof HTTP server", "error", err.Error()) - } - }() -} diff --git a/internal/cmd/commands/dev/dev.go b/internal/cmd/commands/dev/dev.go index ed3db765c..f75789d99 100644 --- a/internal/cmd/commands/dev/dev.go +++ b/internal/cmd/commands/dev/dev.go @@ -87,6 +87,7 @@ type Command struct { flagWorkerProxyListenAddr string flagWorkerPublicAddr string flagOpsListenAddr string + flagDebug bool flagUiPassthroughDir string flagRecoveryKey string flagDatabaseUrl string @@ -263,6 +264,13 @@ func (c *Command) Flags() *base.FlagSets { Usage: "Address to bind to for \"ops\" purpose. If this begins with a forward slash, it will be assumed to be a Unix domain socket path.", }) + f.BoolVar(&base.BoolVar{ + Name: "debug", + Target: &c.flagDebug, + Usage: "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.", + Hidden: true, + }) + f.BoolVar(&base.BoolVar{ Name: "controller-only", Target: &c.flagControllerOnly, @@ -731,8 +739,6 @@ func (c *Command) Run(args []string) int { return base.CommandCliError } - base.StartPprof(c.Context) - if c.flagRecoveryKey != "" { c.Config.DevRecoveryKey = c.flagRecoveryKey } @@ -1002,7 +1008,7 @@ func (c *Command) Run(args []string) int { return base.CommandCliError } - opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...) + opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...) if err != nil { c.UI.Error(fmt.Errorf("Failed to start ops listeners: %w", err).Error()) return base.CommandCliError diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go index 81e2dfdb3..86d43f0cb 100644 --- a/internal/cmd/commands/server/server.go +++ b/internal/cmd/commands/server/server.go @@ -62,6 +62,7 @@ type Command struct { flagLogLevel string flagLogFormat string flagCombineLogs bool + flagDebug bool flagSkipPlugins bool flagSkipAliasTargetCreation bool flagWorkerDnsServer string @@ -148,7 +149,12 @@ func (c *Command) Flags() *base.FlagSets { Target: &c.flagWorkerAuthCaReinitialize, Hidden: true, }) - + f.BoolVar(&base.BoolVar{ + Name: "debug", + Target: &c.flagDebug, + Usage: "Enable debug mode. Currently this exposes pprof endpoints on the ops listener.", + Hidden: true, + }) f.BoolVar(&base.BoolVar{ Name: "skip-plugins", Target: &c.flagSkipPlugins, @@ -217,7 +223,6 @@ func (c *Command) Run(args []string) int { c.WorkerAuthDebuggingEnabled.Store(c.Config.EnableWorkerAuthDebugging) base.StartMemProfiler(c.Context) - base.StartPprof(c.Context) // Note: the checks directly after this must remain where they are because // they rely on the state of configured KMSes. @@ -548,7 +553,7 @@ func (c *Command) Run(args []string) int { return base.CommandCliError } - opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.Listeners...) + opsServer, err := ops.NewServer(c.Context, c.Logger, c.controller, c.worker, c.flagDebug, c.Listeners...) if err != nil { c.UI.Error(err.Error()) return base.CommandCliError diff --git a/internal/cmd/ops/server.go b/internal/cmd/ops/server.go index ca1087ba6..f6de5e3af 100644 --- a/internal/cmd/ops/server.go +++ b/internal/cmd/ops/server.go @@ -11,6 +11,7 @@ import ( "fmt" "net" "net/http" + "net/http/pprof" "os" "time" @@ -39,7 +40,7 @@ type opsBundle struct { // NewServer iterates through all the listeners and sets up HTTP Servers for each, along with individual handlers. // If Controller is set-up, NewServer will set-up a health endpoint for it. -func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, listeners ...*base.ServerListener) (*Server, error) { +func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w *worker.Worker, enableDebug bool, listeners ...*base.ServerListener) (*Server, error) { const op = "ops.NewServer()" if l == nil { return nil, fmt.Errorf("%s: missing logger", op) @@ -57,7 +58,7 @@ func NewServer(ctx context.Context, l hclog.Logger, c *controller.Controller, w return nil, fmt.Errorf("%s: missing ops listener", op) } - h, err := createOpsHandler(ctx, ln.Config, c, w) + h, err := createOpsHandler(ctx, ln.Config, c, w, enableDebug) if err != nil { return nil, err } @@ -131,7 +132,7 @@ func (s *Server) WaitIfHealthExists(d time.Duration, ui cli.Ui) { <-time.After(d) } -func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker) (http.Handler, error) { +func createOpsHandler(_ context.Context, lncfg *listenerutil.ListenerConfig, c *controller.Controller, w *worker.Worker, enableDebug bool) (http.Handler, error) { mux := http.NewServeMux() var h http.Handler var err error @@ -156,6 +157,20 @@ func createOpsHandler(ctx context.Context, lncfg *listenerutil.ListenerConfig, c mux.Handle("/health", h) } mux.Handle("/metrics", promhttp.Handler()) + if enableDebug { + // Turn on pprof endpoints if debug is enabled. + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.Handle("/debug/pprof/block", pprof.Handler("block")) + mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) + mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) + } return cleanhttp.PrintablePathCheckHandler(mux, nil), nil } diff --git a/internal/cmd/ops/server_test.go b/internal/cmd/ops/server_test.go index 800c4f215..58a90685e 100644 --- a/internal/cmd/ops/server_test.go +++ b/internal/cmd/ops/server_test.go @@ -106,7 +106,7 @@ func TestNewServer(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, tt.listeners...) + s, err := NewServer(context.Background(), tt.logger, tt.c, tt.w, false, tt.listeners...) if tt.expErr { require.EqualError(t, err, tt.expErrMsg) require.Nil(t, s) @@ -297,7 +297,7 @@ func TestNewServerIntegration(t *testing.T) { err := bs.SetupListeners(nil, &configutil.SharedConfig{Listeners: tt.listeners}, []string{"ops"}) require.NoError(t, err) - s, err := NewServer(context.Background(), hclog.Default(), nil, nil, bs.Listeners...) + s, err := NewServer(context.Background(), hclog.Default(), nil, nil, false, bs.Listeners...) if tt.expErr { require.EqualError(t, err, tt.expErrMsg) require.Nil(t, s) @@ -597,7 +597,7 @@ func TestHealthEndpointLifecycle(t *testing.T) { require.NoError(t, err) // Controller has started and is set onto our Command object, start ops. - opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, tc.Config().Listeners...) + opsServer, err := NewServer(tc.Context(), hclog.Default(), tc.Controller(), nil, false, tc.Config().Listeners...) require.NoError(t, err) opsServer.Start() @@ -692,6 +692,7 @@ func TestCreateOpsHandler(t *testing.T) { name string setupController bool setupWorker bool + enableDebug bool lncfg *listenerutil.ListenerConfig expErr bool expErrMsg string @@ -800,6 +801,28 @@ func TestCreateOpsHandler(t *testing.T) { expErr: true, expErrMsg: "controller.(Controller).GetHealthHandler: received nil listener config", }, + { + name: "pprof disabled by debug flag", + enableDebug: false, + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, addr string) { + rsp, err := http.Get("http://" + addr + "/debug/pprof/") + require.NoError(t, err) + require.Equal(t, http.StatusNotFound, rsp.StatusCode) + require.NoError(t, rsp.Body.Close()) + }, + }, + { + name: "pprof enabled by debug flag", + enableDebug: true, + lncfg: &listenerutil.ListenerConfig{}, + assertions: func(t *testing.T, addr string) { + rsp, err := http.Get("http://" + addr + "/debug/pprof/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.NoError(t, rsp.Body.Close()) + }, + }, } for _, tt := range tests { @@ -816,7 +839,7 @@ func TestCreateOpsHandler(t *testing.T) { w = tc.Worker() } - h, err := createOpsHandler(ctx, tt.lncfg, c, w) + h, err := createOpsHandler(ctx, tt.lncfg, c, w, tt.enableDebug) if tt.expErr { require.EqualError(t, err, tt.expErrMsg) require.Nil(t, h) diff --git a/testing/TRACING.md b/testing/TRACING.md index f9fa37a32..71550b4e4 100644 --- a/testing/TRACING.md +++ b/testing/TRACING.md @@ -1,19 +1,19 @@ # Tracing in Boundary Boundary includes a small number of runtime tracing user regions, which can be used to see where Boundary spends its time during execution. -To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint is as simple as building with the `pprof` build tag or running +To create a trace, we first need to expose the pprof endpoint. It is disabled by default. Exposing the pprof endpoint requires enabling the runtime `-debug` flag on a process with an `ops` listener. ``` -make build-pprof +make build +boundary dev -debug -ops-listen-address=127.0.0.1:9203 ``` -This will create a new HTTP endpoint on `localhost:6060` of the running binary. As such, it's only accessible to the users on the same machine. -Remember to remove this code again once you're done testing. +This will expose the pprof endpoints on the configured ops listener. With the example above, that means `127.0.0.1:9203`, so it's only accessible to users on the same machine. To create a trace, one can use any tool that allows creating HTTP requests, e.g. `curl`. To create a 3 second trace: ``` -$ curl -o trace.out http://localhost:6060/debug/pprof/trace?seconds=3 +$ curl -o trace.out http://127.0.0.1:9203/debug/pprof/trace?seconds=3 ``` Traces are most interesting if they contain some request handling, so it is recommended to prepare some HTTP requests that trigger the behavior you want to understand that you can run while the trace is being collected.