Remove ALPN Muxer (#1965)

We never ended up using this and it makes it much harder to reason about the listeners.

An explanation: for the moment I separated out all of the listeners in the struct, even though one should suffice, because when I did it that way first I was hitting some really weird and hard to find intermittent behavior where it seemed like the listeners were getting crossed. So I re-did it this way with each listener very explicitly referenced. I'm happy to make a follow-up PR that attempts to combine them all into a single listener without hitting the issues I did, if desired, but at least doing it via a separate PR means that it won't be something with removing the ALPN muxer that is at fault.

internal/cmd/base/listener.go

@@ -11,8 +11,8 @@ import (
 	// We must import sha512 so that it registers with the runtime so that
 	// certificates that use it can be parsed.
 	_ "crypto/sha512"
+	"crypto/tls"
 
-	"github.com/hashicorp/boundary/internal/libs/alpnmux"
 	"github.com/hashicorp/go-secure-stdlib/listenerutil"
 	"github.com/hashicorp/go-secure-stdlib/reloadutil"
 	"github.com/mitchellh/cli"
@@ -21,11 +21,13 @@ import (
 )
 
 type ServerListener struct {
-	Mux          *alpnmux.ALPNMux
-	Config       *listenerutil.ListenerConfig
-	HTTPServer   *http.Server
-	GrpcServer   *grpc.Server
-	ALPNListener net.Listener
+	Config          *listenerutil.ListenerConfig
+	HTTPServer      *http.Server
+	GrpcServer      *grpc.Server
+	ApiListener     net.Listener
+	ClusterListener net.Listener
+	ProxyListener   net.Listener
+	OpsListener     net.Listener
 }
 
 type WorkerAuthInfo struct {
@@ -47,7 +49,7 @@ var BuiltinListeners = map[string]ListenerFactory{
 
 // New creates a new listener of the given type with the given
 // configuration. The type is looked up in the BuiltinListeners map.
-func NewListener(l *listenerutil.ListenerConfig, ui cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) {
+func NewListener(l *listenerutil.ListenerConfig, ui cli.Ui) (net.Listener, map[string]string, reloadutil.ReloadFunc, error) {
 	f, ok := BuiltinListeners[l.Type]
 	if !ok {
 		return nil, nil, nil, fmt.Errorf("unknown listener type: %q", l.Type)
@@ -58,8 +60,23 @@ func NewListener(l *listenerutil.ListenerConfig, ui cli.Ui) (*alpnmux.ALPNMux, m
 	}
 	purpose := l.Purpose[0]
 
+	finalAddr, ln, err := f(purpose, l, ui)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	ln, err = listenerWrapProxy(ln, l)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	props := map[string]string{
+		"addr": finalAddr,
+	}
+
 	switch purpose {
 	case "cluster":
+		// We handle our own cluster authentication
 		l.TLSDisable = true
 	case "proxy":
 		// TODO: Eventually we'll support bringing your own cert, and we'd only
@@ -78,24 +95,8 @@ func NewListener(l *listenerutil.ListenerConfig, ui cli.Ui) (*alpnmux.ALPNMux, m
 		}
 	}
 
-	finalAddr, ln, err := f(purpose, l, ui)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	ln, err = listenerWrapProxy(ln, l)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	props := map[string]string{
-		"addr": finalAddr,
-	}
-
-	alpnMux := alpnmux.New(ln)
-
 	if l.TLSDisable {
-		return alpnMux, props, nil, nil
+		return ln, props, nil, nil
 	}
 
 	// Don't request a client cert unless they've explicitly configured it to do
@@ -107,23 +108,15 @@ func NewListener(l *listenerutil.ListenerConfig, ui cli.Ui) (*alpnmux.ALPNMux, m
 	if err != nil {
 		return nil, nil, nil, err
 	}
-	// Register no proto, "http/1.1", and "h2", with same TLS config
-	if _, err = alpnMux.RegisterProto("", tlsConfig); err != nil {
-		return nil, nil, nil, err
-	}
-	if _, err = alpnMux.RegisterProto("http/1.1", tlsConfig); err != nil {
-		return nil, nil, nil, err
-	}
-	if _, err = alpnMux.RegisterProto("h2", tlsConfig); err != nil {
-		return nil, nil, nil, err
-	}
 
-	return alpnMux, props, reloadFunc, nil
+	return tls.NewListener(ln, tlsConfig), props, reloadFunc, nil
 }
 
 func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.Ui) (string, net.Listener, error) {
 	if l.Address == "" {
 		switch purpose {
+		case "api":
+			l.Address = "127.0.0.1:9200"
 		case "cluster":
 			l.Address = "127.0.0.1:9201"
 		case "proxy":
@@ -131,7 +124,7 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U
 		case "ops":
 			l.Address = "127.0.0.1:9203"
 		default:
-			l.Address = "127.0.0.1:9200"
+			return "", nil, errors.New("no purpose provided for listener and no address given")
 		}
 	}
 
@@ -139,6 +132,8 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U
 	if err != nil {
 		if strings.Contains(err.Error(), "missing port") {
 			switch purpose {
+			case "api":
+				port = "9200"
 			case "cluster":
 				port = "9201"
 			case "proxy":
@@ -146,7 +141,7 @@ func tcpListenerFactory(purpose string, l *listenerutil.ListenerConfig, ui cli.U
 			case "ops":
 				port = "9203"
 			default:
-				port = "9200"
+				return "", nil, errors.New("no purpose provided for listener and no port discoverable")
 			}
 			host = l.Address
 		} else {

internal/cmd/base/servers.go

@@ -381,7 +381,18 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig, allo
 	// we ignore errors
 	b.ShutdownFuncs = append(b.ShutdownFuncs, func() error {
 		for _, ln := range b.Listeners {
-			ln.Mux.Close()
+			if ln.ProxyListener != nil {
+				ln.ProxyListener.Close()
+			}
+			if ln.ClusterListener != nil {
+				ln.ClusterListener.Close()
+			}
+			if ln.ApiListener != nil {
+				ln.ApiListener.Close()
+			}
+			if ln.OpsListener != nil {
+				ln.OpsListener.Close()
+			}
 		}
 		return nil
 	})
@@ -414,7 +425,7 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig, allo
 			}
 		}
 
-		lnMux, props, reloadFunc, err := NewListener(lnConfig, ui)
+		ln, props, reloadFunc, err := NewListener(lnConfig, ui)
 		if err != nil {
 			return fmt.Errorf("Error initializing listener of type %s: %w", lnConfig.Type, err)
 		}
@@ -460,10 +471,22 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig, allo
 		}
 		props["max_request_duration"] = lnConfig.MaxRequestDuration.String()
 
-		b.Listeners = append(b.Listeners, &ServerListener{
-			Mux:    lnMux,
+		serverListener := &ServerListener{
 			Config: lnConfig,
-		})
+		}
+
+		switch purpose {
+		case "api":
+			serverListener.ApiListener = ln
+		case "cluster":
+			serverListener.ClusterListener = ln
+		case "proxy":
+			serverListener.ProxyListener = ln
+		case "ops":
+			serverListener.OpsListener = ln
+		}
+
+		b.Listeners = append(b.Listeners, serverListener)
 
 		props["purpose"] = strings.Join(lnConfig.Purpose, ",")
 

internal/cmd/ops/server.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
-	"github.com/hashicorp/boundary/internal/libs/alpnmux"
 	"github.com/hashicorp/boundary/internal/servers/controller"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-hclog"
@@ -51,6 +50,9 @@ func NewServer(l hclog.Logger, c *controller.Controller, listeners ...*base.Serv
 		if ln.Config.Purpose[0] != "ops" {
 			continue
 		}
+		if ln.OpsListener == nil {
+			return nil, fmt.Errorf("%s: missing ops listener", op)
+		}
 
 		h, err := createOpsHandler(ln.Config, c)
 		if err != nil {
@@ -60,11 +62,7 @@ func NewServer(l hclog.Logger, c *controller.Controller, listeners ...*base.Serv
 		b := &opsBundle{ln: ln, h: h}
 		b.ln.HTTPServer = createHttpServer(l, b.h, b.ln.Config)
 
-		funcs, err := getStartFn(b.ln)
-		if err != nil {
-			return nil, err
-		}
-		b.startFn = funcs
+		b.startFn = []func(){func() { go b.ln.HTTPServer.Serve(b.ln.OpsListener) }}
 
 		bundles = append(bundles, b)
 	}
@@ -89,7 +87,7 @@ func (s *Server) Shutdown() error {
 
 	var closeErrors *multierror.Error
 	for _, b := range s.bundles {
-		if b == nil || b.ln == nil || b.ln.Config == nil || b.ln.Mux == nil || b.ln.HTTPServer == nil {
+		if b == nil || b.ln == nil || b.ln.Config == nil || b.ln.OpsListener == nil || b.ln.HTTPServer == nil {
 			return fmt.Errorf("%s: missing bundle, listener or its fields", op)
 		}
 
@@ -101,7 +99,7 @@ func (s *Server) Shutdown() error {
 			multierror.Append(closeErrors, fmt.Errorf("%s: failed to shutdown http server: %w", op, err))
 		}
 
-		err = b.ln.Mux.Close()
+		err = b.ln.OpsListener.Close()
 		err = listenerCloseErrorCheck(b.ln.Config.Type, err)
 		if err != nil {
 			multierror.Append(closeErrors, fmt.Errorf("%s: failed to close listener mux: %w", op, err))
@@ -169,34 +167,6 @@ func createHttpServer(l hclog.Logger, h http.Handler, lncfg *listenerutil.Listen
 	return s
 }
 
-func getStartFn(ln *base.ServerListener) ([]func(), error) {
-	const op = "getStartFn()"
-
-	funcs := make([]func(), 0)
-	switch ln.Config.TLSDisable {
-	case true:
-		l, err := ln.Mux.RegisterProto(alpnmux.NoProto, nil)
-		if err != nil {
-			return nil, fmt.Errorf("%s: error getting non-tls listener: %w", op, err)
-		}
-		if l == nil {
-			return nil, fmt.Errorf("%s: could not get non-tls listener", op)
-		}
-		funcs = append(funcs, func() { go ln.HTTPServer.Serve(l) })
-
-	default:
-		for _, v := range []string{"", "http/1.1", "h2"} {
-			l := ln.Mux.GetListener(v)
-			if l == nil {
-				return nil, fmt.Errorf("%s: could not get tls proto %q listener", op, v)
-			}
-			funcs = append(funcs, func() { go ln.HTTPServer.Serve(l) })
-		}
-	}
-
-	return funcs, nil
-}
-
 func listenerCloseErrorCheck(lnType string, err error) error {
 	if errors.Is(err, net.ErrClosed) {
 		// Ignore net.ErrClosed - The listener was already closed,

internal/cmd/ops/server_test.go

@@ -18,7 +18,6 @@ import (
 	"time"
 
 	"github.com/hashicorp/boundary/internal/cmd/base"
-	"github.com/hashicorp/boundary/internal/libs/alpnmux"
 	"github.com/hashicorp/boundary/internal/servers/controller"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/health"
 	"github.com/hashicorp/go-hclog"
@@ -280,7 +279,7 @@ func TestNewServerIntegration(t *testing.T) {
 
 			addrs := make([]string, 0, len(s.bundles))
 			for _, b := range s.bundles {
-				addrs = append(addrs, b.ln.Mux.Addr().String())
+				addrs = append(addrs, b.ln.OpsListener.Addr().String())
 			}
 			if tt.assertions != nil {
 				tt.assertions(t, addrs)
@@ -334,7 +333,6 @@ func TestShutdown(t *testing.T) {
 					bundles: []*opsBundle{
 						{
 							ln: &base.ServerListener{
-								Mux:        &alpnmux.ALPNMux{},
 								HTTPServer: &http.Server{},
 							},
 						},
@@ -368,7 +366,6 @@ func TestShutdown(t *testing.T) {
 					bundles: []*opsBundle{
 						{
 							ln: &base.ServerListener{
-								Mux:    &alpnmux.ALPNMux{},
 								Config: &listenerutil.ListenerConfig{},
 							},
 						},
@@ -390,10 +387,9 @@ func TestShutdown(t *testing.T) {
 					bundles: []*opsBundle{
 						{
 							ln: &base.ServerListener{
-								ALPNListener: l1,
-								HTTPServer:   s1,
-								Mux:          alpnmux.New(l1),
-								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+								HTTPServer:  s1,
+								OpsListener: l1,
+								Config:      &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
 							},
 						},
 					},
@@ -401,10 +397,10 @@ func TestShutdown(t *testing.T) {
 			},
 			assertions: func(t *testing.T, s *Server) {
 				// The HTTP Server must be closed.
-				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed)
+				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.OpsListener), http.ErrServerClosed)
 
 				// The underlying listener must be closed.
-				require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed)
+				require.ErrorIs(t, s.bundles[0].ln.OpsListener.Close(), net.ErrClosed)
 			},
 			expErr: false,
 		},
@@ -426,10 +422,9 @@ func TestShutdown(t *testing.T) {
 					bundles: []*opsBundle{
 						{
 							ln: &base.ServerListener{
-								ALPNListener: l1,
-								HTTPServer:   s1,
-								Mux:          alpnmux.New(l1),
-								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+								OpsListener: l1,
+								HTTPServer:  s1,
+								Config:      &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
 							},
 						},
 					},
@@ -437,10 +432,10 @@ func TestShutdown(t *testing.T) {
 			},
 			assertions: func(t *testing.T, s *Server) {
 				// The HTTP Server must be closed.
-				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed)
+				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.OpsListener), http.ErrServerClosed)
 
 				// The underlying listener must be closed.
-				require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed)
+				require.ErrorIs(t, s.bundles[0].ln.OpsListener.Close(), net.ErrClosed)
 			},
 		},
 		{
@@ -466,18 +461,16 @@ func TestShutdown(t *testing.T) {
 					bundles: []*opsBundle{
 						{
 							ln: &base.ServerListener{
-								ALPNListener: l1,
-								HTTPServer:   s1,
-								Mux:          alpnmux.New(l1),
-								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+								OpsListener: l1,
+								HTTPServer:  s1,
+								Config:      &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
 							},
 						},
 						{
 							ln: &base.ServerListener{
-								ALPNListener: l2,
-								HTTPServer:   s2,
-								Mux:          alpnmux.New(l2),
-								Config:       &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
+								OpsListener: l2,
+								HTTPServer:  s2,
+								Config:      &listenerutil.ListenerConfig{Type: "tcp", Purpose: []string{"ops"}},
 							},
 						},
 					},
@@ -485,12 +478,12 @@ func TestShutdown(t *testing.T) {
 			},
 			assertions: func(t *testing.T, s *Server) {
 				// The HTTP Server must be closed.
-				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.ALPNListener), http.ErrServerClosed)
-				require.ErrorIs(t, s.bundles[1].ln.HTTPServer.Serve(s.bundles[1].ln.ALPNListener), http.ErrServerClosed)
+				require.ErrorIs(t, s.bundles[0].ln.HTTPServer.Serve(s.bundles[0].ln.OpsListener), http.ErrServerClosed)
+				require.ErrorIs(t, s.bundles[1].ln.HTTPServer.Serve(s.bundles[1].ln.OpsListener), http.ErrServerClosed)
 
 				// The underlying listener must be closed.
-				require.ErrorIs(t, s.bundles[0].ln.Mux.Close(), net.ErrClosed)
-				require.ErrorIs(t, s.bundles[1].ln.Mux.Close(), net.ErrClosed)
+				require.ErrorIs(t, s.bundles[0].ln.OpsListener.Close(), net.ErrClosed)
+				require.ErrorIs(t, s.bundles[1].ln.OpsListener.Close(), net.ErrClosed)
 			},
 		},
 	}
@@ -556,7 +549,7 @@ func TestHealthEndpointLifecycle(t *testing.T) {
 	opsServer.Start()
 
 	// Assert the ops endpoint is up and returning 200 OK.
-	rsp, err := http.Get("http://" + tc.Config().Listeners[0].Mux.Addr().String() + "/health")
+	rsp, err := http.Get("http://" + tc.Config().Listeners[0].OpsListener.Addr().String() + "/health")
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, rsp.StatusCode)
 
@@ -564,7 +557,7 @@ func TestHealthEndpointLifecycle(t *testing.T) {
 	tc.Controller().HealthService.StartServiceUnavailableReplies()
 
 	// Assert we're receiving 503 Service Unavailable now instead of 200 OK.
-	rsp, err = http.Get("http://" + tc.Config().Listeners[0].Mux.Addr().String() + "/health")
+	rsp, err = http.Get("http://" + tc.Config().Listeners[0].OpsListener.Addr().String() + "/health")
 	require.NoError(t, err)
 	require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode)
 }