refactor (controller): Use a grpc server for the grpc-gateway (#1576)

* refactor (controller): Use a grpc server for the grpc-gateway

This starts a grpc server for the grpc-gateway.  The new gateway
server uses an in-memory listener which also supports "dialing", so
the http proxy can connect to the in-memory listener.

This change allows us to use all grpc server features and was in part
motivated by a need for an interceptor of inbound requests.  With that
said, having a "proper" grpc server for the gateway will provide a lot
more flexibility and capabilities moving forward.

The request context will still be verified via middleware in the
http proxy (see wrapHandlerWithCommonFuncs).  This verified context
will be serialized into a protobuf and sent along to the grpc-gateway
via metadata where an interceptor will add the verified request
info to the request's context (see requestCtxInterceptor).  The
interceptor will verify the serialized request info via a shared ticket
with it's companion in-memory http proxy.  FYI: the ticket is simply
random base62 string (see db.NewPrivateId).

This change also necessitated a minor change to services.  We need to
return an empty response instead of nil, nil when there are no errors
and the response is empty.

* refactor (handlers): Rename gRPC Gateway filter

Originally, this was called an OutgoingInterceptor, which is an
unfortunate name since it's not an interceptor.  Now that we're
refactoring the gateway stuff and have proper interceptors, renaming
this should remove ambiguity

* refactor: Rework how 204 with empty msg is handled

Recent changes to grpc gateway has necessitated a change to how
204s are handled by the gateway

* tests (bats): Check status codes for delete operations

internal/servers/controller/auth/auth.go

@@ -11,7 +11,9 @@ import (
 	"github.com/hashicorp/boundary/api/recovery"
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/errors"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/gen/controller/tokens"
+
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	"github.com/hashicorp/boundary/internal/perms"
@@ -27,7 +29,7 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-type TokenFormat int
+type TokenFormat uint32
 
 const (
 	// We weren't given one or couldn't parse it
@@ -51,22 +53,6 @@ type key int
 
 var verifierKey key
 
-// RequestInfo contains request parameters necessary for checking authn/authz
-type RequestInfo struct {
-	Path           string
-	Method         string
-	PublicId       string
-	EncryptedToken string
-	Token          string
-	TokenFormat    TokenFormat
-
-	// The following are useful for tests
-	scopeIdOverride      string
-	userIdOverride       string
-	DisableAuthzFailures bool
-	DisableAuthEntirely  bool
-}
-
 type VerifyResults struct {
 	UserId      string
 	AuthTokenId string
@@ -108,7 +94,7 @@ type verifier struct {
 	authTokenRepoFn common.AuthTokenRepoFactory
 	serversRepoFn   common.ServersRepoFactory
 	kms             *kms.Kms
-	requestInfo     RequestInfo
+	requestInfo     *authpb.RequestInfo
 	res             *perms.Resource
 	act             action.Type
 	ctx             context.Context
@@ -124,7 +110,7 @@ func NewVerifierContext(ctx context.Context,
 	authTokenRepoFn common.AuthTokenRepoFactory,
 	serversRepoFn common.ServersRepoFactory,
 	kms *kms.Kms,
-	requestInfo RequestInfo) context.Context {
+	requestInfo *authpb.RequestInfo) context.Context {
 	return context.WithValue(ctx, verifierKey, &verifier{
 		iamRepoFn:       iamRepoFn,
 		authTokenRepoFn: authTokenRepoFn,
@@ -174,7 +160,7 @@ func Verify(ctx context.Context, opt ...Option) (ret VerifyResults) {
 	// without fuss
 	if v.requestInfo.DisableAuthEntirely {
 		const op = "auth.(disabled).lookupScope"
-		ret.Scope.Id = v.requestInfo.scopeIdOverride
+		ret.Scope.Id = v.requestInfo.ScopeIdOverride
 		if ret.Scope.Id == "" {
 			ret.Scope.Id = opts.withScopeId
 		}
@@ -214,7 +200,7 @@ func Verify(ctx context.Context, opt ...Option) (ret VerifyResults) {
 				ParentScopeId: scp.GetParentId(),
 			}
 		}
-		ret.UserId = v.requestInfo.userIdOverride
+		ret.UserId = v.requestInfo.UserIdOverride
 		if reqInfo != nil {
 			reqInfo.UserId = ret.UserId
 		}
@@ -291,40 +277,40 @@ func Verify(ctx context.Context, opt ...Option) (ret VerifyResults) {
 func (v *verifier) decryptToken(ctx context.Context) {
 	const op = "auth.(verifier).decryptToken"
 	switch v.requestInfo.TokenFormat {
-	case AuthTokenTypeUnknown:
+	case uint32(AuthTokenTypeUnknown):
 		// Nothing to decrypt
 		return
 
-	case AuthTokenTypeBearer, AuthTokenTypeSplitCookie:
+	case uint32(AuthTokenTypeBearer), uint32(AuthTokenTypeSplitCookie):
 		if v.kms == nil {
 			event.WriteError(ctx, op, stderrors.New("no KMS object available to authz system"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		tokenRepo, err := v.authTokenRepoFn()
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("failed to get authtoken repo"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		at, err := tokenRepo.LookupAuthToken(v.ctx, v.requestInfo.PublicId)
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("failed to look up auth token by public ID"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		if at == nil {
 			event.WriteError(ctx, op, stderrors.New("nil result from looking up auth token by public ID"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		tokenWrapper, err := v.kms.GetWrapper(v.ctx, at.GetScopeId(), kms.KeyPurposeTokens)
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("unable to get wrapper for tokens; continuing as anonymous user"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
@@ -333,62 +319,62 @@ func (v *verifier) decryptToken(ctx context.Context) {
 		case globals.ServiceTokenV1:
 		default:
 			event.WriteError(ctx, op, stderrors.New("unknown token encryption version; continuing as anonymous user"), event.WithInfo("version", version))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		marshaledToken, err := base58.FastBase58Decoding(v.requestInfo.EncryptedToken[len(globals.ServiceTokenV1):])
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("error unmarshaling base58 token; continuing as anonymous user"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		blobInfo := new(wrapping.EncryptedBlobInfo)
 		if err := proto.Unmarshal(marshaledToken, blobInfo); err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("error decoding encrypted token; continuing as anonymous user"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		s1Bytes, err := tokenWrapper.Decrypt(v.ctx, blobInfo, []byte(v.requestInfo.PublicId))
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("error decrypting encrypted token; continuing as anonymous user"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		var s1Info tokens.S1TokenInfo
 		if err := proto.Unmarshal(s1Bytes, &s1Info); err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("error unmarshaling token info; continuing as anonymous user"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
-		if v.requestInfo.TokenFormat == AuthTokenTypeUnknown || s1Info.Token == "" || v.requestInfo.PublicId == "" {
+		if v.requestInfo.TokenFormat == uint32(AuthTokenTypeUnknown) || s1Info.Token == "" || v.requestInfo.PublicId == "" {
 			event.WriteError(ctx, op, stderrors.New("after parsing, could not find valid token; continuing as anonymous user"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 
 		v.requestInfo.Token = s1Info.Token
 		return
 
-	case AuthTokenTypeRecoveryKms:
+	case uint32(AuthTokenTypeRecoveryKms):
 		if v.kms == nil {
 			event.WriteError(ctx, op, stderrors.New("no KMS object available to authz system"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		wrapper := v.kms.GetExternalWrappers().Recovery()
 		if wrapper == nil {
 			event.WriteError(ctx, op, stderrors.New("no recovery KMS is available"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		info, err := recovery.ParseRecoveryToken(v.ctx, wrapper, v.requestInfo.EncryptedToken)
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("decrypt recovery token: error parsing and validating recovery token"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		// If we add the validity period to the creation time (which we've
@@ -396,18 +382,18 @@ func (v *verifier) decryptToken(ctx context.Context) {
 		// it's before now, it's expired and might be a replay.
 		if info.CreationTime.Add(globals.RecoveryTokenValidityPeriod).Before(time.Now()) {
 			event.WriteError(ctx, op, stderrors.New("recovery token has expired (possible replay attack)"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		repo, err := v.serversRepoFn()
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("decrypt recovery token: error fetching servers repo"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		if err := repo.AddRecoveryNonce(v.ctx, info.Nonce); err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("decrypt recovery token: error adding nonce to database (possible replay attack)"))
-			v.requestInfo.TokenFormat = AuthTokenTypeUnknown
+			v.requestInfo.TokenFormat = uint32(AuthTokenTypeUnknown)
 			return
 		}
 		event.WriteError(ctx, op, stderrors.New("recovery KMS was used to authorize a call"), event.WithInfo("url", v.requestInfo.Path, "method", v.requestInfo.Method))
@@ -426,15 +412,15 @@ func (v verifier) performAuthCheck(ctx context.Context) (aclResults perms.ACLRes
 
 	// Validate the token and fetch the corresponding user ID
 	switch v.requestInfo.TokenFormat {
-	case AuthTokenTypeUnknown:
+	case uint32(AuthTokenTypeUnknown):
 		// Nothing; remain as the anonymous user
 
-	case AuthTokenTypeRecoveryKms:
+	case uint32(AuthTokenTypeRecoveryKms):
 		// We validated the encrypted token in decryptToken and handled the
 		// nonces there, so just set the user
 		userId = "u_recovery"
 
-	case AuthTokenTypeBearer, AuthTokenTypeSplitCookie:
+	case uint32(AuthTokenTypeBearer), uint32(AuthTokenTypeSplitCookie):
 		if v.requestInfo.Token == "" {
 			// This will end up staying as the anonymous user
 			break
@@ -500,7 +486,7 @@ func (v verifier) performAuthCheck(ctx context.Context) (aclResults perms.ACLRes
 	}
 
 	// At this point we don't need to look up grants since it's automatically allowed
-	if v.requestInfo.TokenFormat == AuthTokenTypeRecoveryKms {
+	if v.requestInfo.TokenFormat == uint32(AuthTokenTypeRecoveryKms) {
 		aclResults.AuthenticationFinished = true
 		aclResults.Authorized = true
 		retErr = nil
@@ -561,7 +547,7 @@ func (r *VerifyResults) FetchActionSetForType(ctx context.Context, typ resource.
 func (r *VerifyResults) fetchActions(id string, typ resource.Type, availableActions action.ActionSet, opt ...Option) action.ActionSet {
 	switch {
 	case r.v.requestInfo.DisableAuthEntirely,
-		r.v.requestInfo.TokenFormat == AuthTokenTypeRecoveryKms:
+		r.v.requestInfo.TokenFormat == uint32(AuthTokenTypeRecoveryKms):
 		// TODO: See if we can be better about what we return with the anonymous
 		// user and recovery KMS, perhaps given exclusionary options for each
 		return availableActions
@@ -598,7 +584,7 @@ func (r *VerifyResults) fetchActions(id string, typ resource.Type, availableActi
 
 func (r *VerifyResults) FetchOutputFields(res perms.Resource, act action.Type) perms.OutputFieldsMap {
 	switch {
-	case r.v.requestInfo.TokenFormat == AuthTokenTypeRecoveryKms:
+	case r.v.requestInfo.TokenFormat == uint32(AuthTokenTypeRecoveryKms):
 		return perms.OutputFieldsMap{"*": true}
 	case r.v.requestInfo.DisableAuthEntirely:
 		return nil
@@ -611,7 +597,7 @@ func (r *VerifyResults) FetchOutputFields(res perms.Resource, act action.Type) p
 // split cookies and parses it. If it cannot be parsed successfully, the issue
 // is logged and we return blank, so logic will continue as the anonymous user.
 // The public ID and _encrypted_ token are returned along with the token format.
-func GetTokenFromRequest(ctx context.Context, kmsCache *kms.Kms, req *http.Request) (string, string, TokenFormat) {
+func GetTokenFromRequest(ctx context.Context, kmsCache *kms.Kms, req *http.Request) (string, string, uint32) {
 	const op = "auth.GetTokenFromRequest"
 	// First, get the token, either from the authorization header or from split
 	// cookies
@@ -643,21 +629,21 @@ func GetTokenFromRequest(ctx context.Context, kmsCache *kms.Kms, req *http.Reque
 		// We didn't find auth info or a client screwed up and put in a blank
 		// header instead of nothing at all, so return blank which will indicate
 		// the anonymouse user
-		return "", "", AuthTokenTypeUnknown
+		return "", "", uint32(AuthTokenTypeUnknown)
 	}
 
 	if strings.HasPrefix(fullToken, "r_") {
-		return "", fullToken, AuthTokenTypeRecoveryKms
+		return "", fullToken, uint32(AuthTokenTypeRecoveryKms)
 	}
 
 	splitFullToken := strings.Split(fullToken, "_")
 	if len(splitFullToken) != 3 {
 		event.WriteError(ctx, op, stderrors.New("unexpected number of segments in token; continuing as anonymous user"), event.WithInfo("expected", 3, "found", len(splitFullToken)))
-		return "", "", AuthTokenTypeUnknown
+		return "", "", uint32(AuthTokenTypeUnknown)
 	}
 
 	publicId := strings.Join(splitFullToken[0:2], "_")
 	encryptedToken := splitFullToken[2]
 
-	return publicId, encryptedToken, receivedTokenType
+	return publicId, encryptedToken, uint32(receivedTokenType)
 }

internal/servers/controller/auth/auth_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/hashicorp/boundary/internal/authtoken"
 	"github.com/hashicorp/boundary/internal/db"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/servers"
@@ -107,17 +108,17 @@ func TestAuthTokenAuthenticator(t *testing.T) {
 			}
 
 			// Add values for authn/authz checking
-			requestInfo := RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:   req.URL.Path,
 				Method: req.Method,
 			}
 			requestInfo.PublicId, requestInfo.EncryptedToken, requestInfo.TokenFormat = GetTokenFromRequest(context.TODO(), kms, req)
-			assert.Equal(t, tc.tokenFormat, requestInfo.TokenFormat)
+			assert.Equal(t, uint32(tc.tokenFormat), requestInfo.TokenFormat)
 
 			if tc.userId == "" {
 				return
 			}
-			ctx := NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 
 			v, ok := ctx.Value(verifierKey).(*verifier)
 			require.True(t, ok)

internal/servers/controller/auth/testing.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"context"
 
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/requests"
 	"github.com/hashicorp/boundary/internal/servers/controller/common"
 )
@@ -11,16 +12,16 @@ import (
 // auth checking entirely disabled. Supported options: WithScopeId an WithUserId
 // are used directly; WithKms is passed through into the verifier context.
 func DisabledAuthTestContext(iamRepoFn common.IamRepoFactory, scopeId string, opt ...Option) context.Context {
-	reqInfo := RequestInfo{DisableAuthEntirely: true}
+	reqInfo := authpb.RequestInfo{DisableAuthEntirely: true}
 	opts := getOpts(opt...)
-	reqInfo.scopeIdOverride = opts.withScopeId
-	if reqInfo.scopeIdOverride == "" {
-		reqInfo.scopeIdOverride = scopeId
+	reqInfo.ScopeIdOverride = opts.withScopeId
+	if reqInfo.ScopeIdOverride == "" {
+		reqInfo.ScopeIdOverride = scopeId
 	}
-	reqInfo.userIdOverride = opts.withUserId
-	if reqInfo.userIdOverride == "" {
-		reqInfo.userIdOverride = "u_auth"
+	reqInfo.UserIdOverride = opts.withUserId
+	if reqInfo.UserIdOverride == "" {
+		reqInfo.UserIdOverride = "u_auth"
 	}
 	requestContext := context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
-	return NewVerifierContext(requestContext, iamRepoFn, nil, nil, opts.withKms, reqInfo)
+	return NewVerifierContext(requestContext, iamRepoFn, nil, nil, opts.withKms, &reqInfo)
 }

internal/servers/controller/common/scopeids/scope_ids_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/hashicorp/boundary/internal/authtoken"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/servers"
@@ -229,9 +230,9 @@ func TestListingScopeIds(t *testing.T) {
 				authTokenRepoFn,
 				serversRepoFn,
 				kms,
-				auth.RequestInfo{
+				&authpb.RequestInfo{
 					Token:       at.GetToken(),
-					TokenFormat: auth.AuthTokenTypeBearer,
+					TokenFormat: uint32(auth.AuthTokenTypeBearer),
 					PublicId:    at.GetPublicId(),
 				})
 

internal/servers/controller/controller.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
 	"github.com/hashicorp/boundary/internal/authtoken"
@@ -26,6 +28,7 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/mlock"
 	"github.com/patrickmn/go-cache"
 	ua "go.uber.org/atomic"
+	"google.golang.org/grpc"
 )
 
 type Controller struct {
@@ -44,6 +47,12 @@ type Controller struct {
 	// Used for testing and tracking worker health
 	workerStatusUpdateTimes *sync.Map
 
+	// grpc gateway server
+	gatewayServer   *grpc.Server
+	gatewayTicket   string
+	gatewayListener gatewayListener
+	gatewayMux      *runtime.ServeMux
+
 	// Repo factory methods
 	AuthTokenRepoFn       common.AuthTokenRepoFactory
 	VaultCredentialRepoFn common.VaultCredentialRepoFactory
@@ -172,7 +181,7 @@ func (c *Controller) Start() error {
 	if err := c.scheduler.Start(c.baseContext, &c.schedulerWg); err != nil {
 		return fmt.Errorf("error starting scheduler: %w", err)
 	}
-	if err := c.startListeners(); err != nil {
+	if err := c.startListeners(c.baseContext); err != nil {
 		return fmt.Errorf("error starting controller listeners: %w", err)
 	}
 

internal/servers/controller/gateway.go

@@ -0,0 +1,67 @@
+package controller
+
+import (
+	"context"
+	"math"
+	"net"
+
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"github.com/hashicorp/boundary/globals"
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/servers/controller/common"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/test/bufconn"
+)
+
+// newGatewayListener will create an in-memory listener
+func newGatewayListener() (gatewayListener, string) {
+	buffer := globals.DefaultMaxRequestSize // seems like a reasonable size for the ring buffer, but then happily change the size if more info becomes available
+	return bufconn.Listen(int(buffer)), ""
+}
+
+const gatewayTarget = ""
+
+type gatewayListener interface {
+	net.Listener
+	Dial() (net.Conn, error)
+}
+
+func gatewayDialOptions(lis gatewayListener) []grpc.DialOption {
+	return []grpc.DialOption{
+		grpc.WithInsecure(),
+		grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) {
+			return lis.Dial()
+		}),
+	}
+}
+
+func newGatewayMux() *runtime.ServeMux {
+	return runtime.NewServeMux(
+		runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.HTTPBodyMarshaler{
+			Marshaler: handlers.JSONMarshaler(),
+		}),
+		runtime.WithErrorHandler(handlers.ErrorHandler()),
+		runtime.WithForwardResponseOption(handlers.OutgoingResponseFilter),
+	)
+}
+
+func newGatewayServer(ctx context.Context, iamRepoFn common.IamRepoFactory, authTokenRepoFn common.AuthTokenRepoFactory, serversRepoFn common.ServersRepoFactory, kms *kms.Kms) (*grpc.Server, string, error) {
+	const op = "controller.newGatewayServer"
+	ticket, err := db.NewPrivateId("gwticket")
+	if err != nil {
+		return nil, "", errors.Wrap(ctx, err, op, errors.WithMsg("unable to generate gateway ticket"))
+	}
+	requestCtxInterceptor, err := requestCtxInterceptor(ctx, iamRepoFn, authTokenRepoFn, serversRepoFn, kms, ticket)
+	if err != nil {
+		return nil, "", err
+	}
+	return grpc.NewServer(
+		grpc.MaxRecvMsgSize(math.MaxInt32),
+		grpc.MaxSendMsgSize(math.MaxInt32),
+		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(requestCtxInterceptor, errorInterceptor(ctx), statusCodeInterceptor(ctx))),
+	), ticket, nil
+}

internal/servers/controller/handler.go

@@ -13,35 +13,36 @@ import (
 	"strings"
 	"time"
 
-	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/observability/event"
-	"github.com/hashicorp/boundary/internal/requests"
 	"github.com/hashicorp/boundary/internal/servers/common"
 	"github.com/hashicorp/boundary/internal/servers/controller/auth"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/accounts"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/authmethods"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/authtokens"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/credentiallibraries"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/credentialstores"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/groups"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_catalogs"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_sets"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/hosts"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/managed_groups"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/roles"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/scopes"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/sessions"
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers/targets"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers/users"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-secure-stdlib/listenerutil"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
+	"github.com/mr-tron/base58"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/authtokens"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/groups"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/host_catalogs"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/hosts"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/roles"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/scopes"
-	"github.com/hashicorp/boundary/internal/servers/controller/handlers/users"
 )
 
 type HandlerProperties struct {
@@ -78,127 +79,169 @@ func handleGrpcGateway(c *Controller, props HandlerProperties) (http.Handler, er
 	// Register*ServiceHandlerServer methods ignore the passed in ctx. Using it
 	// now however in case this changes in the future.
 	ctx := props.CancelCtx
-	mux := runtime.NewServeMux(
-		runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.HTTPBodyMarshaler{
-			Marshaler: handlers.JSONMarshaler(),
-		}),
-		runtime.WithErrorHandler(handlers.ErrorHandler()),
-		runtime.WithForwardResponseOption(handlers.OutgoingInterceptor),
-	)
-	hcs, err := host_catalogs.NewService(c.StaticHostRepoFn, c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create host catalog handler service: %w", err)
+
+	currentServices := c.gatewayServer.GetServiceInfo()
+	dialOptions := gatewayDialOptions(c.gatewayListener)
+
+	if _, ok := currentServices[services.HostCatalogService_ServiceDesc.ServiceName]; !ok {
+		hcs, err := host_catalogs.NewService(c.StaticHostRepoFn, c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create host catalog handler service: %w", err)
+		}
+		services.RegisterHostCatalogServiceServer(c.gatewayServer, hcs)
+		if err := services.RegisterHostCatalogServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register host catalog service handler: %w", err)
+		}
 	}
-	if err := services.RegisterHostCatalogServiceHandlerServer(ctx, mux, hcs); err != nil {
-		return nil, fmt.Errorf("failed to register host catalog service handler: %w", err)
+	if _, ok := currentServices[services.HostSetService_ServiceDesc.ServiceName]; !ok {
+		hss, err := host_sets.NewService(c.StaticHostRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create host set handler service: %w", err)
+		}
+		services.RegisterHostSetServiceServer(c.gatewayServer, hss)
+		if err := services.RegisterHostSetServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register host set service handler: %w", err)
+		}
 	}
-	hss, err := host_sets.NewService(c.StaticHostRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create host set handler service: %w", err)
+	if _, ok := currentServices[services.HostService_ServiceDesc.ServiceName]; !ok {
+		hs, err := hosts.NewService(c.StaticHostRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create host handler service: %w", err)
+		}
+		services.RegisterHostServiceServer(c.gatewayServer, hs)
+		if err := services.RegisterHostServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register host service handler: %w", err)
+		}
 	}
-	if err := services.RegisterHostSetServiceHandlerServer(ctx, mux, hss); err != nil {
-		return nil, fmt.Errorf("failed to register host set service handler: %w", err)
+	if _, ok := currentServices[services.AccountService_ServiceDesc.ServiceName]; !ok {
+		accts, err := accounts.NewService(c.PasswordAuthRepoFn, c.OidcRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create account handler service: %w", err)
+		}
+		services.RegisterAccountServiceServer(c.gatewayServer, accts)
+		if err := services.RegisterAccountServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register account service handler: %w", err)
+		}
 	}
-	hs, err := hosts.NewService(c.StaticHostRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create host handler service: %w", err)
+	if _, ok := currentServices[services.AuthMethodService_ServiceDesc.ServiceName]; !ok {
+		authMethods, err := authmethods.NewService(c.kms, c.PasswordAuthRepoFn, c.OidcRepoFn, c.IamRepoFn, c.AuthTokenRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create auth method handler service: %w", err)
+		}
+		services.RegisterAuthMethodServiceServer(c.gatewayServer, authMethods)
+		if err := services.RegisterAuthMethodServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register auth method service handler: %w", err)
+		}
 	}
-	if err := services.RegisterHostServiceHandlerServer(ctx, mux, hs); err != nil {
-		return nil, fmt.Errorf("failed to register host service handler: %w", err)
+	if _, ok := currentServices[services.AuthTokenService_ServiceDesc.ServiceName]; !ok {
+		authtoks, err := authtokens.NewService(c.AuthTokenRepoFn, c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create auth token handler service: %w", err)
+		}
+		services.RegisterAuthTokenServiceServer(c.gatewayServer, authtoks)
+		if err := services.RegisterAuthTokenServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register auth token service handler: %w", err)
+		}
 	}
-	accts, err := accounts.NewService(c.PasswordAuthRepoFn, c.OidcRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create account handler service: %w", err)
+	if _, ok := currentServices[services.ScopeService_ServiceDesc.ServiceName]; !ok {
+		os, err := scopes.NewService(c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create scope handler service: %w", err)
+		}
+		services.RegisterScopeServiceServer(c.gatewayServer, os)
+		if err := services.RegisterScopeServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register scope service handler: %w", err)
+		}
 	}
-	if err := services.RegisterAccountServiceHandlerServer(ctx, mux, accts); err != nil {
-		return nil, fmt.Errorf("failed to register account service handler: %w", err)
+	if _, ok := currentServices[services.UserService_ServiceDesc.ServiceName]; !ok {
+		us, err := users.NewService(c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create user handler service: %w", err)
+		}
+		services.RegisterUserServiceServer(c.gatewayServer, us)
+		if err := services.RegisterUserServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register user service handler: %w", err)
+		}
 	}
-	authMethods, err := authmethods.NewService(c.kms, c.PasswordAuthRepoFn, c.OidcRepoFn, c.IamRepoFn, c.AuthTokenRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create auth method handler service: %w", err)
+	if _, ok := currentServices[services.TargetService_ServiceDesc.ServiceName]; !ok {
+		ts, err := targets.NewService(
+			c.kms,
+			c.TargetRepoFn,
+			c.IamRepoFn,
+			c.ServersRepoFn,
+			c.SessionRepoFn,
+			c.StaticHostRepoFn,
+			c.VaultCredentialRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create target handler service: %w", err)
+		}
+		services.RegisterTargetServiceServer(c.gatewayServer, ts)
+		if err := services.RegisterTargetServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register target service handler: %w", err)
+		}
 	}
-	if err := services.RegisterAuthMethodServiceHandlerServer(ctx, mux, authMethods); err != nil {
-		return nil, fmt.Errorf("failed to register auth method service handler: %w", err)
+	if _, ok := currentServices[services.GroupService_ServiceDesc.ServiceName]; !ok {
+		gs, err := groups.NewService(c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create group handler service: %w", err)
+		}
+		services.RegisterGroupServiceServer(c.gatewayServer, gs)
+		if err := services.RegisterGroupServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register group service handler: %w", err)
+		}
 	}
-	authtoks, err := authtokens.NewService(c.AuthTokenRepoFn, c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create auth token handler service: %w", err)
+	if _, ok := currentServices[services.RoleService_ServiceDesc.ServiceName]; !ok {
+		rs, err := roles.NewService(c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create role handler service: %w", err)
+		}
+		services.RegisterRoleServiceServer(c.gatewayServer, rs)
+		if err := services.RegisterRoleServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register role service handler: %w", err)
+		}
 	}
-	if err := services.RegisterAuthTokenServiceHandlerServer(ctx, mux, authtoks); err != nil {
-		return nil, fmt.Errorf("failed to register auth token service handler: %w", err)
+	if _, ok := currentServices[services.SessionService_ServiceDesc.ServiceName]; !ok {
+		ss, err := sessions.NewService(c.SessionRepoFn, c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create session handler service: %w", err)
+		}
+		services.RegisterSessionServiceServer(c.gatewayServer, ss)
+		if err := services.RegisterSessionServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register session service handler: %w", err)
+		}
 	}
-	os, err := scopes.NewService(c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create scope handler service: %w", err)
+	if _, ok := currentServices[services.ManagedGroupService_ServiceDesc.ServiceName]; !ok {
+		mgs, err := managed_groups.NewService(c.OidcRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create managed groups handler service: %w", err)
+		}
+		services.RegisterManagedGroupServiceServer(c.gatewayServer, mgs)
+		if err := services.RegisterManagedGroupServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register managed groups service handler: %w", err)
+		}
 	}
-	if err := services.RegisterScopeServiceHandlerServer(ctx, mux, os); err != nil {
-		return nil, fmt.Errorf("failed to register scope service handler: %w", err)
+	if _, ok := currentServices[services.CredentialStoreService_ServiceDesc.ServiceName]; !ok {
+		cs, err := credentialstores.NewService(c.VaultCredentialRepoFn, c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create credential store handler service: %w", err)
+		}
+		services.RegisterCredentialStoreServiceServer(c.gatewayServer, cs)
+		if err := services.RegisterCredentialStoreServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register credential store service handler: %w", err)
+		}
 	}
-	us, err := users.NewService(c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create user handler service: %w", err)
-	}
-	if err := services.RegisterUserServiceHandlerServer(ctx, mux, us); err != nil {
-		return nil, fmt.Errorf("failed to register user service handler: %w", err)
-	}
-	ts, err := targets.NewService(
-		c.kms,
-		c.TargetRepoFn,
-		c.IamRepoFn,
-		c.ServersRepoFn,
-		c.SessionRepoFn,
-		c.StaticHostRepoFn,
-		c.VaultCredentialRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create target handler service: %w", err)
-	}
-	if err := services.RegisterTargetServiceHandlerServer(ctx, mux, ts); err != nil {
-		return nil, fmt.Errorf("failed to register target service handler: %w", err)
-	}
-	gs, err := groups.NewService(c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create group handler service: %w", err)
-	}
-	if err := services.RegisterGroupServiceHandlerServer(ctx, mux, gs); err != nil {
-		return nil, fmt.Errorf("failed to register group service handler: %w", err)
-	}
-	rs, err := roles.NewService(c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create role handler service: %w", err)
-	}
-	if err := services.RegisterRoleServiceHandlerServer(ctx, mux, rs); err != nil {
-		return nil, fmt.Errorf("failed to register role service handler: %w", err)
-	}
-	ss, err := sessions.NewService(c.SessionRepoFn, c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create session handler service: %w", err)
-	}
-	if err := services.RegisterSessionServiceHandlerServer(ctx, mux, ss); err != nil {
-		return nil, fmt.Errorf("failed to register session service handler: %w", err)
-	}
-	mgs, err := managed_groups.NewService(c.OidcRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create managed groups handler service: %w", err)
-	}
-	if err := services.RegisterManagedGroupServiceHandlerServer(ctx, mux, mgs); err != nil {
-		return nil, fmt.Errorf("failed to register managed groups service handler: %w", err)
-	}
-	cs, err := credentialstores.NewService(c.VaultCredentialRepoFn, c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create credential store handler service: %w", err)
-	}
-	if err := services.RegisterCredentialStoreServiceHandlerServer(ctx, mux, cs); err != nil {
-		return nil, fmt.Errorf("failed to register credential store service handler: %w", err)
-	}
-	cl, err := credentiallibraries.NewService(c.VaultCredentialRepoFn, c.IamRepoFn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create credential library handler service: %w", err)
-	}
-	if err := services.RegisterCredentialLibraryServiceHandlerServer(ctx, mux, cl); err != nil {
-		return nil, fmt.Errorf("failed to register credential library service handler: %w", err)
+	if _, ok := currentServices[services.CredentialLibraryService_ServiceDesc.ServiceName]; !ok {
+		cl, err := credentiallibraries.NewService(c.VaultCredentialRepoFn, c.IamRepoFn)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create credential library handler service: %w", err)
+		}
+		services.RegisterCredentialLibraryServiceServer(c.gatewayServer, cl)
+		if err := services.RegisterCredentialLibraryServiceHandlerFromEndpoint(ctx, c.gatewayMux, gatewayTarget, dialOptions); err != nil {
+			return nil, fmt.Errorf("failed to register credential library service handler: %w", err)
+		}
 	}
 
-	return mux, nil
+	return c.gatewayMux, nil
 }
 
 func wrapHandlerWithCommonFuncs(h http.Handler, c *Controller, props HandlerProperties) http.Handler {
@@ -236,29 +279,27 @@ func wrapHandlerWithCommonFuncs(h http.Handler, c *Controller, props HandlerProp
 		}
 
 		// Add values for authn/authz checking
-		requestInfo := auth.RequestInfo{
+		requestInfo := authpb.RequestInfo{
 			Path:                 r.URL.Path,
 			Method:               r.Method,
 			DisableAuthzFailures: disableAuthzFailures,
 		}
 
 		requestInfo.PublicId, requestInfo.EncryptedToken, requestInfo.TokenFormat = auth.GetTokenFromRequest(ctx, c.kms, r)
-		ctx = auth.NewVerifierContext(ctx, c.IamRepoFn, c.AuthTokenRepoFn, c.ServersRepoFn, c.kms, requestInfo)
 
-		// Add general request information to the context. The information from
-		// the auth verifier context is pretty specifically curated to
-		// authentication/authorization verification so this is more
-		// general-purpose.
-		//
-		// We could use requests.NewRequestContext but this saves an immediate
-		// lookup.
-		ctx = context.WithValue(ctx, requests.ContextRequestInformationKey, &requests.RequestContext{
-			Path:   r.URL.Path,
-			Method: r.Method,
-		})
-
-		// Set the context back on the request
-		r = r.WithContext(ctx)
+		// Serialize the request info to send it across the wire to the
+		// grpc-gateway via an http header
+		requestInfo.Ticket = c.gatewayTicket // allows the grpc-gateway to verify the request info came from it's in-memory companion http proxy
+		marshalledRequestInfo, err := proto.Marshal(&requestInfo)
+		if err != nil {
+			event.WriteError(ctx, op, err, event.WithInfoMsg("error marshaling request info"))
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		// Use the default grpc-gateway mapping rule to pass the request info as
+		// metadata.
+		// See: https://pkg.go.dev/github.com/grpc-ecosystem/grpc-gateway/runtime#DefaultHeaderMatcher
+		r.Header.Set("Grpc-Metadata-"+requestInfoMdKey, base58.FastBase58Encoding(marshalledRequestInfo))
 
 		h.ServeHTTP(w, r)
 	})

internal/servers/controller/handlers/authmethods/authmethod_service.go

@@ -389,7 +389,7 @@ func (s Service) DeleteAuthMethod(ctx context.Context, req *pbs.DeleteAuthMethod
 	if err != nil {
 		return nil, err
 	}
-	return nil, nil
+	return &pbs.DeleteAuthMethodResponse{}, nil
 }
 
 // Authenticate implements the interface pbs.AuthenticationServiceServer.

internal/servers/controller/handlers/authmethods/authmethod_service_test.go

@@ -428,12 +428,14 @@ func TestDelete(t *testing.T) {
 			req: &pbs.DeleteAuthMethodRequest{
 				Id: pwam.GetPublicId(),
 			},
+			res: &pbs.DeleteAuthMethodResponse{},
 		},
 		{
 			name: "Delete an Existing OIDC AuthMethod",
 			req: &pbs.DeleteAuthMethodRequest{
 				Id: oidcam.GetPublicId(),
 			},
+			res: &pbs.DeleteAuthMethodResponse{},
 		},
 		{
 			name: "Delete bad auth_method id",

internal/servers/controller/handlers/authmethods/oidc_test.go

@@ -13,6 +13,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
+
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
 	"github.com/hashicorp/boundary/internal/authtoken"
@@ -206,9 +208,9 @@ func TestList_FilterNonPublic(t *testing.T) {
 					atRepoFn,
 					serversRepoFn,
 					kmsCache,
-					auth.RequestInfo{
+					&authpb.RequestInfo{
 						Token:       at.GetToken(),
-						TokenFormat: auth.AuthTokenTypeBearer,
+						TokenFormat: uint32(auth.AuthTokenTypeBearer),
 						PublicId:    at.GetPublicId(),
 					})
 			}(),

internal/servers/controller/handlers/authtokens/authtoken_service_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/boundary/internal/authtoken"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/requests"
@@ -87,15 +88,15 @@ func TestGetSelf(t *testing.T) {
 			require, assert := require.New(t), assert.New(t)
 			// Setup the auth request information
 			req := httptest.NewRequest("GET", fmt.Sprintf("http://127.0.0.1/v1/auth-tokens/%s", tc.readId), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.token.GetPublicId(),
 				Token:       tc.token.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			ctx = context.WithValue(ctx, requests.ContextRequestInformationKey, &requests.RequestContext{})
 			got, err := a.GetAuthToken(ctx, &pbs.GetAuthTokenRequest{Id: tc.readId})
 			if tc.err != nil {
@@ -236,15 +237,15 @@ func TestList_Self(t *testing.T) {
 			require, assert := require.New(t), assert.New(t)
 			// Setup the auth request information
 			req := httptest.NewRequest("GET", fmt.Sprintf("http://127.0.0.1/v1/auth-tokens?scope_id=%s", o.GetPublicId()), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.requester.GetPublicId(),
 				Token:       tc.requester.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			got, err := a.ListAuthTokens(ctx, &pbs.ListAuthTokensRequest{ScopeId: o.GetPublicId()})
 			require.NoError(err)
 			require.Len(got.Items, 1)
@@ -482,15 +483,15 @@ func TestDeleteSelf(t *testing.T) {
 			require := require.New(t)
 			// Setup the auth request information
 			req := httptest.NewRequest("DELETE", fmt.Sprintf("http://127.0.0.1/v1/auth-tokens/%s", tc.deleteId), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.token.GetPublicId(),
 				Token:       tc.token.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			got, err := a.DeleteAuthToken(ctx, &pbs.DeleteAuthTokenRequest{Id: tc.deleteId})
 			if tc.err != nil {
 				require.EqualError(err, tc.err.Error())

internal/servers/controller/handlers/credentiallibraries/credentiallibrary_service_test.go

@@ -555,6 +555,7 @@ func TestDelete(t *testing.T) {
 		name string
 		id   string
 		err  error
+		res  *pbs.DeleteCredentialLibraryResponse
 	}{
 		{
 			name: "success",
@@ -574,7 +575,7 @@ func TestDelete(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			got, gErr := s.DeleteCredentialLibrary(auth.DisabledAuthTestContext(iamRepoFn, prj.GetPublicId()), &pbs.DeleteCredentialLibraryRequest{Id: tc.id})
-			assert.Nil(t, got)
+			assert.EqualValuesf(t, tc.res, got, "DeleteCredentialLibrary got response %q, wanted %q", got, tc.res)
 			if tc.err != nil {
 				require.Error(t, gErr)
 				assert.True(t, errors.Is(gErr, tc.err))

internal/servers/controller/handlers/credentialstores/credentialstore_service_test.go

@@ -674,6 +674,7 @@ func TestDelete(t *testing.T) {
 		name string
 		id   string
 		err  error
+		res  *pbs.DeleteCredentialStoreResponse
 	}{
 		{
 			name: "success",
@@ -693,7 +694,7 @@ func TestDelete(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			got, gErr := s.DeleteCredentialStore(auth.DisabledAuthTestContext(iamRepoFn, prj.GetPublicId()), &pbs.DeleteCredentialStoreRequest{Id: tc.id})
-			assert.Nil(t, got)
+			assert.EqualValuesf(t, tc.res, got, "DeleteCredentialStore got response %q, wanted %q", got, tc.res)
 			if tc.err != nil {
 				require.Error(t, gErr)
 				assert.True(t, errors.Is(gErr, tc.err))

internal/servers/controller/handlers/errors.go

@@ -5,31 +5,46 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/hashicorp/boundary/internal/errors"
 	pb "github.com/hashicorp/boundary/internal/gen/controller/api"
+	pberrors "github.com/hashicorp/boundary/internal/gen/errors"
 	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/mr-tron/base58"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 const (
 	genericUniquenessMsg = "Invalid request.  Request attempted to make second resource with the same field value that must be unique."
 	genericNotFoundMsg   = "Unable to find requested resource."
+
+	// domainErrHeader defines an http header for encoded domain errors from the
+	// grpc server.
+	domainErrHeader = "x-domain-err"
+	// domainErrMetadataHeader defines an http header for encoded domain errors from the
+	// grpc server via metadata
+	domainErrMetadataHeader = "Grpc-Metadata-X-Domain-Err"
+
+	apiErrHeader         = "x-api-err"
+	apiErrMetadataHeader = "Grpc-Metadata-X-Api-Err"
 )
 
-type apiError struct {
-	status int32
-	inner  *pb.Error
+type ApiError struct {
+	Status int32
+	Inner  *pb.Error
 }
 
-func (e *apiError) Error() string {
-	res := fmt.Sprintf("Status: %d, Kind: %q, Error: %q", e.status, e.inner.GetKind(), e.inner.GetMessage())
+func (e *ApiError) Error() string {
+	res := fmt.Sprintf("Status: %d, Kind: %q, Error: %q", e.Status, e.Inner.GetKind(), e.Inner.GetMessage())
 	var dets []string
-	for _, rf := range e.inner.GetDetails().GetRequestFields() {
+	for _, rf := range e.Inner.GetDetails().GetRequestFields() {
 		dets = append(dets, fmt.Sprintf("{name: %q, desc: %q}", rf.GetName(), rf.GetDescription()))
 	}
 	if len(dets) > 0 {
@@ -39,19 +54,19 @@ func (e *apiError) Error() string {
 	return res
 }
 
-func (e *apiError) Is(target error) bool {
-	var tApiErr *apiError
+func (e *ApiError) Is(target error) bool {
+	var tApiErr *ApiError
 	if !errors.As(target, &tApiErr) {
 		return false
 	}
-	return tApiErr.inner.Kind == e.inner.Kind && tApiErr.status == e.status
+	return tApiErr.Inner.Kind == e.Inner.Kind && tApiErr.Status == e.Status
 }
 
 // ApiErrorWithCode returns an api error with the provided code.
 func ApiErrorWithCode(c codes.Code) error {
-	return &apiError{
-		status: int32(runtime.HTTPStatusFromCode(c)),
-		inner: &pb.Error{
+	return &ApiError{
+		Status: int32(runtime.HTTPStatusFromCode(c)),
+		Inner: &pb.Error{
 			Kind: c.String(),
 		},
 	}
@@ -59,9 +74,9 @@ func ApiErrorWithCode(c codes.Code) error {
 
 // ApiErrorWithCodeAndMessage returns an api error with the provided code and message.
 func ApiErrorWithCodeAndMessage(c codes.Code, msg string, args ...interface{}) error {
-	return &apiError{
-		status: int32(runtime.HTTPStatusFromCode(c)),
-		inner: &pb.Error{
+	return &ApiError{
+		Status: int32(runtime.HTTPStatusFromCode(c)),
+		Inner: &pb.Error{
 			Kind:    c.String(),
 			Message: fmt.Sprintf(msg, args...),
 		},
@@ -70,9 +85,9 @@ func ApiErrorWithCodeAndMessage(c codes.Code, msg string, args ...interface{}) e
 
 // NotFoundError returns an ApiError indicating a resource couldn't be found.
 func NotFoundError() error {
-	return &apiError{
-		status: http.StatusNotFound,
-		inner: &pb.Error{
+	return &ApiError{
+		Status: http.StatusNotFound,
+		Inner: &pb.Error{
 			Kind:    codes.NotFound.String(),
 			Message: "Resource not found.",
 		},
@@ -80,19 +95,19 @@ func NotFoundError() error {
 }
 
 // NotFoundErrorf returns an ApiError indicating a resource couldn't be found.
-func NotFoundErrorf(msg string, a ...interface{}) *apiError {
-	return &apiError{
-		status: http.StatusNotFound,
-		inner: &pb.Error{
+func NotFoundErrorf(msg string, a ...interface{}) *ApiError {
+	return &ApiError{
+		Status: http.StatusNotFound,
+		Inner: &pb.Error{
 			Kind:    codes.NotFound.String(),
 			Message: fmt.Sprintf(msg, a...),
 		},
 	}
 }
 
-var unauthorizedError = &apiError{
-	status: http.StatusForbidden,
-	inner: &pb.Error{
+var unauthorizedError = &ApiError{
+	Status: http.StatusForbidden,
+	Inner: &pb.Error{
 		Kind:    codes.PermissionDenied.String(),
 		Message: "Forbidden.",
 	},
@@ -102,9 +117,9 @@ func ForbiddenError() error {
 	return unauthorizedError
 }
 
-var unauthenticatedError = &apiError{
-	status: http.StatusUnauthorized,
-	inner: &pb.Error{
+var unauthenticatedError = &ApiError{
+	Status: http.StatusUnauthorized,
+	Inner: &pb.Error{
 		Kind:    codes.Unauthenticated.String(),
 		Message: "Unauthenticated, or invalid token.",
 	},
@@ -114,47 +129,49 @@ func UnauthenticatedError() error {
 	return unauthenticatedError
 }
 
-func InvalidArgumentErrorf(msg string, fields map[string]string) *apiError {
+func InvalidArgumentErrorf(msg string, fields map[string]string) *ApiError {
 	const op = "handlers.InvalidArgumentErrorf"
 	ctx := context.TODO()
 	err := ApiErrorWithCodeAndMessage(codes.InvalidArgument, msg)
-	var apiErr *apiError
+	var apiErr *ApiError
 	if !errors.As(err, &apiErr) {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("Unable to build invalid argument api error."))
 	}
 
 	if len(fields) > 0 {
-		apiErr.inner.Details = &pb.ErrorDetails{}
+		apiErr.Inner.Details = &pb.ErrorDetails{}
 	}
 	for k, v := range fields {
-		apiErr.inner.Details.RequestFields = append(apiErr.inner.Details.RequestFields, &pb.FieldError{Name: k, Description: v})
+		apiErr.Inner.Details.RequestFields = append(apiErr.Inner.Details.RequestFields, &pb.FieldError{Name: k, Description: v})
 	}
-	sort.Slice(apiErr.inner.GetDetails().GetRequestFields(), func(i, j int) bool {
-		return apiErr.inner.Details.RequestFields[i].GetName() < apiErr.inner.Details.RequestFields[j].GetName()
+	sort.Slice(apiErr.Inner.GetDetails().GetRequestFields(), func(i, j int) bool {
+		return apiErr.Inner.Details.RequestFields[i].GetName() < apiErr.Inner.Details.RequestFields[j].GetName()
 	})
 	return apiErr
 }
 
+var statusRegEx = regexp.MustCompile("Status: ([0-9]+), Kind: \"(.*)\", Error: \"(.*)\"")
+
 // Converts a known errors into an error that can presented to an end user over the API.
-func backendErrorToApiError(inErr error) *apiError {
+func backendErrorToApiError(inErr error) *ApiError {
 	stErr := status.Convert(inErr)
 
 	switch {
 	case errors.Is(inErr, runtime.ErrNotMatch):
 		// grpc gateway uses this error when the path was not matched, but the error uses codes.Unimplemented which doesn't match the intention.
 		// Overwrite the error to match our expected behavior.
-		return &apiError{
-			status: http.StatusNotFound,
-			inner: &pb.Error{
+		return &ApiError{
+			Status: http.StatusNotFound,
+			Inner: &pb.Error{
 				Kind:    codes.NotFound.String(),
 				Message: http.StatusText(http.StatusNotFound),
 			},
 		}
 	case status.Code(inErr) == codes.Unimplemented:
 		// Instead of returning a 501 we always want to return a 405 when a method isn't implemented.
-		return &apiError{
-			status: http.StatusMethodNotAllowed,
-			inner: &pb.Error{
+		return &ApiError{
+			Status: http.StatusMethodNotAllowed,
+			Inner: &pb.Error{
 				Kind:    codes.Unimplemented.String(),
 				Message: stErr.Message(),
 			},
@@ -176,11 +193,23 @@ func backendErrorToApiError(inErr error) *apiError {
 		statusCode = int32(domainErr.Code)
 	}
 
+	// perhaps the error is from the grpc gateway, so match against the known
+	// apiError msg format.
+	if found := statusRegEx.FindStringSubmatch(stErr.Message()); len(found) == 4 {
+		u32, err := strconv.ParseInt(found[1], 10, 32)
+		if err == nil { // notice it's testing for NO err
+			return &ApiError{
+				Status: int32(u32),
+				Inner:  &pb.Error{Kind: found[2], Message: found[3]},
+			}
+		}
+	}
+
 	// TODO: Don't return potentially sensitive information (like which user id an account
 	//  is already associated with when attempting to re-associate it).
-	return &apiError{
-		status: statusCode,
-		inner:  &pb.Error{Kind: codes.Internal.String(), Message: inErr.Error()},
+	return &ApiError{
+		Status: statusCode,
+		Inner:  &pb.Error{Kind: codes.Internal.String(), Message: inErr.Error()},
 	}
 }
 
@@ -189,7 +218,70 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 	const errorFallback = `{"error": "failed to marshal error message"}`
 	return func(ctx context.Context, _ *runtime.ServeMux, mar runtime.Marshaler, w http.ResponseWriter, r *http.Request, inErr error) {
 		// API specified error, otherwise we need to translate repo/db errors.
-		var apiErr *apiError
+
+		// the grpc server will encoded domain errors into the x-domain-err, so
+		// let's check there first. (see: controller.errorInterceptor)
+		md, ok := runtime.ServerMetadataFromContext(ctx)
+		if ok {
+			defer func() {
+				// make sure we don't leak the headers that were used as a comm
+				// channel between the grpc server and the http proxy
+				delete(md.HeaderMD, domainErrHeader)
+				delete(w.Header(), domainErrMetadataHeader)
+
+				delete(md.HeaderMD, apiErrHeader)
+				delete(w.Header(), apiErrMetadataHeader)
+			}()
+			domainErrHdrs := md.HeaderMD.Get(domainErrHeader)
+			apiErrHdrs := md.HeaderMD.Get(apiErrHeader)
+
+			switch {
+			case len(domainErrHdrs) > 0:
+				decoded, err := base58.FastBase58Decoding(domainErrHdrs[0])
+				if err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to decode domain err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				var pbErr pberrors.Err
+				if err := proto.Unmarshal(decoded, &pbErr); err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to marshal domain err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				inErr = errors.FromPbErrors(&pbErr)
+			case len(apiErrHdrs) > 0:
+				decoded, err := base58.FastBase58Decoding(apiErrHdrs[0])
+				if err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to decode api err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				var pbApiErr pberrors.ApiError
+				if err := proto.Unmarshal(decoded, &pbApiErr); err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to marshal api err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				inErr = &ApiError{
+					Status: pbApiErr.Status,
+					Inner:  pbApiErr.ApiError,
+				}
+			}
+		}
+		var apiErr *ApiError
 		isApiErr := errors.As(inErr, &apiErr)
 		if !isApiErr {
 			if err := backendErrorToApiError(inErr); err != nil && !errors.As(err, &apiErr) {
@@ -197,13 +289,13 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 			}
 		}
 
-		if apiErr.status == http.StatusInternalServerError {
+		if apiErr.Status == http.StatusInternalServerError {
 			event.WriteError(ctx, op, inErr, event.WithInfoMsg("internal error returned"))
 		}
 
-		buf, merr := mar.Marshal(apiErr.inner)
+		buf, merr := mar.Marshal(apiErr.Inner)
 		if merr != nil {
-			event.WriteError(ctx, op, merr, event.WithInfoMsg("failed to marshal error response", "response", fmt.Sprintf("%#v", apiErr.inner)))
+			event.WriteError(ctx, op, merr, event.WithInfoMsg("failed to marshal error response", "response", fmt.Sprintf("%#v", apiErr.Inner)))
 			w.WriteHeader(http.StatusInternalServerError)
 			if _, err := io.WriteString(w, errorFallback); err != nil {
 				event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
@@ -211,8 +303,8 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 			return
 		}
 
-		w.Header().Set("Content-Type", mar.ContentType(apiErr.inner))
-		w.WriteHeader(int(apiErr.status))
+		w.Header().Set("Content-Type", mar.ContentType(apiErr.Inner))
+		w.WriteHeader(int(apiErr.Status))
 		if _, err := w.Write(buf); err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("failed to send response chunk"))
 			return
@@ -221,5 +313,5 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 }
 
 func ToApiError(e error) *pb.Error {
-	return backendErrorToApiError(e).inner
+	return backendErrorToApiError(e).Inner
 }

internal/servers/controller/handlers/errors_test.go

@@ -32,14 +32,14 @@ func TestApiErrorHandler(t *testing.T) {
 	testCases := []struct {
 		name     string
 		err      error
-		expected apiError
+		expected ApiError
 	}{
 		{
 			name: "Not Found",
 			err:  NotFoundErrorf("Test"),
-			expected: apiError{
-				status: http.StatusNotFound,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusNotFound,
+				Inner: &pb.Error{
 					Kind:    "NotFound",
 					Message: "Test",
 				},
@@ -51,9 +51,9 @@ func TestApiErrorHandler(t *testing.T) {
 				"k1": "v1",
 				"k2": "v2",
 			}),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "Test",
 					Details: &pb.ErrorDetails{
@@ -74,9 +74,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "GrpcGateway Routing Error",
 			err:  runtime.ErrNotMatch,
-			expected: apiError{
-				status: http.StatusNotFound,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusNotFound,
+				Inner: &pb.Error{
 					Kind:    "NotFound",
 					Message: http.StatusText(http.StatusNotFound),
 				},
@@ -85,9 +85,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Unimplemented error",
 			err:  status.Error(codes.Unimplemented, "Test"),
-			expected: apiError{
-				status: http.StatusMethodNotAllowed,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusMethodNotAllowed,
+				Inner: &pb.Error{
 					Kind:    "Unimplemented",
 					Message: "Test",
 				},
@@ -96,9 +96,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Unknown error",
 			err:  stderrors.New("Some random error"),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "Some random error",
 				},
@@ -107,9 +107,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db invalid public id",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidPublicId)),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "invalid public id, parameter violation: error #102",
 				},
@@ -118,9 +118,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db invalid parameter",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidParameter)),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "invalid parameter, parameter violation: error #100",
 				},
@@ -129,9 +129,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db invalid field mask",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidFieldMask)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "Error in provided request",
 					Details: &pb.ErrorDetails{RequestFields: []*pb.FieldError{{Name: "update_mask", Description: "Invalid update mask provided."}}},
@@ -141,9 +141,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db empty field mask",
 			err:  errors.E(ctx, errors.WithCode(errors.EmptyFieldMask)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "Error in provided request",
 					Details: &pb.ErrorDetails{RequestFields: []*pb.FieldError{{Name: "update_mask", Description: "Invalid update mask provided."}}},
@@ -153,9 +153,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db not unqiue",
 			err:  errors.E(ctx, errors.WithCode(errors.NotUnique)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: genericUniquenessMsg,
 				},
@@ -164,9 +164,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db record not found",
 			err:  errors.E(ctx, errors.WithCode(errors.RecordNotFound)),
-			expected: apiError{
-				status: http.StatusNotFound,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusNotFound,
+				Inner: &pb.Error{
 					Kind:    "NotFound",
 					Message: genericNotFoundMsg,
 				},
@@ -175,9 +175,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db multiple records",
 			err:  errors.E(ctx, errors.WithCode(errors.MultipleRecords)),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "multiple records, search issue: error #1101",
 				},
@@ -186,9 +186,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error account already associated",
 			err:  errors.E(ctx, errors.WithCode(errors.AccountAlreadyAssociated)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "account already associated with another user, parameter violation: error #114",
 				},
@@ -197,9 +197,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Wrapped domain error",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidAddress), errors.WithMsg("test msg"), errors.WithWrap(errors.E(ctx, errors.WithCode(errors.NotNull), errors.WithMsg("inner msg")))),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "test msg: parameter violation: error #101: inner msg: integrity violation: error #1001",
 				},
@@ -208,9 +208,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Forbidden domain error",
 			err:  errors.E(ctx, errors.WithCode(errors.Forbidden), errors.WithMsg("test msg")),
-			expected: apiError{
-				status: http.StatusForbidden,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusForbidden,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "test msg: unknown: error #403",
 				},
@@ -219,9 +219,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Wrapped forbidden domain error",
 			err:  fmt.Errorf("got error: %w", errors.E(ctx, errors.WithCode(errors.Forbidden), errors.WithMsg("test msg"))),
-			expected: apiError{
-				status: http.StatusForbidden,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusForbidden,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "got error: test msg: unknown: error #403",
 				},
@@ -234,7 +234,7 @@ func TestApiErrorHandler(t *testing.T) {
 			w := httptest.NewRecorder()
 			tested(ctx, mux, outMarsh, w, req, tc.err)
 			resp := w.Result()
-			assert.EqualValues(tc.expected.status, resp.StatusCode)
+			assert.EqualValues(tc.expected.Status, resp.StatusCode)
 
 			got, err := ioutil.ReadAll(resp.Body)
 			require.NoError(err)
@@ -243,8 +243,8 @@ func TestApiErrorHandler(t *testing.T) {
 			err = inMarsh.Unmarshal(got, gotErr)
 			require.NoError(err)
 
-			assert.Equal(tc.expected.status, int32(resp.StatusCode))
-			assert.Empty(cmp.Diff(tc.expected.inner, gotErr, protocmp.Transform()))
+			assert.Equal(tc.expected.Status, int32(resp.StatusCode))
+			assert.Empty(cmp.Diff(tc.expected.Inner, gotErr, protocmp.Transform()))
 		})
 	}
 }

internal/servers/controller/handlers/managed_groups/managed_group_service_test.go

@@ -417,6 +417,7 @@ func TestDelete_twice(t *testing.T) {
 	o, _ := iam.TestScopes(t, iam.TestRepo(t, conn, wrap))
 
 	databaseWrapper, err := kmsCache.GetWrapper(context.Background(), o.PublicId, kms.KeyPurposeDatabase)
+	require.NoError(err)
 
 	oidcAm := oidc.TestAuthMethod(
 		t, conn, databaseWrapper, o.PublicId, oidc.ActivePrivateState,

internal/servers/controller/handlers/outgoing_interceptor_test.go

@@ -1,26 +0,0 @@
-package handlers
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
-	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authtokens"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/protobuf/types/known/structpb"
-)
-
-func TestOutgoingSplitCookie(t *testing.T) {
-	rec := httptest.NewRecorder()
-	attrs, err := ProtoToStruct(&pb.AuthToken{Token: "t_abc_1234567890"})
-	require.NoError(t, err)
-	attrs.GetFields()["token_type"] = structpb.NewStringValue("cookie")
-	require.NoError(t, OutgoingInterceptor(context.Background(), rec, &pbs.AuthenticateResponse{Attributes: attrs}))
-	assert.ElementsMatch(t, rec.Result().Cookies(), []*http.Cookie{
-		{Name: HttpOnlyCookieName, Value: "34567890", HttpOnly: true, Raw: "wt-http-token-cookie=34567890; HttpOnly"},
-		{Name: JsVisibleCookieName, Value: "t_abc_12", Raw: "wt-js-token-cookie=t_abc_12"},
-	})
-}

internal/servers/controller/handlers/outgoing_response_filter.go

@@ -4,30 +4,66 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
+	"github.com/hashicorp/boundary/internal/errors"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	authtokenpb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authtokens"
 
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/proto"
 )
 
 const (
-	HttpOnlyCookieName    = "wt-http-token-cookie"
-	JsVisibleCookieName   = "wt-js-token-cookie"
-	tokenTypeField        = "token_type"
-	finalRedirectUrlField = "final_redirect_url"
-	statusField           = "status"
+	HttpOnlyCookieName       = "wt-http-token-cookie"
+	JsVisibleCookieName      = "wt-js-token-cookie"
+	tokenTypeField           = "token_type"
+	finalRedirectUrlField    = "final_redirect_url"
+	statusField              = "status"
+	StatusCodeHeader         = "x-http-code"
+	statusCodeMetadataHeader = "Grpc-Metadata-X-Http-Code"
 )
 
-func OutgoingInterceptor(ctx context.Context, w http.ResponseWriter, m proto.Message) error {
-	const op = "handlers.OutgoingInterceptor"
-	m = m.ProtoReflect().Interface()
-	if !m.ProtoReflect().IsValid() {
-		// This would be the case if it's a nil pointer
-		w.WriteHeader(http.StatusNoContent)
-		return nil
+// SetStatusCode allows a grpc service handler to set the outgoing http status
+// code
+func SetStatusCode(ctx context.Context, code int) error {
+	const op = "handlers.SetHttpStatusCode"
+	if http.StatusText(code) == "" {
+		return errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("unknown status code: %d", code))
+	}
+	if err := grpc.SetHeader(ctx, metadata.Pairs(StatusCodeHeader, strconv.Itoa(code))); err != nil {
+		return errors.Wrap(ctx, err, op, errors.WithCode(errors.Internal))
+	}
+	return nil
+}
+
+// OutgoingResponseFilter is a gRPC gateway WithForwardResponseOption.  It's
+// basically a filter that can manipulate the http response and has acesss to
+// the outgoing proto msg
+func OutgoingResponseFilter(ctx context.Context, w http.ResponseWriter, m proto.Message) error {
+	const op = "handlers.OutgoingResponseFilter"
+
+	if md, ok := runtime.ServerMetadataFromContext(ctx); ok {
+
+		// set http status codes based on metadata set by the grpc service
+		if statusCodes := md.HeaderMD.Get(StatusCodeHeader); len(statusCodes) > 0 {
+			defer func() {
+				// delete the headers to not expose any grpc-metadata in http response
+				delete(md.HeaderMD, StatusCodeHeader)
+				delete(w.Header(), statusCodeMetadataHeader)
+			}()
+			lastStatus, err := strconv.Atoi(statusCodes[len(statusCodes)-1])
+			if err != nil {
+				return fmt.Errorf("%s: unable to convert status code %s: %w", op, statusCodes[len(statusCodes)-1], err)
+			}
+			w.WriteHeader(lastStatus)
+			return nil
+		}
 	}
 
 	// add the outgoing resp proto.  See: common.flushGatedEvents is where the

internal/servers/controller/handlers/outgoing_response_filter_test.go

@@ -0,0 +1,72 @@
+package handlers
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	emptypb "github.com/hashicorp/boundary/internal/gen/controller/api"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authtokens"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func TestOutgoingSplitCookie(t *testing.T) {
+	rec := httptest.NewRecorder()
+	attrs, err := ProtoToStruct(&pb.AuthToken{Token: "t_abc_1234567890"})
+	require.NoError(t, err)
+	attrs.GetFields()["token_type"] = structpb.NewStringValue("cookie")
+	require.NoError(t, OutgoingResponseFilter(context.Background(), rec, &pbs.AuthenticateResponse{Attributes: attrs}))
+	assert.ElementsMatch(t, rec.Result().Cookies(), []*http.Cookie{
+		{Name: HttpOnlyCookieName, Value: "34567890", HttpOnly: true, Raw: "wt-http-token-cookie=34567890; HttpOnly"},
+		{Name: JsVisibleCookieName, Value: "t_abc_12", Raw: "wt-js-token-cookie=t_abc_12"},
+	})
+}
+
+func TestOutgoingResponseFilter_StatusCode(t *testing.T) {
+	tests := []struct {
+		name           string
+		md             runtime.ServerMetadata
+		msg            proto.Message
+		wantErr        bool
+		wantStatusCode int
+		wantContent    bool
+	}{
+		{
+			name:           "204",
+			md:             runtime.ServerMetadata{HeaderMD: metadata.Pairs(StatusCodeHeader, "204")},
+			wantStatusCode: 204,
+		},
+		{
+			name:           "418",
+			md:             runtime.ServerMetadata{HeaderMD: metadata.Pairs(StatusCodeHeader, "418")},
+			wantStatusCode: 418,
+			wantContent:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			rec := httptest.NewRecorder()
+			ctx := runtime.NewServerMetadataContext(context.Background(), tt.md)
+			err := OutgoingResponseFilter(ctx, rec, &emptypb.EmptyResponse{})
+			if tt.wantErr {
+				require.Error(err)
+				return
+			}
+			require.NoError(err)
+			resp := rec.Result()
+			if tt.wantStatusCode != 0 {
+				assert.Equal(tt.wantStatusCode, resp.StatusCode)
+			}
+		})
+	}
+
+}

internal/servers/controller/handlers/sessions/session_service_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/errors"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
@@ -211,15 +212,15 @@ func TestList_Self(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			// Setup the auth request information
 			req := httptest.NewRequest("GET", fmt.Sprintf("http://127.0.0.1/v1/sessions?scope_id=%s", pWithSessions.GetPublicId()), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.requester.GetPublicId(),
 				Token:       tc.requester.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			got, err := s.ListSessions(ctx, &pbs.ListSessionsRequest{ScopeId: pWithSessions.GetPublicId()})
 			require.NoError(t, err)
 			assert.Equal(t, tc.count, len(got.GetItems()), got.GetItems())

internal/servers/controller/handlers/targets/target_service_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/boundary/internal/credential/vault"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	spbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
@@ -2500,9 +2501,9 @@ func TestAuthorizeSession(t *testing.T) {
 		atRepoFn,
 		serversRepoFn,
 		kms,
-		auth.RequestInfo{
+		&authpb.RequestInfo{
 			Token:       at.GetToken(),
-			TokenFormat: auth.AuthTokenTypeBearer,
+			TokenFormat: uint32(auth.AuthTokenTypeBearer),
 			PublicId:    at.GetPublicId(),
 		})
 
@@ -2674,9 +2675,9 @@ func TestAuthorizeSession_Errors(t *testing.T) {
 		atRepoFn,
 		serversRepoFn,
 		kms,
-		auth.RequestInfo{
+		&authpb.RequestInfo{
 			Token:       at.GetToken(),
-			TokenFormat: auth.AuthTokenTypeBearer,
+			TokenFormat: uint32(auth.AuthTokenTypeBearer),
 			PublicId:    at.GetPublicId(),
 		})
 	r := iam.TestRole(t, conn, proj.GetPublicId())

internal/servers/controller/handlers/verifiers_test.go

@@ -15,10 +15,10 @@ import (
 
 func errorIncludesFields(t *testing.T, err error, wantFields []string) {
 	t.Helper()
-	var apiErr *apiError
+	var apiErr *ApiError
 	require.True(t, errors.As(err, &apiErr))
 	var gotFields []string
-	for _, d := range apiErr.inner.GetDetails().GetRequestFields() {
+	for _, d := range apiErr.Inner.GetDetails().GetRequestFields() {
 		gotFields = append(gotFields, d.GetName())
 	}
 	assert.ElementsMatch(t, gotFields, wantFields)

internal/servers/controller/interceptor.go

@@ -0,0 +1,207 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"reflect"
+
+	"github.com/hashicorp/boundary/internal/errors"
+	pb "github.com/hashicorp/boundary/internal/gen/controller/api"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
+	pberrors "github.com/hashicorp/boundary/internal/gen/errors"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/requests"
+	"github.com/hashicorp/boundary/internal/servers/controller/auth"
+	"github.com/hashicorp/boundary/internal/servers/controller/common"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
+
+	"github.com/mr-tron/base58"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	requestInfoMdKey = "request-info"
+
+	// domainErrHeader defines an http header for encoded domain errors from the
+	// grpc server.
+	domainErrHeader = "x-domain-err"
+
+	// apiErrHeader defines an http header for encoded api errors from the
+	// grpc server.
+	apiErrHeader = "x-api-err"
+)
+
+// requestCtxInterceptor creates an unary server interceptor that pulls grpc
+// metadata into a ctx for the request.  The metadata must be set in an upstream
+// http handler/middleware by marshalling a RequestInfo protobuf into the
+// requestInfoMdKey header (see: controller.wrapHandlerWithCommonFuncs).
+func requestCtxInterceptor(
+	ctx context.Context,
+	iamRepoFn common.IamRepoFactory,
+	authTokenRepoFn common.AuthTokenRepoFactory,
+	serversRepoFn common.ServersRepoFactory,
+	kms *kms.Kms,
+	ticket string,
+) (grpc.UnaryServerInterceptor, error) {
+	const op = "controller.requestCtxInterceptor"
+	if iamRepoFn == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing iam repo function")
+	}
+	if authTokenRepoFn == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing auth token repo function")
+	}
+	if serversRepoFn == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing servers repo function")
+	}
+	if kms == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing kms")
+	}
+	if ticket == "" {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing ticket")
+	}
+	// Authorization unary interceptor function to handle authorize per RPC call
+	return func(interceptorCtx context.Context,
+		req interface{},
+		_ *grpc.UnaryServerInfo,
+		handler grpc.UnaryHandler) (interface{}, error) {
+		md, ok := metadata.FromIncomingContext(interceptorCtx)
+		if !ok {
+			return nil, errors.New(interceptorCtx, errors.Internal, op, "No metadata")
+		}
+
+		values := md.Get(requestInfoMdKey)
+		if len(values) == 0 {
+			return nil, errors.New(interceptorCtx, errors.Internal, op, "Missing request metadata")
+		}
+		if len(values) > 1 {
+			return nil, errors.New(interceptorCtx, errors.Internal, op, fmt.Sprintf("expected 1 value for %s metadata and got %d", requestInfoMdKey, len(values)))
+		}
+
+		decoded, err := base58.FastBase58Decoding(values[0])
+		if err != nil {
+			return nil, errors.Wrap(interceptorCtx, err, op, errors.WithCode(errors.Internal), errors.WithMsg("unable to decode request info"))
+		}
+		var requestInfo authpb.RequestInfo
+		if err := proto.Unmarshal(decoded, &requestInfo); err != nil {
+			return nil, errors.Wrap(interceptorCtx, err, op, errors.WithCode(errors.Internal), errors.WithMsg("unable to unmarshal request info"))
+		}
+		switch {
+		case requestInfo.Ticket == "":
+			return nil, errors.New(interceptorCtx, errors.Internal, op, "Invalid context (missing ticket)")
+		case requestInfo.Ticket != ticket:
+			return nil, errors.New(interceptorCtx, errors.Internal, op, "Invalid context (bad ticket)")
+		}
+
+		interceptorCtx = auth.NewVerifierContext(interceptorCtx, iamRepoFn, authTokenRepoFn, serversRepoFn, kms, &requestInfo)
+
+		// Add general request information to the context. The information from
+		// the auth verifier context is pretty specifically curated to
+		// authentication/authorization verification so this is more
+		// general-purpose.
+		//
+		// We could use requests.NewRequestContext but this saves an immediate
+		// lookup.
+		interceptorCtx = context.WithValue(interceptorCtx, requests.ContextRequestInformationKey, &requests.RequestContext{
+			Path:   requestInfo.Path,
+			Method: requestInfo.Method,
+		})
+
+		// Calls the handler
+		h, err := handler(interceptorCtx, req)
+
+		return h, err // not convinced we want to wrap every error and turn them into domain errors...
+	}, nil
+}
+
+func errorInterceptor(
+	_ context.Context,
+) grpc.UnaryServerInterceptor {
+	const op = "controller.errorInterceptor"
+	return func(interceptorCtx context.Context,
+		req interface{},
+		_ *grpc.UnaryServerInfo,
+		handler grpc.UnaryHandler) (interface{}, error) {
+
+		// call the handler...
+		h, handlerErr := handler(interceptorCtx, req)
+
+		// if there's an error and it's a domain err, then encode it into the
+		if handlerErr != nil {
+			var domainErr *errors.Err
+			isDomainErr := errors.As(handlerErr, &domainErr)
+			if isDomainErr {
+				pbErr := errors.ToPbErrors(domainErr)
+				var buf []byte
+				var err error
+				if buf, err = proto.Marshal(pbErr); err != nil {
+					return h, status.Errorf(codes.Internal, "unable to marshal domain error: %s", err)
+				}
+				if err := grpc.SetHeader(interceptorCtx, metadata.Pairs(domainErrHeader, base58.FastBase58Encoding(buf))); err != nil {
+					return h, status.Errorf(codes.Internal, "unable to set domain error header: %s", err)
+				}
+				return h, handlerErr
+			}
+
+			var apiErr *handlers.ApiError
+			isApiError := errors.As(handlerErr, &apiErr)
+			if isApiError {
+				pbErr := &pberrors.ApiError{
+					ApiError: apiErr.Inner,
+					Status:   apiErr.Status,
+				}
+				var buf []byte
+				var err error
+				if buf, err = proto.Marshal(pbErr); err != nil {
+					return h, status.Errorf(codes.Internal, "unable to marshal api error: %s", err)
+				}
+				if err := grpc.SetHeader(interceptorCtx, metadata.Pairs(apiErrHeader, base58.FastBase58Encoding(buf))); err != nil {
+					return h, status.Errorf(codes.Internal, "unable to set api error header: %s", err)
+				}
+				return h, handlerErr
+			}
+		}
+		return h, handlerErr
+	}
+}
+
+func statusCodeInterceptor(
+	_ context.Context,
+) grpc.UnaryServerInterceptor {
+	const op = "controller.statusCodeInterceptor"
+	return func(interceptorCtx context.Context,
+		req interface{},
+		_ *grpc.UnaryServerInfo,
+		handler grpc.UnaryHandler) (interface{}, error) {
+
+		// call the handler...
+		h, handlerErr := handler(interceptorCtx, req)
+
+		// if a service handler returns nil, nil then we want to single a 204
+		// response to the proxy with no resp msg
+		if isNil(h) && handlerErr == nil {
+			if err := handlers.SetStatusCode(interceptorCtx, http.StatusNoContent); err != nil {
+				return &pb.EmptyResponse{}, err
+			}
+			return &pb.EmptyResponse{}, nil
+		}
+
+		return h, handlerErr
+	}
+}
+
+func isNil(i interface{}) bool {
+	if i == nil {
+		return true
+	}
+	switch reflect.TypeOf(i).Kind() {
+	case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice:
+		return reflect.ValueOf(i).IsNil()
+	}
+	return false
+}

internal/servers/controller/interceptor_test.go

@@ -0,0 +1,455 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"github.com/hashicorp/boundary/internal/authtoken"
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/errors"
+	pb_api "github.com/hashicorp/boundary/internal/gen/controller/api"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
+	pberrors "github.com/hashicorp/boundary/internal/gen/errors"
+	"github.com/hashicorp/boundary/internal/gen/testing/interceptor"
+	"github.com/hashicorp/boundary/internal/iam"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/servers"
+	"github.com/hashicorp/boundary/internal/servers/controller/auth"
+	"github.com/hashicorp/boundary/internal/servers/controller/common"
+	"github.com/hashicorp/boundary/internal/servers/controller/handlers"
+	"github.com/mr-tron/base58"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/proto"
+)
+
+func Test_requestCtxInterceptor(t *testing.T) {
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	kmsCache := kms.TestKms(t, conn, wrapper)
+	iamRepo := iam.TestRepo(t, conn, wrapper)
+
+	iamRepoFn := func() (*iam.Repository, error) {
+		return iamRepo, nil
+	}
+	atRepoFn := func() (*authtoken.Repository, error) {
+		return authtoken.NewRepository(rw, rw, kmsCache)
+	}
+	serversRepoFn := func() (*servers.Repository, error) {
+		return servers.NewRepository(rw, rw, kmsCache)
+	}
+
+	validGatewayTicket := "valid-ticket"
+
+	o, _ := iam.TestScopes(t, iamRepo)
+	at := authtoken.TestAuthToken(t, conn, kmsCache, o.GetPublicId())
+	encToken, err := authtoken.EncryptToken(context.Background(), kmsCache, o.GetPublicId(), at.GetPublicId(), at.GetToken())
+	require.NoError(t, err)
+	tokValue := at.GetPublicId() + "_" + encToken
+
+	newReqCtx := func(gwTicket string) context.Context {
+		req := httptest.NewRequest("GET", "http://127.0.0.1/v1/scopes/o_1", nil)
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", tokValue))
+		// Add values for authn/authz checking
+		requestInfo := authpb.RequestInfo{
+			Path:   req.URL.Path,
+			Method: req.Method,
+		}
+		requestInfo.PublicId, requestInfo.EncryptedToken, requestInfo.TokenFormat = auth.GetTokenFromRequest(context.TODO(), kmsCache, req)
+		requestInfo.Ticket = gwTicket // allows the grpc-gateway to verify the request info came from it's in-memory companion http proxy
+		marshalledRequestInfo, err := proto.Marshal(&requestInfo)
+		require.NoError(t, err)
+		md := metadata.Pairs(requestInfoMdKey, base58.FastBase58Encoding(marshalledRequestInfo))
+		mdCtx := metadata.NewIncomingContext(context.Background(), md)
+
+		md, ok := metadata.FromIncomingContext(mdCtx)
+		require.True(t, ok)
+		require.NotNil(t, md)
+
+		return mdCtx
+	}
+
+	factoryCtx := context.Background()
+
+	returnCtxHandler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return ctx, nil
+	}
+
+	tests := []struct {
+		name                   string
+		requestCtx             context.Context
+		iamRepoFn              common.IamRepoFactory
+		authTokenRepoFn        common.AuthTokenRepoFactory
+		serversRepoFn          common.ServersRepoFactory
+		kms                    *kms.Kms
+		ticket                 string
+		wantFactoryErr         bool
+		wantFactoryErrMatch    *errors.Template
+		wantFactoryErrContains string
+		wantRequestErr         bool
+		wantRequestErrMatch    *errors.Template
+		wantRequestErrContains string
+	}{
+		{
+			name:                   "missing-iam-repo",
+			requestCtx:             newReqCtx(validGatewayTicket),
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			wantFactoryErr:         true,
+			wantFactoryErrMatch:    errors.T(errors.InvalidParameter),
+			wantFactoryErrContains: "missing iam repo",
+		},
+		{
+			name:                   "missing-at-repo",
+			requestCtx:             newReqCtx(validGatewayTicket),
+			iamRepoFn:              iamRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			wantFactoryErr:         true,
+			wantFactoryErrMatch:    errors.T(errors.InvalidParameter),
+			wantFactoryErrContains: "missing auth token repo",
+		},
+		{
+			name:                   "missing-servers-repo",
+			requestCtx:             newReqCtx(validGatewayTicket),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			kms:                    kmsCache,
+			wantFactoryErr:         true,
+			wantFactoryErrMatch:    errors.T(errors.InvalidParameter),
+			wantFactoryErrContains: "missing servers repo function",
+		},
+		{
+			name:                   "missing-kms",
+			requestCtx:             newReqCtx(validGatewayTicket),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			wantFactoryErr:         true,
+			wantFactoryErrMatch:    errors.T(errors.InvalidParameter),
+			wantFactoryErrContains: "missing kms",
+		},
+		{
+			name:                   "missing-factory-ticket",
+			requestCtx:             newReqCtx(validGatewayTicket),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			wantFactoryErr:         true,
+			wantFactoryErrMatch:    errors.T(errors.InvalidParameter),
+			wantFactoryErrContains: "missing ticket",
+		},
+		{
+			name:                   "missing-metadata",
+			requestCtx:             context.Background(),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 validGatewayTicket,
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "No metadata",
+		},
+		{
+			name: "missing-request-info-metadata",
+			requestCtx: func() context.Context {
+				md := metadata.Pairs("greeter-md", "hello")
+				return metadata.NewIncomingContext(context.Background(), md)
+			}(),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 validGatewayTicket,
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "Missing request metadata",
+		},
+		{
+			name: "too-many-request-info-metadata",
+			requestCtx: func() context.Context {
+				md := metadata.Pairs(requestInfoMdKey, "first", requestInfoMdKey, "second")
+				return metadata.NewIncomingContext(context.Background(), md)
+			}(),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 validGatewayTicket,
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "expected 1 value",
+		},
+		{
+			name: "request-info-metadata-not-encoded",
+			requestCtx: func() context.Context {
+				md := metadata.Pairs(requestInfoMdKey, "hello")
+				return metadata.NewIncomingContext(context.Background(), md)
+			}(),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 validGatewayTicket,
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "unable to decode request info",
+		},
+		{
+			name: "request-info-metadata-not-proto",
+			requestCtx: func() context.Context {
+				md := metadata.Pairs(requestInfoMdKey, base58.FastBase58Encoding([]byte("hello")))
+				return metadata.NewIncomingContext(context.Background(), md)
+			}(),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 validGatewayTicket,
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "unable to unmarshal request info",
+		},
+		{
+			name: "request-info-metadata-not-proto",
+			requestCtx: func() context.Context {
+				md := metadata.Pairs(requestInfoMdKey, base58.FastBase58Encoding([]byte("hello")))
+				return metadata.NewIncomingContext(context.Background(), md)
+			}(),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 validGatewayTicket,
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "unable to unmarshal request info",
+		},
+		{
+			name:                   "missing-request-ticket",
+			requestCtx:             newReqCtx(""),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 "validGatewayTicket",
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "Invalid context (missing ticket)",
+		},
+		{
+			name:                   "bad-ticket",
+			requestCtx:             newReqCtx("bad-ticket"),
+			iamRepoFn:              iamRepoFn,
+			authTokenRepoFn:        atRepoFn,
+			serversRepoFn:          serversRepoFn,
+			kms:                    kmsCache,
+			ticket:                 "validGatewayTicket",
+			wantRequestErr:         true,
+			wantRequestErrMatch:    errors.T(errors.Internal),
+			wantRequestErrContains: "Invalid context (bad ticket)",
+		},
+		{
+			name:            "valid",
+			requestCtx:      newReqCtx(validGatewayTicket),
+			iamRepoFn:       iamRepoFn,
+			authTokenRepoFn: atRepoFn,
+			serversRepoFn:   serversRepoFn,
+			kms:             kmsCache,
+			ticket:          validGatewayTicket,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			interceptor, err := requestCtxInterceptor(factoryCtx, tt.iamRepoFn, tt.authTokenRepoFn, tt.serversRepoFn, tt.kms, tt.ticket)
+			if tt.wantFactoryErr {
+				require.Error(err)
+				assert.Nil(interceptor)
+				if tt.wantFactoryErrMatch != nil {
+					assert.Truef(errors.Match(tt.wantFactoryErrMatch, err), "want err code: %q got: %q", tt.wantFactoryErrMatch.Code, err)
+				}
+				if tt.wantFactoryErrContains != "" {
+					assert.Contains(err.Error(), tt.wantFactoryErrContains)
+				}
+				return
+			}
+			require.NoError(err)
+			assert.NotNil(interceptor)
+
+			info := &grpc.UnaryServerInfo{
+				FullMethod: "FakeMethod",
+			}
+			retCtx, err := interceptor(tt.requestCtx, nil, info, returnCtxHandler)
+			if tt.wantRequestErr {
+				require.Error(err)
+				assert.Nil(retCtx)
+				if tt.wantRequestErrMatch != nil {
+					assert.Truef(errors.Match(tt.wantRequestErrMatch, err), "want err code: %q got: %q", tt.wantRequestErrMatch.Code, err)
+				}
+				if tt.wantRequestErrContains != "" {
+					assert.Contains(err.Error(), tt.wantRequestErrContains)
+				}
+				return
+			}
+			require.NoError(err)
+			verifyResults := auth.Verify(retCtx.(context.Context))
+			assert.NotEmpty(verifyResults)
+		})
+	}
+}
+
+func Test_errorInterceptor(t *testing.T) {
+	ctx := context.Background()
+	tests := []struct {
+		name           string
+		factoryCtx     context.Context
+		wantRequestErr bool
+		wantDomainErr  bool
+		wantApiErr     bool
+	}{
+		{
+			name:           "domain-error",
+			factoryCtx:     ctx,
+			wantRequestErr: true,
+			wantDomainErr:  true,
+		},
+		{
+			name:           "api-error",
+			factoryCtx:     ctx,
+			wantRequestErr: true,
+			wantApiErr:     true,
+		},
+		{
+			name:       "success",
+			factoryCtx: ctx,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			require.Falsef(tt.wantApiErr && tt.wantDomainErr, "tests can't require both wantApiErr and wantDomainErr")
+			errInterceptor := errorInterceptor(tt.factoryCtx)
+			client := startTestGreeterService(t, &testGreeter{}, errInterceptor)
+			var header metadata.MD
+			resp, err := client.SayHello(
+				context.Background(),
+				&interceptor.SayHelloRequest{Name: tt.name},
+				grpc.Header(&header),
+			)
+			if tt.wantRequestErr {
+				require.Error(err)
+				domainHdr := header.Get(domainErrHeader)
+				apiErrHdr := header.Get(apiErrHeader)
+				require.Error(err)
+				assert.Nil(resp)
+				if tt.wantDomainErr {
+					require.Len(domainHdr, 1)
+					assert.Len(apiErrHdr, 0)
+					decoded, err := base58.FastBase58Decoding(domainHdr[0])
+					require.NoError(err)
+					var pbErr pberrors.Err
+					err = proto.Unmarshal(decoded, &pbErr)
+					require.NoError(err)
+				}
+				if tt.wantApiErr {
+					require.Len(apiErrHdr, 1)
+					assert.Len(domainHdr, 0)
+					decoded, err := base58.FastBase58Decoding(apiErrHdr[0])
+					require.NoError(err)
+					var pbErr pberrors.ApiError
+					err = proto.Unmarshal(decoded, &pbErr)
+					require.NoError(err)
+				}
+				return
+			}
+			require.NoError(err)
+		})
+	}
+
+}
+
+func Test_statusCodeInterceptor(t *testing.T) {
+	ctx := context.Background()
+	tests := []struct {
+		name           string
+		wantStatusCode int
+		wantErr        bool
+	}{
+		{
+			name:           "nil-nil",
+			wantStatusCode: http.StatusNoContent,
+		},
+		{
+			name:    "nil-err",
+			wantErr: true,
+		},
+		{
+			name: "hello",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			statusInterceptor := statusCodeInterceptor(ctx)
+			client := startTestGreeterService(t, &testGreeter{}, statusInterceptor)
+			var header metadata.MD
+			_, err := client.SayHello(
+				context.Background(),
+				&interceptor.SayHelloRequest{Name: tt.name},
+				grpc.Header(&header),
+			)
+
+			if tt.wantErr {
+				assert.Error(err)
+			}
+			statusHdr := header.Get(handlers.StatusCodeHeader)
+			if tt.wantStatusCode > 0 {
+				require.Len(statusHdr, 1)
+				code, err := strconv.Atoi(statusHdr[0])
+				require.NoError(err)
+				assert.Equal(tt.wantStatusCode, code)
+			} else {
+				require.Len(statusHdr, 0)
+			}
+		})
+	}
+
+}
+
+type testGreeter struct {
+	interceptor.UnimplementedGreeterServiceServer
+}
+
+func (g *testGreeter) SayHello(ctx context.Context, req *interceptor.SayHelloRequest) (*interceptor.SayHelloResponse, error) {
+	const op = "SayHello"
+	switch req.GetName() {
+	case "api-error":
+		return &interceptor.SayHelloResponse{}, &handlers.ApiError{
+			Status: http.StatusInternalServerError,
+			Inner: &pb_api.Error{
+				Kind:    "internal",
+				Op:      op,
+				Message: "api error msg",
+			},
+		}
+	case "domain-error":
+		return &interceptor.SayHelloResponse{Message: "hello"}, errors.New(ctx, errors.Internal, op, "domain error msg")
+	case "nil-nil":
+		return nil, nil
+	case "nil-err":
+		return nil, errors.New(ctx, errors.Internal, op, "nil response error msg")
+	default:
+		return &interceptor.SayHelloResponse{Message: "hello"}, nil
+	}
+}

internal/servers/controller/listeners.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/libs/alpnmux"
@@ -20,10 +21,16 @@ import (
 	"google.golang.org/grpc"
 )
 
-func (c *Controller) startListeners() error {
+func (c *Controller) startListeners(ctx context.Context) error {
 	servers := make([]func(), 0, len(c.conf.Listeners))
 
 	configureForAPI := func(ln *base.ServerListener) error {
+		var err error
+		if c.gatewayServer, c.gatewayTicket, err = newGatewayServer(ctx, c.IamRepoFn, c.AuthTokenRepoFn, c.ServersRepoFn, c.kms); err != nil {
+			return err
+		}
+		c.gatewayMux = newGatewayMux()
+
 		handler, err := c.handler(HandlerProperties{
 			ListenerConfig: ln.Config,
 			CancelCtx:      c.baseContext,
@@ -134,6 +141,11 @@ func (c *Controller) startListeners() error {
 		return nil
 	}
 
+	c.gatewayListener, _ = newGatewayListener()
+	servers = append(servers, func() {
+		go c.gatewayServer.Serve(c.gatewayListener)
+	})
+
 	for _, ln := range c.conf.Listeners {
 		var err error
 		for _, purpose := range ln.Config.Purpose {
@@ -184,6 +196,21 @@ func (c *Controller) stopListeners(serversOnly bool) error {
 			}
 		}()
 	}
+
+	if c.gatewayServer != nil {
+		serverWg.Add(1)
+		go func() {
+			defer serverWg.Done()
+			shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, globals.DefaultMaxRequestDuration)
+			defer shutdownKillCancel()
+			go func() {
+				<-shutdownKill.Done()
+				c.gatewayServer.Stop()
+			}()
+			c.gatewayServer.GracefulStop()
+		}()
+	}
+
 	serverWg.Wait()
 	if serversOnly {
 		return nil

internal/servers/controller/testing.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/authmethods"
 	"github.com/hashicorp/boundary/api/authtokens"
@@ -18,6 +19,7 @@ import (
 	"github.com/hashicorp/boundary/internal/cmd/config"
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/db/schema"
+	"github.com/hashicorp/boundary/internal/gen/testing/interceptor"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/intglobals"
 	"github.com/hashicorp/boundary/internal/kms"
@@ -27,6 +29,9 @@ import (
 	wrapping "github.com/hashicorp/go-kms-wrapping"
 	"github.com/hashicorp/go-secure-stdlib/base62"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/test/bufconn"
 )
 
 const (
@@ -716,3 +721,39 @@ func (tc *TestController) WaitForNextWorkerStatusUpdate(workerId string) error {
 	event.WriteSysEvent(ctx, op, "waiting for next status report from worker received successfully", "worker", workerId)
 	return nil
 }
+
+// startTestGreeterService is intended to facilitate the testing of
+// interceptors.  You provide a greeter service that produces appropriate
+// responses to test your interceptor(s).  This test function will start up a
+// test greeter service wrapped with the  specified interceptors and return an
+// initialized client for the service. The test service will be stopped/cleaned
+// up after the test (or subtests) have completed.
+func startTestGreeterService(t *testing.T, greeter interceptor.GreeterServiceServer, interceptors ...grpc.UnaryServerInterceptor) interceptor.GreeterServiceClient {
+	t.Helper()
+	require := require.New(t)
+	dialCtx := context.Background()
+
+	buffer := 1024 * 1024
+	listener := bufconn.Listen(buffer)
+
+	s := grpc.NewServer(
+		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(interceptors...)),
+	)
+	interceptor.RegisterGreeterServiceServer(s, greeter)
+	go func() {
+		err := s.Serve(listener)
+		require.NoError(err)
+	}()
+
+	conn, _ := grpc.DialContext(dialCtx, "", grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) {
+		return listener.Dial()
+	}), grpc.WithInsecure())
+
+	t.Cleanup(func() {
+		listener.Close()
+		s.Stop()
+	})
+
+	client := interceptor.NewGreeterServiceClient(conn)
+	return client
+}