refactor (controller): Use a grpc server for the grpc-gateway (#1576)

* refactor (controller): Use a grpc server for the grpc-gateway

This starts a grpc server for the grpc-gateway.  The new gateway
server uses an in-memory listener which also supports "dialing", so
the http proxy can connect to the in-memory listener.

This change allows us to use all grpc server features and was in part
motivated by a need for an interceptor of inbound requests.  With that
said, having a "proper" grpc server for the gateway will provide a lot
more flexibility and capabilities moving forward.

The request context will still be verified via middleware in the
http proxy (see wrapHandlerWithCommonFuncs).  This verified context
will be serialized into a protobuf and sent along to the grpc-gateway
via metadata where an interceptor will add the verified request
info to the request's context (see requestCtxInterceptor).  The
interceptor will verify the serialized request info via a shared ticket
with it's companion in-memory http proxy.  FYI: the ticket is simply
random base62 string (see db.NewPrivateId).

This change also necessitated a minor change to services.  We need to
return an empty response instead of nil, nil when there are no errors
and the response is empty.

* refactor (handlers): Rename gRPC Gateway filter

Originally, this was called an OutgoingInterceptor, which is an
unfortunate name since it's not an interceptor.  Now that we're
refactoring the gateway stuff and have proper interceptors, renaming
this should remove ambiguity

* refactor: Rework how 204 with empty msg is handled

Recent changes to grpc gateway has necessitated a change to how
204s are handled by the gateway

* tests (bats): Check status codes for delete operations

internal/servers/controller/handlers/authmethods/authmethod_service.go

@@ -389,7 +389,7 @@ func (s Service) DeleteAuthMethod(ctx context.Context, req *pbs.DeleteAuthMethod
 	if err != nil {
 		return nil, err
 	}
-	return nil, nil
+	return &pbs.DeleteAuthMethodResponse{}, nil
 }
 
 // Authenticate implements the interface pbs.AuthenticationServiceServer.

internal/servers/controller/handlers/authmethods/authmethod_service_test.go

@@ -428,12 +428,14 @@ func TestDelete(t *testing.T) {
 			req: &pbs.DeleteAuthMethodRequest{
 				Id: pwam.GetPublicId(),
 			},
+			res: &pbs.DeleteAuthMethodResponse{},
 		},
 		{
 			name: "Delete an Existing OIDC AuthMethod",
 			req: &pbs.DeleteAuthMethodRequest{
 				Id: oidcam.GetPublicId(),
 			},
+			res: &pbs.DeleteAuthMethodResponse{},
 		},
 		{
 			name: "Delete bad auth_method id",

internal/servers/controller/handlers/authmethods/oidc_test.go

@@ -13,6 +13,8 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
+
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
 	"github.com/hashicorp/boundary/internal/authtoken"
@@ -206,9 +208,9 @@ func TestList_FilterNonPublic(t *testing.T) {
 					atRepoFn,
 					serversRepoFn,
 					kmsCache,
-					auth.RequestInfo{
+					&authpb.RequestInfo{
 						Token:       at.GetToken(),
-						TokenFormat: auth.AuthTokenTypeBearer,
+						TokenFormat: uint32(auth.AuthTokenTypeBearer),
 						PublicId:    at.GetPublicId(),
 					})
 			}(),

internal/servers/controller/handlers/authtokens/authtoken_service_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/boundary/internal/authtoken"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/requests"
@@ -87,15 +88,15 @@ func TestGetSelf(t *testing.T) {
 			require, assert := require.New(t), assert.New(t)
 			// Setup the auth request information
 			req := httptest.NewRequest("GET", fmt.Sprintf("http://127.0.0.1/v1/auth-tokens/%s", tc.readId), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.token.GetPublicId(),
 				Token:       tc.token.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			ctx = context.WithValue(ctx, requests.ContextRequestInformationKey, &requests.RequestContext{})
 			got, err := a.GetAuthToken(ctx, &pbs.GetAuthTokenRequest{Id: tc.readId})
 			if tc.err != nil {
@@ -236,15 +237,15 @@ func TestList_Self(t *testing.T) {
 			require, assert := require.New(t), assert.New(t)
 			// Setup the auth request information
 			req := httptest.NewRequest("GET", fmt.Sprintf("http://127.0.0.1/v1/auth-tokens?scope_id=%s", o.GetPublicId()), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.requester.GetPublicId(),
 				Token:       tc.requester.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			got, err := a.ListAuthTokens(ctx, &pbs.ListAuthTokensRequest{ScopeId: o.GetPublicId()})
 			require.NoError(err)
 			require.Len(got.Items, 1)
@@ -482,15 +483,15 @@ func TestDeleteSelf(t *testing.T) {
 			require := require.New(t)
 			// Setup the auth request information
 			req := httptest.NewRequest("DELETE", fmt.Sprintf("http://127.0.0.1/v1/auth-tokens/%s", tc.deleteId), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.token.GetPublicId(),
 				Token:       tc.token.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			got, err := a.DeleteAuthToken(ctx, &pbs.DeleteAuthTokenRequest{Id: tc.deleteId})
 			if tc.err != nil {
 				require.EqualError(err, tc.err.Error())

internal/servers/controller/handlers/credentiallibraries/credentiallibrary_service_test.go

@@ -555,6 +555,7 @@ func TestDelete(t *testing.T) {
 		name string
 		id   string
 		err  error
+		res  *pbs.DeleteCredentialLibraryResponse
 	}{
 		{
 			name: "success",
@@ -574,7 +575,7 @@ func TestDelete(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			got, gErr := s.DeleteCredentialLibrary(auth.DisabledAuthTestContext(iamRepoFn, prj.GetPublicId()), &pbs.DeleteCredentialLibraryRequest{Id: tc.id})
-			assert.Nil(t, got)
+			assert.EqualValuesf(t, tc.res, got, "DeleteCredentialLibrary got response %q, wanted %q", got, tc.res)
 			if tc.err != nil {
 				require.Error(t, gErr)
 				assert.True(t, errors.Is(gErr, tc.err))

internal/servers/controller/handlers/credentialstores/credentialstore_service_test.go

@@ -674,6 +674,7 @@ func TestDelete(t *testing.T) {
 		name string
 		id   string
 		err  error
+		res  *pbs.DeleteCredentialStoreResponse
 	}{
 		{
 			name: "success",
@@ -693,7 +694,7 @@ func TestDelete(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			got, gErr := s.DeleteCredentialStore(auth.DisabledAuthTestContext(iamRepoFn, prj.GetPublicId()), &pbs.DeleteCredentialStoreRequest{Id: tc.id})
-			assert.Nil(t, got)
+			assert.EqualValuesf(t, tc.res, got, "DeleteCredentialStore got response %q, wanted %q", got, tc.res)
 			if tc.err != nil {
 				require.Error(t, gErr)
 				assert.True(t, errors.Is(gErr, tc.err))

internal/servers/controller/handlers/errors.go

@@ -5,31 +5,46 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/hashicorp/boundary/internal/errors"
 	pb "github.com/hashicorp/boundary/internal/gen/controller/api"
+	pberrors "github.com/hashicorp/boundary/internal/gen/errors"
 	"github.com/hashicorp/boundary/internal/observability/event"
+	"github.com/mr-tron/base58"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 const (
 	genericUniquenessMsg = "Invalid request.  Request attempted to make second resource with the same field value that must be unique."
 	genericNotFoundMsg   = "Unable to find requested resource."
+
+	// domainErrHeader defines an http header for encoded domain errors from the
+	// grpc server.
+	domainErrHeader = "x-domain-err"
+	// domainErrMetadataHeader defines an http header for encoded domain errors from the
+	// grpc server via metadata
+	domainErrMetadataHeader = "Grpc-Metadata-X-Domain-Err"
+
+	apiErrHeader         = "x-api-err"
+	apiErrMetadataHeader = "Grpc-Metadata-X-Api-Err"
 )
 
-type apiError struct {
-	status int32
-	inner  *pb.Error
+type ApiError struct {
+	Status int32
+	Inner  *pb.Error
 }
 
-func (e *apiError) Error() string {
-	res := fmt.Sprintf("Status: %d, Kind: %q, Error: %q", e.status, e.inner.GetKind(), e.inner.GetMessage())
+func (e *ApiError) Error() string {
+	res := fmt.Sprintf("Status: %d, Kind: %q, Error: %q", e.Status, e.Inner.GetKind(), e.Inner.GetMessage())
 	var dets []string
-	for _, rf := range e.inner.GetDetails().GetRequestFields() {
+	for _, rf := range e.Inner.GetDetails().GetRequestFields() {
 		dets = append(dets, fmt.Sprintf("{name: %q, desc: %q}", rf.GetName(), rf.GetDescription()))
 	}
 	if len(dets) > 0 {
@@ -39,19 +54,19 @@ func (e *apiError) Error() string {
 	return res
 }
 
-func (e *apiError) Is(target error) bool {
-	var tApiErr *apiError
+func (e *ApiError) Is(target error) bool {
+	var tApiErr *ApiError
 	if !errors.As(target, &tApiErr) {
 		return false
 	}
-	return tApiErr.inner.Kind == e.inner.Kind && tApiErr.status == e.status
+	return tApiErr.Inner.Kind == e.Inner.Kind && tApiErr.Status == e.Status
 }
 
 // ApiErrorWithCode returns an api error with the provided code.
 func ApiErrorWithCode(c codes.Code) error {
-	return &apiError{
-		status: int32(runtime.HTTPStatusFromCode(c)),
-		inner: &pb.Error{
+	return &ApiError{
+		Status: int32(runtime.HTTPStatusFromCode(c)),
+		Inner: &pb.Error{
 			Kind: c.String(),
 		},
 	}
@@ -59,9 +74,9 @@ func ApiErrorWithCode(c codes.Code) error {
 
 // ApiErrorWithCodeAndMessage returns an api error with the provided code and message.
 func ApiErrorWithCodeAndMessage(c codes.Code, msg string, args ...interface{}) error {
-	return &apiError{
-		status: int32(runtime.HTTPStatusFromCode(c)),
-		inner: &pb.Error{
+	return &ApiError{
+		Status: int32(runtime.HTTPStatusFromCode(c)),
+		Inner: &pb.Error{
 			Kind:    c.String(),
 			Message: fmt.Sprintf(msg, args...),
 		},
@@ -70,9 +85,9 @@ func ApiErrorWithCodeAndMessage(c codes.Code, msg string, args ...interface{}) e
 
 // NotFoundError returns an ApiError indicating a resource couldn't be found.
 func NotFoundError() error {
-	return &apiError{
-		status: http.StatusNotFound,
-		inner: &pb.Error{
+	return &ApiError{
+		Status: http.StatusNotFound,
+		Inner: &pb.Error{
 			Kind:    codes.NotFound.String(),
 			Message: "Resource not found.",
 		},
@@ -80,19 +95,19 @@ func NotFoundError() error {
 }
 
 // NotFoundErrorf returns an ApiError indicating a resource couldn't be found.
-func NotFoundErrorf(msg string, a ...interface{}) *apiError {
-	return &apiError{
-		status: http.StatusNotFound,
-		inner: &pb.Error{
+func NotFoundErrorf(msg string, a ...interface{}) *ApiError {
+	return &ApiError{
+		Status: http.StatusNotFound,
+		Inner: &pb.Error{
 			Kind:    codes.NotFound.String(),
 			Message: fmt.Sprintf(msg, a...),
 		},
 	}
 }
 
-var unauthorizedError = &apiError{
-	status: http.StatusForbidden,
-	inner: &pb.Error{
+var unauthorizedError = &ApiError{
+	Status: http.StatusForbidden,
+	Inner: &pb.Error{
 		Kind:    codes.PermissionDenied.String(),
 		Message: "Forbidden.",
 	},
@@ -102,9 +117,9 @@ func ForbiddenError() error {
 	return unauthorizedError
 }
 
-var unauthenticatedError = &apiError{
-	status: http.StatusUnauthorized,
-	inner: &pb.Error{
+var unauthenticatedError = &ApiError{
+	Status: http.StatusUnauthorized,
+	Inner: &pb.Error{
 		Kind:    codes.Unauthenticated.String(),
 		Message: "Unauthenticated, or invalid token.",
 	},
@@ -114,47 +129,49 @@ func UnauthenticatedError() error {
 	return unauthenticatedError
 }
 
-func InvalidArgumentErrorf(msg string, fields map[string]string) *apiError {
+func InvalidArgumentErrorf(msg string, fields map[string]string) *ApiError {
 	const op = "handlers.InvalidArgumentErrorf"
 	ctx := context.TODO()
 	err := ApiErrorWithCodeAndMessage(codes.InvalidArgument, msg)
-	var apiErr *apiError
+	var apiErr *ApiError
 	if !errors.As(err, &apiErr) {
 		event.WriteError(ctx, op, err, event.WithInfoMsg("Unable to build invalid argument api error."))
 	}
 
 	if len(fields) > 0 {
-		apiErr.inner.Details = &pb.ErrorDetails{}
+		apiErr.Inner.Details = &pb.ErrorDetails{}
 	}
 	for k, v := range fields {
-		apiErr.inner.Details.RequestFields = append(apiErr.inner.Details.RequestFields, &pb.FieldError{Name: k, Description: v})
+		apiErr.Inner.Details.RequestFields = append(apiErr.Inner.Details.RequestFields, &pb.FieldError{Name: k, Description: v})
 	}
-	sort.Slice(apiErr.inner.GetDetails().GetRequestFields(), func(i, j int) bool {
-		return apiErr.inner.Details.RequestFields[i].GetName() < apiErr.inner.Details.RequestFields[j].GetName()
+	sort.Slice(apiErr.Inner.GetDetails().GetRequestFields(), func(i, j int) bool {
+		return apiErr.Inner.Details.RequestFields[i].GetName() < apiErr.Inner.Details.RequestFields[j].GetName()
 	})
 	return apiErr
 }
 
+var statusRegEx = regexp.MustCompile("Status: ([0-9]+), Kind: \"(.*)\", Error: \"(.*)\"")
+
 // Converts a known errors into an error that can presented to an end user over the API.
-func backendErrorToApiError(inErr error) *apiError {
+func backendErrorToApiError(inErr error) *ApiError {
 	stErr := status.Convert(inErr)
 
 	switch {
 	case errors.Is(inErr, runtime.ErrNotMatch):
 		// grpc gateway uses this error when the path was not matched, but the error uses codes.Unimplemented which doesn't match the intention.
 		// Overwrite the error to match our expected behavior.
-		return &apiError{
-			status: http.StatusNotFound,
-			inner: &pb.Error{
+		return &ApiError{
+			Status: http.StatusNotFound,
+			Inner: &pb.Error{
 				Kind:    codes.NotFound.String(),
 				Message: http.StatusText(http.StatusNotFound),
 			},
 		}
 	case status.Code(inErr) == codes.Unimplemented:
 		// Instead of returning a 501 we always want to return a 405 when a method isn't implemented.
-		return &apiError{
-			status: http.StatusMethodNotAllowed,
-			inner: &pb.Error{
+		return &ApiError{
+			Status: http.StatusMethodNotAllowed,
+			Inner: &pb.Error{
 				Kind:    codes.Unimplemented.String(),
 				Message: stErr.Message(),
 			},
@@ -176,11 +193,23 @@ func backendErrorToApiError(inErr error) *apiError {
 		statusCode = int32(domainErr.Code)
 	}
 
+	// perhaps the error is from the grpc gateway, so match against the known
+	// apiError msg format.
+	if found := statusRegEx.FindStringSubmatch(stErr.Message()); len(found) == 4 {
+		u32, err := strconv.ParseInt(found[1], 10, 32)
+		if err == nil { // notice it's testing for NO err
+			return &ApiError{
+				Status: int32(u32),
+				Inner:  &pb.Error{Kind: found[2], Message: found[3]},
+			}
+		}
+	}
+
 	// TODO: Don't return potentially sensitive information (like which user id an account
 	//  is already associated with when attempting to re-associate it).
-	return &apiError{
-		status: statusCode,
-		inner:  &pb.Error{Kind: codes.Internal.String(), Message: inErr.Error()},
+	return &ApiError{
+		Status: statusCode,
+		Inner:  &pb.Error{Kind: codes.Internal.String(), Message: inErr.Error()},
 	}
 }
 
@@ -189,7 +218,70 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 	const errorFallback = `{"error": "failed to marshal error message"}`
 	return func(ctx context.Context, _ *runtime.ServeMux, mar runtime.Marshaler, w http.ResponseWriter, r *http.Request, inErr error) {
 		// API specified error, otherwise we need to translate repo/db errors.
-		var apiErr *apiError
+
+		// the grpc server will encoded domain errors into the x-domain-err, so
+		// let's check there first. (see: controller.errorInterceptor)
+		md, ok := runtime.ServerMetadataFromContext(ctx)
+		if ok {
+			defer func() {
+				// make sure we don't leak the headers that were used as a comm
+				// channel between the grpc server and the http proxy
+				delete(md.HeaderMD, domainErrHeader)
+				delete(w.Header(), domainErrMetadataHeader)
+
+				delete(md.HeaderMD, apiErrHeader)
+				delete(w.Header(), apiErrMetadataHeader)
+			}()
+			domainErrHdrs := md.HeaderMD.Get(domainErrHeader)
+			apiErrHdrs := md.HeaderMD.Get(apiErrHeader)
+
+			switch {
+			case len(domainErrHdrs) > 0:
+				decoded, err := base58.FastBase58Decoding(domainErrHdrs[0])
+				if err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to decode domain err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				var pbErr pberrors.Err
+				if err := proto.Unmarshal(decoded, &pbErr); err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to marshal domain err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				inErr = errors.FromPbErrors(&pbErr)
+			case len(apiErrHdrs) > 0:
+				decoded, err := base58.FastBase58Decoding(apiErrHdrs[0])
+				if err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to decode api err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				var pbApiErr pberrors.ApiError
+				if err := proto.Unmarshal(decoded, &pbApiErr); err != nil {
+					event.WriteError(ctx, op, err, event.WithInfoMsg("failed to marshal api err header"))
+					w.WriteHeader(http.StatusInternalServerError)
+					if _, err := io.WriteString(w, errorFallback); err != nil {
+						event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
+					}
+					return
+				}
+				inErr = &ApiError{
+					Status: pbApiErr.Status,
+					Inner:  pbApiErr.ApiError,
+				}
+			}
+		}
+		var apiErr *ApiError
 		isApiErr := errors.As(inErr, &apiErr)
 		if !isApiErr {
 			if err := backendErrorToApiError(inErr); err != nil && !errors.As(err, &apiErr) {
@@ -197,13 +289,13 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 			}
 		}
 
-		if apiErr.status == http.StatusInternalServerError {
+		if apiErr.Status == http.StatusInternalServerError {
 			event.WriteError(ctx, op, inErr, event.WithInfoMsg("internal error returned"))
 		}
 
-		buf, merr := mar.Marshal(apiErr.inner)
+		buf, merr := mar.Marshal(apiErr.Inner)
 		if merr != nil {
-			event.WriteError(ctx, op, merr, event.WithInfoMsg("failed to marshal error response", "response", fmt.Sprintf("%#v", apiErr.inner)))
+			event.WriteError(ctx, op, merr, event.WithInfoMsg("failed to marshal error response", "response", fmt.Sprintf("%#v", apiErr.Inner)))
 			w.WriteHeader(http.StatusInternalServerError)
 			if _, err := io.WriteString(w, errorFallback); err != nil {
 				event.WriteError(ctx, op, err, event.WithInfoMsg("failed to write response"))
@@ -211,8 +303,8 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 			return
 		}
 
-		w.Header().Set("Content-Type", mar.ContentType(apiErr.inner))
-		w.WriteHeader(int(apiErr.status))
+		w.Header().Set("Content-Type", mar.ContentType(apiErr.Inner))
+		w.WriteHeader(int(apiErr.Status))
 		if _, err := w.Write(buf); err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("failed to send response chunk"))
 			return
@@ -221,5 +313,5 @@ func ErrorHandler() runtime.ErrorHandlerFunc {
 }
 
 func ToApiError(e error) *pb.Error {
-	return backendErrorToApiError(e).inner
+	return backendErrorToApiError(e).Inner
 }

internal/servers/controller/handlers/errors_test.go

@@ -32,14 +32,14 @@ func TestApiErrorHandler(t *testing.T) {
 	testCases := []struct {
 		name     string
 		err      error
-		expected apiError
+		expected ApiError
 	}{
 		{
 			name: "Not Found",
 			err:  NotFoundErrorf("Test"),
-			expected: apiError{
-				status: http.StatusNotFound,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusNotFound,
+				Inner: &pb.Error{
 					Kind:    "NotFound",
 					Message: "Test",
 				},
@@ -51,9 +51,9 @@ func TestApiErrorHandler(t *testing.T) {
 				"k1": "v1",
 				"k2": "v2",
 			}),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "Test",
 					Details: &pb.ErrorDetails{
@@ -74,9 +74,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "GrpcGateway Routing Error",
 			err:  runtime.ErrNotMatch,
-			expected: apiError{
-				status: http.StatusNotFound,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusNotFound,
+				Inner: &pb.Error{
 					Kind:    "NotFound",
 					Message: http.StatusText(http.StatusNotFound),
 				},
@@ -85,9 +85,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Unimplemented error",
 			err:  status.Error(codes.Unimplemented, "Test"),
-			expected: apiError{
-				status: http.StatusMethodNotAllowed,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusMethodNotAllowed,
+				Inner: &pb.Error{
 					Kind:    "Unimplemented",
 					Message: "Test",
 				},
@@ -96,9 +96,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Unknown error",
 			err:  stderrors.New("Some random error"),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "Some random error",
 				},
@@ -107,9 +107,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db invalid public id",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidPublicId)),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "invalid public id, parameter violation: error #102",
 				},
@@ -118,9 +118,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db invalid parameter",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidParameter)),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "invalid parameter, parameter violation: error #100",
 				},
@@ -129,9 +129,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db invalid field mask",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidFieldMask)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "Error in provided request",
 					Details: &pb.ErrorDetails{RequestFields: []*pb.FieldError{{Name: "update_mask", Description: "Invalid update mask provided."}}},
@@ -141,9 +141,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db empty field mask",
 			err:  errors.E(ctx, errors.WithCode(errors.EmptyFieldMask)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "Error in provided request",
 					Details: &pb.ErrorDetails{RequestFields: []*pb.FieldError{{Name: "update_mask", Description: "Invalid update mask provided."}}},
@@ -153,9 +153,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db not unqiue",
 			err:  errors.E(ctx, errors.WithCode(errors.NotUnique)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: genericUniquenessMsg,
 				},
@@ -164,9 +164,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db record not found",
 			err:  errors.E(ctx, errors.WithCode(errors.RecordNotFound)),
-			expected: apiError{
-				status: http.StatusNotFound,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusNotFound,
+				Inner: &pb.Error{
 					Kind:    "NotFound",
 					Message: genericNotFoundMsg,
 				},
@@ -175,9 +175,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error Db multiple records",
 			err:  errors.E(ctx, errors.WithCode(errors.MultipleRecords)),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "multiple records, search issue: error #1101",
 				},
@@ -186,9 +186,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Domain error account already associated",
 			err:  errors.E(ctx, errors.WithCode(errors.AccountAlreadyAssociated)),
-			expected: apiError{
-				status: http.StatusBadRequest,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusBadRequest,
+				Inner: &pb.Error{
 					Kind:    "InvalidArgument",
 					Message: "account already associated with another user, parameter violation: error #114",
 				},
@@ -197,9 +197,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Wrapped domain error",
 			err:  errors.E(ctx, errors.WithCode(errors.InvalidAddress), errors.WithMsg("test msg"), errors.WithWrap(errors.E(ctx, errors.WithCode(errors.NotNull), errors.WithMsg("inner msg")))),
-			expected: apiError{
-				status: http.StatusInternalServerError,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusInternalServerError,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "test msg: parameter violation: error #101: inner msg: integrity violation: error #1001",
 				},
@@ -208,9 +208,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Forbidden domain error",
 			err:  errors.E(ctx, errors.WithCode(errors.Forbidden), errors.WithMsg("test msg")),
-			expected: apiError{
-				status: http.StatusForbidden,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusForbidden,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "test msg: unknown: error #403",
 				},
@@ -219,9 +219,9 @@ func TestApiErrorHandler(t *testing.T) {
 		{
 			name: "Wrapped forbidden domain error",
 			err:  fmt.Errorf("got error: %w", errors.E(ctx, errors.WithCode(errors.Forbidden), errors.WithMsg("test msg"))),
-			expected: apiError{
-				status: http.StatusForbidden,
-				inner: &pb.Error{
+			expected: ApiError{
+				Status: http.StatusForbidden,
+				Inner: &pb.Error{
 					Kind:    "Internal",
 					Message: "got error: test msg: unknown: error #403",
 				},
@@ -234,7 +234,7 @@ func TestApiErrorHandler(t *testing.T) {
 			w := httptest.NewRecorder()
 			tested(ctx, mux, outMarsh, w, req, tc.err)
 			resp := w.Result()
-			assert.EqualValues(tc.expected.status, resp.StatusCode)
+			assert.EqualValues(tc.expected.Status, resp.StatusCode)
 
 			got, err := ioutil.ReadAll(resp.Body)
 			require.NoError(err)
@@ -243,8 +243,8 @@ func TestApiErrorHandler(t *testing.T) {
 			err = inMarsh.Unmarshal(got, gotErr)
 			require.NoError(err)
 
-			assert.Equal(tc.expected.status, int32(resp.StatusCode))
-			assert.Empty(cmp.Diff(tc.expected.inner, gotErr, protocmp.Transform()))
+			assert.Equal(tc.expected.Status, int32(resp.StatusCode))
+			assert.Empty(cmp.Diff(tc.expected.Inner, gotErr, protocmp.Transform()))
 		})
 	}
 }

internal/servers/controller/handlers/managed_groups/managed_group_service_test.go

@@ -417,6 +417,7 @@ func TestDelete_twice(t *testing.T) {
 	o, _ := iam.TestScopes(t, iam.TestRepo(t, conn, wrap))
 
 	databaseWrapper, err := kmsCache.GetWrapper(context.Background(), o.PublicId, kms.KeyPurposeDatabase)
+	require.NoError(err)
 
 	oidcAm := oidc.TestAuthMethod(
 		t, conn, databaseWrapper, o.PublicId, oidc.ActivePrivateState,

internal/servers/controller/handlers/outgoing_interceptor_test.go

@@ -1,26 +0,0 @@
-package handlers
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
-	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authtokens"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/protobuf/types/known/structpb"
-)
-
-func TestOutgoingSplitCookie(t *testing.T) {
-	rec := httptest.NewRecorder()
-	attrs, err := ProtoToStruct(&pb.AuthToken{Token: "t_abc_1234567890"})
-	require.NoError(t, err)
-	attrs.GetFields()["token_type"] = structpb.NewStringValue("cookie")
-	require.NoError(t, OutgoingInterceptor(context.Background(), rec, &pbs.AuthenticateResponse{Attributes: attrs}))
-	assert.ElementsMatch(t, rec.Result().Cookies(), []*http.Cookie{
-		{Name: HttpOnlyCookieName, Value: "34567890", HttpOnly: true, Raw: "wt-http-token-cookie=34567890; HttpOnly"},
-		{Name: JsVisibleCookieName, Value: "t_abc_12", Raw: "wt-js-token-cookie=t_abc_12"},
-	})
-}

internal/servers/controller/handlers/outgoing_response_filter.go

@@ -4,30 +4,66 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
+	"github.com/hashicorp/boundary/internal/errors"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	"github.com/hashicorp/boundary/internal/observability/event"
 	authtokenpb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authtokens"
 
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/proto"
 )
 
 const (
-	HttpOnlyCookieName    = "wt-http-token-cookie"
-	JsVisibleCookieName   = "wt-js-token-cookie"
-	tokenTypeField        = "token_type"
-	finalRedirectUrlField = "final_redirect_url"
-	statusField           = "status"
+	HttpOnlyCookieName       = "wt-http-token-cookie"
+	JsVisibleCookieName      = "wt-js-token-cookie"
+	tokenTypeField           = "token_type"
+	finalRedirectUrlField    = "final_redirect_url"
+	statusField              = "status"
+	StatusCodeHeader         = "x-http-code"
+	statusCodeMetadataHeader = "Grpc-Metadata-X-Http-Code"
 )
 
-func OutgoingInterceptor(ctx context.Context, w http.ResponseWriter, m proto.Message) error {
-	const op = "handlers.OutgoingInterceptor"
-	m = m.ProtoReflect().Interface()
-	if !m.ProtoReflect().IsValid() {
-		// This would be the case if it's a nil pointer
-		w.WriteHeader(http.StatusNoContent)
-		return nil
+// SetStatusCode allows a grpc service handler to set the outgoing http status
+// code
+func SetStatusCode(ctx context.Context, code int) error {
+	const op = "handlers.SetHttpStatusCode"
+	if http.StatusText(code) == "" {
+		return errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("unknown status code: %d", code))
+	}
+	if err := grpc.SetHeader(ctx, metadata.Pairs(StatusCodeHeader, strconv.Itoa(code))); err != nil {
+		return errors.Wrap(ctx, err, op, errors.WithCode(errors.Internal))
+	}
+	return nil
+}
+
+// OutgoingResponseFilter is a gRPC gateway WithForwardResponseOption.  It's
+// basically a filter that can manipulate the http response and has acesss to
+// the outgoing proto msg
+func OutgoingResponseFilter(ctx context.Context, w http.ResponseWriter, m proto.Message) error {
+	const op = "handlers.OutgoingResponseFilter"
+
+	if md, ok := runtime.ServerMetadataFromContext(ctx); ok {
+
+		// set http status codes based on metadata set by the grpc service
+		if statusCodes := md.HeaderMD.Get(StatusCodeHeader); len(statusCodes) > 0 {
+			defer func() {
+				// delete the headers to not expose any grpc-metadata in http response
+				delete(md.HeaderMD, StatusCodeHeader)
+				delete(w.Header(), statusCodeMetadataHeader)
+			}()
+			lastStatus, err := strconv.Atoi(statusCodes[len(statusCodes)-1])
+			if err != nil {
+				return fmt.Errorf("%s: unable to convert status code %s: %w", op, statusCodes[len(statusCodes)-1], err)
+			}
+			w.WriteHeader(lastStatus)
+			return nil
+		}
 	}
 
 	// add the outgoing resp proto.  See: common.flushGatedEvents is where the

internal/servers/controller/handlers/outgoing_response_filter_test.go

@@ -0,0 +1,72 @@
+package handlers
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	emptypb "github.com/hashicorp/boundary/internal/gen/controller/api"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authtokens"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func TestOutgoingSplitCookie(t *testing.T) {
+	rec := httptest.NewRecorder()
+	attrs, err := ProtoToStruct(&pb.AuthToken{Token: "t_abc_1234567890"})
+	require.NoError(t, err)
+	attrs.GetFields()["token_type"] = structpb.NewStringValue("cookie")
+	require.NoError(t, OutgoingResponseFilter(context.Background(), rec, &pbs.AuthenticateResponse{Attributes: attrs}))
+	assert.ElementsMatch(t, rec.Result().Cookies(), []*http.Cookie{
+		{Name: HttpOnlyCookieName, Value: "34567890", HttpOnly: true, Raw: "wt-http-token-cookie=34567890; HttpOnly"},
+		{Name: JsVisibleCookieName, Value: "t_abc_12", Raw: "wt-js-token-cookie=t_abc_12"},
+	})
+}
+
+func TestOutgoingResponseFilter_StatusCode(t *testing.T) {
+	tests := []struct {
+		name           string
+		md             runtime.ServerMetadata
+		msg            proto.Message
+		wantErr        bool
+		wantStatusCode int
+		wantContent    bool
+	}{
+		{
+			name:           "204",
+			md:             runtime.ServerMetadata{HeaderMD: metadata.Pairs(StatusCodeHeader, "204")},
+			wantStatusCode: 204,
+		},
+		{
+			name:           "418",
+			md:             runtime.ServerMetadata{HeaderMD: metadata.Pairs(StatusCodeHeader, "418")},
+			wantStatusCode: 418,
+			wantContent:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			rec := httptest.NewRecorder()
+			ctx := runtime.NewServerMetadataContext(context.Background(), tt.md)
+			err := OutgoingResponseFilter(ctx, rec, &emptypb.EmptyResponse{})
+			if tt.wantErr {
+				require.Error(err)
+				return
+			}
+			require.NoError(err)
+			resp := rec.Result()
+			if tt.wantStatusCode != 0 {
+				assert.Equal(tt.wantStatusCode, resp.StatusCode)
+			}
+		})
+	}
+
+}

internal/servers/controller/handlers/sessions/session_service_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/errors"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
@@ -211,15 +212,15 @@ func TestList_Self(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			// Setup the auth request information
 			req := httptest.NewRequest("GET", fmt.Sprintf("http://127.0.0.1/v1/sessions?scope_id=%s", pWithSessions.GetPublicId()), nil)
-			requestInfo := auth.RequestInfo{
+			requestInfo := authpb.RequestInfo{
 				Path:        req.URL.Path,
 				Method:      req.Method,
-				TokenFormat: auth.AuthTokenTypeBearer,
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
 				PublicId:    tc.requester.GetPublicId(),
 				Token:       tc.requester.GetToken(),
 			}
 
-			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, requestInfo)
+			ctx := auth.NewVerifierContext(context.Background(), iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
 			got, err := s.ListSessions(ctx, &pbs.ListSessionsRequest{ScopeId: pWithSessions.GetPublicId()})
 			require.NoError(t, err)
 			assert.Equal(t, tc.count, len(got.GetItems()), got.GetItems())

internal/servers/controller/handlers/targets/target_service_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/boundary/internal/credential/vault"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	spbs "github.com/hashicorp/boundary/internal/gen/controller/servers/services"
 	"github.com/hashicorp/boundary/internal/host/static"
 	"github.com/hashicorp/boundary/internal/iam"
@@ -2500,9 +2501,9 @@ func TestAuthorizeSession(t *testing.T) {
 		atRepoFn,
 		serversRepoFn,
 		kms,
-		auth.RequestInfo{
+		&authpb.RequestInfo{
 			Token:       at.GetToken(),
-			TokenFormat: auth.AuthTokenTypeBearer,
+			TokenFormat: uint32(auth.AuthTokenTypeBearer),
 			PublicId:    at.GetPublicId(),
 		})
 
@@ -2674,9 +2675,9 @@ func TestAuthorizeSession_Errors(t *testing.T) {
 		atRepoFn,
 		serversRepoFn,
 		kms,
-		auth.RequestInfo{
+		&authpb.RequestInfo{
 			Token:       at.GetToken(),
-			TokenFormat: auth.AuthTokenTypeBearer,
+			TokenFormat: uint32(auth.AuthTokenTypeBearer),
 			PublicId:    at.GetPublicId(),
 		})
 	r := iam.TestRole(t, conn, proj.GetPublicId())

internal/servers/controller/handlers/verifiers_test.go

@@ -15,10 +15,10 @@ import (
 
 func errorIncludesFields(t *testing.T, err error, wantFields []string) {
 	t.Helper()
-	var apiErr *apiError
+	var apiErr *ApiError
 	require.True(t, errors.As(err, &apiErr))
 	var gotFields []string
-	for _, d := range apiErr.inner.GetDetails().GetRequestFields() {
+	for _, d := range apiErr.Inner.GetDetails().GetRequestFields() {
 		gotFields = append(gotFields, d.GetName())
 	}
 	assert.ElementsMatch(t, gotFields, wantFields)