Move the server daemons into their own package (#2061)

* Move the server daemons into their own package

With this change the internal/server package now is dedicated to the server domain (workers and controllers) and not to the server daemons.
The hope is that this will make it easier to keep the daemons decoupled from the representation of the servers in the database.
This commit is contained in:
Todd authored and GitHub committed 2022-05-09 12:45:10 -07:00
1 parent 8ed560fc3e
commit b58dada40d
177 files changed
+275 -275

No files matched your search

@@ -0,0 +1,272 @@
// Package metric provides functions to initialize the controller specific
// collectors and hooks to measure metrics and update the relevant collectors.
package metric
import (
"fmt"
"net/http"
"path"
"regexp"
"strconv"
"strings"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/gen/controller/api/services"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
grpcpb "google.golang.org/genproto/googleapis/api/annotations"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/types/descriptorpb"
)
var (
pathRegex map[*regexp.Regexp]string
expectedPathsToMethods map[string][]string
)
func init() {
expectedPathsToMethods = apiPathsAndMethods()
pathRegex = make(map[*regexp.Regexp]string)
for p := range expectedPathsToMethods {
pathRegex[buildRegexFromPath(p)] = p
}
}
// gatherPathInfo populates the provided map with the paths associated with
// the provided HttpRule proto option as well as the path information contained
// in any embedded AdditionalBindings in that HttpRule.
func gatherPathInfo(rule *grpcpb.HttpRule, paths map[string][]string) {
switch r := rule.GetPattern().(type) {
case *grpcpb.HttpRule_Get:
paths[r.Get] = append(paths[r.Get], http.MethodGet)
case *grpcpb.HttpRule_Post:
paths[r.Post] = append(paths[r.Post], http.MethodPost)
case *grpcpb.HttpRule_Patch:
paths[r.Patch] = append(paths[r.Patch], http.MethodPatch)
case *grpcpb.HttpRule_Put:
paths[r.Put] = append(paths[r.Put], http.MethodPut)
case *grpcpb.HttpRule_Delete:
paths[r.Delete] = append(paths[r.Delete], http.MethodDelete)
default:
panic("unknown rule of ")
}
for _, additional := range rule.AdditionalBindings {
gatherPathInfo(additional, paths)
}
}
func gatherServicePathsAndMethods(fd protoreflect.FileDescriptor, paths map[string][]string) error {
for j := 0; j < fd.Services().Len(); j++ {
sd := fd.Services().Get(j)
for i := 0; i < sd.Methods().Len(); i++ {
r := sd.Methods().Get(i)
opts := r.Options().(*descriptorpb.MethodOptions)
httpRule := proto.GetExtension(opts, grpcpb.E_Http).(*grpcpb.HttpRule)
if proto.Equal(httpRule, &grpcpb.HttpRule{}) || httpRule == nil {
return fmt.Errorf("empty or no http rule found on service method %q", r.FullName())
}
gatherPathInfo(httpRule, paths)
}
}
return nil
}
// apiPathsAndMethods provides the pathing information for all services
// registered to the controller API. This does not include any paths that are
// defined outside of the protobufs such as anything using the OPTION method
// or any other paths registered like the dev UI passthrough path.
func apiPathsAndMethods() map[string][]string {
paths := make(map[string][]string)
protoregistry.GlobalFiles.RangeFilesByPackage(
services.File_controller_api_services_v1_user_service_proto.Package(),
func(f protoreflect.FileDescriptor) bool {
if err := gatherServicePathsAndMethods(f, paths); err != nil {
panic(err)
}
return true
})
return paths
}
func buildRegexFromPath(p string) *regexp.Regexp {
// We only care about how grpc-gateway will route to specific handlers.
// As long as there is at least 1 character that is part of a path segment
// (not a '/', '?', or ':' we have identified an id for the sake of routing.
const idRegexp = "[^\\/\\?\\:]+"
// Replace any tag in the form of {id} or {auth_method_id} with the above
// regex so we can match paths to that when measuring requests.
pWithId := string(regexp.MustCompile("\\{[^\\}]*id\\}").ReplaceAll([]byte(p), []byte(idRegexp)))
// Escape everything except for our id regexp.
var seg []string
for _, s := range strings.Split(pWithId, idRegexp) {
seg = append(seg, regexp.QuoteMeta(s))
}
escapedPathRegex := strings.Join(seg, idRegexp)
return regexp.MustCompile(fmt.Sprintf("^%s$", escapedPathRegex))
}
const (
invalidPathValue = "invalid"
labelHttpCode = "code"
labelHttpPath = "path"
labelHttpMethod = "method"
apiSubSystem = "controller_api"
)
var (
// 100 bytes, 1kb, 10kb, 100kb, 1mb, 10mb, 100mb, 1gb
msgSizeBuckets = prometheus.ExponentialBuckets(100, 10, 8)
// httpRequestLatency collects measurements of how long it takes
// the boundary system to reply to a request to the controller api
// from the time that boundary received the request.
httpRequestLatency prometheus.ObserverVec = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: globals.MetricNamespace,
Subsystem: apiSubSystem,
Name: "http_request_duration_seconds",
Help: "Histogram of latencies for HTTP requests.",
Buckets: prometheus.DefBuckets,
},
[]string{labelHttpCode, labelHttpPath, labelHttpMethod},
)
// httpRequestSize collections measurements of how large each request
// to the boundary controller api is.
httpRequestSize prometheus.ObserverVec = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: globals.MetricNamespace,
Subsystem: apiSubSystem,
Name: "http_request_size_bytes",
Help: "Histogram of request sizes for HTTP requests.",
Buckets: msgSizeBuckets,
},
[]string{labelHttpCode, labelHttpPath, labelHttpMethod},
)
// httpRequestSize collections measurements of how large each response
// from the boundary controller api is.
httpResponseSize prometheus.ObserverVec = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: globals.MetricNamespace,
Subsystem: apiSubSystem,
Name: "http_response_size_bytes",
Help: "Histogram of response sizes for HTTP responses.",
Buckets: msgSizeBuckets,
},
[]string{labelHttpCode, labelHttpPath, labelHttpMethod},
)
)
var universalStatusCodes = []int{
http.StatusUnauthorized,
http.StatusForbidden,
http.StatusNotFound,
http.StatusMethodNotAllowed,
http.StatusBadRequest,
http.StatusInternalServerError,
http.StatusGatewayTimeout,
}
// Codes which are only currently used in the authentication flow
var authenticationStatusCodes = []int{
http.StatusAccepted,
http.StatusFound,
}
var expectedStatusCodesPerMethod = map[string][]int{
http.MethodGet: append(universalStatusCodes,
http.StatusOK),
http.MethodPost: append(universalStatusCodes,
append(authenticationStatusCodes, http.StatusOK)...),
http.MethodPatch: append(universalStatusCodes,
http.StatusOK),
// delete methods always returns no content instead of a StatusOK
http.MethodDelete: append(universalStatusCodes,
http.StatusNoContent),
http.MethodOptions: {
http.StatusNoContent,
http.StatusForbidden,
http.StatusMethodNotAllowed,
},
}
// pathLabel maps the requested path to the label value recorded for metrics
func pathLabel(incomingPath string) string {
if incomingPath == "" || incomingPath[0] != '/' {
incomingPath = fmt.Sprintf("/%s", incomingPath)
}
incomingPath = path.Clean(incomingPath)
for r, ep := range pathRegex {
if r.Match([]byte(incomingPath)) {
return ep
}
}
return invalidPathValue
}
// InstrumentApiHandler provides a handler which measures api
// 1. The response size
// 2. The request size
// 3. The request latency
// and attaches status code, method, and path labels for each of these
// measurements.
func InstrumentApiHandler(wrapped http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
l := prometheus.Labels{
labelHttpPath: pathLabel(req.URL.Path),
}
promhttp.InstrumentHandlerDuration(
httpRequestLatency.MustCurryWith(l),
promhttp.InstrumentHandlerRequestSize(
httpRequestSize.MustCurryWith(l),
promhttp.InstrumentHandlerResponseSize(
httpResponseSize.MustCurryWith(l),
wrapped,
),
),
).ServeHTTP(rw, req)
})
}
// InitializeApiCollectors registers the api collectors to the default
// prometheus register and initializes them to 0 for all possible label
// combinations.
func InitializeApiCollectors(r prometheus.Registerer) {
if r == nil {
return
}
r.MustRegister(httpResponseSize, httpRequestSize, httpRequestLatency)
for p, methods := range expectedPathsToMethods {
for _, m := range methods {
for _, sc := range expectedStatusCodesPerMethod[m] {
l := prometheus.Labels{labelHttpCode: strconv.Itoa(sc), labelHttpPath: p, labelHttpMethod: strings.ToLower(m)}
httpResponseSize.With(l)
httpRequestSize.With(l)
httpRequestLatency.With(l)
}
}
}
// When an invalid path is found, any method is possible both we expect
// an error response.
p := invalidPathValue
for m := range expectedStatusCodesPerMethod {
for _, sc := range []int{http.StatusNotFound, http.StatusMethodNotAllowed} {
l := prometheus.Labels{labelHttpCode: strconv.Itoa(sc), labelHttpPath: p, labelHttpMethod: strings.ToLower(m)}
httpResponseSize.With(l)
httpRequestSize.With(l)
httpRequestLatency.With(l)
}
}
}
@@ -0,0 +1,150 @@
package metric
import (
"net/http"
"testing"
"github.com/hashicorp/boundary/internal/gen/testing/protooptions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuildRegexFromPath(t *testing.T) {
cases := []struct {
path string
match []string
dont []string
}{
{
path: "/v1/pathsomething/{id}:test-thing",
match: []string{
"/v1/pathsomething/a4s_aKsdFh723018djsa:test-thing",
"/v1/pathsomething/h_1234567890:test-thing",
"/v1/pathsomething/{id}:test-thing",
"/v1/pathsomething/{any_old_id}:test-thing",
"/v1/pathsomething/not-an-id:test-thing",
},
dont: []string{
"/v1/pathsomething/a4s_aKsdFh723018djsa:other-thing",
"/v1/pathsomething:test-thing",
"/v1/pathsomething/:test-thing:test-thing",
"/v1/pathsomething/:other-thing",
},
},
{
path: "/v1/pathsomething/{auth_method_id}:authenticate",
match: []string{
"/v1/pathsomething/a4s_aKsdFh723018djsa:authenticate",
"/v1/pathsomething/am_1234567890:authenticate",
"/v1/pathsomething/{id}:authenticate",
"/v1/pathsomething/{auth_method}:authenticate",
},
dont: []string{
"/v1/pathsomething:authenticate",
"/v1/pathsomething:authenticate:authenticate",
"/v1/pathsomething/:authenticate:authenticate",
"/v1/pathsomething/am_1234567890/:authenticate",
"/v1/pathsomething/?whatabout=:authenticate",
},
},
{
path: "/v1/pathsomething/{non_id_tag}:test",
match: []string{
"/v1/pathsomething/{non_id_tag}:test",
},
dont: []string{
"/v1/pathsomething/h_1234567890:test",
"/v1/pathsomething/.*:test",
},
},
{
path: "/v1/pathsomething/a?",
match: []string{
"/v1/pathsomething/a?",
},
dont: []string{
"/v1/pathsomething/",
"/v1/pathsomething/a",
"/v1/pathsomething/aa",
},
},
}
for _, tc := range cases {
t.Run(tc.path, func(t *testing.T) {
r := buildRegexFromPath(tc.path)
for _, m := range tc.match {
assert.True(t, r.Match([]byte(m)), "Couldn't match %q", m)
}
for _, d := range tc.dont {
assert.False(t, r.Match([]byte(d)), "Matched %q", d)
}
})
}
}
func TestPathLabel(t *testing.T) {
cases := []struct {
in string
want string
}{
{
in: "/v1/accounts",
want: "/v1/accounts",
},
{
in: "v1/accounts",
want: "/v1/accounts",
},
{
in: "/v1/hosts",
want: "/v1/hosts",
},
{
in: "/v2/accounts",
want: "invalid",
},
{
in: "/v1/accounts/a_1234567890",
want: "/v1/accounts/{id}",
},
{
in: "/v1/accounts/mistake",
want: "/v1/accounts/{id}",
},
{
in: "/v1/accounts/mistyped_id",
want: "/v1/accounts/{id}",
},
{
in: "v1/accounts/a_1234567890",
want: "/v1/accounts/{id}",
},
{
in: "/v1/accounts/a_1234567890:set-password",
want: "/v1/accounts/{id}:set-password",
},
{
in: "v1/accounts/a_1234567890:set-password",
want: "/v1/accounts/{id}:set-password",
},
{
// mistype the custom action
in: "/v1/accounts/a_1234567890:set-passwords",
want: "invalid",
},
}
for _, tc := range cases {
t.Run(tc.in, func(t *testing.T) {
assert.Equal(t, tc.want, pathLabel(tc.in))
})
}
}
func TestServicePathsAndMethods(t *testing.T) {
paths := make(map[string][]string)
require.NoError(t, gatherServicePathsAndMethods(protooptions.File_testing_options_v1_service_proto, paths))
assert.Equal(t, map[string][]string{
"/v1/test/{id}": {http.MethodGet},
"/v2/test": {http.MethodGet},
}, paths)
}
@@ -0,0 +1,133 @@
package metric
import (
"context"
"strings"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/errors"
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/stats"
"google.golang.org/grpc/status"
)
const (
labelGRpcCode = "grpc_code"
labelGRpcService = "grpc_service"
labelGRpcMethod = "grpc_method"
clusterSubSystem = "controller_cluster"
)
// gRpcRequestLatency collects measurements of how long it takes
// the boundary system to reply to a request to the controller cluster
// from the time that boundary received the request.
var gRpcRequestLatency prometheus.ObserverVec = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: globals.MetricNamespace,
Subsystem: clusterSubSystem,
Name: "grpc_request_duration_seconds",
Help: "Histogram of latencies for gRPC requests.",
Buckets: prometheus.DefBuckets,
},
[]string{labelGRpcCode, labelGRpcService, labelGRpcMethod},
)
// statusFromError retrieves the *status.Status from the provided error. It'll
// attempt to unwrap the *status.Error, which is something status.FromError
// does not do.
func statusFromError(err error) *status.Status {
if s, ok := status.FromError(err); ok {
return s
}
type gRPCStatus interface {
GRPCStatus() *status.Status
}
var unwrappedStatus gRPCStatus
if ok := errors.As(err, &unwrappedStatus); ok {
return unwrappedStatus.GRPCStatus()
}
return status.New(codes.Unknown, "Unknown Code")
}
func splitMethodName(fullMethodName string) (string, string) {
fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash
if i := strings.Index(fullMethodName, "/"); i >= 0 {
return fullMethodName[:i], fullMethodName[i+1:]
}
return "unknown", "unknown"
}
type metricMethodNameContextKey struct{}
type statsHandler struct{}
func (sh statsHandler) TagRPC(ctx context.Context, i *stats.RPCTagInfo) context.Context {
return context.WithValue(ctx, metricMethodNameContextKey{}, i.FullMethodName)
}
func (sh statsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
return ctx
}
func (sh statsHandler) HandleConn(context.Context, stats.ConnStats) {
}
func (sh statsHandler) HandleRPC(ctx context.Context, s stats.RPCStats) {
switch v := s.(type) {
case *stats.End:
// Accept the ok, but ignore it. This code doesn't need to panic
// and if "fullName" is an empty string splitMethodName will
// set service and method to "unknown".
fullName, _ := ctx.Value(metricMethodNameContextKey{}).(string)
service, method := splitMethodName(fullName)
l := prometheus.Labels{
labelGRpcMethod: method,
labelGRpcService: service,
labelGRpcCode: statusFromError(v.Error).Code().String(),
}
gRpcRequestLatency.With(l).Observe(v.EndTime.Sub(v.BeginTime).Seconds())
}
}
var allCodes = []codes.Code{
codes.OK, codes.InvalidArgument, codes.PermissionDenied,
codes.FailedPrecondition,
// Codes which can be generated by the gRPC framework
codes.Canceled, codes.Unknown, codes.DeadlineExceeded,
codes.ResourceExhausted, codes.Unimplemented, codes.Internal,
codes.Unavailable, codes.Unauthenticated,
}
// InstrumentClusterStatsHandler returns a gRPC stats.Handler which observes
// cluster specific metrics. Use with the cluster gRPC server.
func InstrumentClusterStatsHandler() statsHandler {
return statsHandler{}
}
// InitializeClusterCollectors registers the cluster metrics to the default
// prometheus register and initializes them to 0 for all possible label
// combinations.
func InitializeClusterCollectors(r prometheus.Registerer, server *grpc.Server) {
if r == nil {
return
}
r.MustRegister(gRpcRequestLatency)
for serviceName, info := range server.GetServiceInfo() {
for _, mInfo := range info.Methods {
for _, c := range allCodes {
l := prometheus.Labels{
labelGRpcMethod: mInfo.Name,
labelGRpcService: serviceName,
labelGRpcCode: c.String(),
}
gRpcRequestLatency.With(l)
}
}
}
}
@@ -0,0 +1,169 @@
package metric
import (
"context"
"fmt"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/stats"
"google.golang.org/grpc/status"
)
func TestStatsHandler(t *testing.T) {
bkpLatency := gRpcRequestLatency
defer func() {
gRpcRequestLatency = bkpLatency
}()
handler := InstrumentClusterStatsHandler()
cases := []struct {
name string
stats []stats.RPCStats
fullMethodName string
wantedLabels prometheus.Labels
wantedLatency float64
}{
{
name: "basic",
fullMethodName: "/some.service.path/method",
stats: []stats.RPCStats{
&stats.End{
BeginTime: time.Time{}.Add(time.Second),
EndTime: time.Time{}.Add(5 * time.Second),
},
},
wantedLabels: map[string]string{
labelGRpcCode: "OK",
labelGRpcMethod: "method",
labelGRpcService: "some.service.path",
},
wantedLatency: (4 * time.Second).Seconds(),
},
{
name: "ignored stats",
fullMethodName: "/some.service.path/method",
stats: []stats.RPCStats{
&stats.Begin{
BeginTime: time.Time{},
IsTransparentRetryAttempt: true,
},
&stats.InPayload{
Length: 5,
WireLength: 15,
RecvTime: time.Time{}.Add(time.Second).Add(500 * time.Millisecond),
},
&stats.OutPayload{
Length: 5,
WireLength: 15,
SentTime: time.Time{}.Add(2 * time.Second),
},
&stats.End{
BeginTime: time.Time{}.Add(time.Second),
EndTime: time.Time{}.Add(5 * time.Second),
},
},
wantedLabels: map[string]string{
labelGRpcCode: "OK",
labelGRpcMethod: "method",
labelGRpcService: "some.service.path",
},
wantedLatency: (4 * time.Second).Seconds(),
},
{
name: "bad method name",
fullMethodName: "",
stats: []stats.RPCStats{
&stats.End{
BeginTime: time.Time{}.Add(time.Second),
EndTime: time.Time{}.Add(5 * time.Second),
},
},
wantedLabels: map[string]string{
labelGRpcCode: "OK",
labelGRpcMethod: "unknown",
labelGRpcService: "unknown",
},
wantedLatency: (4 * time.Second).Seconds(),
},
{
name: "error code",
fullMethodName: "/some.service.path/method",
stats: []stats.RPCStats{
&stats.End{
BeginTime: time.Time{}.Add(time.Second),
EndTime: time.Time{}.Add(5 * time.Second),
Error: status.Error(codes.Canceled, "test"),
},
},
wantedLabels: map[string]string{
labelGRpcCode: "Canceled",
labelGRpcMethod: "method",
labelGRpcService: "some.service.path",
},
wantedLatency: (4 * time.Second).Seconds(),
},
{
name: "wrapped error",
fullMethodName: "/some.service.path/method",
stats: []stats.RPCStats{
&stats.End{
BeginTime: time.Time{}.Add(time.Second),
EndTime: time.Time{}.Add(5 * time.Second),
Error: fmt.Errorf("%w", status.Error(codes.InvalidArgument, "test")),
},
},
wantedLabels: map[string]string{
labelGRpcCode: "InvalidArgument",
labelGRpcMethod: "method",
labelGRpcService: "some.service.path",
},
wantedLatency: (4 * time.Second).Seconds(),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
testableLatency := &testableObserverVec{}
gRpcRequestLatency = testableLatency
ctx := context.Background()
ctx = handler.TagRPC(ctx, &stats.RPCTagInfo{
FullMethodName: tc.fullMethodName,
})
for _, i := range tc.stats {
handler.HandleRPC(ctx, i)
}
assert.Len(t, testableLatency.observations, 1)
assert.Equal(t, testableLatency.observations[0].observation, tc.wantedLatency)
assert.Equal(t, testableLatency.observations[0].labels, tc.wantedLabels)
})
}
}
// testableObserverVec allows us to assert which observations are being made
// with which labels.
type testableObserverVec struct {
observations []*testableObserver
prometheus.ObserverVec
}
func (v *testableObserverVec) With(l prometheus.Labels) prometheus.Observer {
ret := &testableObserver{labels: l}
v.observations = append(v.observations, ret)
return ret
}
type testableObserver struct {
labels prometheus.Labels
observation float64
}
func (o *testableObserver) Observe(f float64) {
o.observation = f
}