From d5262e8a154e5f57d7158ec0c935311e1115d93e Mon Sep 17 00:00:00 2001 From: Damian Debkowski Date: Mon, 25 Mar 2024 15:45:01 -0700 Subject: [PATCH] feat: parse generic aws api errors --- internal/errors/error.go | 160 ++++++++++++++ internal/errors/errror_test.go | 382 +++++++++++++++++++++++++++++++++ internal/errors/testing.go | 34 +++ 3 files changed, 576 insertions(+) create mode 100644 internal/errors/errror_test.go create mode 100644 internal/errors/testing.go diff --git a/internal/errors/error.go b/internal/errors/error.go index 92f8a6e..af36cab 100644 --- a/internal/errors/error.go +++ b/internal/errors/error.go @@ -4,12 +4,40 @@ package errors import ( + "bytes" + "errors" "fmt" + "net/http" "sort" "strings" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/aws/aws-sdk-go-v2/aws/retry" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/smithy-go" + pb "github.com/hashicorp/boundary/sdk/pbs/plugin" +) + +const ( + // awsErrorAccessDenied is returned when the credentials do not have + // the required permissions to do a specific action on a resource. + // This error will persist until the IAM Role Policy is updated by + // the owner. This error cannot be resolved by retrying the request. + awsErrorAccessDenied = "AccessDenied" + + // awsErrorInvalidAccessKeyId is returned when the credential does + // not exist. This error will persist until the credentials attached + // to the storage bucket is updated. This error can occur when the + // given credentials are deleted, deactivated, or rotated. This + // error cannot be resolved by retrying the request. + awsErrorInvalidAccessKeyId = "InvalidAccessKeyId" + + // awsErrorExpiredToken is returned when the provided credential is + // expired. This error can only be returned by a dynamic credential + // type. This error can resolve itself by retrying the request. + awsErrorExpiredToken = "ExpiredToken" ) // InvalidArgumentError returns an grpc invalid argument status error. @@ -24,3 +52,135 @@ func InvalidArgumentError(msg string, f map[string]string) error { } return status.Error(codes.InvalidArgument, msg) } + +// ParseAWSError converts an aws service error into a RPC status. This method +// will handle parsing the following aws error codes: throttling, connections, +// and credentials. This method will fallback to parsing the http status code +// when it cannot match an aws error code. This method does not handle specific +// service type errors such as S3 or EC2. +func ParseAWSError(err error, msg string) (st *status.Status) { + if err == nil { + return nil + } + + // find the service name of the aws api + serviceName := "unknown" + var oe *smithy.OperationError + if errors.As(err, &oe) { + serviceName = oe.Service() + } + + // The unknown plugin error code will be used + // when the aws service returns an error that + // has failed to be parsed. By default, this + // error code is retryable. + plgErr := &pb.PluginError{ + Code: pb.ERROR_ERROR_UNKNOWN, + Message: err.Error(), + Nonretryable: false, + } + + defer func() { + if st == nil { + statusMsg := fmt.Sprintf("aws service %s: unknown error: %s", serviceName, msg) + st = status.New(codes.Unknown, statusMsg) + } + st, err = st.WithDetails(plgErr) + if err != nil { + st = status.New(codes.Internal, err.Error()) + } + }() + + // evaluate against aws throttling error codes + throttleErr := retry.ThrottleErrorCode{ + Codes: retry.DefaultThrottleErrorCodes, + } + if throttleErr.IsErrorThrottle(err).Bool() { + plgErr.Code = pb.ERROR_ERROR_THROTTLING + statusMsg := fmt.Sprintf("aws service %s: throttling error: %s", serviceName, msg) + return status.New(codes.Unavailable, statusMsg) + } + // evaluate against aws connection error codes + connectionErr := retry.IsErrorRetryables([]retry.IsErrorRetryable{ + retry.NoRetryCanceledError{}, + retry.RetryableError{}, + retry.RetryableConnectionError{}, + retry.RetryableHTTPStatusCode{ + Codes: retry.DefaultRetryableHTTPStatusCodes, + }, + retry.RetryableErrorCode{ + Codes: retry.DefaultRetryableErrorCodes, + }, + }) + if connectionErr.IsErrorRetryable(err).Bool() { + plgErr.Code = pb.ERROR_ERROR_TIMEOUT + statusMsg := fmt.Sprintf("aws service %s: connectivity error: %s", serviceName, msg) + return status.New(codes.DeadlineExceeded, statusMsg) + } + // evaluate against aws credentials error codes + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg) + switch apiErr.ErrorCode() { + case awsErrorAccessDenied: + fallthrough + case awsErrorInvalidAccessKeyId: + plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL + plgErr.Nonretryable = true + return status.New(codes.PermissionDenied, statusMsg) + case awsErrorExpiredToken: + plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL + return status.New(codes.PermissionDenied, statusMsg) + } + } + + // default to evaluating the http status code + if httpErr, ok := err.(*awshttp.ResponseError); ok { + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(httpErr.Response.Body) + if err != nil { + return status.New(codes.Internal, err.Error()) + } + defer httpErr.Response.Body.Close() + plgErr.Message = buf.String() + switch httpErr.HTTPStatusCode() { + case http.StatusBadRequest: + plgErr.Code = pb.ERROR_ERROR_BAD_REQUEST + plgErr.Nonretryable = true + statusMsg := fmt.Sprintf("aws service %s: bad request error: %s", serviceName, msg) + return status.New(codes.InvalidArgument, statusMsg) + case http.StatusUnauthorized: + plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL + statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg) + return status.New(codes.PermissionDenied, statusMsg) + case http.StatusForbidden: + plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL + plgErr.Nonretryable = true + statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg) + return status.New(codes.PermissionDenied, statusMsg) + case http.StatusNotFound: + plgErr.Code = pb.ERROR_ERROR_BAD_REQUEST + plgErr.Nonretryable = true + statusMsg := fmt.Sprintf("aws service %s: resource not found error: %s", serviceName, msg) + return status.New(codes.NotFound, statusMsg) + case http.StatusTooManyRequests: + plgErr.Code = pb.ERROR_ERROR_THROTTLING + statusMsg := fmt.Sprintf("aws service %s: throttling error: %s", serviceName, msg) + return status.New(codes.Unavailable, statusMsg) + case http.StatusRequestTimeout: + fallthrough + case http.StatusInternalServerError: + fallthrough + case http.StatusBadGateway: + fallthrough + case http.StatusServiceUnavailable: + fallthrough + case http.StatusGatewayTimeout: + plgErr.Code = pb.ERROR_ERROR_TIMEOUT + statusMsg := fmt.Sprintf("aws service %s: connectivity error: %s", serviceName, msg) + return status.New(codes.DeadlineExceeded, statusMsg) + } + } + + return st +} diff --git a/internal/errors/errror_test.go b/internal/errors/errror_test.go new file mode 100644 index 0000000..d8c0e66 --- /dev/null +++ b/internal/errors/errror_test.go @@ -0,0 +1,382 @@ +package errors + +import ( + "fmt" + "testing" + + "net/http" + + pb "github.com/hashicorp/boundary/sdk/pbs/plugin" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +func Test_ParseAWSError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + expectedStatusCode codes.Code + expectedStatusMsg string + expectedPluginError *pb.PluginError + }{ + { + name: "nil-error", + expectedStatusCode: codes.OK, + }, + { + name: "default-to-unknwon", + err: fmt.Errorf("what can this error be?"), + expectedStatusCode: codes.Unknown, + expectedStatusMsg: "aws service unknown: unknown error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_UNKNOWN, + Message: "what can this error be?", + Nonretryable: false, + }, + }, + { + name: "http-status-code-bad-request", + err: TestAwsHttpResponseError(http.StatusBadRequest, "bad request"), + expectedStatusCode: codes.InvalidArgument, + expectedStatusMsg: "aws service unknown: bad request error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_BAD_REQUEST, + Message: "bad request", + Nonretryable: true, + }, + }, + { + name: "http-status-code-unauthorized", + err: TestAwsHttpResponseError(http.StatusUnauthorized, "unauthorized request"), + expectedStatusCode: codes.PermissionDenied, + expectedStatusMsg: "aws service unknown: invalid credentials error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_INVALID_CREDENTIAL, + Message: "unauthorized request", + Nonretryable: false, + }, + }, + { + name: "http-status-code-forbidden", + err: TestAwsHttpResponseError(http.StatusForbidden, "forbidden request"), + expectedStatusCode: codes.PermissionDenied, + expectedStatusMsg: "aws service unknown: invalid credentials error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_INVALID_CREDENTIAL, + Message: "forbidden request", + Nonretryable: true, + }, + }, + { + name: "http-status-code-not-found", + err: TestAwsHttpResponseError(http.StatusNotFound, "not found request"), + expectedStatusCode: codes.NotFound, + expectedStatusMsg: "aws service unknown: resource not found error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_BAD_REQUEST, + Message: "not found request", + Nonretryable: true, + }, + }, + { + name: "http-status-code-too-many-requests", + err: TestAwsHttpResponseError(http.StatusTooManyRequests, "too many requests"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "too many requests", + Nonretryable: false, + }, + }, + { + name: "http-status-code-request-timeout", + err: TestAwsHttpResponseError(http.StatusRequestTimeout, "request timeout"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "request timeout", + Nonretryable: false, + }, + }, + { + name: "http-status-code-internal-server-error", + err: TestAwsHttpResponseError(http.StatusInternalServerError, "internal server error"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "https response error StatusCode: 500", + Nonretryable: false, + }, + }, + { + name: "http-status-code-bad-gateway", + err: TestAwsHttpResponseError(http.StatusBadGateway, "bad gateway"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "https response error StatusCode: 502", + Nonretryable: false, + }, + }, + { + name: "http-status-code-service-unavailable", + err: TestAwsHttpResponseError(http.StatusServiceUnavailable, "service unavailable"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "https response error StatusCode: 503", + Nonretryable: false, + }, + }, + { + name: "http-status-code-gateway-timeout", + err: TestAwsHttpResponseError(http.StatusGatewayTimeout, "gateway timeout"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "https response error StatusCode: 504", + Nonretryable: false, + }, + }, + { + name: "aws-throttling", + err: TestAwsError("Throttling", "throttling error"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "throttling error", + Nonretryable: false, + }, + }, + { + name: "aws-throttling-exception", + err: TestAwsError("ThrottlingException", "throttling exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "throttling exception", + Nonretryable: false, + }, + }, + { + name: "aws-throttled-exception", + err: TestAwsError("ThrottledException", "throttled exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "throttled exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-throttled-exception", + err: TestAwsError("RequestThrottledException", "request throttled exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "request throttled exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-too-many-requests-exception", + err: TestAwsError("TooManyRequestsException", "too many requests exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "too many requests exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-provisioned-throughput-exceeded-exception", + err: TestAwsError("ProvisionedThroughputExceededException", "provisioned throughput exceeded exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "provisioned throughput exceeded exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-transaction-in-progress-exception", + err: TestAwsError("TransactionInProgressException", "transaction in progress exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "transaction in progress exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-limit-exceeded", + err: TestAwsError("RequestLimitExceeded", "request limit exceeded"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "request limit exceeded", + Nonretryable: false, + }, + }, + { + name: "aws-request-bandwidth-limit-exceeded", + err: TestAwsError("BandwidthLimitExceeded", "bandwidth limit exceeded"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "bandwidth limit exceeded", + Nonretryable: false, + }, + }, + { + name: "aws-request-limit-exceeded-exception", + err: TestAwsError("LimitExceededException", "limit exceeded exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "limit exceeded exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-throttled", + err: TestAwsError("RequestThrottled", "request throttled"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "request throttled", + Nonretryable: false, + }, + }, + { + name: "aws-request-slow-down", + err: TestAwsError("SlowDown", "slow down"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "slow down", + Nonretryable: false, + }, + }, + { + name: "aws-request-prior-request-not-complete", + err: TestAwsError("PriorRequestNotComplete", "prior request not complete"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "prior request not complete", + Nonretryable: false, + }, + }, + { + name: "aws-request-ec2-throttled-exception", + err: TestAwsError("EC2ThrottledException", "ec2 throttled exception"), + expectedStatusCode: codes.Unavailable, + expectedStatusMsg: "aws service unknown: throttling error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_THROTTLING, + Message: "ec2 throttled exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-timeout", + err: TestAwsError("RequestTimeout", "request timeout"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "request timeout", + Nonretryable: false, + }, + }, + { + name: "aws-request-timeout-exception", + err: TestAwsError("RequestTimeoutException", "request timeout exception"), + expectedStatusCode: codes.DeadlineExceeded, + expectedStatusMsg: "aws service unknown: connectivity error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_TIMEOUT, + Message: "request timeout exception", + Nonretryable: false, + }, + }, + { + name: "aws-request-access-denied", + err: TestAwsError(awsErrorAccessDenied, "not authorized to perform action"), + expectedStatusCode: codes.PermissionDenied, + expectedStatusMsg: "aws service unknown: invalid credentials error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_INVALID_CREDENTIAL, + Message: "not authorized to perform action", + Nonretryable: true, + }, + }, + { + name: "aws-request-invalid-access-key-id", + err: TestAwsError(awsErrorInvalidAccessKeyId, "The AWS Access Key Id you provided does not exist in our records."), + expectedStatusCode: codes.PermissionDenied, + expectedStatusMsg: "aws service unknown: invalid credentials error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_INVALID_CREDENTIAL, + Message: "The AWS Access Key Id you provided does not exist in our records.", + Nonretryable: true, + }, + }, + { + name: "aws-request-expired-token", + err: TestAwsError(awsErrorExpiredToken, "The security token included in the request is expired."), + expectedStatusCode: codes.PermissionDenied, + expectedStatusMsg: "aws service unknown: invalid credentials error: test", + expectedPluginError: &pb.PluginError{ + Code: pb.ERROR_ERROR_INVALID_CREDENTIAL, + Message: "The security token included in the request is expired.", + Nonretryable: false, + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + actualStatus := ParseAWSError(tt.err, "test") + if tt.expectedStatusCode == codes.OK { + require.Nil(actualStatus) + return + } + + require.Equal(tt.expectedStatusCode, actualStatus.Code()) + require.Equal(tt.expectedStatusMsg, actualStatus.Message()) + require.Len(actualStatus.Details(), 1) + for _, detail := range actualStatus.Details() { + switch errDetail := detail.(type) { + case *pb.PluginError: + require.Equal(tt.expectedPluginError.Code, errDetail.Code) + require.Contains(errDetail.Message, tt.expectedPluginError.Message) + require.Equal(tt.expectedPluginError.Nonretryable, errDetail.Nonretryable) + } + } + }) + } +} diff --git a/internal/errors/testing.go b/internal/errors/testing.go new file mode 100644 index 0000000..895647f --- /dev/null +++ b/internal/errors/testing.go @@ -0,0 +1,34 @@ +package errors + +import ( + "io" + "net/http" + "strings" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/smithy-go" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +// TestAwsHttpResponseError returns an aws http response error +func TestAwsHttpResponseError(code int, msg string) error { + return &awshttp.ResponseError{ + ResponseError: &smithyhttp.ResponseError{ + Response: &smithyhttp.Response{ + Response: &http.Response{ + StatusCode: code, + Body: io.NopCloser(strings.NewReader(msg)), + }, + }, + }, + } +} + +// TestAwsError returns an generic api error +func TestAwsError(code, msg string) error { + return &smithy.GenericAPIError{ + Code: code, + Message: msg, + Fault: smithy.FaultServer, + } +}