feat: parse generic aws api errors

internal/errors/error.go

@@ -4,12 +4,40 @@
 package errors
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
+	"net/http"
 	"sort"
 	"strings"
 
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/smithy-go"
+	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+)
+
+const (
+	// awsErrorAccessDenied is returned when the credentials do not have
+	// the required permissions to do a specific action on a resource.
+	// This error will persist until the IAM Role Policy is updated by
+	// the owner. This error cannot be resolved by retrying the request.
+	awsErrorAccessDenied = "AccessDenied"
+
+	// awsErrorInvalidAccessKeyId is returned when the credential does
+	// not exist. This error will persist until the credentials attached
+	// to the storage bucket is updated. This error can occur when the
+	// given credentials are deleted, deactivated, or rotated. This
+	// error cannot be resolved by retrying the request.
+	awsErrorInvalidAccessKeyId = "InvalidAccessKeyId"
+
+	// awsErrorExpiredToken is returned when the provided credential is
+	// expired. This error can only be returned by a dynamic credential
+	// type. This error can resolve itself by retrying the request.
+	awsErrorExpiredToken = "ExpiredToken"
 )
 
 // InvalidArgumentError returns an grpc invalid argument status error.
@@ -24,3 +52,135 @@ func InvalidArgumentError(msg string, f map[string]string) error {
 	}
 	return status.Error(codes.InvalidArgument, msg)
 }
+
+// ParseAWSError converts an aws service error into a RPC status. This method
+// will handle parsing the following aws error codes: throttling, connections,
+// and credentials. This method will fallback to parsing the http status code
+// when it cannot match an aws error code. This method does not handle specific
+// service type errors such as S3 or EC2.
+func ParseAWSError(err error, msg string) (st *status.Status) {
+	if err == nil {
+		return nil
+	}
+
+	// find the service name of the aws api
+	serviceName := "unknown"
+	var oe *smithy.OperationError
+	if errors.As(err, &oe) {
+		serviceName = oe.Service()
+	}
+
+	// The unknown plugin error code will be used
+	// when the aws service returns an error that
+	// has failed to be parsed. By default, this
+	// error code is retryable.
+	plgErr := &pb.PluginError{
+		Code:         pb.ERROR_ERROR_UNKNOWN,
+		Message:      err.Error(),
+		Nonretryable: false,
+	}
+
+	defer func() {
+		if st == nil {
+			statusMsg := fmt.Sprintf("aws service %s: unknown error: %s", serviceName, msg)
+			st = status.New(codes.Unknown, statusMsg)
+		}
+		st, err = st.WithDetails(plgErr)
+		if err != nil {
+			st = status.New(codes.Internal, err.Error())
+		}
+	}()
+
+	// evaluate against aws throttling error codes
+	throttleErr := retry.ThrottleErrorCode{
+		Codes: retry.DefaultThrottleErrorCodes,
+	}
+	if throttleErr.IsErrorThrottle(err).Bool() {
+		plgErr.Code = pb.ERROR_ERROR_THROTTLING
+		statusMsg := fmt.Sprintf("aws service %s: throttling error: %s", serviceName, msg)
+		return status.New(codes.Unavailable, statusMsg)
+	}
+	// evaluate against aws connection error codes
+	connectionErr := retry.IsErrorRetryables([]retry.IsErrorRetryable{
+		retry.NoRetryCanceledError{},
+		retry.RetryableError{},
+		retry.RetryableConnectionError{},
+		retry.RetryableHTTPStatusCode{
+			Codes: retry.DefaultRetryableHTTPStatusCodes,
+		},
+		retry.RetryableErrorCode{
+			Codes: retry.DefaultRetryableErrorCodes,
+		},
+	})
+	if connectionErr.IsErrorRetryable(err).Bool() {
+		plgErr.Code = pb.ERROR_ERROR_TIMEOUT
+		statusMsg := fmt.Sprintf("aws service %s: connectivity error: %s", serviceName, msg)
+		return status.New(codes.DeadlineExceeded, statusMsg)
+	}
+	// evaluate against aws credentials error codes
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) {
+		statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg)
+		switch apiErr.ErrorCode() {
+		case awsErrorAccessDenied:
+			fallthrough
+		case awsErrorInvalidAccessKeyId:
+			plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL
+			plgErr.Nonretryable = true
+			return status.New(codes.PermissionDenied, statusMsg)
+		case awsErrorExpiredToken:
+			plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL
+			return status.New(codes.PermissionDenied, statusMsg)
+		}
+	}
+
+	// default to evaluating the http status code
+	if httpErr, ok := err.(*awshttp.ResponseError); ok {
+		buf := new(bytes.Buffer)
+		_, err = buf.ReadFrom(httpErr.Response.Body)
+		if err != nil {
+			return status.New(codes.Internal, err.Error())
+		}
+		defer httpErr.Response.Body.Close()
+		plgErr.Message = buf.String()
+		switch httpErr.HTTPStatusCode() {
+		case http.StatusBadRequest:
+			plgErr.Code = pb.ERROR_ERROR_BAD_REQUEST
+			plgErr.Nonretryable = true
+			statusMsg := fmt.Sprintf("aws service %s: bad request error: %s", serviceName, msg)
+			return status.New(codes.InvalidArgument, statusMsg)
+		case http.StatusUnauthorized:
+			plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL
+			statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg)
+			return status.New(codes.PermissionDenied, statusMsg)
+		case http.StatusForbidden:
+			plgErr.Code = pb.ERROR_ERROR_INVALID_CREDENTIAL
+			plgErr.Nonretryable = true
+			statusMsg := fmt.Sprintf("aws service %s: invalid credentials error: %s", serviceName, msg)
+			return status.New(codes.PermissionDenied, statusMsg)
+		case http.StatusNotFound:
+			plgErr.Code = pb.ERROR_ERROR_BAD_REQUEST
+			plgErr.Nonretryable = true
+			statusMsg := fmt.Sprintf("aws service %s: resource not found error: %s", serviceName, msg)
+			return status.New(codes.NotFound, statusMsg)
+		case http.StatusTooManyRequests:
+			plgErr.Code = pb.ERROR_ERROR_THROTTLING
+			statusMsg := fmt.Sprintf("aws service %s: throttling error: %s", serviceName, msg)
+			return status.New(codes.Unavailable, statusMsg)
+		case http.StatusRequestTimeout:
+			fallthrough
+		case http.StatusInternalServerError:
+			fallthrough
+		case http.StatusBadGateway:
+			fallthrough
+		case http.StatusServiceUnavailable:
+			fallthrough
+		case http.StatusGatewayTimeout:
+			plgErr.Code = pb.ERROR_ERROR_TIMEOUT
+			statusMsg := fmt.Sprintf("aws service %s: connectivity error: %s", serviceName, msg)
+			return status.New(codes.DeadlineExceeded, statusMsg)
+		}
+	}
+
+	return st
+}

internal/errors/errror_test.go

@@ -0,0 +1,382 @@
+package errors
+
+import (
+	"fmt"
+	"testing"
+
+	"net/http"
+
+	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+)
+
+func Test_ParseAWSError(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name                string
+		err                 error
+		expectedStatusCode  codes.Code
+		expectedStatusMsg   string
+		expectedPluginError *pb.PluginError
+	}{
+		{
+			name:               "nil-error",
+			expectedStatusCode: codes.OK,
+		},
+		{
+			name:               "default-to-unknwon",
+			err:                fmt.Errorf("what can this error be?"),
+			expectedStatusCode: codes.Unknown,
+			expectedStatusMsg:  "aws service unknown: unknown error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_UNKNOWN,
+				Message:      "what can this error be?",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-bad-request",
+			err:                TestAwsHttpResponseError(http.StatusBadRequest, "bad request"),
+			expectedStatusCode: codes.InvalidArgument,
+			expectedStatusMsg:  "aws service unknown: bad request error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_BAD_REQUEST,
+				Message:      "bad request",
+				Nonretryable: true,
+			},
+		},
+		{
+			name:               "http-status-code-unauthorized",
+			err:                TestAwsHttpResponseError(http.StatusUnauthorized, "unauthorized request"),
+			expectedStatusCode: codes.PermissionDenied,
+			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_INVALID_CREDENTIAL,
+				Message:      "unauthorized request",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-forbidden",
+			err:                TestAwsHttpResponseError(http.StatusForbidden, "forbidden request"),
+			expectedStatusCode: codes.PermissionDenied,
+			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_INVALID_CREDENTIAL,
+				Message:      "forbidden request",
+				Nonretryable: true,
+			},
+		},
+		{
+			name:               "http-status-code-not-found",
+			err:                TestAwsHttpResponseError(http.StatusNotFound, "not found request"),
+			expectedStatusCode: codes.NotFound,
+			expectedStatusMsg:  "aws service unknown: resource not found error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_BAD_REQUEST,
+				Message:      "not found request",
+				Nonretryable: true,
+			},
+		},
+		{
+			name:               "http-status-code-too-many-requests",
+			err:                TestAwsHttpResponseError(http.StatusTooManyRequests, "too many requests"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "too many requests",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-request-timeout",
+			err:                TestAwsHttpResponseError(http.StatusRequestTimeout, "request timeout"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "request timeout",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-internal-server-error",
+			err:                TestAwsHttpResponseError(http.StatusInternalServerError, "internal server error"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "https response error StatusCode: 500",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-bad-gateway",
+			err:                TestAwsHttpResponseError(http.StatusBadGateway, "bad gateway"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "https response error StatusCode: 502",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-service-unavailable",
+			err:                TestAwsHttpResponseError(http.StatusServiceUnavailable, "service unavailable"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "https response error StatusCode: 503",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "http-status-code-gateway-timeout",
+			err:                TestAwsHttpResponseError(http.StatusGatewayTimeout, "gateway timeout"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "https response error StatusCode: 504",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-throttling",
+			err:                TestAwsError("Throttling", "throttling error"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "throttling error",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-throttling-exception",
+			err:                TestAwsError("ThrottlingException", "throttling exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "throttling exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-throttled-exception",
+			err:                TestAwsError("ThrottledException", "throttled exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "throttled exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-throttled-exception",
+			err:                TestAwsError("RequestThrottledException", "request throttled exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "request throttled exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-too-many-requests-exception",
+			err:                TestAwsError("TooManyRequestsException", "too many requests exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "too many requests exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-provisioned-throughput-exceeded-exception",
+			err:                TestAwsError("ProvisionedThroughputExceededException", "provisioned throughput exceeded exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "provisioned throughput exceeded exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-transaction-in-progress-exception",
+			err:                TestAwsError("TransactionInProgressException", "transaction in progress exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "transaction in progress exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-limit-exceeded",
+			err:                TestAwsError("RequestLimitExceeded", "request limit exceeded"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "request limit exceeded",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-bandwidth-limit-exceeded",
+			err:                TestAwsError("BandwidthLimitExceeded", "bandwidth limit exceeded"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "bandwidth limit exceeded",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-limit-exceeded-exception",
+			err:                TestAwsError("LimitExceededException", "limit exceeded exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "limit exceeded exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-throttled",
+			err:                TestAwsError("RequestThrottled", "request throttled"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "request throttled",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-slow-down",
+			err:                TestAwsError("SlowDown", "slow down"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "slow down",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-prior-request-not-complete",
+			err:                TestAwsError("PriorRequestNotComplete", "prior request not complete"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "prior request not complete",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-ec2-throttled-exception",
+			err:                TestAwsError("EC2ThrottledException", "ec2 throttled exception"),
+			expectedStatusCode: codes.Unavailable,
+			expectedStatusMsg:  "aws service unknown: throttling error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_THROTTLING,
+				Message:      "ec2 throttled exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-timeout",
+			err:                TestAwsError("RequestTimeout", "request timeout"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "request timeout",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-timeout-exception",
+			err:                TestAwsError("RequestTimeoutException", "request timeout exception"),
+			expectedStatusCode: codes.DeadlineExceeded,
+			expectedStatusMsg:  "aws service unknown: connectivity error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_TIMEOUT,
+				Message:      "request timeout exception",
+				Nonretryable: false,
+			},
+		},
+		{
+			name:               "aws-request-access-denied",
+			err:                TestAwsError(awsErrorAccessDenied, "not authorized to perform action"),
+			expectedStatusCode: codes.PermissionDenied,
+			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_INVALID_CREDENTIAL,
+				Message:      "not authorized to perform action",
+				Nonretryable: true,
+			},
+		},
+		{
+			name:               "aws-request-invalid-access-key-id",
+			err:                TestAwsError(awsErrorInvalidAccessKeyId, "The AWS Access Key Id you provided does not exist in our records."),
+			expectedStatusCode: codes.PermissionDenied,
+			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_INVALID_CREDENTIAL,
+				Message:      "The AWS Access Key Id you provided does not exist in our records.",
+				Nonretryable: true,
+			},
+		},
+		{
+			name:               "aws-request-expired-token",
+			err:                TestAwsError(awsErrorExpiredToken, "The security token included in the request is expired."),
+			expectedStatusCode: codes.PermissionDenied,
+			expectedStatusMsg:  "aws service unknown: invalid credentials error: test",
+			expectedPluginError: &pb.PluginError{
+				Code:         pb.ERROR_ERROR_INVALID_CREDENTIAL,
+				Message:      "The security token included in the request is expired.",
+				Nonretryable: false,
+			},
+		},
+	}
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			require := require.New(t)
+
+			actualStatus := ParseAWSError(tt.err, "test")
+			if tt.expectedStatusCode == codes.OK {
+				require.Nil(actualStatus)
+				return
+			}
+
+			require.Equal(tt.expectedStatusCode, actualStatus.Code())
+			require.Equal(tt.expectedStatusMsg, actualStatus.Message())
+			require.Len(actualStatus.Details(), 1)
+			for _, detail := range actualStatus.Details() {
+				switch errDetail := detail.(type) {
+				case *pb.PluginError:
+					require.Equal(tt.expectedPluginError.Code, errDetail.Code)
+					require.Contains(errDetail.Message, tt.expectedPluginError.Message)
+					require.Equal(tt.expectedPluginError.Nonretryable, errDetail.Nonretryable)
+				}
+			}
+		})
+	}
+}

internal/errors/testing.go

@@ -0,0 +1,34 @@
+package errors
+
+import (
+	"io"
+	"net/http"
+	"strings"
+
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/smithy-go"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+// TestAwsHttpResponseError returns an aws http response error
+func TestAwsHttpResponseError(code int, msg string) error {
+	return &awshttp.ResponseError{
+		ResponseError: &smithyhttp.ResponseError{
+			Response: &smithyhttp.Response{
+				Response: &http.Response{
+					StatusCode: code,
+					Body:       io.NopCloser(strings.NewReader(msg)),
+				},
+			},
+		},
+	}
+}
+
+// TestAwsError returns an generic api error
+func TestAwsError(code, msg string) error {
+	return &smithy.GenericAPIError{
+		Code:    code,
+		Message: msg,
+		Fault:   smithy.FaultServer,
+	}
+}