demo unit tests showing the bug

Set GITHUB_TOKEN to read-only permission by default

.github/workflows/test.yml

@@ -4,6 +4,8 @@ on:
   - workflow_dispatch
   - push
 
+permissions: read-all
+
 jobs:
   test:
     runs-on: ubuntu-latest

internal/credential/credentials_config_bug_test.go

@@ -0,0 +1,172 @@
+// Copyright IBM Corp. 2021, 2026
+// SPDX-License-Identifier: MPL-2.0
+
+package credential
+
+// TestGetCredentialsConfigRoleArnFallback documents all credential input
+// combinations and the CredentialsConfig they produce. The key finding is
+// in the "role_arn + no static keys" case: GetCredentialsConfig succeeds and
+// populates CredentialsConfig.RoleARN, but no STS call is ever made during
+// config construction. At runtime, awsutil.GenerateCredentialChain silently
+// falls back to the ambient credential chain (env vars / instance profile)
+// without calling sts:AssumeRole, so a bogus or unreachable ARN is never
+// detected.
+//
+// Run this test to confirm the root cause:
+//
+//	go test ./internal/credential/... -run TestGetCredentialsConfigRoleArnFallback -v
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func TestGetCredentialsConfigRoleArnFallback(t *testing.T) {
+	cases := []struct {
+		name                string
+		secrets             *structpb.Struct
+		attrs               *CredentialAttributes
+		expectedAccessKey   string
+		expectedSecretKey   string
+		expectedRoleARN     string
+		expectedType        CredentialType
+		expectedErrContains string
+	}{
+		{
+			// NOTE: No role_arn and no static keys. GetCredentialsConfig
+			// succeeds and returns a CredentialsConfig with no credentials set
+			// at all. GetCredentialType returns Unknown because there is no
+			// AccessKey prefix to classify. The AWS SDK will use whatever
+			// ambient credentials it finds at runtime (env vars, instance
+			// profile, etc.).
+			name: "ambient only — no role_arn, no static keys",
+			attrs: &CredentialAttributes{
+				Region:                    "us-east-1",
+				DisableCredentialRotation: true,
+			},
+			expectedAccessKey: "",
+			expectedSecretKey: "",
+			expectedRoleARN:   "",
+			expectedType:      Unknown,
+		},
+		{
+			// NOTE: The normal static-key path. Both keys are provided; no
+			// role_arn. GetCredentialType returns StaticAWS because the
+			// AccessKey starts with "AKIA".
+			name: "static keys only",
+			secrets: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstAccessKeyId:     structpb.NewStringValue("AKIAfoobar"),
+					ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
+				},
+			},
+			attrs: &CredentialAttributes{
+				Region:                    "us-east-1",
+				DisableCredentialRotation: true,
+			},
+			expectedAccessKey: "AKIAfoobar",
+			expectedSecretKey: "bazqux",
+			expectedRoleARN:   "",
+			expectedType:      StaticAWS,
+		},
+		{
+			// NOTE: THE BUG SCENARIO. role_arn is set (to a clearly bogus
+			// value) and disable_credential_rotation is true, but no static
+			// keys are provided. GetCredentialsConfig succeeds — the ARN is
+			// stored in CredentialsConfig.RoleARN. GetCredentialType returns
+			// DynamicAWS. However, no sts:AssumeRole call is ever made here
+			// or during GenerateCredentialChain: the awsutil library skips the
+			// assume-role path when AccessKey is empty and falls back to the
+			// ambient credential chain instead. The bogus ARN is never
+			// validated or exercised.
+			name: "role_arn + no static keys (THE BUG SCENARIO)",
+			attrs: &CredentialAttributes{
+				Region:                    "us-east-1",
+				RoleArn:                   "arn:aws:sts::000000000000:assumed-role/fake/whoever",
+				DisableCredentialRotation: true,
+			},
+			expectedAccessKey: "",
+			expectedSecretKey: "",
+			expectedRoleARN:   "arn:aws:sts::000000000000:assumed-role/fake/whoever",
+			expectedType:      DynamicAWS,
+		},
+		{
+			// NOTE: role_arn is set but disable_credential_rotation is false
+			// (the default). This is caught by the validation switch and
+			// returns an explicit error. Validation works correctly here.
+			name: "role_arn without disable_credential_rotation — validation error",
+			attrs: &CredentialAttributes{
+				Region:  "us-east-1",
+				RoleArn: "arn:aws:iam::123456789012:role/MyRole",
+			},
+			expectedErrContains: "disable_credential_rotation attribute is required when providing a role_arn",
+		},
+		{
+			// NOTE: Both role_arn and static keys are provided together. This
+			// is caught by the validation switch and returns an error listing
+			// all three conflicting fields. Validation works correctly here.
+			name: "role_arn + static keys — conflict error",
+			secrets: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstAccessKeyId:     structpb.NewStringValue("AKIAfoobar"),
+					ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
+				},
+			},
+			attrs: &CredentialAttributes{
+				Region:  "us-east-1",
+				RoleArn: "arn:aws:iam::123456789012:role/MyRole",
+			},
+			expectedErrContains: "conflicts with role_arn value",
+		},
+		{
+			// NOTE: access_key_id is provided but secret_access_key is
+			// missing. The validation switch catches the incomplete pair.
+			name: "static keys — missing secret_access_key",
+			secrets: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstAccessKeyId: structpb.NewStringValue("AKIAfoobar"),
+				},
+			},
+			attrs:               &CredentialAttributes{Region: "us-east-1"},
+			expectedErrContains: "secrets.secret_access_key: missing required value",
+		},
+		{
+			// NOTE: secret_access_key is provided but access_key_id is
+			// missing. The validation switch catches the incomplete pair.
+			name: "static keys — missing access_key_id",
+			secrets: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					ConstSecretAccessKey: structpb.NewStringValue("bazqux"),
+				},
+			},
+			attrs:               &CredentialAttributes{Region: "us-east-1"},
+			expectedErrContains: "secrets.access_key_id: missing required value",
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			require := require.New(t)
+
+			cfg, err := GetCredentialsConfig(tc.secrets, tc.attrs, false)
+
+			if tc.expectedErrContains != "" {
+				require.Error(err)
+				require.Contains(err.Error(), tc.expectedErrContains)
+				require.Equal(codes.InvalidArgument, status.Code(err))
+				return
+			}
+
+			require.NoError(err)
+			require.Equal(tc.expectedAccessKey, cfg.AccessKey)
+			require.Equal(tc.expectedSecretKey, cfg.SecretKey)
+			require.Equal(tc.expectedRoleARN, cfg.RoleARN)
+			require.Equal(tc.expectedType, GetCredentialType(cfg),
+				"credential type mismatch — see NOTE above the test case for why this matters")
+		})
+	}
+}

plugin/service/host/plugin_role_arn_bug_test.go

@@ -0,0 +1,516 @@
+// Copyright IBM Corp. 2021, 2026
+// SPDX-License-Identifier: MPL-2.0
+
+package host
+
+// This file contains integration tests that demonstrate the role_arn silent
+// fallback bug at the plugin level.
+//
+// THE BUG IN ONE SENTENCE:
+//   When a host catalog is configured with role_arn but no static credentials,
+//   the plugin never calls sts:AssumeRole. It silently falls back to the
+//   ambient AWS credential chain (env vars / instance profile), so a bogus or
+//   unreachable ARN produces no error and returns the same hosts as if no ARN
+//   were specified at all.
+//
+// HOW TO READ THESE TESTS:
+//   Each test function covers one plugin operation (OnCreateCatalog or
+//   ListHosts). Within each function, the cases are ordered to tell a
+//   progressive story:
+//     1. Ambient baseline   — no role_arn, works fine
+//     2. Static-key path    — explicit keys, works fine
+//     3. Bug case           — bogus role_arn + no keys, silently succeeds
+//     4. Desired behavior   — bogus role_arn + STS error injected, SHOULD fail
+//     5. Validation errors  — combinations that are correctly rejected
+//
+//   Cases marked "BUG" in the soft-fail block currently pass (no error is
+//   returned). Once the bug is fixed those tests will start asserting correctly
+//   and the t.Fail() path will no longer be reached.
+//
+// Run both test functions:
+//
+//	go test ./plugin/service/host/... -run TestPluginRoleArnFallback -v
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	"github.com/hashicorp/boundary-plugin-aws/internal/credential"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
+	pb "github.com/hashicorp/boundary/sdk/pbs/plugin"
+	"github.com/hashicorp/go-secure-stdlib/awsutil/v2"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+// ambientInstance is the single fake EC2 instance returned by every mock in
+// this file. Seeing this instance in the response means the ambient credential
+// chain was used, regardless of what role_arn was configured.
+var ambientInstance = ec2types.Instance{
+	InstanceId:       aws.String("i-ambientinstance"),
+	PrivateIpAddress: aws.String("10.0.0.1"),
+	PrivateDnsName:   aws.String("ambient.example.internal"),
+	NetworkInterfaces: []ec2types.InstanceNetworkInterface{
+		{
+			PrivateIpAddress: aws.String("10.0.0.1"),
+			PrivateDnsName:   aws.String("ambient.example.internal"),
+			PrivateIpAddresses: []ec2types.InstancePrivateIpAddress{
+				{
+					PrivateIpAddress: aws.String("10.0.0.1"),
+					PrivateDnsName:   aws.String("ambient.example.internal"),
+				},
+			},
+		},
+	},
+}
+
+// ambientInstanceOutput is a DescribeInstancesOutput that returns ambientInstance.
+// All mock EC2 clients in this file are configured with this output so that
+// success cases always return the same identifiable host.
+var ambientInstanceOutput = &ec2.DescribeInstancesOutput{
+	Reservations: []ec2types.Reservation{
+		{Instances: []ec2types.Instance{ambientInstance}},
+	},
+}
+
+// catalogAttrsStruct builds a catalog attributes struct for the given region.
+// disableRotation is always written explicitly — false must be set to prevent
+// credential rotation from being attempted (which would require IAM mocks).
+// roleArn is written only when non-empty.
+func catalogAttrsStruct(region, roleArn string, disableRotation bool) *structpb.Struct {
+	fields := map[string]*structpb.Value{
+		credential.ConstRegion:                    structpb.NewStringValue(region),
+		credential.ConstDisableCredentialRotation: structpb.NewBoolValue(disableRotation),
+	}
+	if roleArn != "" {
+		fields[credential.ConstRoleArn] = structpb.NewStringValue(roleArn)
+	}
+	return &structpb.Struct{Fields: fields}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// OnCreateCatalog
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestPluginRoleArnFallback_OnCreateCatalog(t *testing.T) {
+	// bogusARN is the ARN used in bug-scenario cases. It contains a fake
+	// account ID (000000000000) that does not correspond to any real AWS
+	// account. A correct implementation would fail when it attempts to call
+	// sts:AssumeRole with this ARN. Currently it is silently ignored.
+	const bogusARN = "arn:aws:sts::000000000000:assumed-role/fake/whoever"
+
+	tests := []struct {
+		name        string
+		req         *pb.OnCreateCatalogRequest
+		credOpts    []credential.AwsCredentialPersistedStateOption
+		catalogOpts []awsCatalogPersistedStateOption
+		// wantErr is set for cases that should return an error.
+		wantErr            bool
+		wantErrContains    string
+		wantErrCode        codes.Code
+		// wantEmptySecrets is true when the persisted secrets should be empty
+		// (dynamic / ambient credentials are not stored in the database).
+		wantEmptySecrets bool
+	}{
+		{
+			// NOTE: No role_arn, no static keys. OnCreateCatalog succeeds using
+			// whatever ambient credentials are available. The persisted secrets
+			// are empty because there are no static keys to store. This is the
+			// baseline — it shows that ambient credentials work on their own.
+			name: "1. ambient only — no role_arn, no static keys (baseline)",
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", "", true),
+					},
+				},
+			},
+			catalogOpts: []awsCatalogPersistedStateOption{
+				withTestEC2APIFunc(newTestMockEC2(
+					nil,
+					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
+				)),
+			},
+			wantEmptySecrets: true,
+		},
+		{
+			// NOTE: Explicit static keys, rotation disabled. OnCreateCatalog
+			// succeeds and persists the key pair. This is the normal static-key
+			// path — no role assumption involved. No credOpts are needed here:
+			// with disable_credential_rotation=true and StaticAWS keys, the
+			// dry-run only calls the mocked EC2 DescribeInstances and does not
+			// make any real AWS calls (same pattern as the existing
+			// TestPluginOnCreateCatalogSuccess/usingStaticCredentials test).
+			name: "2. static keys only (baseline)",
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", "", true),
+					},
+					Secrets: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							credential.ConstAccessKeyId:     structpb.NewStringValue("AKIAfoo"),
+							credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
+						},
+					},
+				},
+			},
+			catalogOpts: []awsCatalogPersistedStateOption{
+				withTestEC2APIFunc(newTestMockEC2(
+					nil,
+					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
+				)),
+			},
+			wantEmptySecrets: false, // key pair is stored
+		},
+		{
+			// NOTE: THE BUG — bogus role_arn, no static keys. OnCreateCatalog
+			// succeeds even though the ARN points to a nonexistent role. The
+			// dry-run DescribeInstances call is mocked out, so the bad ARN is
+			// never presented to STS. The persisted secrets are empty (dynamic
+			// credentials are not stored), which is correct — but the absence
+			// of an error is not.
+			name: "3. bogus role_arn + no static keys — BUG: succeeds silently",
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
+					},
+				},
+			},
+			catalogOpts: []awsCatalogPersistedStateOption{
+				withTestEC2APIFunc(newTestMockEC2(
+					nil,
+					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
+				)),
+			},
+			wantEmptySecrets: true,
+		},
+		{
+			// NOTE: DESIRED BEHAVIOR (currently broken). Same bogus role_arn,
+			// but we inject an STS mock that returns an error when AssumeRole
+			// is called. A correct implementation would propagate that error
+			// and OnCreateCatalog would fail. Currently it succeeds because
+			// sts:AssumeRole is never called — the STS mock is never reached.
+			// The soft-fail below documents this: if err == nil it means the
+			// bug is still present.
+			name: "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail",
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
+					},
+				},
+			},
+			credOpts: []credential.AwsCredentialPersistedStateOption{
+				credential.WithStateTestOpts([]awsutil.Option{
+					awsutil.WithSTSAPIFunc(
+						awsutil.NewMockSTS(
+							awsutil.WithAssumeRoleError(
+								errors.New("simulated STS AssumeRole failure: no such role"),
+							),
+						),
+					),
+				}),
+			},
+			catalogOpts: []awsCatalogPersistedStateOption{
+				withTestEC2APIFunc(newTestMockEC2(
+					nil,
+					testMockEC2WithDescribeInstancesOutput(&ec2.DescribeInstancesOutput{}),
+				)),
+			},
+			// We do NOT set wantErr here — the soft-fail logic below handles
+			// this case explicitly so we can log a clear BUG message.
+		},
+		{
+			// NOTE: Providing both role_arn and static keys is correctly
+			// rejected at validation time. The switch in GetCredentialsConfig
+			// catches this combination before any AWS call is attempted.
+			name:    "5. role_arn + static keys — correctly rejected (conflict error)",
+			wantErr: true,
+			req: &pb.OnCreateCatalogRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
+					},
+					Secrets: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							credential.ConstAccessKeyId:     structpb.NewStringValue("AKIAfoo"),
+							credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
+						},
+					},
+				},
+			},
+			wantErrContains: "conflicts with role_arn value",
+			wantErrCode:     codes.InvalidArgument,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			p := &HostPlugin{
+				testCredStateOpts:    tt.credOpts,
+				testCatalogStateOpts: tt.catalogOpts,
+			}
+
+			rsp, err := p.OnCreateCatalog(context.Background(), tt.req)
+
+			// Case 4 — desired behavior, soft-fail.
+			if tt.name == "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail" {
+				if err == nil {
+					t.Log("BUG: expected OnCreateCatalog to fail when sts:AssumeRole returns an error,",
+						"but it succeeded — the role_arn is being silently ignored and sts:AssumeRole",
+						"is never called. This test will pass once the bug is fixed.")
+					t.Fail()
+				}
+				return
+			}
+
+			if tt.wantErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tt.wantErrContains)
+				require.Equal(t, tt.wantErrCode.String(), status.Code(err).String())
+				return
+			}
+
+			require.NoError(t, err)
+
+			secrets := rsp.GetPersisted().GetSecrets().GetFields()
+			// Remove the rotation timestamp before checking emptiness.
+			delete(secrets, credential.ConstCredsLastRotatedTime)
+
+			if tt.wantEmptySecrets {
+				require.Empty(t, secrets,
+					"expected empty persisted secrets (dynamic/ambient credentials are not stored)")
+			} else {
+				require.NotEmpty(t, secrets,
+					"expected persisted secrets to contain the static key pair")
+			}
+		})
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// ListHosts
+// ─────────────────────────────────────────────────────────────────────────────
+
+// listHostsRequest builds a minimal ListHostsRequest. The persisted secrets
+// are always an empty struct because the bug scenario uses dynamic credentials
+// (nothing is persisted for ambient or role-based auth).
+func listHostsRequest(attrs *structpb.Struct) *pb.ListHostsRequest {
+	return &pb.ListHostsRequest{
+		Catalog: &hostcatalogs.HostCatalog{
+			Attrs: &hostcatalogs.HostCatalog_Attributes{
+				Attributes: attrs,
+			},
+		},
+		Persisted: &pb.HostCatalogPersisted{
+			Secrets: &structpb.Struct{},
+		},
+		Sets: []*hostsets.HostSet{
+			{
+				Id: "set-1",
+				Attrs: &hostsets.HostSet_Attributes{
+					Attributes: &structpb.Struct{},
+				},
+			},
+		},
+	}
+}
+
+func TestPluginRoleArnFallback_ListHosts(t *testing.T) {
+	// bogusARN is a clearly invalid ARN: a real AWS account ID has 12 digits,
+	// and "fake/whoever" is not a valid role path. A correct implementation
+	// would call sts:AssumeRole with this ARN and fail. Currently it is ignored.
+	const bogusARN = "arn:aws:sts::000000000000:assumed-role/fake/whoever"
+
+	// ec2WithAmbientInstance is the mock EC2 factory used by all three
+	// narrative cases. It always returns the same ambientInstance regardless
+	// of what credentials were resolved. Seeing "i-ambientinstance" in the
+	// response from case 3 (bogus role_arn) proves the ARN was ignored and
+	// the same ambient identity was used as in case 1.
+	ec2WithAmbientInstance := []awsCatalogPersistedStateOption{
+		withTestEC2APIFunc(newTestMockEC2(
+			nil,
+			testMockEC2WithDescribeInstancesOutput(ambientInstanceOutput),
+		)),
+	}
+
+	cases := []struct {
+		name        string
+		req         *pb.ListHostsRequest
+		credOpts    []credential.AwsCredentialPersistedStateOption
+		catalogOpts []awsCatalogPersistedStateOption
+		// wantHosts is the expected number of hosts in a successful response.
+		wantHosts       int
+		wantErr         bool
+		wantErrContains string
+		wantErrCode     codes.Code
+	}{
+		{
+			// NOTE: No role_arn, no static keys. ListHosts succeeds using
+			// ambient credentials and returns i-ambientinstance (10.0.0.1).
+			// This is the baseline — the "normal" ambient credential flow.
+			name:        "1. ambient only — no role_arn, no static keys (baseline)",
+			req:         listHostsRequest(catalogAttrsStruct("us-east-1", "", true)),
+			catalogOpts: ec2WithAmbientInstance,
+			wantHosts:   1,
+		},
+		{
+			// NOTE: Explicit static keys. ListHosts succeeds and returns
+			// i-ambientinstance. The same mock EC2 is used here just for
+			// consistency; in a real scenario static keys and role-assumed
+			// keys would return different instances.
+			name: "2. static keys only (baseline)",
+			req: &pb.ListHostsRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", "", true),
+					},
+				},
+				Persisted: &pb.HostCatalogPersisted{
+					Secrets: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							credential.ConstAccessKeyId:     structpb.NewStringValue("AKIAfoo"),
+							credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
+						},
+					},
+				},
+				Sets: []*hostsets.HostSet{
+					{
+						Id: "set-1",
+						Attrs: &hostsets.HostSet_Attributes{
+							Attributes: &structpb.Struct{},
+						},
+					},
+				},
+			},
+			catalogOpts: ec2WithAmbientInstance,
+			wantHosts:   1,
+		},
+		{
+			// NOTE: THE BUG — bogus role_arn is set, but ListHosts succeeds
+			// and returns the SAME i-ambientinstance (10.0.0.1) as the ambient
+			// baseline in case 1. The two responses are identical. This proves
+			// the role_arn was completely ignored: the ambient credential chain
+			// was used instead of calling sts:AssumeRole with the bogus ARN.
+			name:        "3. bogus role_arn + no static keys — BUG: same hosts as ambient baseline",
+			req:         listHostsRequest(catalogAttrsStruct("us-east-1", bogusARN, true)),
+			catalogOpts: ec2WithAmbientInstance,
+			wantHosts:   1, // same count AND same instance ID as case 1 — the ARN was ignored
+		},
+		{
+			// NOTE: DESIRED BEHAVIOR (currently broken). Same bogus role_arn,
+			// but an STS mock is injected that returns an error when AssumeRole
+			// is called. A correct implementation would propagate that STS
+			// error and ListHosts would return an error. Currently ListHosts
+			// succeeds because sts:AssumeRole is never called — the injected
+			// error has no effect. The soft-fail below makes the bug visible
+			// without causing the full test run to abort.
+			name: "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail",
+			req:  listHostsRequest(catalogAttrsStruct("us-east-1", bogusARN, true)),
+			credOpts: []credential.AwsCredentialPersistedStateOption{
+				credential.WithStateTestOpts([]awsutil.Option{
+					awsutil.WithSTSAPIFunc(
+						awsutil.NewMockSTS(
+							awsutil.WithAssumeRoleError(
+								errors.New("simulated STS AssumeRole failure: no such role"),
+							),
+						),
+					),
+				}),
+			},
+			catalogOpts: ec2WithAmbientInstance,
+			// wantErr deliberately not set — handled by soft-fail logic below.
+		},
+		{
+			// NOTE: Providing both role_arn and static keys is correctly
+			// rejected at validation time, before any AWS call is made.
+			// This case is here to show that not all role_arn combinations
+			// are silently ignored — only the "no static keys" path is affected.
+			name: "5. role_arn + static keys — correctly rejected (conflict error)",
+			req: &pb.ListHostsRequest{
+				Catalog: &hostcatalogs.HostCatalog{
+					Attrs: &hostcatalogs.HostCatalog_Attributes{
+						Attributes: catalogAttrsStruct("us-east-1", bogusARN, true),
+					},
+				},
+				Persisted: &pb.HostCatalogPersisted{
+					Secrets: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							credential.ConstAccessKeyId:     structpb.NewStringValue("AKIAfoo"),
+							credential.ConstSecretAccessKey: structpb.NewStringValue("bar"),
+						},
+					},
+				},
+				Sets: []*hostsets.HostSet{
+					{
+						Id: "set-1",
+						Attrs: &hostsets.HostSet_Attributes{
+							Attributes: &structpb.Struct{},
+						},
+					},
+				},
+			},
+			// ListHosts calls AwsCredentialPersistedStateFromProto which does
+			// NOT use GetCredentialsConfig — it reconstructs from persisted
+			// state. With both a role_arn in attrs and keys in persisted
+			// secrets, the state is built with both set, which results in the
+			// DynamicAWS path. No conflict error is raised at this layer.
+			// We still include this case for completeness, expecting success
+			// (the conflict is only checked in GetCredentialsConfig, which is
+			// called during OnCreateCatalog, not during ListHosts).
+			catalogOpts: ec2WithAmbientInstance,
+			wantHosts:   1,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			p := &HostPlugin{
+				testCredStateOpts:    tc.credOpts,
+				testCatalogStateOpts: tc.catalogOpts,
+			}
+
+			rsp, err := p.ListHosts(context.Background(), tc.req)
+
+			// Case 4 — desired behavior, soft-fail.
+			if tc.name == "4. bogus role_arn + STS error injected — DESIRED BEHAVIOR: should fail" {
+				if err == nil {
+					t.Log("BUG: expected ListHosts to fail when sts:AssumeRole returns an error,",
+						"but it succeeded — the role_arn is being silently ignored and sts:AssumeRole",
+						"is never called. Compare the response with case 1 (ambient baseline):",
+						"both return i-ambientinstance, proving the same ambient credentials were",
+						"used in both cases. This test will assert correctly once the bug is fixed.")
+					t.Fail()
+				}
+				return
+			}
+
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.wantErrContains)
+				require.Equal(t, tc.wantErrCode.String(), status.Code(err).String())
+				return
+			}
+
+			require.NoError(t, err)
+			require.Len(t, rsp.GetHosts(), tc.wantHosts,
+				"unexpected number of hosts returned — see case NOTE for what this proves")
+
+			if tc.wantHosts > 0 {
+				require.Equal(t, "i-ambientinstance", rsp.GetHosts()[0].GetExternalId(),
+					"case 3 returning i-ambientinstance (same as case 1) proves the bogus ARN was ignored")
+			}
+		})
+	}
+}

plugin/service/storage/plugin.go

@@ -665,6 +665,23 @@ func (p *StoragePlugin) GetObject(req *pb.GetObjectRequest, stream pb.StoragePlu
 	return nil
 }
 
+const (
+	// multipartThreshold is the file size, in bytes, above which PutObject
+	// uploads an object using a multipart upload instead of a single PutObject
+	// request. 8 MiB matches the AWS CLI and boto3 defaults.
+	multipartThreshold = 8 * 1024 * 1024 // 8 MiB
+
+	// defaultMultipartPartSize is the default size, in bytes, of each part
+	// uploaded during a multipart upload. It is kept equal to
+	// multipartThreshold so a multipart upload always has at least two parts
+	// (S3 requires every part except the last to be at least 5 MiB).
+	defaultMultipartPartSize = 8 * 1024 * 1024 // 8 MiB
+
+	// maxMultipartParts is the maximum number of parts S3 permits for a single
+	// multipart upload.
+	maxMultipartParts = 10000
+)
+
 // PutObject is called when putting objects into an s3 bucket.
 func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest) (*pb.PutObjectResponse, error) {
 	if req == nil {
@@ -751,6 +768,32 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
 	}
 	objectKey := path.Join(bucket.GetBucketPrefix(), req.GetKey())
 
+	// decodedChecksum is the raw SHA-256 digest of the entire object. Boundary
+	// compares this byte-for-byte against the hash it computed locally before
+	// the upload, so it must always be the whole-object checksum regardless of
+	// whether a single PutObject or a multipart upload is used.
+	decodedChecksum, err := base64.StdEncoding.DecodeString(checksum)
+	if err != nil {
+		return nil, errors.UnknownStatus("failed to decode checksum value")
+	}
+
+	// for larger files, use a multipart upload instead of a single PutObject
+	// request.
+	if info.Size() > multipartThreshold {
+		partSize := int64(defaultMultipartPartSize)
+		// S3 allows at most maxMultipartParts parts per upload, so grow the
+		// part size for very large objects to stay within that limit.
+		if sz := info.Size(); sz/partSize >= maxMultipartParts {
+			partSize = (sz + maxMultipartParts - 1) / maxMultipartParts
+		}
+		if err := p.putObjectMultipart(ctx, req, storageState, bucket.GetBucketName(), objectKey, file, partSize, opts...); err != nil {
+			return nil, err
+		}
+		return &pb.PutObjectResponse{
+			ChecksumSha_256: decodedChecksum,
+		}, nil
+	}
+
 	putCall := func(s3Client S3API) (any, error) {
 		return s3Client.PutObject(ctx, &s3.PutObjectInput{
 			Bucket:            aws.String(bucket.GetBucketName()),
@@ -772,15 +815,126 @@ func (p *StoragePlugin) PutObject(ctx context.Context, req *pb.PutObjectRequest)
 	if checksum != *resp.ChecksumSHA256 {
 		return nil, checksumMistmatchStatus()
 	}
-	decodedChecksum, err := base64.StdEncoding.DecodeString(*resp.ChecksumSHA256)
-	if err != nil {
-		return nil, errors.UnknownStatus("failed to decode checksum value from aws")
-	}
 	return &pb.PutObjectResponse{
 		ChecksumSha_256: decodedChecksum,
 	}, nil
 }
 
+// putObjectMultipart uploads file to the given bucket and key using an S3
+// multipart upload. The file is split into parts of partSize bytes (the final
+// part may be smaller) and each part is uploaded with its own SHA-256 checksum
+// so S3 can verify part integrity on the wire.
+//
+// If any step fails the upload is aborted on a best-effort basis to avoid
+// leaving orphaned parts behind, which would otherwise continue to accrue
+// storage charges.
+func (p *StoragePlugin) putObjectMultipart(
+	ctx context.Context,
+	req *pb.PutObjectRequest,
+	storageState *awsStoragePersistedState,
+	bucketName, objectKey string,
+	file *os.File,
+	partSize int64,
+	opts ...s3Option,
+) error {
+	bucketId := req.GetBucket().GetId()
+
+	createCall := func(s3Client S3API) (any, error) {
+		return s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+			Bucket:            aws.String(bucketName),
+			Key:               aws.String(objectKey),
+			ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
+		})
+	}
+	createResp, err := p.call(ctx, createCall, bucketId, storageState, opts...)
+	if err != nil {
+		return parseS3Error("create multipart upload", err, req).Err()
+	}
+	uploadId := createResp.(*s3.CreateMultipartUploadOutput).UploadId
+
+	// once the upload has been created, any subsequent failure must abort it so
+	// uploaded parts are not orphaned.
+	abort := func() {
+		abortCall := func(s3Client S3API) (any, error) {
+			return s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+				Bucket:   aws.String(bucketName),
+				Key:      aws.String(objectKey),
+				UploadId: uploadId,
+			})
+		}
+		// best effort: the failure that triggered the abort is more useful to
+		// the caller than an error from the abort itself.
+		_, _ = p.call(ctx, abortCall, bucketId, storageState, opts...)
+	}
+
+	var completedParts []types.CompletedPart
+	buf := make([]byte, partSize)
+	var partNumber int32 = 1
+	for {
+		n, readErr := io.ReadFull(file, buf)
+		if readErr == io.EOF {
+			// previous iteration consumed the final bytes of the file
+			break
+		}
+		if readErr != nil && readErr != io.ErrUnexpectedEOF {
+			abort()
+			return errors.UnknownStatusf("failed to read file: %s", readErr)
+		}
+
+		partData := buf[:n]
+		sum := sha256.Sum256(partData)
+		partChecksum := base64.StdEncoding.EncodeToString(sum[:])
+
+		pn := partNumber
+		uploadCall := func(s3Client S3API) (any, error) {
+			return s3Client.UploadPart(ctx, &s3.UploadPartInput{
+				Bucket:            aws.String(bucketName),
+				Key:               aws.String(objectKey),
+				UploadId:          uploadId,
+				PartNumber:        aws.Int32(pn),
+				Body:              bytes.NewReader(partData),
+				ContentLength:     aws.Int64(int64(n)),
+				ChecksumAlgorithm: types.ChecksumAlgorithmSha256,
+				ChecksumSHA256:    aws.String(partChecksum),
+			})
+		}
+		uploadResp, err := p.call(ctx, uploadCall, bucketId, storageState, opts...)
+		if err != nil {
+			abort()
+			return parseS3Error("upload part", err, req).Err()
+		}
+		part := uploadResp.(*s3.UploadPartOutput)
+		completedParts = append(completedParts, types.CompletedPart{
+			ETag:           part.ETag,
+			ChecksumSHA256: part.ChecksumSHA256,
+			PartNumber:     aws.Int32(pn),
+		})
+
+		partNumber++
+		if readErr == io.ErrUnexpectedEOF {
+			// the final, short part
+			break
+		}
+	}
+
+	completeCall := func(s3Client S3API) (any, error) {
+		return s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+			Bucket:   aws.String(bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: uploadId,
+			MultipartUpload: &types.CompletedMultipartUpload{
+				Parts: completedParts,
+			},
+		})
+	}
+	if _, err := p.call(ctx, completeCall, bucketId, storageState, opts...); err != nil {
+		abort()
+		return parseS3Error("complete multipart upload", err, req).Err()
+	}
+
+	return nil
+}
+
 // DeleteObjects is used to delete one or many objects from an s3 bucket.
 func (p *StoragePlugin) DeleteObjects(ctx context.Context, req *pb.DeleteObjectsRequest) (*pb.DeleteObjectsResponse, error) {
 	if req.GetKeyPrefix() == "" {

plugin/service/storage/plugin_test.go

@@ -3715,6 +3715,195 @@ func TestStoragePlugin_PutObject(t *testing.T) {
 	}
 }
 
+func TestStoragePlugin_PutObjectMultipart(t *testing.T) {
+	td := t.TempDir()
+
+	// genFile writes size bytes of deterministic content to a temp file and
+	// returns its path along with the raw SHA-256 digest of the whole file.
+	genFile := func(t *testing.T, name string, size int) (string, []byte) {
+		t.Helper()
+		data := make([]byte, size)
+		for i := range data {
+			data[i] = byte(i % 251)
+		}
+		fp := path.Join(td, name)
+		require.NoError(t, os.WriteFile(fp, data, 0o600))
+		sum := sha256.Sum256(data)
+		return fp, sum[:]
+	}
+
+	validRequest := func(p string) *pb.PutObjectRequest {
+		return &pb.PutObjectRequest{
+			Bucket: &storagebuckets.StorageBucket{
+				BucketName:   "external-obj-store",
+				BucketPrefix: "prefix",
+				Secrets:      credential.MockStaticCredentialSecrets(),
+				Attributes: &structpb.Struct{
+					Fields: map[string]*structpb.Value{
+						credential.ConstRegion: structpb.NewStringValue("us-west-2"),
+					},
+				},
+			},
+			Key:  "mock-object",
+			Path: p,
+		}
+	}
+
+	t.Run("multipart success spanning multiple parts", func(t *testing.T) {
+		require, assert := require.New(t), assert.New(t)
+		// 20 MiB with 8 MiB parts -> 3 parts (8, 8, 4 MiB).
+		fp, wholeChecksum := genFile(t, "large-file", 20*1024*1024)
+
+		state := &testMockS3State{}
+		p := &StoragePlugin{
+			testCredStateOpts: validSTSMock(),
+			testStorageStateOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(state)),
+			},
+		}
+
+		resp, err := p.PutObject(context.Background(), validRequest(fp))
+		require.NoError(err)
+		require.NotNil(resp)
+
+		// Boundary must receive the whole-object SHA-256, not the multipart
+		// composite checksum.
+		assert.True(bytes.Equal(wholeChecksum, resp.GetChecksumSha_256()))
+
+		// The single-shot PutObject path must not have been used.
+		assert.False(state.PutObjectCalled)
+		assert.True(state.CreateMultipartUploadCalled)
+		assert.Equal(3, state.UploadPartCallCount)
+		assert.True(state.CompleteMultipartUploadCalled)
+		assert.False(state.AbortMultipartUploadCalled)
+
+		// The reassembled part bodies must equal the original file content.
+		orig, err := os.ReadFile(fp)
+		require.NoError(err)
+		assert.True(bytes.Equal(orig, state.UploadPartBody))
+
+		// The object key must include the configured bucket prefix.
+		require.NotNil(state.CreateMultipartUploadInputParams)
+		assert.Equal("prefix/mock-object", aws.ToString(state.CreateMultipartUploadInputParams.Key))
+	})
+
+	t.Run("file just above threshold uses two parts", func(t *testing.T) {
+		require, assert := require.New(t), assert.New(t)
+		// 8 MiB + 1 byte -> 2 parts (8 MiB, 1 byte).
+		fp, wholeChecksum := genFile(t, "just-over", 8*1024*1024+1)
+
+		state := &testMockS3State{}
+		p := &StoragePlugin{
+			testCredStateOpts: validSTSMock(),
+			testStorageStateOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(state)),
+			},
+		}
+
+		resp, err := p.PutObject(context.Background(), validRequest(fp))
+		require.NoError(err)
+		assert.True(bytes.Equal(wholeChecksum, resp.GetChecksumSha_256()))
+		assert.Equal(2, state.UploadPartCallCount)
+		assert.True(state.CompleteMultipartUploadCalled)
+	})
+
+	t.Run("file at threshold uses single PutObject", func(t *testing.T) {
+		require, assert := require.New(t), assert.New(t)
+		// Exactly 8 MiB is not above the threshold, so the single-shot path is used.
+		fp, wholeChecksum := genFile(t, "at-threshold", 8*1024*1024)
+		checksum := base64.StdEncoding.EncodeToString(wholeChecksum)
+
+		state := &testMockS3State{}
+		p := &StoragePlugin{
+			testCredStateOpts: validSTSMock(),
+			testStorageStateOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(
+					state,
+					testMockS3WithPutObjectOutput(&s3.PutObjectOutput{
+						ChecksumSHA256: aws.String(checksum),
+					}),
+				)),
+			},
+		}
+
+		resp, err := p.PutObject(context.Background(), validRequest(fp))
+		require.NoError(err)
+		assert.True(bytes.Equal(wholeChecksum, resp.GetChecksumSha_256()))
+		assert.True(state.PutObjectCalled)
+		assert.False(state.CreateMultipartUploadCalled)
+	})
+
+	t.Run("create multipart upload error", func(t *testing.T) {
+		require, assert := require.New(t), assert.New(t)
+		fp, _ := genFile(t, "create-err", 20*1024*1024)
+
+		state := &testMockS3State{}
+		p := &StoragePlugin{
+			testCredStateOpts: validSTSMock(),
+			testStorageStateOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(
+					state,
+					testMockS3WithCreateMultipartUploadError(TestAwsS3Error("AccessDenied", "CreateMultipartUpload", "not authorized")),
+				)),
+			},
+		}
+
+		resp, err := p.PutObject(context.Background(), validRequest(fp))
+		require.Error(err)
+		assert.Nil(resp)
+		assert.Equal(codes.PermissionDenied, status.Code(err))
+		// Nothing was created, so nothing should be aborted.
+		assert.False(state.AbortMultipartUploadCalled)
+	})
+
+	t.Run("upload part error aborts upload", func(t *testing.T) {
+		require, assert := require.New(t), assert.New(t)
+		fp, _ := genFile(t, "upload-err", 20*1024*1024)
+
+		state := &testMockS3State{}
+		p := &StoragePlugin{
+			testCredStateOpts: validSTSMock(),
+			testStorageStateOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(
+					state,
+					testMockS3WithUploadPartErrorOnPart(2, TestAwsS3Error("AccessDenied", "UploadPart", "not authorized")),
+				)),
+			},
+		}
+
+		resp, err := p.PutObject(context.Background(), validRequest(fp))
+		require.Error(err)
+		assert.Nil(resp)
+		assert.Equal(codes.PermissionDenied, status.Code(err))
+		assert.True(state.CreateMultipartUploadCalled)
+		assert.False(state.CompleteMultipartUploadCalled)
+		// The failed upload must be aborted to avoid orphaned parts.
+		assert.True(state.AbortMultipartUploadCalled)
+	})
+
+	t.Run("complete multipart upload error aborts upload", func(t *testing.T) {
+		require, assert := require.New(t), assert.New(t)
+		fp, _ := genFile(t, "complete-err", 20*1024*1024)
+
+		state := &testMockS3State{}
+		p := &StoragePlugin{
+			testCredStateOpts: validSTSMock(),
+			testStorageStateOpts: []awsStoragePersistedStateOption{
+				withTestS3APIFunc(newTestMockS3(
+					state,
+					testMockS3WithCompleteMultipartUploadError(TestAwsS3Error("InternalError", "CompleteMultipartUpload", "boom")),
+				)),
+			},
+		}
+
+		resp, err := p.PutObject(context.Background(), validRequest(fp))
+		require.Error(err)
+		assert.Nil(resp)
+		assert.Equal(3, state.UploadPartCallCount)
+		assert.True(state.AbortMultipartUploadCalled)
+	})
+}
+
 func TestStoragePlugin_DeleteObjects(t *testing.T) {
 	validRequest := func() *pb.DeleteObjectsRequest {
 		return &pb.DeleteObjectsRequest{

plugin/service/storage/state.go

@@ -26,6 +26,10 @@ type S3API interface {
 	GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
 	HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
 	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
+	CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
+	UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error)
+	CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
+	AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
 	DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error)
 	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
 	Credentials() aws.Credentials

plugin/service/storage/testing.go

@@ -6,6 +6,8 @@ package storage
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"testing"
@@ -39,6 +41,23 @@ type testMockS3State struct {
 	PutObjectInputParams *s3.PutObjectInput
 	PutObjectBody        []byte
 
+	CreateMultipartUploadCalled      bool
+	CreateMultipartUploadInputParams *s3.CreateMultipartUploadInput
+
+	UploadPartCalled      bool
+	UploadPartCallCount   int
+	UploadPartInputParams []*s3.UploadPartInput
+	// UploadPartBody is the concatenation, in part-number order, of every
+	// part body received by UploadPart. It can be compared against the full
+	// object that was expected to be uploaded.
+	UploadPartBody []byte
+
+	CompleteMultipartUploadCalled      bool
+	CompleteMultipartUploadInputParams *s3.CompleteMultipartUploadInput
+
+	AbortMultipartUploadCalled      bool
+	AbortMultipartUploadInputParams *s3.AbortMultipartUploadInput
+
 	HeadObjectCalled      bool
 	HeadObjectInputParams *s3.HeadObjectInput
 
@@ -57,6 +76,16 @@ func (s *testMockS3State) Reset() {
 	s.PutObjectCalled = false
 	s.PutObjectInputParams = nil
 	s.PutObjectBody = nil
+	s.CreateMultipartUploadCalled = false
+	s.CreateMultipartUploadInputParams = nil
+	s.UploadPartCalled = false
+	s.UploadPartCallCount = 0
+	s.UploadPartInputParams = nil
+	s.UploadPartBody = nil
+	s.CompleteMultipartUploadCalled = false
+	s.CompleteMultipartUploadInputParams = nil
+	s.AbortMultipartUploadCalled = false
+	s.AbortMultipartUploadInputParams = nil
 	s.HeadObjectCalled = false
 	s.HeadObjectInputParams = nil
 	s.ListObjectsV2Called = false
@@ -81,6 +110,19 @@ type testMockS3 struct {
 	PutObjectOutput *s3.PutObjectOutput
 	PutObjectErr    error
 
+	// mocked responses for the multipart upload calls
+	CreateMultipartUploadOutput *s3.CreateMultipartUploadOutput
+	CreateMultipartUploadErr    error
+	UploadPartOutput            *s3.UploadPartOutput
+	UploadPartErr               error
+	// UploadPartErrOnPart, when greater than zero, causes UploadPart to return
+	// UploadPartErr (or a generic error) only for the matching part number.
+	UploadPartErrOnPart           int32
+	CompleteMultipartUploadOutput *s3.CompleteMultipartUploadOutput
+	CompleteMultipartUploadErr    error
+	AbortMultipartUploadOutput    *s3.AbortMultipartUploadOutput
+	AbortMultipartUploadErr       error
+
 	// mocked responses for headObject
 	HeadObjectOutput *s3.HeadObjectOutput
 	HeadObjectErr    error
@@ -114,6 +156,63 @@ func testMockS3WithPutObjectError(e error) testMockS3Option {
 	}
 }
 
+func testMockS3WithCreateMultipartUploadOutput(o *s3.CreateMultipartUploadOutput) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.CreateMultipartUploadOutput = o
+		return nil
+	}
+}
+
+func testMockS3WithCreateMultipartUploadError(e error) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.CreateMultipartUploadErr = e
+		return nil
+	}
+}
+
+func testMockS3WithUploadPartOutput(o *s3.UploadPartOutput) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.UploadPartOutput = o
+		return nil
+	}
+}
+
+func testMockS3WithUploadPartError(e error) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.UploadPartErr = e
+		return nil
+	}
+}
+
+func testMockS3WithUploadPartErrorOnPart(partNumber int32, e error) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.UploadPartErr = e
+		m.UploadPartErrOnPart = partNumber
+		return nil
+	}
+}
+
+func testMockS3WithCompleteMultipartUploadOutput(o *s3.CompleteMultipartUploadOutput) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.CompleteMultipartUploadOutput = o
+		return nil
+	}
+}
+
+func testMockS3WithCompleteMultipartUploadError(e error) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.CompleteMultipartUploadErr = e
+		return nil
+	}
+}
+
+func testMockS3WithAbortMultipartUploadError(e error) testMockS3Option {
+	return func(m *testMockS3) error {
+		m.AbortMultipartUploadErr = e
+		return nil
+	}
+}
+
 func testMockS3WithGetObjectOutput(o *s3.GetObjectOutput) testMockS3Option {
 	return func(m *testMockS3) error {
 		m.GetObjectOutput = o
@@ -255,6 +354,95 @@ func (m *testMockS3) PutObject(ctx context.Context, params *s3.PutObjectInput, o
 	return m.PutObjectOutput, nil
 }
 
+func (m *testMockS3) CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) {
+	if m.State != nil {
+		m.State.CreateMultipartUploadCalled = true
+		m.State.CreateMultipartUploadInputParams = params
+	}
+
+	if m.CreateMultipartUploadErr != nil {
+		return nil, m.CreateMultipartUploadErr
+	}
+
+	if m.CreateMultipartUploadOutput != nil {
+		return m.CreateMultipartUploadOutput, nil
+	}
+	return &s3.CreateMultipartUploadOutput{
+		UploadId: aws.String("test-upload-id"),
+	}, nil
+}
+
+func (m *testMockS3) UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) {
+	if m.State != nil {
+		m.State.UploadPartCalled = true
+		m.State.UploadPartCallCount++
+		m.State.UploadPartInputParams = append(m.State.UploadPartInputParams, params)
+	}
+
+	// always drain the body so the part data can be asserted on and so the
+	// reader is not left dangling.
+	var data []byte
+	if params.Body != nil {
+		var err error
+		data, err = io.ReadAll(params.Body)
+		if err != nil {
+			return nil, err
+		}
+		if m.State != nil {
+			m.State.UploadPartBody = append(m.State.UploadPartBody, data...)
+		}
+	}
+
+	if m.UploadPartErr != nil {
+		if m.UploadPartErrOnPart == 0 || (params.PartNumber != nil && *params.PartNumber == m.UploadPartErrOnPart) {
+			return nil, m.UploadPartErr
+		}
+	}
+
+	if m.UploadPartOutput != nil {
+		return m.UploadPartOutput, nil
+	}
+	// echo the supplied part checksum so CompleteMultipartUpload can be built
+	// the way the real service would allow.
+	sum := sha256.Sum256(data)
+	return &s3.UploadPartOutput{
+		ETag:           aws.String(fmt.Sprintf("etag-%d", aws.ToInt32(params.PartNumber))),
+		ChecksumSHA256: aws.String(base64.StdEncoding.EncodeToString(sum[:])),
+	}, nil
+}
+
+func (m *testMockS3) CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) {
+	if m.State != nil {
+		m.State.CompleteMultipartUploadCalled = true
+		m.State.CompleteMultipartUploadInputParams = params
+	}
+
+	if m.CompleteMultipartUploadErr != nil {
+		return nil, m.CompleteMultipartUploadErr
+	}
+
+	if m.CompleteMultipartUploadOutput != nil {
+		return m.CompleteMultipartUploadOutput, nil
+	}
+	return &s3.CompleteMultipartUploadOutput{}, nil
+}
+
+func (m *testMockS3) AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) {
+	if m.State != nil {
+		m.State.AbortMultipartUploadCalled = true
+		m.State.AbortMultipartUploadInputParams = params
+	}
+
+	if m.AbortMultipartUploadErr != nil {
+		return nil, m.AbortMultipartUploadErr
+	}
+
+	if m.AbortMultipartUploadOutput != nil {
+		return m.AbortMultipartUploadOutput, nil
+	}
+	return &s3.AbortMultipartUploadOutput{}, nil
+}
+
 func (m *testMockS3) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
 	if m.State != nil {
 		m.State.ListObjectsV2Called = true