Matt Romanelli
f81151b58c
**Problem** * While adding e2e tests for AssumeRole functionality, it was noticed that some of the static credential e2e tests were flaky. * The tests would sometimes fail with an AuthFailure. **Solution** * It was discovered that this error happened sometimes after a static credential rotation and that AuthFailure occurs if EC2 is invoked with the wrong credentials. * Due to IAM's eventual consistency model, it can take some time (up to several seconds) for changes, including rotated credentials, to get propagated to all AWS services, including EC2. * The tests were intermittently failing because sometimes the credentials would be propagated in time before invoking EC2 and other times they were not propagated in time. * The fix is to add a retry mechanism when calling DescribeInstances. A 30 second window is defined in which the retries can take place. * The retries happen only for AuthFailures. Any other failure will not trigger a retry. **Testing** * After making this change, the intermittent failures were no longer seen for the e2e tests * The e2e tests cover updating/creating catalogs and sets as well us updating the host list. All pass. * New unit tests were added to test the retry mechanism. Support had to be added to mock a series of DescribeInstances calls vs a single call * New and existing unit tests passed
131 lines
3.5 KiB
Go
131 lines
3.5 KiB
Go
// Copyright IBM Corp. 2021, 2026
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package host
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
|
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
|
|
)
|
|
|
|
const (
|
|
testOptionErr = "test option error"
|
|
testGetUserErr = "test error for GetUser"
|
|
testGetCallerIdentityErr = "test error for GetCallerIdentity"
|
|
testDeleteAccessKeyErr = "test error for DeleteAccessKey"
|
|
testDescribeInstancesError = "DescribeInstances error"
|
|
)
|
|
|
|
type testMockEC2State struct {
|
|
DescribeInstancesCalled bool
|
|
DescribeInstancesInputParams *ec2.DescribeInstancesInput
|
|
DescribeInstancesCallCount int
|
|
DescribeInstancesInputs []*ec2.DescribeInstancesInput
|
|
ClientRegions []string
|
|
}
|
|
|
|
func (s *testMockEC2State) Reset() {
|
|
s.DescribeInstancesCalled = false
|
|
s.DescribeInstancesInputParams = nil
|
|
s.DescribeInstancesCallCount = 0
|
|
s.DescribeInstancesInputs = nil
|
|
s.ClientRegions = nil
|
|
}
|
|
|
|
type testMockEC2 struct {
|
|
EC2API
|
|
|
|
State *testMockEC2State
|
|
Region string
|
|
|
|
DescribeInstancesOutput *ec2.DescribeInstancesOutput
|
|
DescribeInstancesError error
|
|
|
|
// DescribeInstancesErrorSequence, if set, returns each error in order on
|
|
// successive calls. Once exhausted, DescribeInstancesOutput is returned.
|
|
DescribeInstancesErrorSequence []error
|
|
}
|
|
|
|
type testMockEC2Option func(m *testMockEC2) error
|
|
|
|
func testMockEC2WithDescribeInstancesOutput(o *ec2.DescribeInstancesOutput) testMockEC2Option {
|
|
return func(m *testMockEC2) error {
|
|
m.DescribeInstancesOutput = o
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func testMockEC2WithDescribeInstancesError(e error) testMockEC2Option {
|
|
return func(m *testMockEC2) error {
|
|
m.DescribeInstancesError = e
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// testMockEC2WithDescribeInstancesErrorSequence sets a sequence of errors to
|
|
// return on successive DescribeInstances calls. Once the sequence is exhausted,
|
|
// the mock returns DescribeInstancesOutput (nil error).
|
|
func testMockEC2WithDescribeInstancesErrorSequence(errs ...error) testMockEC2Option {
|
|
return func(m *testMockEC2) error {
|
|
m.DescribeInstancesErrorSequence = errs
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func newTestMockEC2(state *testMockEC2State, opts ...testMockEC2Option) ec2APIFunc {
|
|
return func(cfgs ...aws.Config) (EC2API, error) {
|
|
m := &testMockEC2{
|
|
State: state,
|
|
}
|
|
|
|
for _, opt := range opts {
|
|
if err := opt(m); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
for _, cfg := range cfgs {
|
|
// Last region takes precedence
|
|
if cfg.Region != "" {
|
|
m.Region = cfg.Region
|
|
}
|
|
}
|
|
|
|
if state != nil {
|
|
state.ClientRegions = append(state.ClientRegions, m.Region)
|
|
}
|
|
|
|
return m, nil
|
|
}
|
|
}
|
|
|
|
func (m *testMockEC2) DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
|
|
if m.State != nil {
|
|
m.State.DescribeInstancesCalled = true
|
|
m.State.DescribeInstancesInputParams = input
|
|
m.State.DescribeInstancesCallCount++
|
|
m.State.DescribeInstancesInputs = append(m.State.DescribeInstancesInputs, input)
|
|
}
|
|
|
|
if len(m.DescribeInstancesErrorSequence) > 0 {
|
|
err := m.DescribeInstancesErrorSequence[0]
|
|
m.DescribeInstancesErrorSequence = m.DescribeInstancesErrorSequence[1:]
|
|
return nil, err
|
|
}
|
|
|
|
if m.DescribeInstancesError != nil {
|
|
return nil, m.DescribeInstancesError
|
|
}
|
|
|
|
return m.DescribeInstancesOutput, nil
|
|
}
|
|
|
|
type ec2FilterSorter []types.Filter
|
|
|
|
func (s ec2FilterSorter) Len() int { return len(s) }
|
|
func (s ec2FilterSorter) Less(i, j int) bool { return *s[i].Name < *s[j].Name }
|
|
func (s ec2FilterSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|