threshold/internal/credential/vault/jobs_test.go

package vault

import (
	"context"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/boundary/internal/authtoken"
	"github.com/hashicorp/boundary/internal/db"
	"github.com/hashicorp/boundary/internal/errors"
	"github.com/hashicorp/boundary/internal/host/static"
	"github.com/hashicorp/boundary/internal/iam"
	"github.com/hashicorp/boundary/internal/kms"
	"github.com/hashicorp/boundary/internal/scheduler"
	"github.com/hashicorp/boundary/internal/session"
	"github.com/hashicorp/boundary/internal/target"
	"github.com/hashicorp/go-hclog"
	wrapping "github.com/hashicorp/go-kms-wrapping"
	vault "github.com/hashicorp/vault/api"
	"github.com/jinzhu/gorm"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const testUpdateTokenStatusExpirationQuery = `
update credential_vault_token
   set status = ?,
       last_renewal_time = now(),
       expiration_time   = wt_add_seconds_to_now(?)
 where token_hmac = ?;
`

func testVaultToken(t *testing.T,
	conn *gorm.DB,
	wrapper wrapping.Wrapper,
	v *TestVaultServer,
	cs *CredentialStore,
	status TokenStatus,
	expiration time.Duration) *Token {
	t.Helper()
	assert, require := assert.New(t), require.New(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	rw := db.New(conn)

	secret, _ := v.CreateToken(t)
	inToken, err := newToken(cs.PublicId, []byte(secret.Auth.ClientToken), []byte(secret.Auth.Accessor), expiration)
	require.NoError(err)
	inToken.Status = string(status)

	databaseWrapper, err := kmsCache.GetWrapper(context.Background(), cs.ScopeId, kms.KeyPurposeDatabase)
	require.NoError(err)

	require.NoError(inToken.encrypt(context.Background(), databaseWrapper))

	query := insertTokenQuery
	queryValues := []interface{}{
		inToken.TokenHmac,
		inToken.CtToken,
		inToken.StoreId,
		inToken.KeyId,
		inToken.Status,
	}
	expire := int(expiration.Seconds())
	if expire < 0 {
		// last_renewal_time must be before expiration_time, if we are testing a expiration
		// in the past set last_renewal_time to 1 second before that
		query = strings.Replace(query,
			"$6, -- last_renewal_time",
			"wt_add_seconds_to_now($6),  -- last_renewal_time",
			-1)
		queryValues = append(queryValues, expire-1, expire)
	} else {
		queryValues = append(queryValues, "now()", expire)
	}

	numRows, err := rw.Exec(context.Background(), query, queryValues)
	assert.Equal(1, numRows)
	require.NoError(err)

	outToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &outToken, "token_hmac = ?", inToken.TokenHmac))
	require.NoError(outToken.decrypt(context.Background(), databaseWrapper))

	return outToken
}

func testVaultCred(t *testing.T,
	conn *gorm.DB,
	v *TestVaultServer,
	cl *CredentialLibrary,
	sess *session.Session,
	token *Token,
	status CredentialStatus,
	expiration time.Duration) (*vault.Secret, *Credential) {
	t.Helper()
	assert, require := assert.New(t), require.New(t)
	rw := db.New(conn)

	client := v.client(t)
	var secret *vault.Secret
	var err error
	switch Method(cl.HttpMethod) {
	case MethodGet:
		secret, err = client.get(cl.VaultPath)
	case MethodPost:
		secret, err = client.post(cl.VaultPath, cl.HttpRequestBody)
	}
	require.NoError(err)
	require.NotNil(secret)

	id, err := newCredentialId()
	require.NoError(err)

	query := insertCredentialWithExpirationQuery
	queryValues := []interface{}{
		id,
		cl.GetPublicId(),
		sess.GetPublicId(),
		token.GetTokenHmac(),
		secret.LeaseID,
		true,
		status,
	}
	expire := int(expiration.Seconds())
	if expire < 0 {
		// last_renewal_time must be before expiration_time, if we are testing a expiration
		// in the past set last_renewal_time to 1 second before that
		query = strings.Replace(query,
			"$8, -- last_renewal_time",
			"wt_add_seconds_to_now($8),  -- last_renewal_time",
			-1)
		queryValues = append(queryValues, expire-1, expire)
	} else {
		queryValues = append(queryValues, "now()", expire)
	}

	numRows, err := rw.Exec(context.Background(), query, queryValues)
	assert.Equal(1, numRows)
	assert.NoError(err)

	outCred := allocCredential()
	require.NoError(rw.LookupWhere(context.Background(), &outCred, "public_id = ?", id))

	return secret, outCred
}

func TestNewTokenRenewalJob(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	type args struct {
		r      db.Reader
		w      db.Writer
		kms    *kms.Kms
		logger hclog.Logger
	}
	tests := []struct {
		name        string
		args        args
		options     []Option
		wantLimit   int
		wantErr     bool
		wantErrCode errors.Code
	}{
		{
			name:        "nil reader",
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil writer",
			args: args{
				r: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil kms",
			args: args{
				r: rw,
				w: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil logger",
			args: args{
				r:   rw,
				w:   rw,
				kms: kmsCache,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "valid-no-options",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			wantLimit: db.DefaultLimit,
		},
		{
			name: "valid-with-limit",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			options:   []Option{WithLimit(100)},
			wantLimit: 100,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			got, err := newTokenRenewalJob(tt.args.r, tt.args.w, tt.args.kms, tt.args.logger, tt.options...)
			if tt.wantErr {
				require.Error(err)
				assert.Nil(got)
				assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err)
				return
			}
			require.NoError(err)
			require.NotNil(got)
			assert.Equal(tt.args.r, got.reader)
			assert.Equal(tt.args.w, got.writer)
			assert.Equal(tt.args.kms, got.kms)
			require.NotNil(got.logger)
			assert.Equal(tt.wantLimit, got.limit)
		})
	}
}

func TestTokenRenewalJob_RunLimits(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	v := NewTestVaultServer(t)

	count := 5
	tests := []struct {
		name    string
		opts    []Option
		wantLen int
	}{
		{
			name:    "with-no-limits",
			wantLen: count + 1, // +1 for current token
		},
		{
			name:    "with-negative-limit",
			opts:    []Option{WithLimit(-1)},
			wantLen: count + 1, // +1 for current token
		},
		{
			name:    "with-limit",
			opts:    []Option{WithLimit(2)},
			wantLen: 2,
		},
		{
			name:    "with-limit-greater-than-count",
			opts:    []Option{WithLimit(count + 10)},
			wantLen: count + 1, // +1 for current token
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			_, token := v.CreateToken(t)
			in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
			require.NoError(err)
			sche := scheduler.TestScheduler(t, conn, wrapper)
			repo, err := NewRepository(rw, rw, kmsCache, sche)
			require.NoError(err)
			cs, err := repo.CreateCredentialStore(context.Background(), in)
			require.NoError(err)

			// Create additional tokens and alternative between token statuses, revoked and
			// expired tokens should have no impact on number tokens renewed
			for i := 0; i < count*3; i++ {
				var status TokenStatus
				switch i % 3 {
				case 0:
					status = MaintainingToken
				case 1:
					status = RevokedToken
				case 2:
					status = ExpiredToken
				}
				testVaultToken(t, conn, wrapper, v, cs, status, 5*time.Minute)
			}

			// inserting new tokens moves the current token to a maintaining state, move it back to current and set expiration time
			numRows, err := rw.Exec(context.Background(), testUpdateTokenStatusExpirationQuery, []interface{}{CurrentToken, time.Minute.Seconds(), cs.outputToken.TokenHmac})
			require.NoError(err)
			assert.Equal(1, numRows)

			r, err := newTokenRenewalJob(rw, rw, kmsCache, hclog.L(), tt.opts...)
			require.NoError(err)

			err = r.Run(context.Background())
			require.NoError(err)
			assert.Equal(tt.wantLen, r.numTokens)

			// Set all tokens to revoked for next test
			_, err = rw.Exec(context.Background(), "update credential_vault_token set status = 'revoked'", nil)
			assert.NoError(err)
		})
	}
}

func TestTokenRenewalJob_Run(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	v := NewTestVaultServer(t)

	// Create 24 hour token
	_, token := v.CreateToken(t, WithTokenPeriod(24*time.Hour))

	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
	require.NoError(err)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(err)

	r, err := newTokenRenewalJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	err = sche.RegisterJob(context.Background(), r)
	require.NoError(err)

	err = r.Run(context.Background())
	require.NoError(err)
	// No tokens should have been renewed since token expiration is 24 hours by default
	assert.Equal(0, r.numProcessed)

	// Create maintaining, revoked and expired tokens
	maintainToken := testVaultToken(t, conn, wrapper, v, cs, MaintainingToken, time.Minute)
	revokedToken := testVaultToken(t, conn, wrapper, v, cs, RevokedToken, time.Minute)
	expiredToken := testVaultToken(t, conn, wrapper, v, cs, ExpiredToken, time.Minute)

	// inserting new tokens moves the current token to a maintaining state, move it back to current and set expiration time
	count, err := rw.Exec(context.Background(), testUpdateTokenStatusExpirationQuery, []interface{}{CurrentToken, time.Minute.Seconds(), cs.outputToken.TokenHmac})
	require.NoError(err)
	assert.Equal(1, count)

	currentToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &currentToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))
	databaseWrapper, err := kmsCache.GetWrapper(context.Background(), cs.ScopeId, kms.KeyPurposeDatabase)
	require.NoError(err)
	require.NoError(currentToken.decrypt(context.Background(), databaseWrapper))

	// Sleep to move clock
	time.Sleep(time.Second * 2)

	// Get ttls in vault to verify against before running renewal
	lookupToken := v.LookupToken(t, string(currentToken.GetToken()))
	oldCurrentTtl, err := lookupToken.TokenTTL()
	require.NoError(err)
	lookupToken = v.LookupToken(t, string(maintainToken.GetToken()))
	oldMaintainTtl, err := lookupToken.TokenTTL()
	require.NoError(err)
	lookupToken = v.LookupToken(t, string(revokedToken.GetToken()))
	oldRevokedTtl, err := lookupToken.TokenTTL()
	require.NoError(err)
	lookupToken = v.LookupToken(t, string(expiredToken.GetToken()))
	oldExpiredTtl, err := lookupToken.TokenTTL()
	require.NoError(err)

	// Run token renewal again
	err = r.Run(context.Background())
	require.NoError(err)
	// Current and maintaining token should have been processed
	assert.Equal(2, r.numProcessed)

	// Verify current and maintaining token were renewed in vault
	lookupToken = v.LookupToken(t, string(currentToken.GetToken()))
	newTtl, err := lookupToken.TokenTTL()
	require.NoError(err)
	assert.True(oldCurrentTtl < newTtl)
	lookupToken = v.LookupToken(t, string(maintainToken.GetToken()))
	newTtl, err = lookupToken.TokenTTL()
	require.NoError(err)
	assert.True(oldMaintainTtl < newTtl)

	// Verify expired and revoked tokens were not renewed in vault
	lookupToken = v.LookupToken(t, string(revokedToken.GetToken()))
	newTtl, err = lookupToken.TokenTTL()
	require.NoError(err)
	assert.True(oldRevokedTtl >= newTtl)
	lookupToken = v.LookupToken(t, string(expiredToken.GetToken()))
	newTtl, err = lookupToken.TokenTTL()
	require.NoError(err)
	assert.True(oldExpiredTtl >= newTtl)

	// Verify current and maintaining tokens were renewed in repo
	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{currentToken.TokenHmac}))
	assert.True(currentToken.GetExpirationTime().AsTime().Before(repoToken.GetExpirationTime().AsTime()))
	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{maintainToken.TokenHmac}))
	assert.True(maintainToken.GetExpirationTime().AsTime().Before(repoToken.GetExpirationTime().AsTime()))

	// Verify revoked and expired tokens were not renewed in the repo
	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{revokedToken.TokenHmac}))
	assert.Equal(revokedToken.GetExpirationTime().AsTime(), repoToken.GetExpirationTime().AsTime())
	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{expiredToken.TokenHmac}))
	assert.Equal(expiredToken.GetExpirationTime().AsTime(), repoToken.GetExpirationTime().AsTime())
}

func TestTokenRenewalJob_RunExpired(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	v := NewTestVaultServer(t)

	// Create 1s token so it expires in vault before we can renew it
	_, ct := v.CreateToken(t, WithTokenPeriod(time.Second))

	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	assert.NoError(err)
	require.NotNil(in)

	r, err := newTokenRenewalJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	err = sche.RegisterJob(context.Background(), r)
	require.NoError(err)

	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(err)

	// Sleep to move clock and expire token
	time.Sleep(time.Second * 2)

	// Token should have expired in vault, run should now expire in repo
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(1, r.numTokens)

	// Verify token was expired in repo
	token := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &token, "store_id = ?", []interface{}{cs.GetPublicId()}))
	assert.Equal(string(ExpiredToken), token.Status)
}

func TestTokenRenewalJob_NextRunIn(t *testing.T) {
	t.Parallel()

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))

	v := NewTestVaultServer(t)

	type tokenArgs struct {
		e time.Duration
		s TokenStatus
	}
	tests := []struct {
		name            string
		currentTokenExp time.Duration
		tokens          []tokenArgs
		want            time.Duration
		skipCredStore   bool
	}{
		{
			name:          "default-duration",
			skipCredStore: true,
			want:          defaultNextRunIn,
		},
		{
			name:            "1-hour-token",
			currentTokenExp: time.Hour,
			want:            30 * time.Minute,
		},
		{
			name:            "2-hour-token",
			currentTokenExp: 2 * time.Hour,
			want:            time.Hour,
		},
		{
			name:            "multiple-maintaining",
			currentTokenExp: 24 * time.Hour,
			tokens: []tokenArgs{
				{e: 12 * time.Hour, s: MaintainingToken},
				{e: 6 * time.Hour, s: MaintainingToken},
				{e: 8 * time.Hour, s: MaintainingToken},
			},
			// 6 hours is the soonest expiry
			want: 3 * time.Hour,
		},
		{
			name:            "multiple-maintaining-with-single-overdue-renewal",
			currentTokenExp: 24 * time.Hour,
			tokens: []tokenArgs{
				{e: 24 * time.Hour, s: MaintainingToken},
				{e: 6 * time.Hour, s: MaintainingToken},
				{e: -12 * time.Hour, s: MaintainingToken},
				{e: 10 * time.Hour, s: MaintainingToken},
			},
			want: 0,
		},
		{
			name:            "multiple-current-soonest",
			currentTokenExp: 6 * time.Hour,
			tokens: []tokenArgs{
				{e: time.Minute, s: RevokedToken},
				{e: time.Minute, s: ExpiredToken},
				{e: 8 * time.Hour, s: MaintainingToken},
			},
			// 6 hours is the soonest expiry, revoked and expired tokens with sooner
			// expirations should be ignored
			want: 3 * time.Hour,
		},
		{
			name:            "multiple-maintaining-soonest",
			currentTokenExp: 6 * time.Hour,
			tokens: []tokenArgs{
				{e: time.Minute, s: RevokedToken},
				{e: time.Minute, s: ExpiredToken},
				{e: 4 * time.Hour, s: MaintainingToken},
			},
			// 4 hour maintaining token is the soonest expiry
			// revoked and expired tokens with sooner expirations should be ignored
			want: 2 * time.Hour,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)
			r, err := newTokenRenewalJob(rw, rw, kmsCache, hclog.L())
			assert.NoError(err)
			require.NotNil(r)

			if !tt.skipCredStore {
				_, token := v.CreateToken(t)
				in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
				require.NoError(err)
				sche := scheduler.TestScheduler(t, conn, wrapper)
				repo, err := NewRepository(rw, rw, kmsCache, sche)
				require.NoError(err)
				cs, err := repo.CreateCredentialStore(context.Background(), in)
				require.NoError(err)

				for _, token := range tt.tokens {
					testVaultToken(t, conn, wrapper, v, cs, token.s, token.e)
				}

				// inserting new tokens moves the current token to a maintaining state, move it back to current and set expiration time
				count, err := rw.Exec(context.Background(), testUpdateTokenStatusExpirationQuery, []interface{}{CurrentToken, tt.currentTokenExp.Seconds(), cs.outputToken.TokenHmac})
				require.NoError(err)
				assert.Equal(1, count)
			}

			got, err := r.NextRunIn()
			require.NoError(err)
			// Round to time.Minute to account for lost time between creating tokens and determining next run
			assert.Equal(tt.want.Round(time.Minute), got.Round(time.Minute))

			// Set all tokens to revoked for next test
			_, err = rw.Exec(context.Background(), "update credential_vault_token set status = 'revoked'", nil)
			assert.NoError(err)
		})
	}
}

func TestNewTokenRevocationJob(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	type args struct {
		r      db.Reader
		w      db.Writer
		kms    *kms.Kms
		logger hclog.Logger
	}
	tests := []struct {
		name        string
		args        args
		options     []Option
		wantLimit   int
		wantErr     bool
		wantErrCode errors.Code
	}{
		{
			name:        "nil reader",
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil writer",
			args: args{
				r: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil kms",
			args: args{
				r: rw,
				w: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil logger",
			args: args{
				r:   rw,
				w:   rw,
				kms: kmsCache,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "valid-no-options",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			wantLimit: db.DefaultLimit,
		},
		{
			name: "valid-with-limit",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			options:   []Option{WithLimit(100)},
			wantLimit: 100,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			got, err := newTokenRevocationJob(tt.args.r, tt.args.w, tt.args.kms, tt.args.logger, tt.options...)
			if tt.wantErr {
				require.Error(err)
				assert.Nil(got)
				assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err)
				return
			}
			require.NoError(err)
			require.NotNil(got)
			assert.Equal(tt.args.r, got.reader)
			assert.Equal(tt.args.w, got.writer)
			assert.Equal(tt.args.kms, got.kms)
			require.NotNil(got.logger)
			assert.Equal(tt.wantLimit, got.limit)
		})
	}
}

func TestTokenRevocationJob_RunLimits(t *testing.T) {
	t.Parallel()

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))

	v := NewTestVaultServer(t)

	count := 5
	tests := []struct {
		name    string
		opts    []Option
		wantLen int
	}{
		{
			name:    "with-no-limits",
			wantLen: count,
		},
		{
			name:    "with-negative-limit",
			opts:    []Option{WithLimit(-1)},
			wantLen: count,
		},
		{
			name:    "with-limit",
			opts:    []Option{WithLimit(2)},
			wantLen: 2,
		},
		{
			name:    "with-limit-greater-than-count",
			opts:    []Option{WithLimit(count + 5)},
			wantLen: count,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			_, token := v.CreateToken(t)
			in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
			require.NoError(err)
			sche := scheduler.TestScheduler(t, conn, wrapper)
			repo, err := NewRepository(rw, rw, kmsCache, sche)
			require.NoError(err)
			cs, err := repo.CreateCredentialStore(context.Background(), in)
			require.NoError(err)

			for i := 0; i < count*3; i++ {
				var status TokenStatus
				// Alternative between token status, the current token as well as the
				// revoked and expired tokens should have no impact on number tokens revoked
				switch i % 3 {
				case 0:
					status = MaintainingToken
				case 1:
					status = RevokedToken
				case 2:
					status = ExpiredToken
				}
				testVaultToken(t, conn, wrapper, v, cs, status, 5*time.Minute)
			}

			// inserting new tokens moves the current token to a maintaining state, move it back to current and set expiration time
			numRows, err := rw.Exec(context.Background(), testUpdateTokenStatusExpirationQuery, []interface{}{CurrentToken, time.Minute.Seconds(), cs.outputToken.TokenHmac})
			require.NoError(err)
			assert.Equal(1, numRows)

			r, err := newTokenRevocationJob(rw, rw, kmsCache, hclog.L(), tt.opts...)
			require.NoError(err)

			err = r.Run(context.Background())
			require.NoError(err)
			assert.Equal(tt.wantLen, r.numTokens)

			// Set all tokens to revoked for next test
			_, err = rw.Exec(context.Background(), "update credential_vault_token set status = 'revoked'", nil)
			require.NoError(err)
		})
	}
}

func TestTokenRevocationJob_Run(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)

	_, ct := v.CreateToken(t)
	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	require.NoError(err)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(err)

	r, err := newTokenRevocationJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	err = sche.RegisterJob(context.Background(), r)
	require.NoError(err)

	// No tokens should have been revoked since only the current token exists
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(0, r.numProcessed)

	// Create maintaining tokens with and without credentials
	noCredsToken := testVaultToken(t, conn, wrapper, v, cs, MaintainingToken, 5*time.Minute)
	credsToken := testVaultToken(t, conn, wrapper, v, cs, MaintainingToken, 5*time.Minute)
	revokeToken := testVaultToken(t, conn, wrapper, v, cs, RevokeToken, 5*time.Minute)

	// inserting new tokens moves the current token to a maintaining state, move it back to current and set expiration time
	count, err := rw.Exec(context.Background(), testUpdateTokenStatusExpirationQuery, []interface{}{CurrentToken, (5 * time.Minute).Seconds(), cs.outputToken.TokenHmac})
	require.NoError(err)
	assert.Equal(1, count)

	// Create cred lib and session for credential
	libPath := path.Join("database", "creds", "opened")
	cl, err := NewCredentialLibrary(cs.PublicId, libPath, WithMethod(MethodGet))
	require.NoError(err)
	cl.PublicId, err = newCredentialLibraryId()
	require.NoError(err)
	err = rw.Create(context.Background(), cl)
	require.NoError(err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	target.TestCredentialLibrary(t, conn, tar.GetPublicId(), cl.GetPublicId())
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	// Create credential attached to credsToken
	_, cred := testVaultCred(t, conn, v, cl, sess, credsToken, ActiveCredential, 5*time.Minute)

	// Create fake credential attached to revokeToken
	_, revokeCred := testVaultCred(t, conn, v, cl, sess, revokeToken, ActiveCredential, 5*time.Minute)

	// Running should revoke noCredsToken and the revokeToken even though it has active
	// credentials it has been marked for revocation
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(2, r.numProcessed)

	// Verify noCredsToken was revoked in vault
	v.VerifyTokenInvalid(t, string(noCredsToken.GetToken()))

	// Verify noCredsToken was set to revoked in repo
	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{noCredsToken.TokenHmac}))
	assert.Equal(string(RevokedToken), repoToken.Status)

	// Verify revokeToken was revoked in vault
	v.VerifyTokenInvalid(t, string(revokeToken.GetToken()))

	// Verify revokeToken was set to revoked in repo
	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{revokeToken.TokenHmac}))
	assert.Equal(string(RevokedToken), repoToken.Status)

	// Verify revokeCred attached to revokeToken were marked as revoked
	lookupCred := allocCredential()
	lookupCred.PublicId = revokeCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(RevokedCredential), lookupCred.Status)

	// Verify credsToken was not revoked in vault
	lookupToken := v.LookupToken(t, string(credsToken.GetToken()))
	assert.NotNil(lookupToken)

	// Revoke credential in repo
	query, queryValues := cred.updateStatusQuery(RevokedCredential)
	rows, err := rw.Exec(context.Background(), query, queryValues)
	assert.Equal(1, rows)
	assert.NoError(err)

	// Running again should now revoke the credsToken
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(1, r.numProcessed)

	// Verify credsToken was revoked in vault
	v.VerifyTokenInvalid(t, string(credsToken.GetToken()))

	// Verify credsToken was set to revoked in repo
	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{credsToken.TokenHmac}))
	assert.Equal(string(RevokedToken), repoToken.Status)

	err = r.Run(context.Background())
	require.NoError(err)
	// With only the current token remaining no tokens should be revoked
	assert.Equal(0, r.numProcessed)
}

func TestNewCredentialRenewalJob(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	type args struct {
		r      db.Reader
		w      db.Writer
		kms    *kms.Kms
		logger hclog.Logger
	}
	tests := []struct {
		name        string
		args        args
		options     []Option
		wantLimit   int
		wantErr     bool
		wantErrCode errors.Code
	}{
		{
			name:        "nil reader",
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil writer",
			args: args{
				r: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil kms",
			args: args{
				r: rw,
				w: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil logger",
			args: args{
				r:   rw,
				w:   rw,
				kms: kmsCache,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "valid-no-options",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			wantLimit: db.DefaultLimit,
		},
		{
			name: "valid-with-limit",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			options:   []Option{WithLimit(100)},
			wantLimit: 100,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			got, err := newCredentialRenewalJob(tt.args.r, tt.args.w, tt.args.kms, tt.args.logger, tt.options...)
			if tt.wantErr {
				require.Error(err)
				assert.Nil(got)
				assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err)
				return
			}
			require.NoError(err)
			require.NotNil(got)
			assert.Equal(tt.args.r, got.reader)
			assert.Equal(tt.args.w, got.writer)
			assert.Equal(tt.args.kms, got.kms)
			require.NotNil(got.logger)
			assert.Equal(tt.wantLimit, got.limit)
		})
	}
}

func TestCredentialRenewalJob_RunLimits(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)
	_, ct := v.CreateToken(t)
	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	require.NoError(t, err)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(t, err)
	cs, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(t, err)

	libPath := path.Join("database", "creds", "opened")
	cl, err := NewCredentialLibrary(cs.PublicId, libPath, WithMethod(MethodGet))
	require.NoError(t, err)
	cl.PublicId, err = newCredentialLibraryId()
	require.NoError(t, err)
	err = rw.Create(context.Background(), cl)
	require.NoError(t, err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	target.TestCredentialLibrary(t, conn, tar.GetPublicId(), cl.GetPublicId())
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})
	credsToken := testVaultToken(t, conn, wrapper, v, cs, CurrentToken, 5*time.Minute)

	count := 10
	tests := []struct {
		name    string
		opts    []Option
		wantLen int
	}{
		{
			name:    "with-no-limits",
			wantLen: count,
		},
		{
			name:    "with-negative-limit",
			opts:    []Option{WithLimit(-1)},
			wantLen: count,
		},
		{
			name:    "with-limit",
			opts:    []Option{WithLimit(2)},
			wantLen: 2,
		},
		{
			name:    "with-limit-greater-than-count",
			opts:    []Option{WithLimit(count + 5)},
			wantLen: count,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			for i := 0; i < count*4; i++ {
				var status CredentialStatus
				// Alternative between cred status, revoke, revoked and expired should have
				// no impact on number creds renewed.  Only active creds should be renewed
				switch i % 4 {
				case 0:
					status = ActiveCredential
				case 1:
					status = RevokeCredential
				case 2:
					status = RevokedCredential
				case 3:
					status = ExpiredCredential
				}
				testVaultCred(t, conn, v, cl, sess, credsToken, status, 5*time.Minute)
			}

			r, err := newCredentialRenewalJob(rw, rw, kmsCache, hclog.L(), tt.opts...)
			require.NoError(err)

			err = r.Run(context.Background())
			require.NoError(err)
			assert.Equal(tt.wantLen, r.numCreds)

			// Set all credentials to revoked for next test
			_, err = rw.Exec(context.Background(), "update credential_vault_credential set status = 'revoked'", nil)
			assert.NoError(err)
		})
	}
}

func TestCredentialRenewalJob_Run(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	kmsCache := kms.TestKms(t, conn, wrapper)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)

	_, token := v.CreateToken(t, WithPolicies([]string{"default", "boundary-controller", "database"}))
	credStoreIn, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), credStoreIn)
	require.NoError(err)

	libPath := path.Join("database", "creds", "opened")
	libIn, err := NewCredentialLibrary(cs.GetPublicId(), libPath)
	require.NoError(err)
	cl, err := repo.CreateCredentialLibrary(context.Background(), prj.GetPublicId(), libIn)
	require.NoError(err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	csToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &csToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))

	credRenewal, err := newCredentialRenewalJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	err = credRenewal.Run(context.Background())
	require.NoError(err)
	// No credentials should have been renewed
	assert.Equal(0, credRenewal.numCreds)

	_, activeCred := testVaultCred(t, conn, v, cl, sess, csToken, ActiveCredential, 5*time.Minute)
	_, revokeCred := testVaultCred(t, conn, v, cl, sess, csToken, RevokeCredential, 5*time.Minute)
	_, revokedCred := testVaultCred(t, conn, v, cl, sess, csToken, RevokedCredential, 5*time.Minute)
	_, expiredCred := testVaultCred(t, conn, v, cl, sess, csToken, ExpiredCredential, 5*time.Minute)

	secret := v.LookupLease(t, activeCred.ExternalId)
	// Secret should not have a last renewal time
	assert.Nil(secret.Data["last_renewal"])

	// Sleep to move clock
	time.Sleep(2 * time.Second)

	err = credRenewal.Run(context.Background())
	require.NoError(err)
	// The active credential should have been renewed
	assert.Equal(1, credRenewal.numCreds)

	// Active credential expiration time should have been updated
	lookupCred := allocCredential()
	lookupCred.PublicId = activeCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Truef(lookupCred.ExpirationTime.AsTime().After(activeCred.ExpirationTime.AsTime()), "expected expiration time to be updated")

	// Revoke, Revoked and Expired credentials expiration times should not have changed
	lookupCred = allocCredential()
	lookupCred.PublicId = revokeCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(lookupCred.ExpirationTime.AsTime(), revokeCred.ExpirationTime.AsTime())
	lookupCred = allocCredential()
	lookupCred.PublicId = revokedCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(lookupCred.ExpirationTime.AsTime(), revokedCred.ExpirationTime.AsTime())
	lookupCred = allocCredential()
	lookupCred.PublicId = expiredCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(lookupCred.ExpirationTime.AsTime(), expiredCred.ExpirationTime.AsTime())

	// Active credential should have a last renewal time in Vault
	secret = v.LookupLease(t, activeCred.ExternalId)
	assert.NotNil(secret.Data["last_renewal"])

	// Revoke, Revoked and Expired credentials should not have a last renewal time
	secret = v.LookupLease(t, revokeCred.ExternalId)
	assert.Nil(secret.Data["last_renewal"])
	secret = v.LookupLease(t, revokedCred.ExternalId)
	assert.Nil(secret.Data["last_renewal"])
	secret = v.LookupLease(t, expiredCred.ExternalId)
	assert.Nil(secret.Data["last_renewal"])
}

func TestCredentialRenewalJob_RunExpired(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	kmsCache := kms.TestKms(t, conn, wrapper)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)

	_, token := v.CreateToken(t, WithPolicies([]string{"default", "boundary-controller", "database"}))
	credStoreIn, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), credStoreIn)
	require.NoError(err)

	libPath := path.Join("database", "creds", "opened")
	libIn, err := NewCredentialLibrary(cs.GetPublicId(), libPath)
	require.NoError(err)
	cl, err := repo.CreateCredentialLibrary(context.Background(), prj.GetPublicId(), libIn)
	require.NoError(err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))

	credRenewal, err := newCredentialRenewalJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	_, cred := testVaultCred(t, conn, v, cl, sess, repoToken, ActiveCredential, time.Minute)

	vc := v.client(t).cl
	// revoke the credential in Vault outside of the job
	assert.NoError(vc.Sys().Revoke(cred.ExternalId))

	// Credential status should still be active
	lookupCred := allocCredential()
	lookupCred.PublicId = cred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(ActiveCredential), lookupCred.Status)

	err = credRenewal.Run(context.Background())
	require.NoError(err)
	// The active credential should have been processed
	assert.Equal(1, credRenewal.numCreds)

	// Credential status should have been updated to expired
	lookupCred = allocCredential()
	lookupCred.PublicId = cred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(ExpiredCredential), lookupCred.Status)
}

func TestCredentialRenewalJob_NextRunIn(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)

	_, ct := v.CreateToken(t)
	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	require.NoError(t, err)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(t, err)
	cs, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(t, err)

	libPath := path.Join("database", "creds", "opened")
	libIn, err := NewCredentialLibrary(cs.GetPublicId(), libPath)
	require.NoError(t, err)
	cl, err := repo.CreateCredentialLibrary(context.Background(), prj.GetPublicId(), libIn)
	require.NoError(t, err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	target.TestCredentialLibrary(t, conn, tar.GetPublicId(), cl.GetPublicId())
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	type args struct {
		e time.Duration
		s CredentialStatus
	}
	tests := []struct {
		name string
		args []args
		want time.Duration
	}{
		{
			name: "default-duration",
			want: defaultNextRunIn,
		},
		{
			name: "1-hour-active-credential",
			args: []args{
				{s: ActiveCredential, e: time.Hour},
			},
			want: 30 * time.Minute,
		},
		{
			name: "1-hour-revoke-credential",
			args: []args{
				{s: RevokeCredential, e: time.Hour},
			},
			want: defaultNextRunIn,
		},
		{
			name: "1-hour-revoked-credential",
			args: []args{
				{s: RevokedCredential, e: time.Hour},
			},
			want: defaultNextRunIn,
		},
		{
			name: "1-hour-expired-credential",
			args: []args{
				{s: ExpiredCredential, e: time.Hour},
			},
			want: defaultNextRunIn,
		},
		{
			name: "multiple-all-active",
			args: []args{
				{s: ActiveCredential, e: 24 * time.Hour},
				{s: ActiveCredential, e: 6 * time.Hour},
				{s: ActiveCredential, e: 10 * time.Hour},
			},
			// 6 hours is the soonest expiration time
			want: 3 * time.Hour,
		},
		{
			name: "multiple-mixed",
			args: []args{
				{s: RevokeCredential, e: 2 * time.Hour},
				{s: ActiveCredential, e: 48 * time.Hour},
				{s: ActiveCredential, e: 24 * time.Hour},
				{s: RevokedCredential, e: 6 * time.Hour},
				{s: RevokeCredential, e: 8 * time.Hour},
				{s: ExpiredCredential, e: 10 * time.Hour},
			},
			// 24 hours is the soonest active expiration time
			want: 12 * time.Hour,
		},
		{
			name: "overdue-active-renewal",
			args: []args{
				{s: ActiveCredential, e: -12 * time.Hour},
			},
			want: 0,
		},
		{
			name: "non-active-overdue",
			args: []args{
				{s: ActiveCredential, e: 24 * time.Hour},
				{s: ExpiredCredential, e: -12 * time.Hour},
				{s: RevokeCredential, e: -12 * time.Hour},
				{s: RevokedCredential, e: -12 * time.Hour},
			},
			// The active credential is expiring in 24 hours the non active credentials
			// that are overdue should not impact the query
			want: 12 * time.Hour,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)
			r, err := newCredentialRenewalJob(rw, rw, kmsCache, hclog.L())
			assert.NoError(err)
			require.NotNil(r)

			token := testVaultToken(t, conn, wrapper, v, cs, CurrentToken, 5*time.Minute)
			for _, cred := range tt.args {
				testVaultCred(t, conn, v, cl, sess, token, cred.s, cred.e)
			}

			got, err := r.NextRunIn()
			require.NoError(err)
			// Round to time.Minute to account for lost time between creating credentials and determining next run
			assert.Equal(tt.want.Round(time.Minute), got.Round(time.Minute))

			// Set all credentials to revoked for next test
			_, err = rw.Exec(context.Background(), "update credential_vault_credential set status = 'revoked'", nil)
			assert.NoError(err)
		})
	}
}

func TestNewCredentialRevocationJob(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	type args struct {
		r      db.Reader
		w      db.Writer
		kms    *kms.Kms
		logger hclog.Logger
	}
	tests := []struct {
		name        string
		args        args
		options     []Option
		wantLimit   int
		wantErr     bool
		wantErrCode errors.Code
	}{
		{
			name:        "nil reader",
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil writer",
			args: args{
				r: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil kms",
			args: args{
				r: rw,
				w: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil logger",
			args: args{
				r:   rw,
				w:   rw,
				kms: kmsCache,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "valid-no-options",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			wantLimit: db.DefaultLimit,
		},
		{
			name: "valid-with-limit",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			options:   []Option{WithLimit(100)},
			wantLimit: 100,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			got, err := newCredentialRevocationJob(tt.args.r, tt.args.w, tt.args.kms, tt.args.logger, tt.options...)
			if tt.wantErr {
				require.Error(err)
				assert.Nil(got)
				assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err)
				return
			}
			require.NoError(err)
			require.NotNil(got)
			assert.Equal(tt.args.r, got.reader)
			assert.Equal(tt.args.w, got.writer)
			assert.Equal(tt.args.kms, got.kms)
			require.NotNil(got.logger)
			assert.Equal(tt.wantLimit, got.limit)
		})
	}
}

func TestCredentialRevocationJob_RunLimits(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)
	_, ct := v.CreateToken(t, WithPolicies([]string{"default", "boundary-controller", "database"}))
	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	require.NoError(t, err)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(t, err)
	cs, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(t, err)

	libPath := path.Join("database", "creds", "opened")
	cl, err := NewCredentialLibrary(cs.PublicId, libPath, WithMethod(MethodGet))
	require.NoError(t, err)
	cl.PublicId, err = newCredentialLibraryId()
	require.NoError(t, err)
	err = rw.Create(context.Background(), cl)
	require.NoError(t, err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	target.TestCredentialLibrary(t, conn, tar.GetPublicId(), cl.GetPublicId())
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	repoToken := allocToken()
	require.NoError(t, rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))

	count := 10
	tests := []struct {
		name    string
		opts    []Option
		wantLen int
	}{
		{
			name:    "with-no-limits",
			wantLen: count,
		},
		{
			name:    "with-negative-limit",
			opts:    []Option{WithLimit(-1)},
			wantLen: count,
		},
		{
			name:    "with-limit",
			opts:    []Option{WithLimit(2)},
			wantLen: 2,
		},
		{
			name:    "with-limit-greater-than-count",
			opts:    []Option{WithLimit(count + 5)},
			wantLen: count,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			for i := 0; i < count*4; i++ {
				var status CredentialStatus
				// Alternative between cred status, active, revoked and expired should have
				// no impact on number creds revoked.  Only revoke creds should be revoked.
				switch i % 4 {
				case 0:
					status = ActiveCredential
				case 1:
					status = RevokeCredential
				case 2:
					status = RevokedCredential
				case 3:
					status = ExpiredCredential
				}
				testVaultCred(t, conn, v, cl, sess, repoToken, status, 5*time.Minute)
			}

			r, err := newCredentialRevocationJob(rw, rw, kmsCache, hclog.L(), tt.opts...)
			require.NoError(err)

			err = r.Run(context.Background())
			require.NoError(err)
			assert.Equal(tt.wantLen, r.numCreds)

			// Set all credentials to revoked for next test
			_, err = rw.Exec(context.Background(), "update credential_vault_credential set status = 'revoked'", nil)
			assert.NoError(err)
		})
	}
}

func TestCredentialRevocationJob_Run(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	testDb := v.MountDatabase(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	kmsCache := kms.TestKms(t, conn, wrapper)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)

	_, token := v.CreateToken(t, WithPolicies([]string{"default", "boundary-controller", "database"}))
	credStoreIn, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), credStoreIn)
	require.NoError(err)

	libPath := path.Join("database", "creds", "opened")
	libIn, err := NewCredentialLibrary(cs.GetPublicId(), libPath)
	require.NoError(err)
	cl, err := repo.CreateCredentialLibrary(context.Background(), prj.GetPublicId(), libIn)
	require.NoError(err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))

	r, err := newCredentialRevocationJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	err = r.Run(context.Background())
	require.NoError(err)
	// No credentials should have been revoked
	assert.Equal(0, r.numCreds)

	secret1, _ := testVaultCred(t, conn, v, cl, sess, repoToken, ActiveCredential, 5*time.Minute)
	revokeSecret, revokeCred := testVaultCred(t, conn, v, cl, sess, repoToken, RevokeCredential, 5*time.Minute)
	secret2, _ := testVaultCred(t, conn, v, cl, sess, repoToken, RevokedCredential, 5*time.Minute)
	secret3, _ := testVaultCred(t, conn, v, cl, sess, repoToken, ExpiredCredential, 5*time.Minute)

	// Verify the revokeCred has a status of revoke
	lookupCred := allocCredential()
	lookupCred.PublicId = revokeCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(RevokeCredential), lookupCred.Status)

	// Verify revokeCred is valid in testDb
	assert.NoError(testDb.ValidateCredential(t, revokeSecret))

	err = r.Run(context.Background())
	require.NoError(err)
	// The revoke credential should have been revoked
	assert.Equal(1, r.numCreds)

	// revokeCred should now have a status of revoked
	lookupCred = allocCredential()
	lookupCred.PublicId = revokeCred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(RevokedCredential), lookupCred.Status)

	// revokeCred should no longer be valid in test database
	assert.Error(testDb.ValidateCredential(t, revokeSecret))

	// Other creds should still be valid in test database
	assert.NoError(testDb.ValidateCredential(t, secret1))
	assert.NoError(testDb.ValidateCredential(t, secret2))
	assert.NoError(testDb.ValidateCredential(t, secret3))
}

func TestCredentialRevocationJob_RunDeleted(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	testDb := v.MountDatabase(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	kmsCache := kms.TestKms(t, conn, wrapper)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)

	_, token := v.CreateToken(t, WithPolicies([]string{"default", "boundary-controller", "database"}))
	credStoreIn, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), credStoreIn)
	require.NoError(err)

	libPath := path.Join("database", "creds", "opened")
	libIn, err := NewCredentialLibrary(cs.GetPublicId(), libPath)
	require.NoError(err)
	cl, err := repo.CreateCredentialLibrary(context.Background(), prj.GetPublicId(), libIn)
	require.NoError(err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	sess := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))

	r, err := newCredentialRevocationJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	secret, cred := testVaultCred(t, conn, v, cl, sess, repoToken, ActiveCredential, 5*time.Hour)

	err = r.Run(context.Background())
	require.NoError(err)
	// No credentials should have been revoked as expiration is 5 hours from now
	assert.Equal(0, r.numCreds)

	// Deleting the library should set the cred library_id to null, but not revoke the cred
	count, err := rw.Delete(context.Background(), cl)
	require.NoError(err)
	assert.Equal(1, count)

	err = r.Run(context.Background())
	require.NoError(err)
	// No credentials should have been revoked
	assert.Equal(0, r.numCreds)

	// Verify the cred has a status of active with an empty libraryId
	lookupCred := allocCredential()
	lookupCred.PublicId = cred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(ActiveCredential), lookupCred.Status)
	assert.Empty(lookupCred.LibraryId)

	// secret should still be valid in test database
	assert.NoError(testDb.ValidateCredential(t, secret))

	// Deleting the session should set the cred session_id to null and schedule cred for revocation
	count, err = rw.Delete(context.Background(), sess)
	require.NoError(err)
	assert.Equal(1, count)

	// cred should now have a status of revoke and empty sessionId
	lookupCred = allocCredential()
	lookupCred.PublicId = cred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Empty(lookupCred.SessionId)
	assert.Equal(string(RevokeCredential), lookupCred.Status)

	err = r.Run(context.Background())
	require.NoError(err)
	// The revoke credential should have been revoked
	assert.Equal(1, r.numCreds)

	// cred should now have a status of revoked
	lookupCred = allocCredential()
	lookupCred.PublicId = cred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(RevokedCredential), lookupCred.Status)

	// secret should no longer be valid in test database
	assert.Error(testDb.ValidateCredential(t, secret))
}

func TestNewCredentialStoreCleanupJob(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)

	type args struct {
		r      db.Reader
		w      db.Writer
		kms    *kms.Kms
		logger hclog.Logger
	}
	tests := []struct {
		name        string
		args        args
		options     []Option
		wantLimit   int
		wantErr     bool
		wantErrCode errors.Code
	}{
		{
			name:        "nil reader",
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil writer",
			args: args{
				r: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil kms",
			args: args{
				r: rw,
				w: rw,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "nil logger",
			args: args{
				r:   rw,
				w:   rw,
				kms: kmsCache,
			},
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "valid-no-options",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			wantLimit: db.DefaultLimit,
		},
		{
			name: "valid-with-limit",
			args: args{
				r:      rw,
				w:      rw,
				kms:    kmsCache,
				logger: hclog.L(),
			},
			options:   []Option{WithLimit(100)},
			wantLimit: 100,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			got, err := newCredentialStoreCleanupJob(tt.args.r, tt.args.w, tt.args.kms, tt.args.logger, tt.options...)
			if tt.wantErr {
				require.Error(err)
				assert.Nil(got)
				assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err)
				return
			}
			require.NoError(err)
			require.NotNil(got)
			assert.Equal(tt.args.r, got.reader)
			assert.Equal(tt.args.w, got.writer)
			assert.Equal(tt.args.kms, got.kms)
			require.NotNil(got.logger)
			assert.Equal(tt.wantLimit, got.limit)
		})
	}
}

func TestCredentialStoreCleanupJob_Run(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	kmsCache := kms.TestKms(t, conn, wrapper)
	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	v := NewTestVaultServer(t)

	_, ct := v.CreateToken(t)
	in, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	require.NoError(err)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)
	cs1, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(err)

	_, ct = v.CreateToken(t)
	in, err = NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(ct))
	require.NoError(err)
	cs2, err := repo.CreateCredentialStore(context.Background(), in)
	require.NoError(err)

	// Get token hmac for verifications below
	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "store_id = ?", []interface{}{cs1.PublicId}))
	cs1TokenHmac := repoToken.TokenHmac

	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "store_id = ?", []interface{}{cs2.PublicId}))
	cs2TokenHmac := repoToken.TokenHmac

	// create second token on cs2
	secondToken := testVaultToken(t, conn, wrapper, v, cs2, MaintainingToken, time.Hour)

	r, err := newCredentialStoreCleanupJob(rw, rw, kmsCache, hclog.L())
	require.NoError(err)

	err = sche.RegisterJob(context.Background(), r)
	require.NoError(err)

	// No credential stores should have been cleaned up
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(0, r.numStores)

	// Soft delete both credential stores
	count, err := repo.DeleteCredentialStore(context.Background(), cs1.PublicId)
	require.NoError(err)
	assert.Equal(1, count)

	count, err = repo.DeleteCredentialStore(context.Background(), cs2.PublicId)
	require.NoError(err)
	assert.Equal(1, count)

	// Verify tokens have been set to revoke
	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "store_id = ?", []interface{}{cs1.PublicId}))
	assert.Equal(string(RevokeToken), repoToken.Status)

	repoToken = allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "store_id = ?", []interface{}{cs2.PublicId}))
	assert.Equal(string(RevokeToken), repoToken.Status)

	// Both soft deleted credential stores should not be cleaned up yet
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(0, r.numStores)

	// Update cs1 token to be marked as revoked
	count, err = rw.Exec(context.Background(), updateTokenStatusQuery, []interface{}{RevokedToken, cs1TokenHmac})
	require.NoError(err)
	assert.Equal(1, count)

	// cs1 should be deleted
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(1, r.numStores)

	// Lookup of cs1 and its token should fail
	agg := allocPublicStore()
	agg.PublicId = cs1.PublicId
	err = rw.LookupByPublicId(context.Background(), agg)
	require.Error(err)
	assert.True(errors.IsNotFoundError(err))
	repoToken = allocToken()
	err = rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs1TokenHmac})
	require.Error(err)
	assert.True(errors.IsNotFoundError(err))

	// Lookup of cs2 and its token should not error
	_, err = repo.LookupCredentialStore(context.Background(), cs2.PublicId)
	require.NoError(err)
	repoToken = allocToken()
	err = rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs2TokenHmac})
	require.NoError(err)

	// Update cs2 token expiration time
	count, err = rw.Exec(context.Background(), "update credential_vault_token set expiration_time = now() where token_hmac = ?;", []interface{}{cs2TokenHmac})
	require.NoError(err)
	assert.Equal(1, count)

	// cs2 still has a second token not yet revoked/expired
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(0, r.numStores)

	// Lookup of cs2 and its token should not error
	_, err = repo.LookupCredentialStore(context.Background(), cs2.PublicId)
	require.NoError(err)
	repoToken = allocToken()
	err = rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs2TokenHmac})
	require.NoError(err)

	// set secondToken with an expired status
	count, err = rw.Exec(context.Background(), updateTokenStatusQuery, []interface{}{ExpiredToken, secondToken.TokenHmac})
	require.NoError(err)
	assert.Equal(1, count)

	// With no un-expired or un-revoked tokens cs2 should now be deleted
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(1, r.numStores)

	// Lookup of cs2 and its token should fail
	agg = allocPublicStore()
	agg.PublicId = cs2.PublicId
	err = rw.LookupByPublicId(context.Background(), agg)
	require.Error(err)
	assert.True(errors.IsNotFoundError(err))
	repoToken = allocToken()
	err = rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs2TokenHmac})
	require.Error(err)
	assert.True(errors.IsNotFoundError(err))
	err = rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{secondToken.TokenHmac})
	require.Error(err)
	assert.True(errors.IsNotFoundError(err))
}

func TestNewCredentialCleanupJob(t *testing.T) {
	t.Parallel()
	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)

	type args struct {
		w db.Writer
	}
	tests := []struct {
		name        string
		args        args
		options     []Option
		wantLimit   int
		wantErr     bool
		wantErrCode errors.Code
	}{
		{
			name:        "nil writer",
			wantErr:     true,
			wantErrCode: errors.InvalidParameter,
		},
		{
			name: "valid",
			args: args{
				w: rw,
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert, require := assert.New(t), require.New(t)

			got, err := newCredentialCleanupJob(tt.args.w)
			if tt.wantErr {
				require.Error(err)
				assert.Nil(got)
				assert.Truef(errors.Match(errors.T(tt.wantErrCode), err), "Unexpected error %s", err)
				return
			}
			require.NoError(err)
			require.NotNil(got)
			assert.Equal(tt.args.w, got.writer)
		})
	}
}

func TestCredentialCleanupJob_Run(t *testing.T) {
	t.Parallel()
	assert, require := assert.New(t), require.New(t)

	v := NewTestVaultServer(t, WithDockerNetwork(true))
	v.MountDatabase(t)

	conn, _ := db.TestSetup(t, "postgres")
	rw := db.New(conn)
	wrapper := db.TestWrapper(t)
	org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
	kmsCache := kms.TestKms(t, conn, wrapper)
	sche := scheduler.TestScheduler(t, conn, wrapper)
	repo, err := NewRepository(rw, rw, kmsCache, sche)
	require.NoError(err)

	_, token := v.CreateToken(t, WithPolicies([]string{"default", "boundary-controller", "database"}))
	credStoreIn, err := NewCredentialStore(prj.GetPublicId(), v.Addr, []byte(token))
	require.NoError(err)
	cs, err := repo.CreateCredentialStore(context.Background(), credStoreIn)
	require.NoError(err)

	libPath := path.Join("database", "creds", "opened")
	libIn, err := NewCredentialLibrary(cs.GetPublicId(), libPath)
	require.NoError(err)
	cl, err := repo.CreateCredentialLibrary(context.Background(), prj.GetPublicId(), libIn)
	require.NoError(err)

	at := authtoken.TestAuthToken(t, conn, kmsCache, org.GetPublicId())
	uId := at.GetIamUserId()
	hc := static.TestCatalogs(t, conn, prj.GetPublicId(), 1)[0]
	hs := static.TestSets(t, conn, hc.GetPublicId(), 1)[0]
	h := static.TestHosts(t, conn, hc.GetPublicId(), 1)[0]
	static.TestSetMembers(t, conn, hs.GetPublicId(), []*static.Host{h})
	tar := target.TestTcpTarget(t, conn, prj.GetPublicId(), "test", target.WithHostSets([]string{hs.GetPublicId()}))
	sess1 := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})
	sess2 := session.TestSession(t, conn, wrapper, session.ComposedOf{
		UserId:      uId,
		HostId:      h.GetPublicId(),
		TargetId:    tar.GetPublicId(),
		HostSetId:   hs.GetPublicId(),
		AuthTokenId: at.GetPublicId(),
		ScopeId:     prj.GetPublicId(),
		Endpoint:    "tcp://127.0.0.1:22",
	})

	repoToken := allocToken()
	require.NoError(rw.LookupWhere(context.Background(), &repoToken, "token_hmac = ?", []interface{}{cs.outputToken.TokenHmac}))

	r, err := newCredentialCleanupJob(rw)
	require.NoError(err)

	_, sess1Cred1 := testVaultCred(t, conn, v, cl, sess1, repoToken, ActiveCredential, 5*time.Hour)
	_, sess1Cred2 := testVaultCred(t, conn, v, cl, sess1, repoToken, ActiveCredential, 5*time.Hour)
	_, sess1Cred3 := testVaultCred(t, conn, v, cl, sess1, repoToken, ActiveCredential, 5*time.Hour)
	_, sess2Cred := testVaultCred(t, conn, v, cl, sess2, repoToken, ActiveCredential, 5*time.Hour)

	// No credentials should be cleaned up
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(0, r.numCreds)

	// Delete sess1
	count, err := rw.Delete(context.Background(), sess1)
	require.NoError(err)
	assert.Equal(1, count)

	// Credentials are still in the revoke state so none should be deleted yet
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(0, r.numCreds)

	query, queryArgs := sess1Cred1.updateStatusQuery(RevokedCredential)
	count, err = rw.Exec(context.Background(), query, queryArgs)
	require.NoError(err)
	assert.Equal(1, count)

	query, queryArgs = sess1Cred2.updateStatusQuery(ExpiredCredential)
	count, err = rw.Exec(context.Background(), query, queryArgs)
	require.NoError(err)
	assert.Equal(1, count)

	query, queryArgs = sess1Cred3.updateStatusQuery(UnknownCredentialStatus)
	count, err = rw.Exec(context.Background(), query, queryArgs)
	require.NoError(err)
	assert.Equal(1, count)

	query, queryArgs = sess2Cred.updateStatusQuery(RevokedCredential)
	count, err = rw.Exec(context.Background(), query, queryArgs)
	require.NoError(err)
	assert.Equal(1, count)

	// Only the three credentials associated with the deleted session should be deleted
	err = r.Run(context.Background())
	require.NoError(err)
	assert.Equal(3, r.numCreds)

	// Session 1 creds should no longer exist
	lookupCred := allocCredential()
	lookupCred.PublicId = sess1Cred1.PublicId
	require.Error(rw.LookupById(context.Background(), lookupCred))
	lookupCred.PublicId = sess1Cred2.PublicId
	require.Error(rw.LookupById(context.Background(), lookupCred))
	lookupCred.PublicId = sess1Cred3.PublicId
	require.Error(rw.LookupById(context.Background(), lookupCred))

	// Session 2 creds should still exist but be revoked
	lookupCred.PublicId = sess2Cred.PublicId
	require.NoError(rw.LookupById(context.Background(), lookupCred))
	assert.Equal(string(RevokedCredential), lookupCred.Status)
}