Auth Method list pagination (#4165)

* wip

* implement list pagination for auth methods

* remove unused code

* update tests to fix sorting

* address PR comments

* simplify query, add token tests

* change test name

* wip sql testing

* add boolean column to indicate whether auth methods should be returned to unauthenticated users, remove secrets from api response

* is_active_public not null

* remove secrets and unwrapping

* move not null to immediately after data migration

* order public_id descending

* fix(db): Fix auth method migrations (#4193)

* wip

* fix db tests

* fix failing api tests

* reverse order of create time and update time in test

---------

Co-authored-by: Michael Gaffney <mike@gaffney.cc>

internal/daemon/controller/common/common.go

@@ -4,6 +4,7 @@
 package common
 
 import (
+	"github.com/hashicorp/boundary/internal/auth"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
@@ -28,6 +29,7 @@ type (
 	OidcAuthRepoFactory            = oidc.OidcRepoFactory
 	LdapAuthRepoFactory            = ldap.RepoFactory
 	PasswordAuthRepoFactory        func() (*password.Repository, error)
+	AuthMethodRepoFactory          func() (*auth.AuthMethodRepository, error)
 	ServersRepoFactory             func() (*server.Repository, error)
 	StaticRepoFactory              func() (*static.Repository, error)
 	PluginHostRepoFactory          func() (*pluginhost.Repository, error)

internal/daemon/controller/controller.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/hashicorp/boundary/internal/auth"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
@@ -134,6 +135,7 @@ type Controller struct {
 	OidcRepoFn                common.OidcAuthRepoFactory
 	LdapRepoFn                common.LdapAuthRepoFactory
 	PasswordAuthRepoFn        common.PasswordAuthRepoFactory
+	AuthMethodRepoFn          common.AuthMethodRepoFactory
 	ServersRepoFn             common.ServersRepoFactory
 	SessionRepoFn             session.RepositoryFactory
 	ConnectionRepoFn          common.ConnectionRepoFactory
@@ -422,6 +424,9 @@ func New(ctx context.Context, conf *Config) (*Controller, error) {
 	c.PasswordAuthRepoFn = func() (*password.Repository, error) {
 		return password.NewRepository(ctx, dbase, dbase, c.kms)
 	}
+	c.AuthMethodRepoFn = func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, dbase, dbase, c.kms)
+	}
 	c.TargetRepoFn = func(o ...target.Option) (*target.Repository, error) {
 		return target.NewRepository(ctx, dbase, dbase, c.kms, o...)
 	}

internal/daemon/controller/handler.go

@@ -164,7 +164,17 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 		services.RegisterAccountServiceServer(s, accts)
 	}
 	if _, ok := currentServices[services.AuthMethodService_ServiceDesc.ServiceName]; !ok {
-		authMethods, err := authmethods.NewService(c.baseContext, c.kms, c.PasswordAuthRepoFn, c.OidcRepoFn, c.IamRepoFn, c.AuthTokenRepoFn, c.LdapRepoFn)
+		authMethods, err := authmethods.NewService(
+			c.baseContext,
+			c.kms,
+			c.PasswordAuthRepoFn,
+			c.OidcRepoFn,
+			c.IamRepoFn,
+			c.AuthTokenRepoFn,
+			c.LdapRepoFn,
+			c.AuthMethodRepoFn,
+			c.conf.RawConfig.Controller.MaxPageSize,
+		)
 		if err != nil {
 			return fmt.Errorf("failed to create auth method handler service: %w", err)
 		}

internal/daemon/controller/handlers/authmethods/authmethod_service.go

@@ -26,6 +26,8 @@ import (
 	"github.com/hashicorp/boundary/internal/errors"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/listtoken"
+	"github.com/hashicorp/boundary/internal/pagination"
 	"github.com/hashicorp/boundary/internal/perms"
 	"github.com/hashicorp/boundary/internal/requests"
 	"github.com/hashicorp/boundary/internal/types/action"
@@ -57,8 +59,6 @@ const (
 	authMethodIdField = "auth_method_id"
 	tokenTypeField    = "type"
 	isPrimaryField    = "is_primary"
-
-	domain = "auth"
 )
 
 var (
@@ -84,18 +84,31 @@ var (
 type Service struct {
 	pbs.UnsafeAuthMethodServiceServer
 
-	kms        *kms.Kms
-	pwRepoFn   common.PasswordAuthRepoFactory
-	oidcRepoFn common.OidcAuthRepoFactory
-	iamRepoFn  common.IamRepoFactory
-	atRepoFn   common.AuthTokenRepoFactory
-	ldapRepoFn common.LdapAuthRepoFactory
+	kms         *kms.Kms
+	pwRepoFn    common.PasswordAuthRepoFactory
+	oidcRepoFn  common.OidcAuthRepoFactory
+	iamRepoFn   common.IamRepoFactory
+	atRepoFn    common.AuthTokenRepoFactory
+	ldapRepoFn  common.LdapAuthRepoFactory
+	amRepoFn    common.AuthMethodRepoFactory
+	maxPageSize uint
 }
 
 var _ pbs.AuthMethodServiceServer = (*Service)(nil)
 
 // NewService returns a auth method service which handles auth method related requests to boundary.
-func NewService(ctx context.Context, kms *kms.Kms, pwRepoFn common.PasswordAuthRepoFactory, oidcRepoFn common.OidcAuthRepoFactory, iamRepoFn common.IamRepoFactory, atRepoFn common.AuthTokenRepoFactory, ldapRepoFn common.LdapAuthRepoFactory, opt ...handlers.Option) (Service, error) {
+func NewService(
+	ctx context.Context,
+	kms *kms.Kms,
+	pwRepoFn common.PasswordAuthRepoFactory,
+	oidcRepoFn common.OidcAuthRepoFactory,
+	iamRepoFn common.IamRepoFactory,
+	atRepoFn common.AuthTokenRepoFactory,
+	ldapRepoFn common.LdapAuthRepoFactory,
+	amRepoFn common.AuthMethodRepoFactory,
+	maxPageSize uint,
+	opt ...handlers.Option,
+) (Service, error) {
 	const op = "authmethods.NewService"
 	if kms == nil {
 		return Service{}, errors.New(ctx, errors.InvalidParameter, op, "missing kms")
@@ -115,15 +128,32 @@ func NewService(ctx context.Context, kms *kms.Kms, pwRepoFn common.PasswordAuthR
 	if atRepoFn == nil {
 		return Service{}, fmt.Errorf("nil auth token repository provided")
 	}
-	s := Service{kms: kms, pwRepoFn: pwRepoFn, oidcRepoFn: oidcRepoFn, iamRepoFn: iamRepoFn, atRepoFn: atRepoFn, ldapRepoFn: ldapRepoFn}
+	if amRepoFn == nil {
+		return Service{}, fmt.Errorf("nil auth method repository provided")
+	}
+	if maxPageSize == 0 {
+		maxPageSize = uint(globals.DefaultMaxPageSize)
+	}
+	s := Service{
+		kms:         kms,
+		pwRepoFn:    pwRepoFn,
+		oidcRepoFn:  oidcRepoFn,
+		iamRepoFn:   iamRepoFn,
+		atRepoFn:    atRepoFn,
+		ldapRepoFn:  ldapRepoFn,
+		amRepoFn:    amRepoFn,
+		maxPageSize: maxPageSize,
+	}
 
 	return s, nil
 }
 
 // ListAuthMethods implements the interface pbs.AuthMethodServiceServer.
 func (s Service) ListAuthMethods(ctx context.Context, req *pbs.ListAuthMethodsRequest) (*pbs.ListAuthMethodsResponse, error) {
+	const op = "authmethods.(Service).ListAuthMethods"
+
 	if err := validateListRequest(ctx, req); err != nil {
-		return nil, err
+		return nil, errors.Wrap(ctx, err, op)
 	}
 	authResults := s.authResult(ctx, req.GetScopeId(), action.List)
 	if authResults.Error != nil {
@@ -138,6 +168,21 @@ func (s Service) ListAuthMethods(ctx context.Context, req *pbs.ListAuthMethodsRe
 			return nil, authResults.Error
 		}
 	}
+	pageSize := int(s.maxPageSize)
+	// Use the requested page size only if it is smaller than
+	// the configured max.
+	if req.GetPageSize() != 0 && uint(req.GetPageSize()) < s.maxPageSize {
+		pageSize = int(req.GetPageSize())
+	}
+
+	withUnauthenticatedUser := false
+	reqCtx, ok := requests.RequestContextFromCtx(ctx)
+	if !ok {
+		return nil, errors.New(ctx, errors.Internal, op, "no request context found")
+	}
+	if reqCtx.UserId == globals.AnonymousUserId {
+		withUnauthenticatedUser = true
+	}
 
 	scopeIds, scopeInfoMap, err := scopeids.GetListingScopeIds(
 		ctx, s.iamRepoFn, authResults, req.GetScopeId(), resource.AuthMethod, req.GetRecursive())
@@ -149,63 +194,127 @@ func (s Service) ListAuthMethods(ctx context.Context, req *pbs.ListAuthMethodsRe
 		return &pbs.ListAuthMethodsResponse{}, nil
 	}
 
-	ul, err := s.listFromRepo(ctx, scopeIds, authResults)
-	if err != nil {
-		return nil, err
-	}
-	if len(ul) == 0 {
-		return &pbs.ListAuthMethodsResponse{}, nil
+	var filterItemFn func(ctx context.Context, item auth.AuthMethod) (bool, error)
+	switch {
+	case req.GetFilter() != "":
+		// Only use a filter if we need to
+		filter, err := handlers.NewFilter(ctx, req.GetFilter())
+		if err != nil {
+			return nil, err
+		}
+		// TODO: replace the need for this function with some way to convert the `filter`
+		// to a domain type. This would allow filtering to happen in the domain, and we could
+		// remove this callback altogether.
+		filterItemFn = func(ctx context.Context, item auth.AuthMethod) (bool, error) {
+			outputOpts, ok, err := newOutputOpts(ctx, item, scopeInfoMap, authResults)
+			if err != nil {
+				return false, errors.Wrap(ctx, err, op)
+			}
+			if !ok {
+				return false, nil
+			}
+
+			pbItem, err := toAuthMethodProto(ctx, item, outputOpts...)
+			if err != nil {
+				return false, err
+			}
+
+			filterable, err := subtypes.Filterable(ctx, pbItem)
+			if err != nil {
+				return false, err
+			}
+			return filter.Match(filterable), nil
+		}
+	default:
+		filterItemFn = func(ctx context.Context, item auth.AuthMethod) (bool, error) {
+			return true, nil
+		}
 	}
 
-	filter, err := handlers.NewFilter(ctx, req.GetFilter())
+	repo, err := s.amRepoFn()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(ctx, err, op)
 	}
-	finalItems := make([]*pb.AuthMethod, 0, len(ul))
-	res := perms.Resource{
-		Type: resource.AuthMethod,
-	}
-	for _, am := range ul {
-		res.Id = am.GetPublicId()
-		res.ScopeId = am.GetScopeId()
-		authorizedActions := authResults.FetchActionSetForId(ctx, am.GetPublicId(), IdActions[globals.ResourceInfoFromPrefix(am.GetPublicId()).Subtype], requestauth.WithResource(&res)).Strings()
-		if len(authorizedActions) == 0 {
-			continue
-		}
 
-		outputFields := authResults.FetchOutputFields(res, action.List).SelfOrDefaults(authResults.UserId)
-		outputOpts := make([]handlers.Option, 0, 3)
-		outputOpts = append(outputOpts, handlers.WithOutputFields(outputFields))
-		if outputFields.Has(globals.ScopeField) {
-			outputOpts = append(outputOpts, handlers.WithScope(scopeInfoMap[am.GetScopeId()]))
+	grantsHash, err := authResults.GrantsHash(ctx)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+
+	var listResp *pagination.ListResponse[auth.AuthMethod]
+	var sortBy string
+	if req.GetListToken() == "" {
+		sortBy = "created_time"
+		listResp, err = auth.ListAuthMethods(ctx, grantsHash, pageSize, filterItemFn, repo, scopeIds, withUnauthenticatedUser)
+		if err != nil {
+			return nil, err
 		}
-		if outputFields.Has(globals.AuthorizedActionsField) {
-			outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
+	} else {
+		listToken, err := handlers.ParseListToken(ctx, req.GetListToken(), resource.AuthMethod, grantsHash)
+		if err != nil {
+			return nil, err
 		}
-		if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-			collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, am.GetPublicId())
+		switch st := listToken.Subtype.(type) {
+		case *listtoken.PaginationToken:
+			sortBy = "created_time"
+			listResp, err = auth.ListAuthMethodsPage(ctx, grantsHash, pageSize, filterItemFn, listToken, repo, scopeIds, withUnauthenticatedUser)
 			if err != nil {
 				return nil, err
 			}
-			outputOpts = append(outputOpts, handlers.WithAuthorizedCollectionActions(collectionActions))
-		}
-
-		item, err := toAuthMethodProto(ctx, am, outputOpts...)
-		if err != nil {
-			return nil, err
-		}
-
-		// This comes last so that we can use item fields in the filter after
-		// the allowed fields are populated above
-		filterable, err := subtypes.Filterable(ctx, item)
-		if err != nil {
-			return nil, err
-		}
-		if filter.Match(filterable) {
-			finalItems = append(finalItems, item)
+		case *listtoken.StartRefreshToken:
+			sortBy = "updated_time"
+			listResp, err = auth.ListAuthMethodsRefresh(ctx, grantsHash, pageSize, filterItemFn, listToken, repo, scopeIds, withUnauthenticatedUser)
+			if err != nil {
+				return nil, err
+			}
+		case *listtoken.RefreshToken:
+			sortBy = "updated_time"
+			listResp, err = auth.ListAuthMethodsRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listToken, repo, scopeIds, withUnauthenticatedUser)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, handlers.ApiErrorWithCodeAndMessage(codes.InvalidArgument, "unexpected list token subtype: %T", st)
 		}
 	}
-	return &pbs.ListAuthMethodsResponse{Items: finalItems}, nil
+
+	finalItems := make([]*pb.AuthMethod, 0, len(listResp.Items))
+	for _, item := range listResp.Items {
+		outputOpts, ok, err := newOutputOpts(ctx, item, scopeInfoMap, authResults)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op)
+		}
+		if !ok {
+			continue
+		}
+
+		pbItem, err := toAuthMethodProto(ctx, item, outputOpts...)
+		if err != nil {
+			continue
+		}
+		finalItems = append(finalItems, pbItem)
+	}
+	respType := "delta"
+	if listResp.CompleteListing {
+		respType = "complete"
+	}
+	resp := &pbs.ListAuthMethodsResponse{
+		Items:        finalItems,
+		EstItemCount: uint32(listResp.EstimatedItemCount),
+		RemovedIds:   listResp.DeletedIds,
+		ResponseType: respType,
+		SortBy:       sortBy,
+		SortDir:      "desc",
+	}
+
+	if listResp.ListToken != nil {
+		resp.ListToken, err = handlers.MarshalListToken(ctx, listResp.ListToken, pbs.ResourceType_RESOURCE_TYPE_AUTH_METHOD)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return resp, nil
 }
 
 // GetAuthMethod implements the interface pbs.AuthMethodServiceServer.
@@ -499,54 +608,6 @@ func (s Service) getFromRepo(ctx context.Context, id string) (auth.AuthMethod, e
 	return am, nil
 }
 
-func (s Service) listFromRepo(ctx context.Context, scopeIds []string, authResults requestauth.VerifyResults) ([]auth.AuthMethod, error) {
-	const op = "authmethods.(Service).listFromRepo"
-	reqCtx, ok := requests.RequestContextFromCtx(ctx)
-	if !ok {
-		return nil, errors.New(ctx, errors.Internal, op, "no request context found")
-	}
-
-	var outUl []auth.AuthMethod
-
-	oidcRepo, err := s.oidcRepoFn()
-	if err != nil {
-		return nil, errors.Wrap(ctx, err, op)
-	}
-	ol, err := oidcRepo.ListAuthMethods(ctx, scopeIds, oidc.WithUnauthenticatedUser(reqCtx.UserId == globals.AnonymousUserId), oidc.WithLimit(-1))
-	if err != nil {
-		return nil, errors.Wrap(ctx, err, op)
-	}
-	for _, item := range ol {
-		outUl = append(outUl, item)
-	}
-
-	repo, err := s.pwRepoFn()
-	if err != nil {
-		return nil, errors.Wrap(ctx, err, op)
-	}
-	pl, err := repo.ListAuthMethods(ctx, scopeIds)
-	if err != nil {
-		return nil, errors.Wrap(ctx, err, op)
-	}
-	for _, item := range pl {
-		outUl = append(outUl, item)
-	}
-
-	ldapRepo, err := s.ldapRepoFn()
-	if err != nil {
-		return nil, errors.Wrap(ctx, err, op)
-	}
-	ll, err := ldapRepo.ListAuthMethods(ctx, scopeIds, ldap.WithUnauthenticatedUser(ctx, reqCtx.UserId == globals.AnonymousUserId))
-	if err != nil {
-		return nil, errors.Wrap(ctx, err, op)
-	}
-	for _, item := range ll {
-		outUl = append(outUl, item)
-	}
-
-	return outUl, nil
-}
-
 func (s Service) createInRepo(ctx context.Context, scopeId string, item *pb.AuthMethod) (auth.AuthMethod, error) {
 	const op = "authmethods.(Service).createInRepo"
 	var out auth.AuthMethod
@@ -1508,3 +1569,33 @@ func transformAuthenticateResponseAttributes(ctx context.Context, msg proto.Mess
 	}
 	return nil
 }
+
+func newOutputOpts(ctx context.Context, item auth.AuthMethod, scopeInfoMap map[string]*scopes.ScopeInfo, authResults requestauth.VerifyResults) ([]handlers.Option, bool, error) {
+	res := perms.Resource{
+		Type: resource.AuthMethod,
+	}
+	res.Id = item.GetPublicId()
+	res.ScopeId = item.GetScopeId()
+	authorizedActions := authResults.FetchActionSetForId(ctx, item.GetPublicId(), IdActions[globals.ResourceInfoFromPrefix(item.GetPublicId()).Subtype], requestauth.WithResource(&res)).Strings()
+	if len(authorizedActions) == 0 {
+		return nil, false, nil
+	}
+
+	outputFields := authResults.FetchOutputFields(res, action.List).SelfOrDefaults(authResults.UserId)
+	outputOpts := make([]handlers.Option, 0, 3)
+	outputOpts = append(outputOpts, handlers.WithOutputFields(outputFields))
+	if outputFields.Has(globals.ScopeField) {
+		outputOpts = append(outputOpts, handlers.WithScope(scopeInfoMap[item.GetScopeId()]))
+	}
+	if outputFields.Has(globals.AuthorizedActionsField) {
+		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
+	}
+	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
+		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, item.GetPublicId())
+		if err != nil {
+			return nil, false, err
+		}
+		outputOpts = append(outputOpts, handlers.WithAuthorizedCollectionActions(collectionActions))
+	}
+	return outputOpts, true, nil
+}

internal/daemon/controller/handlers/authmethods/authmethod_service_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/hashicorp/boundary/globals"
+	"github.com/hashicorp/boundary/internal/auth"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
@@ -29,8 +30,11 @@ import (
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/errors"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/requests"
+	"github.com/hashicorp/boundary/internal/server"
 	"github.com/hashicorp/boundary/internal/types/action"
 	"github.com/hashicorp/boundary/internal/types/scope"
 	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/authmethods"
@@ -112,6 +116,9 @@ func TestGet(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kmsCache)
 	}
+	authMethodRepoFn := func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
@@ -257,7 +264,7 @@ func TestGet(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
 
-			s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+			s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 			require.NoError(err, "Couldn't create new auth_method service.")
 
 			got, gErr := s.GetAuthMethod(requestauth.DisabledAuthTestContext(iamRepoFn, tc.scopeId), tc.req)
@@ -273,14 +280,14 @@ func TestGet(t *testing.T) {
 			assert.Empty(cmp.Diff(
 				got,
 				tc.res,
-				protocmp.Transform(),
-				protocmp.IgnoreFields(&pb.OidcAuthMethodAttributes{}, "client_secret_hmac"),
 				cmpopts.SortSlices(func(a, b string) bool {
 					return a < b
 				}),
 				cmpopts.SortSlices(func(a, b protocmp.Message) bool {
 					return a.String() < b.String()
 				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pb.OidcAuthMethodAttributes{}, "client_secret_hmac"),
 			), "GetAuthMethod(%q) got response %q, wanted %q", tc.req, got, tc.res)
 		})
 	}
@@ -307,6 +314,9 @@ func TestList(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kmsCache)
 	}
+	authMethodRepoFn := func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	oNoAuthMethods, _ := iam.TestScopes(t, iamRepo)
@@ -435,17 +445,33 @@ func TestList(t *testing.T) {
 		{
 			name: "List Some Auth Methods",
 			req:  &pbs.ListAuthMethodsRequest{ScopeId: oWithAuthMethods.GetPublicId()},
-			res:  &pbs.ListAuthMethodsResponse{Items: cpSorted(wantSomeAuthMethods)},
+			res: &pbs.ListAuthMethodsResponse{
+				Items:        cpSorted(wantSomeAuthMethods),
+				EstItemCount: 5,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "List Other Auth Methods",
 			req:  &pbs.ListAuthMethodsRequest{ScopeId: oWithOtherAuthMethods.GetPublicId()},
-			res:  &pbs.ListAuthMethodsResponse{Items: cpSorted(wantOtherAuthMethods)},
+			res: &pbs.ListAuthMethodsResponse{
+				Items:        cpSorted(wantOtherAuthMethods),
+				EstItemCount: 3,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "List No Auth Methods",
 			req:  &pbs.ListAuthMethodsRequest{ScopeId: oNoAuthMethods.GetPublicId()},
-			res:  &pbs.ListAuthMethodsResponse{},
+			res: &pbs.ListAuthMethodsResponse{
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "Unfound Auth Method",
@@ -459,6 +485,10 @@ func TestList(t *testing.T) {
 				Items: func() []*pb.AuthMethod {
 					return cpSorted(append(wantSomeAuthMethods, wantOtherAuthMethods...))
 				}(),
+				EstItemCount: 8,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
 			},
 		},
 		{
@@ -467,12 +497,22 @@ func TestList(t *testing.T) {
 				ScopeId: "global", Recursive: true,
 				Filter: fmt.Sprintf(`"/item/scope/id"==%q`, oWithAuthMethods.GetPublicId()),
 			},
-			res: &pbs.ListAuthMethodsResponse{Items: cpSorted(wantSomeAuthMethods)},
+			res: &pbs.ListAuthMethodsResponse{
+				Items:        cpSorted(wantSomeAuthMethods),
+				EstItemCount: 5,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "Filter All Auth Methods",
 			req:  &pbs.ListAuthMethodsRequest{ScopeId: oWithAuthMethods.GetPublicId(), Filter: `"/item/id"=="nothingmatchesthis"`},
-			res:  &pbs.ListAuthMethodsResponse{},
+			res: &pbs.ListAuthMethodsResponse{
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "Filter Bad Format",
@@ -483,7 +523,7 @@ func TestList(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
-			s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+			s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 			require.NoError(err, "Couldn't create new auth_method service.")
 
 			// First check with non-anonymous user
@@ -501,18 +541,17 @@ func TestList(t *testing.T) {
 			}
 
 			slices.SortFunc(got.Items, sorterFn)
-			assert.Empty(cmp.Diff(
-				got,
-				tc.res,
-				protocmp.Transform(),
-				protocmp.IgnoreFields(&pb.OidcAuthMethodAttributes{}, "client_secret_hmac"),
-				protocmp.IgnoreFields(&pb.LdapAuthMethodAttributes{}, "bind_password_hmac", "client_certificate_key_hmac"),
+			assert.Empty(cmp.Diff(got, tc.res,
 				cmpopts.SortSlices(func(a, b string) bool {
 					return a < b
 				}),
 				cmpopts.SortSlices(func(a, b protocmp.Message) bool {
 					return a.String() < b.String()
 				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListAuthMethodsResponse{}, "list_token"),
+				protocmp.IgnoreFields(&pb.OidcAuthMethodAttributes{}, "client_secret_hmac"),
+				protocmp.IgnoreFields(&pb.LdapAuthMethodAttributes{}, "bind_password_hmac", "client_certificate_key_hmac"),
 			), "ListAuthMethods() for scope %q got response %q, wanted %q", tc.req.GetScopeId(), got, tc.res)
 
 			// Now check with anonymous user
@@ -550,6 +589,9 @@ func TestDelete(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kmsCache)
 	}
+	authMethodRepoFn := func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
@@ -563,7 +605,7 @@ func TestDelete(t *testing.T) {
 
 	ldapAm := ldap.TestAuthMethod(t, conn, databaseWrapper, o.GetPublicId(), []string{"ldaps://ldap1"})
 
-	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Error when getting new auth_method service.")
 
 	cases := []struct {
@@ -643,12 +685,15 @@ func TestDelete_twice(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kms)
 	}
+	authMethodRepoFn := func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, rw, rw, kms)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
 	am := password.TestAuthMethods(t, conn, o.GetPublicId(), 1)[0]
 
-	s, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	s, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(err, "Error when getting new auth_method service.")
 
 	req := &pbs.DeleteAuthMethodRequest{
@@ -682,6 +727,9 @@ func TestCreate(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, testKms)
 	}
+	authMethodRepoFn := func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, rw, rw, testKms)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
@@ -1478,7 +1526,7 @@ func TestCreate(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
 
-			s, err := authmethods.NewService(ctx, testKms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+			s, err := authmethods.NewService(ctx, testKms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 			require.NoError(err, "Error when getting new auth_method service.")
 
 			got, gErr := s.CreateAuthMethod(requestauth.DisabledAuthTestContext(iamRepoFn, tc.req.GetItem().GetScopeId()), tc.req)
@@ -1550,3 +1598,426 @@ func TestCreate(t *testing.T) {
 		})
 	}
 }
+
+func TestListPagination(t *testing.T) {
+	// Set database read timeout to avoid duplicates in response
+	oldReadTimeout := globals.RefreshReadLookbackDuration
+	globals.RefreshReadLookbackDuration = 0
+	t.Cleanup(func() {
+		globals.RefreshReadLookbackDuration = oldReadTimeout
+	})
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	sqlDB, err := conn.SqlDB(ctx)
+	require.NoError(t, err)
+	wrapper := db.TestWrapper(t)
+	kmsCache := kms.TestKms(t, conn, wrapper)
+	iamRepoFn := func() (*iam.Repository, error) {
+		return iam.TestRepo(t, conn, wrapper), nil
+	}
+	oidcRepoFn := func() (*oidc.Repository, error) {
+		return oidc.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	ldapRepoFn := func() (*ldap.Repository, error) {
+		return ldap.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	pwRepoFn := func() (*password.Repository, error) {
+		return password.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	tokenRepoFn := func() (*authtoken.Repository, error) {
+		return authtoken.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	serversRepoFn := func() (*server.Repository, error) {
+		return server.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	authMethodRepoFn := func() (*auth.AuthMethodRepository, error) {
+		return auth.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
+
+	iamRepo, err := iamRepoFn()
+	require.NoError(t, err)
+	tokenRepo, err := tokenRepoFn()
+	require.NoError(t, err)
+	ldapRepo, err := ldapRepoFn()
+	require.NoError(t, err)
+	oidcRepo, err := oidcRepoFn()
+	require.NoError(t, err)
+
+	orgNoAms, _ := iam.TestScopes(t, iamRepo)
+	org, proj := iam.TestScopes(t, iamRepo)
+	databaseWrapper, err := kmsCache.GetWrapper(context.Background(), org.GetPublicId(), kms.KeyPurposeDatabase)
+	require.NoError(t, err)
+
+	var allAuthMethods []*pb.AuthMethod
+
+	oidcam := oidc.TestAuthMethod(t, conn, databaseWrapper, org.GetPublicId(), oidc.ActivePublicState, "alice_rp", "secret",
+		oidc.WithIssuer(oidc.TestConvertToUrls(t, "https://alice.com")[0]), oidc.WithApiUrl(oidc.TestConvertToUrls(t, "https://api.com")[0]), oidc.WithSigningAlgs(oidc.EdDSA),
+		oidc.WithPrompts(oidc.Consent))
+	iam.TestSetPrimaryAuthMethod(t, iamRepo, org, oidcam.GetPublicId())
+
+	allAuthMethods = append(allAuthMethods, &pb.AuthMethod{
+		Id:          oidcam.GetPublicId(),
+		ScopeId:     org.GetPublicId(),
+		CreatedTime: oidcam.GetCreateTime().GetTimestamp(),
+		UpdatedTime: oidcam.GetUpdateTime().GetTimestamp(),
+		Scope:       &scopepb.ScopeInfo{Id: org.GetPublicId(), Type: scope.Org.String(), ParentScopeId: scope.Global.String()},
+		Version:     2,
+		Type:        oidc.Subtype.String(),
+		Attrs: &pb.AuthMethod_OidcAuthMethodsAttributes{
+			OidcAuthMethodsAttributes: &pb.OidcAuthMethodAttributes{
+				Issuer:           wrapperspb.String("https://alice.com"),
+				ClientId:         wrapperspb.String("alice_rp"),
+				ClientSecretHmac: "<hmac>",
+				State:            string(oidc.ActivePublicState),
+				ApiUrlPrefix:     wrapperspb.String("https://api.com"),
+				CallbackUrl:      fmt.Sprintf(oidc.CallbackEndpoint, "https://api.com"),
+				SigningAlgorithms: []string{
+					string(oidc.EdDSA),
+				},
+				Prompts: []string{string(oidc.Consent)},
+			},
+		},
+		IsPrimary:                   true,
+		AuthorizedActions:           oidcAuthorizedActions,
+		AuthorizedCollectionActions: authorizedCollectionActions,
+	})
+
+	ldapAm := ldap.TestAuthMethod(t, conn, databaseWrapper, org.GetPublicId(), []string{"ldaps://ldap1"},
+		ldap.WithOperationalState(ctx, ldap.ActivePublicState),
+		ldap.WithMaximumPageSize(ctx, 10),
+		ldap.WithDerefAliases(ctx, ldap.DerefAlways),
+	)
+	allAuthMethods = append(allAuthMethods, &pb.AuthMethod{
+		Id:          ldapAm.GetPublicId(),
+		ScopeId:     org.GetPublicId(),
+		CreatedTime: ldapAm.GetCreateTime().GetTimestamp(),
+		UpdatedTime: ldapAm.GetUpdateTime().GetTimestamp(),
+		Scope:       &scopepb.ScopeInfo{Id: org.GetPublicId(), Type: scope.Org.String(), ParentScopeId: scope.Global.String()},
+		Version:     1,
+		Type:        ldap.Subtype.String(),
+		Attrs: &pb.AuthMethod_LdapAuthMethodsAttributes{
+			LdapAuthMethodsAttributes: &pb.LdapAuthMethodAttributes{
+				State:              string(ldap.ActivePublicState),
+				Urls:               []string{"ldaps://ldap1"},
+				MaximumPageSize:    10,
+				DereferenceAliases: wrapperspb.String(string(ldap.DerefAlways)),
+			},
+		},
+		AuthorizedActions:           ldapAuthorizedActions,
+		AuthorizedCollectionActions: authorizedCollectionActions,
+	})
+
+	for _, am := range password.TestAuthMethods(t, conn, org.GetPublicId(), 7) {
+		allAuthMethods = append(allAuthMethods, &pb.AuthMethod{
+			Id:          am.GetPublicId(),
+			ScopeId:     org.GetPublicId(),
+			CreatedTime: am.GetCreateTime().GetTimestamp(),
+			UpdatedTime: am.GetUpdateTime().GetTimestamp(),
+			Scope:       &scopepb.ScopeInfo{Id: org.GetPublicId(), Type: scope.Org.String(), ParentScopeId: scope.Global.String()},
+			Version:     1,
+			Type:        "password",
+			Attrs: &pb.AuthMethod_PasswordAuthMethodAttributes{
+				PasswordAuthMethodAttributes: &pb.PasswordAuthMethodAttributes{
+					MinPasswordLength:  8,
+					MinLoginNameLength: 3,
+				},
+			},
+			AuthorizedActions:           pwAuthorizedActions,
+			AuthorizedCollectionActions: authorizedCollectionActions,
+		})
+	}
+
+	masterAuthMethod := password.TestAuthMethods(t, conn, org.GetPublicId(), 1)[0]
+	acct := password.TestAccount(t, conn, masterAuthMethod.GetPublicId(), "test_user")
+	u := iam.TestUser(t, iamRepo, org.GetPublicId(), iam.WithAccountIds(acct.PublicId))
+	orgRole := iam.TestRole(t, conn, org.GetPublicId())
+	iam.TestRoleGrant(t, conn, orgRole.GetPublicId(), "id=*;type=*;actions=*")
+	iam.TestUserRole(t, conn, orgRole.GetPublicId(), u.GetPublicId())
+	projRole := iam.TestRole(t, conn, proj.GetPublicId())
+	iam.TestRoleGrant(t, conn, projRole.GetPublicId(), "id=*;type=*;actions=*")
+	iam.TestUserRole(t, conn, projRole.GetPublicId(), u.GetPublicId())
+	at, err := tokenRepo.CreateAuthToken(ctx, u, acct.GetPublicId())
+	require.NoError(t, err)
+
+	allAuthMethods = append(allAuthMethods, &pb.AuthMethod{
+		Id:          masterAuthMethod.GetPublicId(),
+		ScopeId:     org.GetPublicId(),
+		CreatedTime: masterAuthMethod.GetCreateTime().GetTimestamp(),
+		UpdatedTime: masterAuthMethod.GetUpdateTime().GetTimestamp(),
+		Scope:       &scopepb.ScopeInfo{Id: org.GetPublicId(), Type: scope.Org.String(), ParentScopeId: scope.Global.String()},
+		Version:     1,
+		Type:        "password",
+		Attrs: &pb.AuthMethod_PasswordAuthMethodAttributes{
+			PasswordAuthMethodAttributes: &pb.PasswordAuthMethodAttributes{
+				MinPasswordLength:  8,
+				MinLoginNameLength: 3,
+			},
+		},
+		AuthorizedActions:           pwAuthorizedActions,
+		AuthorizedCollectionActions: authorizedCollectionActions,
+	})
+
+	// Reverse slice since we're sorting by create time descending
+	slices.Reverse(allAuthMethods)
+
+	// Run analyze to update postgres meta tables
+	_, err = sqlDB.ExecContext(ctx, "analyze")
+	require.NoError(t, err)
+
+	// Test without anon user
+	requestInfo := authpb.RequestInfo{
+		TokenFormat: uint32(requestauth.AuthTokenTypeBearer),
+		PublicId:    at.GetPublicId(),
+		Token:       at.GetToken(),
+	}
+	requestContext := context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
+	ctx = requestauth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kmsCache, &requestInfo)
+
+	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, tokenRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
+	require.NoError(t, err)
+
+	cmpOptions := []cmp.Option{
+		protocmp.Transform(),
+		cmpopts.SortSlices(func(a, b string) bool {
+			return a < b
+		}),
+		cmpopts.SortSlices(func(a, b protocmp.Message) bool {
+			return a.String() < b.String()
+		}),
+		protocmp.IgnoreFields(&pb.OidcAuthMethodAttributes{}, "client_secret_hmac"),
+		protocmp.IgnoreFields(&pb.LdapAuthMethodAttributes{}, "bind_password_hmac", "client_certificate_key_hmac"),
+		protocmp.IgnoreFields(&pbs.ListAuthMethodsResponse{}, "list_token"),
+	}
+
+	// Start paginating, recursively
+	req := &pbs.ListAuthMethodsRequest{
+		ScopeId:   org.PublicId,
+		Filter:    "",
+		ListToken: "",
+		PageSize:  2,
+	}
+	got, err := s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 2)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        allAuthMethods[0:2],
+				ResponseType: "delta",
+				ListToken:    "",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+
+	// Request second page
+	req.ListToken = got.ListToken
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 2)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        allAuthMethods[2:4],
+				ResponseType: "delta",
+				ListToken:    "",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+
+	// Request rest of results
+	req.ListToken = got.ListToken
+	req.PageSize = 10
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 6)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        allAuthMethods[4:],
+				ResponseType: "complete",
+				ListToken:    "",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+
+	// create another auth method
+	am := password.TestAuthMethods(t, conn, org.GetPublicId(), 1)[0]
+	ampb := &pb.AuthMethod{
+		Id:          am.GetPublicId(),
+		ScopeId:     org.GetPublicId(),
+		CreatedTime: am.GetCreateTime().GetTimestamp(),
+		UpdatedTime: am.GetUpdateTime().GetTimestamp(),
+		Scope:       &scopepb.ScopeInfo{Id: org.GetPublicId(), Type: scope.Org.String(), ParentScopeId: scope.Global.String()},
+		Version:     1,
+		Type:        "password",
+		Attrs: &pb.AuthMethod_PasswordAuthMethodAttributes{
+			PasswordAuthMethodAttributes: &pb.PasswordAuthMethodAttributes{
+				MinPasswordLength:  8,
+				MinLoginNameLength: 3,
+			},
+		},
+		AuthorizedActions:           pwAuthorizedActions,
+		AuthorizedCollectionActions: authorizedCollectionActions,
+	}
+	// add to front since it's most recently updated
+	allAuthMethods = append([]*pb.AuthMethod{ampb}, allAuthMethods...)
+
+	// delete a different auth method
+	_, err = oidcRepo.DeleteAuthMethod(ctx, allAuthMethods[len(allAuthMethods)-1].Id)
+	require.NoError(t, err)
+	deletedAM := allAuthMethods[len(allAuthMethods)-1]
+	allAuthMethods = allAuthMethods[:len(allAuthMethods)-1]
+
+	// update a different auth method
+	ldapAm.AuthMethod.Description = "new-description"
+	allAuthMethods[len(allAuthMethods)-1].Description = wrapperspb.String("new-description")
+	updatedAM, _, err := ldapRepo.UpdateAuthMethod(ctx, ldapAm, ldapAm.Version, []string{"description"})
+	require.NoError(t, err)
+	allAuthMethods[len(allAuthMethods)-1].UpdatedTime = updatedAM.UpdateTime.GetTimestamp()
+	allAuthMethods[len(allAuthMethods)-1].Version = updatedAM.GetVersion()
+	// add to front since it's most recently updated
+	allAuthMethods = append([]*pb.AuthMethod{allAuthMethods[len(allAuthMethods)-1]}, allAuthMethods[:len(allAuthMethods)-1]...)
+
+	// request updated results
+	req.ListToken = got.ListToken
+	req.PageSize = 1
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        []*pb.AuthMethod{allAuthMethods[0]},
+				ResponseType: "delta",
+				ListToken:    "",
+				SortBy:       "updated_time",
+				SortDir:      "desc",
+				// should be the deleted auth method
+				RemovedIds:   []string{deletedAM.Id},
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+
+	// get next page
+	req.ListToken = got.ListToken
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        []*pb.AuthMethod{allAuthMethods[1]},
+				ResponseType: "complete",
+				ListToken:    "",
+				SortBy:       "updated_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+
+	// Request new page with filter requiring looping
+	// to fill the page.
+	req.ListToken = ""
+	req.PageSize = 1
+	req.Filter = fmt.Sprintf(`"/item/id"==%q or "/item/id"==%q`, allAuthMethods[len(allAuthMethods)-2].Id, allAuthMethods[len(allAuthMethods)-1].Id)
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        []*pb.AuthMethod{allAuthMethods[len(allAuthMethods)-2]},
+				ResponseType: "delta",
+				ListToken:    "",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				// Should be empty again
+				RemovedIds:   nil,
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+	req.ListToken = got.ListToken
+	// Get the second page
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        []*pb.AuthMethod{allAuthMethods[len(allAuthMethods)-1]},
+				ResponseType: "complete",
+				ListToken:    "",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: 10,
+			},
+			cmpOptions...,
+		),
+	)
+	req.ListToken = got.ListToken
+
+	// List items in the empty scope
+	req = &pbs.ListAuthMethodsRequest{
+		ScopeId:   orgNoAms.PublicId,
+		Filter:    "",
+		ListToken: "",
+		PageSize:  2,
+	}
+	got, err = s.ListAuthMethods(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 0)
+	// Compare without comparing the list token
+	assert.Empty(
+		t,
+		cmp.Diff(
+			got,
+			&pbs.ListAuthMethodsResponse{
+				Items:        nil,
+				ResponseType: "complete",
+				ListToken:    "",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+			},
+			cmpOptions...,
+		),
+	)
+}

internal/daemon/controller/handlers/authmethods/ldap_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/hashicorp/boundary/globals"
+	am "github.com/hashicorp/boundary/internal/auth"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
@@ -67,10 +68,13 @@ func Test_UpdateLdap(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kms)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kms)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
-	tested, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	tested, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Error when getting new auth_method service.")
 
 	defaultScopeInfo := &scopepb.ScopeInfo{Id: o.GetPublicId(), Type: o.GetType(), ParentScopeId: scope.Global.String()}
@@ -915,6 +919,9 @@ func TestAuthenticate_Ldap(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(testCtx, testRw, testRw, testKms)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(testCtx, testRw, testRw, testKms)
+	}
 
 	orgDbWrapper, err := testKms.GetWrapper(testCtx, o.GetPublicId(), kms.KeyPurposeDatabase)
 	require.NoError(t, err)
@@ -1094,7 +1101,7 @@ func TestAuthenticate_Ldap(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
-			s, err := authmethods.NewService(testCtx, testKms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+			s, err := authmethods.NewService(testCtx, testKms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 			require.NoError(err)
 
 			resp, err := s.Authenticate(auth.DisabledAuthTestContext(iamRepoFn, o.GetPublicId()), tc.request)

internal/daemon/controller/handlers/authmethods/oidc_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/hashicorp/boundary/globals"
 	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 
+	am "github.com/hashicorp/boundary/internal/auth"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
@@ -63,6 +64,8 @@ type setup struct {
 	ldapRepoFn                  common.LdapAuthRepoFactory
 	pwRepoFn                    common.PasswordAuthRepoFactory
 	atRepoFn                    common.AuthTokenRepoFactory
+	authMethodRepoFn            common.AuthMethodRepoFactory
+	maxPageSize                 int
 	org                         *iam.Scope
 	proj                        *iam.Scope
 	databaseWrapper             wrapping.Wrapper
@@ -104,12 +107,26 @@ func getSetup(t *testing.T) setup {
 	ret.atRepoFn = func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, ret.rw, ret.rw, ret.kmsCache)
 	}
+	ret.authMethodRepoFn = func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, ret.rw, ret.rw, ret.kmsCache)
+	}
+	ret.maxPageSize = 1000
 
 	ret.org, ret.proj = iam.TestScopes(t, ret.iamRepo)
 	ret.databaseWrapper, err = ret.kmsCache.GetWrapper(ret.ctx, ret.org.PublicId, kms.KeyPurposeDatabase)
 	require.NoError(err)
 
-	ret.authMethodService, err = authmethods.NewService(ret.ctx, ret.kmsCache, ret.pwRepoFn, ret.oidcRepoFn, ret.iamRepoFn, ret.atRepoFn, ret.ldapRepoFn)
+	ret.authMethodService, err = authmethods.NewService(
+		ret.ctx,
+		ret.kmsCache,
+		ret.pwRepoFn,
+		ret.oidcRepoFn,
+		ret.iamRepoFn,
+		ret.atRepoFn,
+		ret.ldapRepoFn,
+		ret.authMethodRepoFn,
+		uint(ret.maxPageSize),
+	)
 	require.NoError(err)
 
 	ret.testProvider = capoidc.StartTestProvider(t)
@@ -168,6 +185,9 @@ func TestList_FilterNonPublic(t *testing.T) {
 	serversRepoFn := func() (*server.Repository, error) {
 		return server.NewRepository(ctx, rw, rw, kmsCache)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
 
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
@@ -195,7 +215,7 @@ func TestList_FilterNonPublic(t *testing.T) {
 			oidc.WithIssuer(oidc.TestConvertToUrls(t, fmt.Sprintf("https://alice%d.com", i))[0]), oidc.WithApiUrl(oidc.TestConvertToUrls(t, "https://api.com")[0]))
 	}
 
-	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Couldn't create new auth_method service.")
 
 	req := &pbs.ListAuthMethodsRequest{
@@ -270,10 +290,13 @@ func TestUpdate_OIDC(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kms)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kms)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
-	tested, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	tested, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Error when getting new auth_method service.")
 
 	defaultScopeInfo := &scopepb.ScopeInfo{Id: o.GetPublicId(), Type: o.GetType(), ParentScopeId: scope.Global.String()}
@@ -1139,6 +1162,9 @@ func TestUpdate_OIDCDryRun(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kmsCache)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
@@ -1192,7 +1218,7 @@ func TestUpdate_OIDCDryRun(t *testing.T) {
 		},
 	}
 
-	tested, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	tested, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Error when getting new auth_method service.")
 	cases := []struct {
 		name    string
@@ -1339,6 +1365,9 @@ func TestChangeState_OIDC(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kmsCache)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kmsCache)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
@@ -1362,7 +1391,7 @@ func TestChangeState_OIDC(t *testing.T) {
 	mismatchedAM := oidc.TestAuthMethod(t, conn, databaseWrapper, o.PublicId, "inactive", "different_client_id", oidc.ClientSecret(tpClientSecret),
 		oidc.WithIssuer(oidc.TestConvertToUrls(t, tp.Addr())[0]), oidc.WithSigningAlgs(oidc.EdDSA), oidc.WithApiUrl(oidc.TestConvertToUrls(t, "https://example.callback:58")[0]), oidc.WithCertificates(tpCert...))
 
-	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	s, err := authmethods.NewService(ctx, kmsCache, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Error when getting new auth_method service.")
 
 	wantTemplate := &pb.AuthMethod{

internal/daemon/controller/handlers/authmethods/password_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/hashicorp/boundary/globals"
+	am "github.com/hashicorp/boundary/internal/auth"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
 	"github.com/hashicorp/boundary/internal/auth/password"
@@ -62,10 +63,13 @@ func TestUpdate_Password(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kms)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kms)
+	}
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 
 	o, _ := iam.TestScopes(t, iamRepo)
-	tested, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	tested, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(t, err, "Error when getting new auth_method service.")
 
 	defaultScopeInfo := &scopepb.ScopeInfo{Id: o.GetPublicId(), Type: o.GetType(), ParentScopeId: scope.Global.String()}
@@ -510,6 +514,9 @@ func TestAuthenticate_Password(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kms)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kms)
+	}
 	am := password.TestAuthMethods(t, conn, o.GetPublicId(), 1)[0]
 
 	iam.TestSetPrimaryAuthMethod(t, iam.TestRepo(t, conn, wrapper), o, am.PublicId)
@@ -652,7 +659,7 @@ func TestAuthenticate_Password(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
-			s, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+			s, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 			require.NoError(err)
 
 			resp, err := s.Authenticate(auth.DisabledAuthTestContext(iamRepoFn, o.GetPublicId()), tc.request)
@@ -718,6 +725,9 @@ func TestAuthenticate_AuthAccountConnectedToIamUser_Password(t *testing.T) {
 	atRepoFn := func() (*authtoken.Repository, error) {
 		return authtoken.NewRepository(ctx, rw, rw, kms)
 	}
+	authMethodRepoFn := func() (*am.AuthMethodRepository, error) {
+		return am.NewAuthMethodRepository(ctx, rw, rw, kms)
+	}
 
 	am := password.TestAuthMethods(t, conn, o.GetPublicId(), 1)[0]
 	acct, err := password.NewAccount(ctx, am.GetPublicId(), password.WithLoginName(testLoginName))
@@ -735,7 +745,7 @@ func TestAuthenticate_AuthAccountConnectedToIamUser_Password(t *testing.T) {
 	iamUser, err := iamRepo.LookupUserWithLogin(context.Background(), acct.GetPublicId())
 	require.NoError(err)
 
-	s, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn)
+	s, err := authmethods.NewService(ctx, kms, pwRepoFn, oidcRepoFn, iamRepoFn, atRepoFn, ldapRepoFn, authMethodRepoFn, 1000)
 	require.NoError(err)
 	resp, err := s.Authenticate(auth.DisabledAuthTestContext(iamRepoFn, o.GetPublicId()), &pbs.AuthenticateRequest{
 		AuthMethodId: am.GetPublicId(),