Add pagination for iam Roles (#4161)

* Add pagination for iam Roles

internal/daemon/controller/handler.go

@@ -245,7 +245,7 @@ func (c *Controller) registerGrpcServices(s *grpc.Server) error {
 		services.RegisterGroupServiceServer(s, gs)
 	}
 	if _, ok := currentServices[services.RoleService_ServiceDesc.ServiceName]; !ok {
-		rs, err := roles.NewService(c.baseContext, c.IamRepoFn)
+		rs, err := roles.NewService(c.baseContext, c.IamRepoFn, c.conf.RawConfig.Controller.MaxPageSize)
 		if err != nil {
 			return fmt.Errorf("failed to create role handler service: %w", err)
 		}

internal/daemon/controller/handlers/roles/role_service.go

@@ -16,12 +16,15 @@ import (
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/iam/store"
+	"github.com/hashicorp/boundary/internal/listtoken"
+	"github.com/hashicorp/boundary/internal/pagination"
 	"github.com/hashicorp/boundary/internal/perms"
 	"github.com/hashicorp/boundary/internal/requests"
 	"github.com/hashicorp/boundary/internal/types/action"
 	"github.com/hashicorp/boundary/internal/types/resource"
 	"github.com/hashicorp/boundary/internal/types/scope"
 	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/roles"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/scopes"
 	"github.com/hashicorp/boundary/version"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 	"google.golang.org/grpc/codes"
@@ -72,22 +75,28 @@ func init() {
 type Service struct {
 	pbs.UnsafeRoleServiceServer
 
-	repoFn common.IamRepoFactory
+	repoFn      common.IamRepoFactory
+	maxPageSize uint
 }
 
 var _ pbs.RoleServiceServer = (*Service)(nil)
 
 // NewService returns a role service which handles role related requests to boundary.
-func NewService(ctx context.Context, repo common.IamRepoFactory) (Service, error) {
+func NewService(ctx context.Context, repo common.IamRepoFactory, maxPageSize uint) (Service, error) {
 	const op = "roles.NewService"
 	if repo == nil {
 		return Service{}, errors.New(ctx, errors.InvalidParameter, op, "missing iam repository")
 	}
-	return Service{repoFn: repo}, nil
+	if maxPageSize == 0 {
+		maxPageSize = uint(globals.DefaultMaxPageSize)
+	}
+	return Service{repoFn: repo, maxPageSize: maxPageSize}, nil
 }
 
 // ListRoles implements the interface pbs.RoleServiceServer.
 func (s Service) ListRoles(ctx context.Context, req *pbs.ListRolesRequest) (*pbs.ListRolesResponse, error) {
+	const op = "roles.(Service).ListRoles"
+
 	if err := validateListRequest(ctx, req); err != nil {
 		return nil, err
 	}
@@ -115,50 +124,117 @@ func (s Service) ListRoles(ctx context.Context, req *pbs.ListRolesRequest) (*pbs
 		return &pbs.ListRolesResponse{}, nil
 	}
 
-	items, err := s.listFromRepo(ctx, scopeIds)
-	if err != nil {
-		return nil, err
-	}
-	if len(items) == 0 {
-		return &pbs.ListRolesResponse{}, nil
+	pageSize := int(s.maxPageSize)
+	// Use the requested page size only if it is smaller than
+	// the configured max.
+	if req.GetPageSize() != 0 && uint(req.GetPageSize()) < s.maxPageSize {
+		pageSize = int(req.GetPageSize())
 	}
 
-	filter, err := handlers.NewFilter(ctx, req.GetFilter())
-	if err != nil {
-		return nil, err
-	}
-	finalItems := make([]*pb.Role, 0, len(items))
-	res := perms.Resource{
-		Type: resource.Role,
-	}
-	for _, item := range items {
-		res.Id = item.GetPublicId()
-		res.ScopeId = item.GetScopeId()
-		authorizedActions := authResults.FetchActionSetForId(ctx, item.GetPublicId(), IdActions, auth.WithResource(&res)).Strings()
-		if len(authorizedActions) == 0 {
-			continue
-		}
-
-		outputFields := authResults.FetchOutputFields(res, action.List).SelfOrDefaults(authResults.UserId)
-		outputOpts := make([]handlers.Option, 0, 3)
-		outputOpts = append(outputOpts, handlers.WithOutputFields(outputFields))
-		if outputFields.Has(globals.ScopeField) {
-			outputOpts = append(outputOpts, handlers.WithScope(scopeInfoMap[item.GetScopeId()]))
-		}
-		if outputFields.Has(globals.AuthorizedActionsField) {
-			outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
-		}
-
-		item, err := toProto(ctx, item, nil, nil, outputOpts...)
+	var filterItemFn func(ctx context.Context, item *iam.Role) (bool, error)
+	switch {
+	case req.GetFilter() != "":
+		// Only use a filter if we need to
+		filter, err := handlers.NewFilter(ctx, req.GetFilter())
 		if err != nil {
 			return nil, err
 		}
-
-		if filter.Match(item) {
-			finalItems = append(finalItems, item)
+		filterItemFn = func(ctx context.Context, item *iam.Role) (bool, error) {
+			outputOpts, ok := newOutputOpts(ctx, item, scopeInfoMap, authResults)
+			if !ok {
+				return false, nil
+			}
+			pbItem, err := toProto(ctx, item, nil, nil, outputOpts...)
+			if err != nil {
+				return false, err
+			}
+			return filter.Match(pbItem), nil
+		}
+	default:
+		filterItemFn = func(ctx context.Context, item *iam.Role) (bool, error) {
+			return true, nil
 		}
 	}
-	return &pbs.ListRolesResponse{Items: finalItems}, nil
+
+	grantsHash, err := authResults.GrantsHash(ctx)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+
+	repo, err := s.repoFn()
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+
+	var listResp *pagination.ListResponse[*iam.Role]
+	var sortBy string
+	if req.GetListToken() == "" {
+		sortBy = "created_time"
+		listResp, err = iam.ListRoles(ctx, grantsHash, pageSize, filterItemFn, repo, scopeIds)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		listToken, err := handlers.ParseListToken(ctx, req.GetListToken(), resource.Role, grantsHash)
+		if err != nil {
+			return nil, err
+		}
+		switch st := listToken.Subtype.(type) {
+		case *listtoken.PaginationToken:
+			sortBy = "created_time"
+			listResp, err = iam.ListRolesPage(ctx, grantsHash, pageSize, filterItemFn, listToken, repo, scopeIds)
+			if err != nil {
+				return nil, err
+			}
+		case *listtoken.StartRefreshToken:
+			sortBy = "updated_time"
+			listResp, err = iam.ListRolesRefresh(ctx, grantsHash, pageSize, filterItemFn, listToken, repo, scopeIds)
+			if err != nil {
+				return nil, err
+			}
+		case *listtoken.RefreshToken:
+			sortBy = "updated_time"
+			listResp, err = iam.ListRolesRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listToken, repo, scopeIds)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, handlers.ApiErrorWithCodeAndMessage(codes.InvalidArgument, "unexpected list token subtype: %T", st)
+		}
+	}
+
+	finalItems := make([]*pb.Role, 0, len(listResp.Items))
+	for _, item := range listResp.Items {
+		outputOpts, ok := newOutputOpts(ctx, item, scopeInfoMap, authResults)
+		if !ok {
+			continue
+		}
+		item, err := toProto(ctx, item, nil, nil, outputOpts...)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op)
+		}
+		finalItems = append(finalItems, item)
+	}
+	respType := "delta"
+	if listResp.CompleteListing {
+		respType = "complete"
+	}
+	resp := &pbs.ListRolesResponse{
+		Items:        finalItems,
+		EstItemCount: uint32(listResp.EstimatedItemCount),
+		RemovedIds:   listResp.DeletedIds,
+		ResponseType: respType,
+		SortBy:       sortBy,
+		SortDir:      "desc",
+	}
+
+	if listResp.ListToken != nil {
+		resp.ListToken, err = handlers.MarshalListToken(ctx, listResp.ListToken, pbs.ResourceType_RESOURCE_TYPE_ROLE)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return resp, nil
 }
 
 // GetRoles implements the interface pbs.RoleServiceServer.
@@ -620,18 +696,6 @@ func (s Service) deleteFromRepo(ctx context.Context, id string) (bool, error) {
 	return rows > 0, nil
 }
 
-func (s Service) listFromRepo(ctx context.Context, scopeIds []string) ([]*iam.Role, error) {
-	repo, err := s.repoFn()
-	if err != nil {
-		return nil, err
-	}
-	rl, err := repo.ListRoles(ctx, scopeIds, iam.WithLimit(-1))
-	if err != nil {
-		return nil, err
-	}
-	return rl, nil
-}
-
 func (s Service) addPrinciplesInRepo(ctx context.Context, roleId string, principalIds []string, version uint32) (*iam.Role, []*iam.PrincipalRole, []*iam.RoleGrant, error) {
 	const op = "roles.(Service).addPrincpleInRepo"
 	repo, err := s.repoFn()
@@ -1159,3 +1223,26 @@ func validateRemoveRoleGrantsRequest(ctx context.Context, req *pbs.RemoveRoleGra
 	}
 	return nil
 }
+
+func newOutputOpts(ctx context.Context, item *iam.Role, scopeInfoMap map[string]*scopes.ScopeInfo, authResults auth.VerifyResults) ([]handlers.Option, bool) {
+	res := perms.Resource{
+		Type: resource.Role,
+	}
+	res.Id = item.GetPublicId()
+	res.ScopeId = item.GetScopeId()
+	authorizedActions := authResults.FetchActionSetForId(ctx, item.GetPublicId(), IdActions, auth.WithResource(&res))
+	if len(authorizedActions) == 0 {
+		return nil, false
+	}
+
+	outputFields := authResults.FetchOutputFields(res, action.List).SelfOrDefaults(authResults.UserId)
+	outputOpts := make([]handlers.Option, 0, 3)
+	outputOpts = append(outputOpts, handlers.WithOutputFields(outputFields))
+	if outputFields.Has(globals.ScopeField) {
+		outputOpts = append(outputOpts, handlers.WithScope(scopeInfoMap[item.GetScopeId()]))
+	}
+	if outputFields.Has(globals.AuthorizedActionsField) {
+		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions.Strings()))
+	}
+	return outputOpts, true
+}

internal/daemon/controller/handlers/roles/role_service_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 	"testing"
 
@@ -15,14 +16,19 @@ import (
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/auth/ldap"
 	"github.com/hashicorp/boundary/internal/auth/oidc"
+	"github.com/hashicorp/boundary/internal/auth/password"
+	"github.com/hashicorp/boundary/internal/authtoken"
 	"github.com/hashicorp/boundary/internal/daemon/controller/auth"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers"
 	"github.com/hashicorp/boundary/internal/daemon/controller/handlers/roles"
 	"github.com/hashicorp/boundary/internal/db"
 	pbs "github.com/hashicorp/boundary/internal/gen/controller/api/services"
+	authpb "github.com/hashicorp/boundary/internal/gen/controller/auth"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/perms"
+	"github.com/hashicorp/boundary/internal/requests"
+	"github.com/hashicorp/boundary/internal/server"
 	"github.com/hashicorp/boundary/internal/types/scope"
 	pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/roles"
 	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/scopes"
@@ -178,7 +184,7 @@ func TestGet(t *testing.T) {
 			req := proto.Clone(toMerge).(*pbs.GetRoleRequest)
 			proto.Merge(req, tc.req)
 
-			s, err := roles.NewService(context.Background(), repoFn)
+			s, err := roles.NewService(context.Background(), repoFn, 1000)
 			require.NoError(err, "Couldn't create new role service.")
 
 			got, gErr := s.GetRole(auth.DisabledAuthTestContext(repoFn, tc.scopeId), req)
@@ -199,6 +205,7 @@ func TestGet(t *testing.T) {
 }
 
 func TestList(t *testing.T) {
+	ctx := context.Background()
 	conn, _ := db.TestSetup(t, "postgres")
 	wrap := db.TestWrapper(t)
 	iamRepo := iam.TestRepo(t, conn, wrap)
@@ -237,6 +244,16 @@ func TestList(t *testing.T) {
 		totalRoles = append(totalRoles, wantProjRoles[i])
 	}
 
+	slices.Reverse(wantOrgRoles)
+	slices.Reverse(wantProjRoles)
+	slices.Reverse(totalRoles)
+
+	// Run analyze to update postgres estimates
+	sqlDb, err := conn.SqlDB(ctx)
+	require.NoError(t, err)
+	_, err = sqlDb.ExecContext(ctx, "analyze")
+	require.NoError(t, err)
+
 	cases := []struct {
 		name string
 		req  *pbs.ListRolesRequest
@@ -246,58 +263,117 @@ func TestList(t *testing.T) {
 		{
 			name: "List Many Role",
 			req:  &pbs.ListRolesRequest{ScopeId: oWithRoles.GetPublicId()},
-			res:  &pbs.ListRolesResponse{Items: wantOrgRoles},
+			res: &pbs.ListRolesResponse{
+				Items:        wantOrgRoles,
+				EstItemCount: uint32(len(wantOrgRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "List No Roles",
 			req:  &pbs.ListRolesRequest{ScopeId: oNoRoles.GetPublicId()},
-			res:  &pbs.ListRolesResponse{},
+			res: &pbs.ListRolesResponse{
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
+		},
+		{
+			name: "Paginate listing",
+			req:  &pbs.ListRolesRequest{ScopeId: "global", Recursive: true, PageSize: 2},
+			res: &pbs.ListRolesResponse{
+				Items:        totalRoles[:2],
+				EstItemCount: uint32(len(totalRoles)),
+				ResponseType: "delta",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "List Many Project Role",
 			req:  &pbs.ListRolesRequest{ScopeId: pWithRoles.GetPublicId()},
-			res:  &pbs.ListRolesResponse{Items: wantProjRoles},
+			res: &pbs.ListRolesResponse{
+				Items:        wantProjRoles,
+				EstItemCount: uint32(len(wantProjRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "List No Project Roles",
 			req:  &pbs.ListRolesRequest{ScopeId: pNoRoles.GetPublicId()},
-			res:  &pbs.ListRolesResponse{},
+			res: &pbs.ListRolesResponse{
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "List proj roles recursively",
 			req:  &pbs.ListRolesRequest{ScopeId: pWithRoles.GetPublicId(), Recursive: true},
 			res: &pbs.ListRolesResponse{
-				Items: wantProjRoles,
+				Items:        wantProjRoles,
+				EstItemCount: uint32(len(wantProjRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
 			},
 		},
 		{
 			name: "List org roles recursively",
 			req:  &pbs.ListRolesRequest{ScopeId: oNoRoles.GetPublicId(), Recursive: true},
 			res: &pbs.ListRolesResponse{
-				Items: wantProjRoles,
+				Items:        wantProjRoles,
+				EstItemCount: uint32(len(wantProjRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
 			},
 		},
 		{
 			name: "List global roles recursively",
 			req:  &pbs.ListRolesRequest{ScopeId: "global", Recursive: true},
 			res: &pbs.ListRolesResponse{
-				Items: totalRoles,
+				Items:        totalRoles,
+				EstItemCount: uint32(len(totalRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
 			},
 		},
 		{
 			name: "Filter to org roles",
 			req:  &pbs.ListRolesRequest{ScopeId: "global", Recursive: true, Filter: fmt.Sprintf(`"/item/scope/id"==%q`, oWithRoles.GetPublicId())},
-			res:  &pbs.ListRolesResponse{Items: wantOrgRoles},
+			res: &pbs.ListRolesResponse{
+				Items:        wantOrgRoles,
+				EstItemCount: uint32(len(wantOrgRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "Filter to proj roles",
 			req:  &pbs.ListRolesRequest{ScopeId: "global", Recursive: true, Filter: fmt.Sprintf(`"/item/scope/id"==%q`, pWithRoles.GetPublicId())},
-			res:  &pbs.ListRolesResponse{Items: wantProjRoles},
+			res: &pbs.ListRolesResponse{
+				Items:        wantProjRoles,
+				EstItemCount: uint32(len(wantProjRoles)),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "Filter to no roles",
 			req:  &pbs.ListRolesRequest{ScopeId: pWithRoles.GetPublicId(), Filter: `"/item/id"=="doesnt match"`},
-			res:  &pbs.ListRolesResponse{},
+			res: &pbs.ListRolesResponse{
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+			},
 		},
 		{
 			name: "Filter Bad Format",
@@ -308,7 +384,7 @@ func TestList(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
-			s, err := roles.NewService(context.Background(), repoFn)
+			s, err := roles.NewService(context.Background(), repoFn, 1000)
 			require.NoError(err, "Couldn't create new role service.")
 
 			// Test the non-anon case
@@ -323,6 +399,7 @@ func TestList(t *testing.T) {
 				got,
 				tc.res,
 				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
 				cmpopts.SortSlices(func(a, b string) bool {
 					return a < b
 				}),
@@ -345,10 +422,351 @@ func TestList(t *testing.T) {
 	}
 }
 
+func roleToProto(r *iam.Role, scope *scopes.ScopeInfo, authorizedActions []string) *pb.Role {
+	return &pb.Role{
+		Id:                r.GetPublicId(),
+		ScopeId:           r.GetScopeId(),
+		Scope:             scope,
+		CreatedTime:       r.GetCreateTime().GetTimestamp(),
+		UpdatedTime:       r.GetUpdateTime().GetTimestamp(),
+		GrantScopeId:      &wrapperspb.StringValue{Value: r.GetGrantScopeId()},
+		Version:           r.GetVersion(),
+		AuthorizedActions: testAuthorizedActions,
+	}
+}
+
+func TestListPagination(t *testing.T) {
+	// Set database read timeout to avoid duplicates in response
+	oldReadTimeout := globals.RefreshReadLookbackDuration
+	globals.RefreshReadLookbackDuration = 0
+	t.Cleanup(func() {
+		globals.RefreshReadLookbackDuration = oldReadTimeout
+	})
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	sqlDB, err := conn.SqlDB(ctx)
+	require.NoError(t, err)
+	wrapper := db.TestWrapper(t)
+	kms := kms.TestKms(t, conn, wrapper)
+	rw := db.New(conn)
+
+	iamRepo := iam.TestRepo(t, conn, wrapper)
+	iamRepoFn := func() (*iam.Repository, error) {
+		return iamRepo, nil
+	}
+	tokenRepoFn := func() (*authtoken.Repository, error) {
+		return authtoken.NewRepository(ctx, rw, rw, kms)
+	}
+	serversRepoFn := func() (*server.Repository, error) {
+		return server.NewRepository(ctx, rw, rw, kms)
+	}
+	tokenRepo, err := tokenRepoFn()
+	require.NoError(t, err)
+
+	// Run analyze to update postgres meta tables
+	_, err = sqlDB.ExecContext(ctx, "analyze")
+	require.NoError(t, err)
+
+	oNoRoles, pWithRoles := iam.TestScopes(t, iamRepo, iam.WithSkipDefaultRoleCreation(true))
+	oWithRoles, pNoRoles := iam.TestScopes(t, iamRepo, iam.WithSkipDefaultRoleCreation(true))
+
+	authMethod := password.TestAuthMethods(t, conn, "global", 1)[0]
+	acct := password.TestAccount(t, conn, authMethod.GetPublicId(), "test_user")
+	u := iam.TestUser(t, iamRepo, "global", iam.WithAccountIds(acct.PublicId))
+
+	var allRoles []*pb.Role
+	for _, scope := range []*iam.Scope{oWithRoles, oNoRoles, pWithRoles, pNoRoles} {
+		allowedRole := iam.TestRole(t, conn, scope.GetPublicId())
+		iam.TestRoleGrant(t, conn, allowedRole.GetPublicId(), "id=*;type=*;actions=*")
+		iam.TestUserRole(t, conn, allowedRole.GetPublicId(), u.GetPublicId())
+		allRoles = append(allRoles, roleToProto(allowedRole,
+			&scopes.ScopeInfo{
+				Id:            scope.GetPublicId(),
+				ParentScopeId: scope.GetParentId(),
+				Type:          scope.GetType(),
+			},
+			testAuthorizedActions,
+		))
+	}
+
+	at, err := tokenRepo.CreateAuthToken(ctx, u, acct.GetPublicId())
+	require.NoError(t, err)
+
+	// Test without anon user
+	requestInfo := authpb.RequestInfo{
+		TokenFormat: uint32(auth.AuthTokenTypeBearer),
+		PublicId:    at.GetPublicId(),
+		Token:       at.GetToken(),
+	}
+	requestContext := context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
+	ctx = auth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
+
+	var safeToDeleteRole string
+	orgScopeInfo := &scopes.ScopeInfo{Id: oWithRoles.GetPublicId(), Type: scope.Org.String(), ParentScopeId: scope.Global.String()}
+	projScopeInfo := &scopes.ScopeInfo{Id: pWithRoles.GetPublicId(), Type: scope.Project.String(), ParentScopeId: pWithRoles.GetParentId()}
+	for i := 0; i < 10; i++ {
+		or := iam.TestRole(t, conn, oWithRoles.GetPublicId())
+		allRoles = append(allRoles, roleToProto(or, orgScopeInfo, testAuthorizedActions))
+		pr := iam.TestRole(t, conn, pWithRoles.GetPublicId())
+		allRoles = append(allRoles, roleToProto(pr, projScopeInfo, testAuthorizedActions))
+		safeToDeleteRole = pr.GetPublicId()
+	}
+	slices.Reverse(allRoles)
+
+	a, err := roles.NewService(ctx, iamRepoFn, 1000)
+	require.NoError(t, err, "Couldn't create new user service.")
+
+	// Run analyze to update postgres estimates
+	_, err = sqlDB.ExecContext(context.Background(), "analyze")
+	require.NoError(t, err)
+
+	itemCount := uint32(len(allRoles))
+	testPageSize := int((itemCount - 2) / 2)
+	// Start paginating, recursively
+	req := &pbs.ListRolesRequest{
+		ScopeId:   "global",
+		Recursive: true,
+		Filter:    "",
+		ListToken: "",
+		PageSize:  uint32(testPageSize),
+	}
+	got, err := a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), testPageSize)
+	// Compare without comparing the list token
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        allRoles[0:testPageSize],
+				ResponseType: "delta",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				// In addition to the added roles, there are the roles added
+				// by the test setup when specifying the permissions of the
+				// requester
+				EstItemCount: itemCount,
+			},
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.Transform(),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+
+	// Request second page
+	req.ListToken = got.ListToken
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), testPageSize)
+	// Compare without comparing the list token
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        allRoles[testPageSize : testPageSize*2],
+				ResponseType: "delta",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+
+	// Request rest of results
+	req.ListToken = got.ListToken
+	req.PageSize = 10
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 2)
+	// Compare without comparing the list token
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        allRoles[testPageSize*2:],
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+
+	// Update 2 roles and see them in the refresh
+	r1 := allRoles[len(allRoles)-1]
+	r1.Description = wrapperspb.String("updated1")
+	resp1, err := a.UpdateRole(ctx, &pbs.UpdateRoleRequest{
+		Id:         r1.GetId(),
+		Item:       &pb.Role{Description: r1.GetDescription(), Version: r1.GetVersion()},
+		UpdateMask: &field_mask.FieldMask{Paths: []string{"description"}},
+	})
+	require.NoError(t, err)
+	r1.UpdatedTime = resp1.GetItem().GetUpdatedTime()
+	r1.Version = resp1.GetItem().GetVersion()
+	allRoles = append([]*pb.Role{r1}, allRoles[:len(allRoles)-1]...)
+
+	r2 := allRoles[len(allRoles)-1]
+	r2.Description = wrapperspb.String("updated2")
+	resp2, err := a.UpdateRole(ctx, &pbs.UpdateRoleRequest{
+		Id:         r2.GetId(),
+		Item:       &pb.Role{Description: r2.GetDescription(), Version: r2.GetVersion()},
+		UpdateMask: &field_mask.FieldMask{Paths: []string{"description"}},
+	})
+	require.NoError(t, err)
+	r2.UpdatedTime = resp2.GetItem().GetUpdatedTime()
+	r2.Version = resp2.GetItem().GetVersion()
+	allRoles = append([]*pb.Role{r2}, allRoles[:len(allRoles)-1]...)
+
+	// Run analyze to update postgres estimates
+	_, err = sqlDB.ExecContext(ctx, "analyze")
+	require.NoError(t, err)
+
+	// Request updated results
+	req.ListToken = got.ListToken
+	req.PageSize = 1
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	// Compare without comparing the list token
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        []*pb.Role{allRoles[0]},
+				ResponseType: "delta",
+				SortBy:       "updated_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+
+	// Get next page
+	req.ListToken = got.ListToken
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	// Compare without comparing the list token
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        []*pb.Role{allRoles[1]},
+				ResponseType: "complete",
+				SortBy:       "updated_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+
+	// Request new page with filter requiring looping
+	// to fill the page.
+	req.ListToken = ""
+	req.PageSize = 1
+	req.Filter = fmt.Sprintf(`"/item/id"==%q or "/item/id"==%q`, allRoles[len(allRoles)-2].Id, allRoles[len(allRoles)-1].Id)
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        []*pb.Role{allRoles[len(allRoles)-2]},
+				ResponseType: "delta",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				// Should be empty again
+				RemovedIds:   nil,
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+	req.ListToken = got.ListToken
+	// Get the second page
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	require.Len(t, got.GetItems(), 1)
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        []*pb.Role{allRoles[len(allRoles)-1]},
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				RemovedIds:   nil,
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+
+	_, err = iamRepo.DeleteRole(ctx, safeToDeleteRole)
+	require.NoError(t, err)
+	req.ListToken = got.ListToken
+	got, err = a.ListRoles(ctx, req)
+	require.NoError(t, err)
+	assert.Empty(t,
+		cmp.Diff(
+			got,
+			&pbs.ListRolesResponse{
+				Items:        nil,
+				ResponseType: "complete",
+				SortBy:       "updated_time",
+				SortDir:      "desc",
+				RemovedIds:   []string{safeToDeleteRole},
+				EstItemCount: itemCount,
+			},
+			protocmp.Transform(),
+			protocmp.SortRepeated(func(a, b string) bool {
+				return strings.Compare(a, b) < 0
+			}),
+			protocmp.IgnoreFields(&pbs.ListRolesResponse{}, "list_token"),
+		),
+	)
+}
+
 func TestDelete(t *testing.T) {
 	or, pr, repoFn := createDefaultRolesAndRepo(t)
 
-	s, err := roles.NewService(context.Background(), repoFn)
+	s, err := roles.NewService(context.Background(), repoFn, 1000)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	cases := []struct {
@@ -414,7 +832,7 @@ func TestDelete_twice(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 	or, pr, repoFn := createDefaultRolesAndRepo(t)
 
-	s, err := roles.NewService(context.Background(), repoFn)
+	s, err := roles.NewService(context.Background(), repoFn, 1000)
 	require.NoError(err, "Error when getting new role service")
 	req := &pbs.DeleteRoleRequest{
 		Id: or.GetPublicId(),
@@ -559,7 +977,7 @@ func TestCreate(t *testing.T) {
 			req := proto.Clone(toMerge).(*pbs.CreateRoleRequest)
 			proto.Merge(req, tc.req)
 
-			s, err := roles.NewService(context.Background(), repoFn)
+			s, err := roles.NewService(context.Background(), repoFn, 1000)
 			require.NoError(err, "Error when getting new role service.")
 
 			got, gErr := s.CreateRole(auth.DisabledAuthTestContext(repoFn, tc.req.GetItem().GetScopeId()), req)
@@ -636,7 +1054,7 @@ func TestUpdate(t *testing.T) {
 	var orVersion uint32 = 1
 	var prVersion uint32 = 1
 
-	tested, err := roles.NewService(ctx, repoFn)
+	tested, err := roles.NewService(ctx, repoFn, 0)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	resetRoles := func(proj bool) {
@@ -1052,7 +1470,7 @@ func TestAddPrincipal(t *testing.T) {
 		return iamRepo, nil
 	}
 	o, p := iam.TestScopes(t, iamRepo)
-	s, err := roles.NewService(ctx, repoFn)
+	s, err := roles.NewService(ctx, repoFn, 0)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	kmsCache := kms.TestKms(t, conn, wrap)
@@ -1257,7 +1675,7 @@ func TestSetPrincipal(t *testing.T) {
 	repoFn := func() (*iam.Repository, error) {
 		return iamRepo, nil
 	}
-	s, err := roles.NewService(context.Background(), repoFn)
+	s, err := roles.NewService(context.Background(), repoFn, 1000)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	o, p := iam.TestScopes(t, iamRepo)
@@ -1459,7 +1877,7 @@ func TestRemovePrincipal(t *testing.T) {
 	repoFn := func() (*iam.Repository, error) {
 		return iamRepo, nil
 	}
-	s, err := roles.NewService(ctx, repoFn)
+	s, err := roles.NewService(ctx, repoFn, 0)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	o, p := iam.TestScopes(t, iamRepo)
@@ -1721,7 +2139,7 @@ func TestAddGrants(t *testing.T) {
 	repoFn := func() (*iam.Repository, error) {
 		return iamRepo, nil
 	}
-	s, err := roles.NewService(context.Background(), repoFn)
+	s, err := roles.NewService(context.Background(), repoFn, 1000)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	addCases := []struct {
@@ -1869,7 +2287,7 @@ func TestSetGrants(t *testing.T) {
 		return iamRepo, nil
 	}
 
-	s, err := roles.NewService(context.Background(), repoFn)
+	s, err := roles.NewService(context.Background(), repoFn, 1000)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	setCases := []struct {
@@ -2014,7 +2432,7 @@ func TestRemoveGrants(t *testing.T) {
 	repoFn := func() (*iam.Repository, error) {
 		return iamRepo, nil
 	}
-	s, err := roles.NewService(context.Background(), repoFn)
+	s, err := roles.NewService(context.Background(), repoFn, 1000)
 	require.NoError(t, err, "Error when getting new role service.")
 
 	removeCases := []struct {

internal/daemon/controller/handlers/scopes/scope_service_test.go

@@ -967,15 +967,18 @@ func TestCreate(t *testing.T) {
 					if withUserId {
 						repo, err := repoFn()
 						require.NoError(err)
-						roles, err := repo.ListRoles(ctx, []string{got.GetItem().GetId()})
+						noopFilter := func(ctx context.Context, item *iam.Role) (bool, error) {
+							return true, nil
+						}
+						roles, err := iam.ListRoles(ctx, []byte("test"), globals.DefaultMaxPageSize, noopFilter, repo, []string{got.GetItem().GetId()})
 						require.NoError(err)
 						switch tc.scopeId {
 						case defaultOrg.PublicId:
-							require.Len(roles, 2)
+							require.Len(roles.Items, 2)
 						case "global":
-							require.Len(roles, 2)
+							require.Len(roles.Items, 2)
 						}
-						for _, role := range roles {
+						for _, role := range roles.Items {
 							switch role.GetName() {
 							case "Administration":
 								assert.Equal(fmt.Sprintf("Role created for administration of scope %s by user %s at its creation time", got.GetItem().GetId(), userId), role.GetDescription())