Merge pull request #5128 from hashicorp/tmessi-cherry-pick-cache-speedup

Cache speed up and additional performance improvements

This includes several changes to improve the performance of the
client cache, including:

* Making the sqlite database for the cache persistent, so the cache does
   not need to be rebuilt each time the boundary client is started.
* Limiting the number of results retrieved when listing resources to 250.
* Preventing redundant refreshes running at the same time for a given
   client.
* Adding indexes to the cache database.

It also refactors the grants query that is run to fetch grants for a
user when processing a controller API request. This greatly reduces the
amount of data that is returned from the database, but required a number
of changes in the application logic to account for different kinds of
grant scopes.

Lastly this makes some performance improvements in the controller
database. It adds several indexes to help with the grants query and with
delete cascades for several resources. And it changes the delete trigger
for sessions to run per statement instead of per row to help with bulk
deleting of sessions.

api/accounts/account.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type AccountListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	authMethodId  string
+	allRemovedIds []string
 }
 
 func (n AccountListResult) GetItems() []*Account {
@@ -313,7 +321,12 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["auth_method_id"] = authMethodId
 
-	req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...)
+	requestPath := "accounts"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -340,96 +353,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.authMethodId = authMethodId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Account, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AccountListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Account that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Account has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Account) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Account) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +451,85 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AccountListResult, opt ...Option) (*AccountListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.authMethodId == "" {
+		return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["auth_method_id"] = currentPage.authMethodId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "accounts"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AccountListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.authMethodId = currentPage.authMethodId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/accounts/option.gen.go

@@ -5,6 +5,7 @@
 package accounts
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -52,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +97,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/aliases/alias.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type AliasListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AliasListResult) GetItems() []*Alias {
@@ -319,7 +327,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...)
+	requestPath := "aliases"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -346,96 +359,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Alias, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AliasListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Alias that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Alias has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Alias) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Alias) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +459,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AliasListResult, opt ...Option) (*AliasListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "aliases"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AliasListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/aliases/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/authmethods/authmethods.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type AuthMethodListResult struct {
 	ListToken    string        `json:"list_token,omitempty"`
 	ResponseType string        `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AuthMethodListResult) GetItems() []*AuthMethod {
@@ -319,7 +327,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...)
+	requestPath := "auth-methods"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -346,96 +359,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*AuthMethod, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AuthMethodListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any AuthMethod that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a AuthMethod has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *AuthMethod) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *AuthMethod) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +459,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthMethodListResult, opt ...Option) (*AuthMethodListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "auth-methods"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AuthMethodListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/authmethods/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/authtokens/authtokens.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -66,6 +67,13 @@ type AuthTokenListResult struct {
 	ListToken    string       `json:"list_token,omitempty"`
 	ResponseType string       `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AuthTokenListResult) GetItems() []*AuthToken {
@@ -204,7 +212,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...)
+	requestPath := "auth-tokens"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -231,96 +244,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*AuthToken, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AuthTokenListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any AuthToken that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a AuthToken has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *AuthToken) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *AuthToken) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -334,4 +344,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthTokenListResult, opt ...Option) (*AuthTokenListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "auth-tokens"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AuthTokenListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/authtokens/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -85,6 +91,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/billing/option.gen.go

@@ -6,6 +6,7 @@ package billing
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -21,12 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -53,6 +57,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -91,6 +98,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithEndTime(inEndTime string) Option {
 	return func(o *options) {
 		o.queryMap["end_time"] = fmt.Sprintf("%v", inEndTime)

api/credentiallibraries/credential_library.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type CredentialLibraryListResult struct {
 	ListToken    string               `json:"list_token,omitempty"`
 	ResponseType string               `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive         bool
+	pageSize          uint32
+	credentialStoreId string
+	allRemovedIds     []string
 }
 
 func (n CredentialLibraryListResult) GetItems() []*CredentialLibrary {
@@ -319,7 +327,12 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["credential_store_id"] = credentialStoreId
 
-	req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...)
+	requestPath := "credential-libraries"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -346,96 +359,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.credentialStoreId = credentialStoreId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*CredentialLibrary, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialLibraryListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any CredentialLibrary that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a CredentialLibrary has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *CredentialLibrary) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *CredentialLibrary) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +457,85 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialLibraryListResult, opt ...Option) (*CredentialLibraryListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.credentialStoreId == "" {
+		return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["credential_store_id"] = currentPage.credentialStoreId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "credential-libraries"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialLibraryListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.credentialStoreId = currentPage.credentialStoreId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentiallibraries/option.gen.go

@@ -5,6 +5,7 @@
 package credentiallibraries
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -52,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +97,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithVaultSSHCertificateCredentialLibraryAdditionalValidPrincipals(inAdditionalValidPrincipals []string) Option {
 	return func(o *options) {
 		raw, ok := o.postMap["attributes"]

api/credentials/credential.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type CredentialListResult struct {
 	ListToken    string        `json:"list_token,omitempty"`
 	ResponseType string        `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive         bool
+	pageSize          uint32
+	credentialStoreId string
+	allRemovedIds     []string
 }
 
 func (n CredentialListResult) GetItems() []*Credential {
@@ -317,7 +325,12 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["credential_store_id"] = credentialStoreId
 
-	req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...)
+	requestPath := "credentials"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -344,96 +357,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.credentialStoreId = credentialStoreId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Credential, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Credential that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Credential has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Credential) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Credential) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,4 +455,85 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialListResult, opt ...Option) (*CredentialListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.credentialStoreId == "" {
+		return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["credential_store_id"] = currentPage.credentialStoreId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "credentials"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.credentialStoreId = currentPage.credentialStoreId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentials/option.gen.go

@@ -5,6 +5,7 @@
 package credentials
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -52,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +97,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/credentialstores/credential_store.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type CredentialStoreListResult struct {
 	ListToken    string             `json:"list_token,omitempty"`
 	ResponseType string             `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n CredentialStoreListResult) GetItems() []*CredentialStore {
@@ -318,7 +326,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...)
+	requestPath := "credential-stores"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -345,96 +358,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*CredentialStore, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialStoreListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any CredentialStore that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a CredentialStore has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *CredentialStore) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *CredentialStore) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -448,4 +458,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialStoreListResult, opt ...Option) (*CredentialStoreListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "credential-stores"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialStoreListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentialstores/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/go.mod

@@ -3,7 +3,7 @@ module github.com/hashicorp/boundary/api
 go 1.23.1
 
 require (
-	github.com/hashicorp/boundary/sdk v0.0.40
+	github.com/hashicorp/boundary/sdk v0.0.48
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-kms-wrapping/v2 v2.0.14
 	github.com/hashicorp/go-retryablehttp v0.7.4
@@ -18,7 +18,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/atomic v1.11.0
 	golang.org/x/time v0.3.0
-	google.golang.org/grpc v1.59.0
+	google.golang.org/grpc v1.61.0
 	google.golang.org/protobuf v1.33.0
 	nhooyr.io/websocket v1.8.10
 )
@@ -36,9 +36,10 @@ require (
 	github.com/mitchellh/pointerstructure v1.2.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

api/go.sum

@@ -21,12 +21,12 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/boundary/sdk v0.0.40 h1:HNJcMWHCjoraPJALTZ9JssSoP/vflew2+VB656nvRlY=
-github.com/hashicorp/boundary/sdk v0.0.40/go.mod h1:+XTDYf9YNeKIbGOPJwy7hlO2Le4zgzCtHCG/u+z4THI=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/boundary/sdk v0.0.48 h1:4HqyX1tS1kuaCa18OSbPGf8ZHJuwdmm1yaxr1u+nxZ4=
+github.com/hashicorp/boundary/sdk v0.0.48/go.mod h1:9iOT7kDM6mYcSkKxNuZlv8rP7U5BG1kXoevjLLL8lNQ=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -88,8 +88,8 @@ github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f h1:E87tDTVS5W
 github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f/go.mod h1:3J2qVK16Lq8V+wfiL2lPeDZ7UWMxk5LemerHa1p6N00=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -117,14 +117,13 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
-github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
@@ -146,8 +145,8 @@ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -157,19 +156,19 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -182,22 +181,22 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA=
-google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 h1:HJMDndgxest5n2y77fnErkM62iUsptE/H8p0dC2Huo4=
-google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405/go.mod h1:oT32Z4o8Zv2xPQTg0pbVaPr0MPOH6f14RgXt7zfIpwg=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
-google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
-google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
+google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg=
+google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k=
+google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe h1:0poefMBYvYbs7g5UkjS6HcxBPaTRAmznle9jnxYoAI8=
+google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s=
+google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0=
+google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

api/groups/group.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type GroupListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n GroupListResult) GetItems() []*Group {
@@ -312,7 +320,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...)
+	requestPath := "groups"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -339,96 +352,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Group, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(GroupListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Group that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Group has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Group) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Group) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -442,6 +452,94 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *GroupListResult, opt ...Option) (*GroupListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "groups"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(GroupListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddMembers(ctx context.Context, id string, version uint32, memberIds []string, opt ...Option) (*GroupUpdateResult, error) {

api/groups/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/hostcatalogs/host_catalog.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -73,6 +74,13 @@ type HostCatalogListResult struct {
 	ListToken    string         `json:"list_token,omitempty"`
 	ResponseType string         `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n HostCatalogListResult) GetItems() []*HostCatalog {
@@ -323,7 +331,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...)
+	requestPath := "host-catalogs"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -350,96 +363,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*HostCatalog, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostCatalogListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any HostCatalog that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a HostCatalog has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *HostCatalog) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *HostCatalog) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -453,4 +463,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostCatalogListResult, opt ...Option) (*HostCatalogListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "host-catalogs"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostCatalogListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/hostcatalogs/option.gen.go

@@ -22,13 +22,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +61,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +102,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/hosts/host.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -74,6 +75,13 @@ type HostListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	hostCatalogId string
+	allRemovedIds []string
 }
 
 func (n HostListResult) GetItems() []*Host {
@@ -319,7 +327,12 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["host_catalog_id"] = hostCatalogId
 
-	req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...)
+	requestPath := "hosts"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -346,96 +359,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.hostCatalogId = hostCatalogId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Host, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Host that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Host has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Host) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Host) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +457,85 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostListResult, opt ...Option) (*HostListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.hostCatalogId == "" {
+		return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "hosts"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.hostCatalogId = currentPage.hostCatalogId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/hosts/option.gen.go

@@ -5,6 +5,7 @@
 package hosts
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -52,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +97,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithStaticHostAddress(inAddress string) Option {
 	return func(o *options) {
 		raw, ok := o.postMap["attributes"]

api/hostsets/host_set.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -72,6 +73,13 @@ type HostSetListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	hostCatalogId string
+	allRemovedIds []string
 }
 
 func (n HostSetListResult) GetItems() []*HostSet {
@@ -317,7 +325,12 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["host_catalog_id"] = hostCatalogId
 
-	req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...)
+	requestPath := "host-sets"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -344,96 +357,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.hostCatalogId = hostCatalogId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*HostSet, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostSetListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any HostSet that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a HostSet has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *HostSet) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *HostSet) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,6 +455,87 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostSetListResult, opt ...Option) (*HostSetListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.hostCatalogId == "" {
+		return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "host-sets"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostSetListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.hostCatalogId = currentPage.hostCatalogId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddHosts(ctx context.Context, id string, version uint32, hostIds []string, opt ...Option) (*HostSetUpdateResult, error) {

api/hostsets/option.gen.go

@@ -5,6 +5,7 @@
 package hostsets
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -52,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +97,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/managedgroups/managedgroups.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ManagedGroupListResult struct {
 	ListToken    string          `json:"list_token,omitempty"`
 	ResponseType string          `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	authMethodId  string
+	allRemovedIds []string
 }
 
 func (n ManagedGroupListResult) GetItems() []*ManagedGroup {
@@ -313,7 +321,12 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["auth_method_id"] = authMethodId
 
-	req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
+	requestPath := "managed-groups"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -340,96 +353,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.authMethodId = authMethodId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*ManagedGroup, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(ManagedGroupListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any ManagedGroup that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a ManagedGroup has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *ManagedGroup) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *ManagedGroup) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +451,85 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *ManagedGroupListResult, opt ...Option) (*ManagedGroupListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.authMethodId == "" {
+		return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["auth_method_id"] = currentPage.authMethodId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "managed-groups"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(ManagedGroupListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.authMethodId = currentPage.authMethodId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/managedgroups/option.gen.go

@@ -5,6 +5,7 @@
 package managedgroups
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
 }
 
 func getDefaultOptions() options {
@@ -52,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +97,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/policies/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/policies/policy.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type PolicyListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n PolicyListResult) GetItems() []*Policy {
@@ -317,7 +325,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...)
+	requestPath := "policies"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -344,96 +357,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Policy, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(PolicyListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Policy that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Policy has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Policy) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Policy) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,4 +457,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *PolicyListResult, opt ...Option) (*PolicyListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "policies"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(PolicyListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/roles/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/roles/role.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -70,6 +71,13 @@ type RoleListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n RoleListResult) GetItems() []*Role {
@@ -315,7 +323,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...)
+	requestPath := "roles"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -342,96 +355,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Role, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(RoleListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Role that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Role has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Role) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Role) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -445,6 +455,94 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *RoleListResult, opt ...Option) (*RoleListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "roles"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(RoleListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddGrantScopes(ctx context.Context, id string, version uint32, grantScopeIds []string, opt ...Option) (*RoleUpdateResult, error) {

api/scopes/option.gen.go

@@ -22,13 +22,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +61,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +102,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/scopes/scope.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ScopeListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n ScopeListResult) GetItems() []*Scope {
@@ -313,7 +321,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...)
+	requestPath := "scopes"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -340,96 +353,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Scope, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(ScopeListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Scope that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Scope has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Scope) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Scope) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +453,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *ScopeListResult, opt ...Option) (*ScopeListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "scopes"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(ScopeListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/sessionrecordings/option.gen.go

@@ -20,13 +20,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -56,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -75,6 +81,28 @@ func WithListToken(listToken string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/sessionrecordings/session_recording.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -73,6 +74,13 @@ type SessionRecordingListResult struct {
 	ListToken    string              `json:"list_token,omitempty"`
 	ResponseType string              `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n SessionRecordingListResult) GetItems() []*SessionRecording {
@@ -211,7 +219,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...)
+	requestPath := "session-recordings"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -238,96 +251,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*SessionRecording, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(SessionRecordingListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any SessionRecording that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a SessionRecording has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *SessionRecording) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *SessionRecording) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -341,4 +351,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionRecordingListResult, opt ...Option) (*SessionRecordingListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "session-recordings"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(SessionRecordingListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/sessions/option.gen.go

@@ -22,13 +22,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +61,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +102,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/sessions/session.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -75,6 +76,13 @@ type SessionListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n SessionListResult) GetItems() []*Session {
@@ -171,7 +179,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...)
+	requestPath := "sessions"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -198,96 +211,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Session, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(SessionListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Session that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Session has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Session) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Session) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -301,4 +311,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionListResult, opt ...Option) (*SessionListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "sessions"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(SessionListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/storagebuckets/option.gen.go

@@ -22,13 +22,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +61,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +102,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/storagebuckets/storage_bucket.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -76,6 +77,13 @@ type StorageBucketListResult struct {
 	ListToken    string           `json:"list_token,omitempty"`
 	ResponseType string           `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n StorageBucketListResult) GetItems() []*StorageBucket {
@@ -321,7 +329,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...)
+	requestPath := "storage-buckets"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -348,96 +361,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*StorageBucket, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(StorageBucketListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any StorageBucket that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a StorageBucket has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *StorageBucket) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *StorageBucket) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -451,4 +461,92 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *StorageBucketListResult, opt ...Option) (*StorageBucketListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "storage-buckets"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(StorageBucketListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/targets/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/targets/target.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -81,6 +82,13 @@ type TargetListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n TargetListResult) GetItems() []*Target {
@@ -331,7 +339,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...)
+	requestPath := "targets"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -358,96 +371,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Target, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(TargetListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Target that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Target has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Target) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Target) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -461,6 +471,94 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *TargetListResult, opt ...Option) (*TargetListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "targets"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(TargetListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddCredentialSources(ctx context.Context, id string, version uint32, opt ...Option) (*TargetUpdateResult, error) {

api/users/custom.go

@@ -5,10 +5,8 @@ package users
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/url"
-	"slices"
 
 	"github.com/hashicorp/boundary/api/aliases"
 )
@@ -25,135 +23,44 @@ func (c *Client) ListResolvableAliases(ctx context.Context, userId string, opt .
 		return nil, fmt.Errorf("nil client")
 	}
 
-	opts, apiOpts := getOpts(opt...)
-	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("users/%s:list-resolvable-aliases", url.PathEscape(userId)), nil, apiOpts...)
-	if err != nil {
-		return nil, fmt.Errorf("error creating List request: %w", err)
-	}
-
-	if len(opts.queryMap) > 0 {
-		q := url.Values{}
-		for k, v := range opts.queryMap {
-			q.Add(k, v)
-		}
-		req.URL.RawQuery = q.Encode()
-	}
-
-	resp, err := c.client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("error performing client request during List call: %w", err)
-	}
-
-	target := new(aliases.AliasListResult)
-	apiErr, err := resp.Decode(target)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding List response: %w", err)
-	}
-	if apiErr != nil {
-		return nil, apiErr
-	}
-	target.Response = resp
-	if target.ResponseType == "complete" || target.ResponseType == "" {
-		return target, nil
-	}
-	// If there are more results, automatically fetch the rest of the results.
-	// idToIndex keeps a map from the ID of an item to its index in target.Items.
-	// This is used to update updated items in-place and remove deleted items
-	// from the result after pagination is done.
-	idToIndex := map[string]int{}
-	for i, item := range target.Items {
-		idToIndex[item.Id] = i
-	}
-	for {
-		req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("users/%s:list-resolvable-aliases", url.PathEscape(userId)), nil, apiOpts...)
-		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
-		}
-
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(aliases.AliasListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
-			if i, ok := idToIndex[item.Id]; ok {
-				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
-			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
-			}
-		}
-		// RemovedIds contain any Alias that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
-			break
-		}
-	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
-	// Remove items that were deleted since the end of the last iteration.
-	// If an Alias has been updated and subsequently removed, we don't want
-	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
-		if i, ok := idToIndex[removedId]; ok {
-			// Remove the item at index i without preserving order
-			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
-			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
-		}
-	}
-	// Since we paginated to the end, we can avoid confusion
-	// for the user by setting the estimated item count to the
-	// length of the items slice. If we don't set this here, it
-	// will equal the value returned in the last response, which is
-	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *aliases.Alias) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
-	// Finally, since we made at least 2 requests to the server to fulfill this
-	// function call, resp.Body and resp.Map will only contain the most recent response.
-	// Overwrite them with the true response.
-	target.GetResponse().Body.Reset()
-	if err := json.NewEncoder(target.GetResponse().Body).Encode(target); err != nil {
-		return nil, fmt.Errorf("error encoding final JSON list response: %w", err)
-	}
-	if err := json.Unmarshal(target.GetResponse().Body.Bytes(), &target.GetResponse().Map); err != nil {
-		return nil, fmt.Errorf("error encoding final map list response: %w", err)
-	}
-	// Note: the HTTP response body is consumed by resp.Decode in the loop,
-	// so it doesn't need to be updated (it will always be, and has always been, empty).
-	return target, nil
+	opts, _ := getOpts(opt...)
+	apiClient := aliases.NewClient(c.client)
+	return apiClient.List(ctx, "global",
+		aliases.WithAutomaticVersioning(opts.withAutomaticVersioning),
+		aliases.WithSkipCurlOutput(opts.withSkipCurlOutput),
+		aliases.WithFilter(opts.withFilter),
+		aliases.WithListToken(opts.withListToken),
+		aliases.WithClientDirectedPagination(opts.withClientDirectedPagination),
+		aliases.WithPageSize(opts.withPageSize),
+		aliases.WithRecursive(opts.withRecursive),
+		aliases.WithResourcePathOverride(fmt.Sprintf("users/%s:list-resolvable-aliases", url.PathEscape(userId))),
+	)
+}
+
+func (c *Client) ListResolvableAliasesNextPage(ctx context.Context, userId string, currentPage *aliases.AliasListResult, opt ...Option) (*aliases.AliasListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListResolvableAliasesNextPage request")
+	}
+	if userId == "" {
+		return nil, fmt.Errorf("empty userId value passed into ListResolvableAliasesNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListResolvableAliasesNextPage request")
+	}
+
+	opts, _ := getOpts(opt...)
+	apiClient := aliases.NewClient(c.client)
+	return apiClient.ListNextPage(ctx, currentPage,
+		aliases.WithAutomaticVersioning(opts.withAutomaticVersioning),
+		aliases.WithSkipCurlOutput(opts.withSkipCurlOutput),
+		aliases.WithFilter(opts.withFilter),
+		aliases.WithListToken(opts.withListToken),
+		aliases.WithClientDirectedPagination(opts.withClientDirectedPagination),
+		aliases.WithPageSize(opts.withPageSize),
+		aliases.WithRecursive(opts.withRecursive),
+		aliases.WithResourcePathOverride(fmt.Sprintf("users/%s:list-resolvable-aliases", url.PathEscape(userId))),
+	)
 }

api/users/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/users/user.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -71,6 +72,13 @@ type UserListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n UserListResult) GetItems() []*User {
@@ -316,7 +324,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...)
+	requestPath := "users"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -343,96 +356,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*User, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(UserListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any User that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a User has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *User) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *User) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -446,6 +456,94 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *UserListResult, opt ...Option) (*UserListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "users"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(UserListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddAccounts(ctx context.Context, id string, version uint32, accountIds []string, opt ...Option) (*UserUpdateResult, error) {

api/workers/option.gen.go

@@ -21,13 +21,16 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withResourcePathOverride     string
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +101,28 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/workers/worker.gen.go

@@ -6,11 +6,9 @@ package workers
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
-	"slices"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -78,6 +76,13 @@ type WorkerListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n WorkerListResult) GetItems() []*Worker {
@@ -372,7 +377,12 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["scope_id"] = scopeId
 
-	req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...)
+	requestPath := "workers"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -399,109 +409,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work
 		return nil, apiErr
 	}
 	target.Response = resp
-	if target.ResponseType == "complete" || target.ResponseType == "" {
-		return target, nil
-	}
-	// If there are more results, automatically fetch the rest of the results.
-	// idToIndex keeps a map from the ID of an item to its index in target.Items.
-	// This is used to update updated items in-place and remove deleted items
-	// from the result after pagination is done.
-	idToIndex := map[string]int{}
-	for i, item := range target.Items {
-		idToIndex[item.Id] = i
-	}
-	for {
-		req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...)
-		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
-		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(WorkerListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
-			if i, ok := idToIndex[item.Id]; ok {
-				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
-			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
-			}
-		}
-		// RemovedIds contain any Worker that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
-			break
-		}
-	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
-	// Remove items that were deleted since the end of the last iteration.
-	// If a Worker has been updated and subsequently removed, we don't want
-	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
-		if i, ok := idToIndex[removedId]; ok {
-			// Remove the item at index i without preserving order
-			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
-			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
-		}
-	}
-	// Since we paginated to the end, we can avoid confusion
-	// for the user by setting the estimated item count to the
-	// length of the items slice. If we don't set this here, it
-	// will equal the value returned in the last response, which is
-	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Worker) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
-	// Finally, since we made at least 2 requests to the server to fulfill this
-	// function call, resp.Body and resp.Map will only contain the most recent response.
-	// Overwrite them with the true response.
-	target.Response.Body.Reset()
-	if err := json.NewEncoder(target.Response.Body).Encode(target); err != nil {
-		return nil, fmt.Errorf("error encoding final JSON list response: %w", err)
-	}
-	if err := json.Unmarshal(target.Response.Body.Bytes(), &target.Response.Map); err != nil {
-		return nil, fmt.Errorf("error encoding final map list response: %w", err)
-	}
-	// Note: the HTTP response body is consumed by resp.Decode in the loop,
-	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
 }
 
 func (c *Client) AddWorkerTags(ctx context.Context, id string, version uint32, apiTags map[string][]string, opt ...Option) (*WorkerUpdateResult, error) {

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
 	github.com/hashicorp/boundary/api v0.0.50
-	github.com/hashicorp/boundary/sdk v0.0.46
+	github.com/hashicorp/boundary/sdk v0.0.48
 	github.com/hashicorp/cap v0.5.1-0.20240315182732-faa330bfb8df
 	github.com/hashicorp/dawdle v0.5.0
 	github.com/hashicorp/eventlogger v0.2.9
@@ -81,7 +81,7 @@ require (
 	nhooyr.io/websocket v1.8.10
 )
 
-require github.com/hashicorp/go-dbw v0.1.4
+require github.com/hashicorp/go-dbw v0.1.5-0.20240909162114-6cee92b3da36
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.2
@@ -100,7 +100,6 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/miekg/dns v1.1.58
 	github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
-	github.com/mitchellh/go-homedir v1.1.0
 	github.com/sevlyar/go-daemon v0.1.6
 	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3
 	golang.org/x/net v0.25.0
@@ -127,6 +126,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/sys/user v0.1.0 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect

go.sum

@@ -201,8 +201,8 @@ github.com/hashicorp/go-bexpr v0.1.13 h1:HNwp7vZrMpRq8VZXj8VF90LbZpRjQQpim1oJF0D
 github.com/hashicorp/go-bexpr v0.1.13/go.mod h1:gN7hRKB3s7yT+YvTdnhZVLTENejvhlkZ8UE4YVBS+Q8=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-dbw v0.1.4 h1:FipsEg7UbWiB8hBQylTYWJobG6jVr0LNoRdo9IClGs4=
-github.com/hashicorp/go-dbw v0.1.4/go.mod h1:/YHbfK7mgG9k09aB74Imw3fEOwno0eTtlFTTYGZ7SFk=
+github.com/hashicorp/go-dbw v0.1.5-0.20240909162114-6cee92b3da36 h1:rPD+2QPhCLq8mKMx2FnIaqR5PTNT+LzhjfacYWuvFzY=
+github.com/hashicorp/go-dbw v0.1.5-0.20240909162114-6cee92b3da36/go.mod h1:/YHbfK7mgG9k09aB74Imw3fEOwno0eTtlFTTYGZ7SFk=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-kms-wrapping/extras/kms/v2 v2.0.0-20231219183231-6bac757bb482 h1:1DqTnLaNk658AEenlF4PNGYd9b1hXE/+0jSOBIGOAms=

internal/alias/target/repository_alias_list_resolvable.go

@@ -10,26 +10,57 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/db/timestamp"
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/perms"
+	"github.com/hashicorp/boundary/internal/types/scope"
 )
 
-// targetAndScopeIdsForDestinations returns the target ids for which there is
-// at least one permission. If all targets in a specific scope are granted
-// permission for an action, then the scope id is in the returned scope id slice.
-func targetAndScopeIdsForDestinations(perms []perms.Permission) ([]string, []string) {
-	var targetIds, scopeIds []string
-	for _, perm := range perms {
-		switch {
-		case perm.All:
-			scopeIds = append(scopeIds, perm.ScopeId)
-		case len(perm.ResourceIds) > 0:
-			targetIds = append(targetIds, perm.ResourceIds...)
+func splitPermissions(permissions []perms.Permission) (directIds, directScopeIds, childAllScopes []string, allDescendants bool) {
+	// First check for all descendants. Since what we are querying for below is
+	// for targets (either IDs, or targets within specific scopes), and targets
+	// are not in global, if this matches we can actually ignore everything
+	// else.
+	for _, perm := range permissions {
+		if perm.GrantScopeId == globals.GrantScopeDescendants && perm.All {
+			allDescendants = true
+			return
 		}
 	}
-	return targetIds, scopeIds
+
+	directIds = make([]string, 0, len(permissions))
+	directScopeIds = make([]string, 0, len(permissions))
+	childAllScopes = make([]string, 0, len(permissions))
+	for _, perm := range permissions {
+		switch {
+		case allDescendants:
+			// See the above check; we don't need any other info
+		case perm.GrantScopeId == scope.Global.String() || strings.HasPrefix(perm.GrantScopeId, globals.OrgPrefix):
+			// There are no targets in global or orgs
+		case perm.RoleScopeId == scope.Global.String() && perm.GrantScopeId == globals.GrantScopeChildren:
+			// A role in global that includes children will include only orgs,
+			// which do not have targets, so ignore
+		case perm.GrantScopeId == globals.GrantScopeChildren && perm.All:
+			// Because of the above check this will match only grants from org
+			// roles. If the grant scope is children and all, we store the scope
+			// ID.
+			childAllScopes = append(childAllScopes, perm.RoleScopeId)
+		case perm.All:
+			// We ignore descendants and if this was a children grant scope and
+			// perm.All it would match the above case. So this is a grant
+			// directly on a scope. Since only projects contain targets, we can
+			// ignore any grant scope ID that doesn't match targets.
+			if strings.HasPrefix(perm.GrantScopeId, globals.ProjectPrefix) {
+				directScopeIds = append(directScopeIds, perm.GrantScopeId)
+			}
+		case len(perm.ResourceIds) > 0:
+			// It's an ID grant
+			directIds = append(directIds, perm.ResourceIds...)
+		}
+	}
+	return
 }
 
 // listResolvableAliases lists aliases which have a destination id set to that
@@ -41,7 +72,8 @@ func (r *Repository) listResolvableAliases(ctx context.Context, permissions []pe
 	case len(permissions) == 0:
 		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing permissions")
 	}
-	toTargetIds, toTargetsInScopeIds := targetAndScopeIdsForDestinations(permissions)
+
+	directIds, directScopeIds, childAllScopes, allDescendants := splitPermissions(permissions)
 
 	opts, err := getOpts(opt...)
 	if err != nil {
@@ -59,16 +91,33 @@ func (r *Repository) listResolvableAliases(ctx context.Context, permissions []pe
 
 	var args []any
 	var destinationIdClauses []string
-	if len(toTargetIds) > 0 {
-		destinationIdClauses = append(destinationIdClauses, "destination_id in @target_ids")
-		args = append(args, sql.Named("target_ids", toTargetIds))
-	}
-	if len(toTargetsInScopeIds) > 0 {
-		destinationIdClauses = append(destinationIdClauses, "destination_id in (select public_id from target where project_id in @target_scope_ids)")
-		args = append(args, sql.Named("target_scope_ids", toTargetsInScopeIds))
-	}
-	if len(destinationIdClauses) == 0 {
-		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "no target ids or scope ids provided")
+
+	switch {
+	case allDescendants:
+		// This matches all targets
+		destinationIdClauses = append(destinationIdClauses, "destination_id in (select public_id from target)")
+	default:
+		// Add orgs with all permissions on children
+		if len(childAllScopes) > 0 {
+			destinationIdClauses = append(destinationIdClauses,
+				"destination_id in "+
+					"(select public_id from target where project_id in "+
+					"(select public_id from iam_scope where parent_id = any(@child_all_scopes)))",
+			)
+			args = append(args, sql.Named("child_all_scopes", "{"+strings.Join(childAllScopes, ",")+"}"))
+		}
+		// Add target ids
+		if len(directIds) > 0 {
+			destinationIdClauses = append(destinationIdClauses, "destination_id = any(@target_ids)")
+			args = append(args, sql.Named("target_ids", "{"+strings.Join(directIds, ",")+"}"))
+		}
+		if len(directScopeIds) > 0 {
+			destinationIdClauses = append(destinationIdClauses, "destination_id in (select public_id from target where project_id = any(@target_scope_ids))")
+			args = append(args, sql.Named("target_scope_ids", "{"+strings.Join(directScopeIds, ",")+"}"))
+		}
+		if len(destinationIdClauses) == 0 && len(childAllScopes) == 0 {
+			return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "no target ids or scope ids provided")
+		}
 	}
 
 	whereClause := fmt.Sprintf("destination_id is not null and (%s)", strings.Join(destinationIdClauses, " or "))
@@ -98,7 +147,8 @@ func (r *Repository) listResolvableAliasesRefresh(ctx context.Context, updatedAf
 	case len(permissions) == 0:
 		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing permissions")
 	}
-	toTargetIds, toTargetsInScopeIds := targetAndScopeIdsForDestinations(permissions)
+
+	directIds, directScopeIds, childAllScopes, allDescendants := splitPermissions(permissions)
 
 	opts, err := getOpts(opt...)
 	if err != nil {
@@ -116,16 +166,33 @@ func (r *Repository) listResolvableAliasesRefresh(ctx context.Context, updatedAf
 
 	var args []any
 	var destinationIdClauses []string
-	if len(toTargetIds) > 0 {
-		destinationIdClauses = append(destinationIdClauses, "destination_id in @target_ids")
-		args = append(args, sql.Named("target_ids", toTargetIds))
-	}
-	if len(toTargetsInScopeIds) > 0 {
-		destinationIdClauses = append(destinationIdClauses, "destination_id in (select public_id from target where project_id in @target_scope_ids)")
-		args = append(args, sql.Named("target_scope_ids", toTargetsInScopeIds))
-	}
-	if len(destinationIdClauses) == 0 {
-		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "no target ids or scope ids provided")
+
+	switch {
+	case allDescendants:
+		// This matches all targets
+		destinationIdClauses = append(destinationIdClauses, "destination_id in (select public_id from target)")
+	default:
+		// Add orgs with all permissions on children
+		if len(childAllScopes) > 0 {
+			destinationIdClauses = append(destinationIdClauses,
+				"destination_id in "+
+					"(select public_id from target where project_id in "+
+					"(select public_id from iam_scope where parent_id = any(@child_all_scopes)))",
+			)
+			args = append(args, sql.Named("child_all_scopes", "{"+strings.Join(childAllScopes, ",")+"}"))
+		}
+		// Add target ids
+		if len(directIds) > 0 {
+			destinationIdClauses = append(destinationIdClauses, "destination_id = any(@target_ids)")
+			args = append(args, sql.Named("target_ids", "{"+strings.Join(directIds, ",")+"}"))
+		}
+		if len(directScopeIds) > 0 {
+			destinationIdClauses = append(destinationIdClauses, "destination_id in (select public_id from target where project_id = any(@target_scope_ids))")
+			args = append(args, sql.Named("target_scope_ids", "{"+strings.Join(directScopeIds, ",")+"}"))
+		}
+		if len(destinationIdClauses) == 0 && len(childAllScopes) == 0 {
+			return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "no target ids or scope ids provided")
+		}
 	}
 
 	whereClause := fmt.Sprintf("update_time > @updated_after_time and destination_id is not null and (%s)",
@@ -162,21 +229,40 @@ func (r *Repository) listRemovedResolvableAliasIds(ctx context.Context, since ti
 		// to be provided.
 		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing permissions")
 	}
-	toTargetIds, toTargetsInScopeIds := targetAndScopeIdsForDestinations(permissions)
+
+	directIds, directScopeIds, childAllScopes, allDescendants := splitPermissions(permissions)
 
 	var args []any
 	var destinationIdClauses []string
-	if len(toTargetIds) > 0 {
-		destinationIdClauses = append(destinationIdClauses, "destination_id not in @target_ids")
-		args = append(args, sql.Named("target_ids", toTargetIds))
-	}
-	if len(toTargetsInScopeIds) > 0 {
-		destinationIdClauses = append(destinationIdClauses, "destination_id not in (select public_id from target where project_id in @target_scope_ids)")
-		args = append(args, sql.Named("target_scope_ids", toTargetsInScopeIds))
-	}
-	if len(destinationIdClauses) == 0 {
-		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "no target ids or scope ids provided")
+
+	switch {
+	case allDescendants:
+		// This matches all targets
+		destinationIdClauses = append(destinationIdClauses, "destination_id not in (select public_id from target)")
+	default:
+		// Add orgs with all permissions on children
+		if len(childAllScopes) > 0 {
+			destinationIdClauses = append(destinationIdClauses,
+				"destination_id not in "+
+					"(select public_id from target where project_id in "+
+					"(select public_id from iam_scope where parent_id = any(@child_all_scopes)))",
+			)
+			args = append(args, sql.Named("child_all_scopes", "{"+strings.Join(childAllScopes, ",")+"}"))
+		}
+		// Add target ids
+		if len(directIds) > 0 {
+			destinationIdClauses = append(destinationIdClauses, "destination_id != all(@target_ids)")
+			args = append(args, sql.Named("target_ids", "{"+strings.Join(directIds, ",")+"}"))
+		}
+		if len(directScopeIds) > 0 {
+			destinationIdClauses = append(destinationIdClauses, "destination_id not in (select public_id from target where project_id = any(@target_scope_ids))")
+			args = append(args, sql.Named("target_scope_ids", "{"+strings.Join(directScopeIds, ",")+"}"))
+		}
+		if len(destinationIdClauses) == 0 && len(childAllScopes) == 0 {
+			return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "no target ids or scope ids provided")
+		}
 	}
+
 	whereClause := fmt.Sprintf("update_time > @updated_after_time and (destination_id is null or (%s))",
 		strings.Join(destinationIdClauses, " and "))
 	args = append(args,

internal/alias/target/service_list_resolvable_ext_test.go

@@ -30,6 +30,11 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
+// NOTE: These tests rely on state from previous tests, so they should be run in
+// order -- running some subtests on their own will result in errors. It might
+// be nice for them to be refactored at some point to start with a known state
+// to avoid this. At the time I'm writing this, I'm not doing that because I'm
+// not sure if this was a purposeful design choice.
 func TestService_ListResolvableAliases(t *testing.T) {
 	fiveDaysAgo := time.Now()
 	// Set database read timeout to avoid duplicates in response
@@ -56,12 +61,12 @@ func TestService_ListResolvableAliases(t *testing.T) {
 	}
 	byIdPerms := []perms.Permission{
 		{
-			ScopeId:     proj.GetPublicId(),
-			Resource:    resource.Target,
-			Action:      action.ListResolvableAliases,
-			ResourceIds: []string{tar.GetPublicId(), "ttcp_unknownid"},
-			OnlySelf:    false,
-			All:         false,
+			GrantScopeId: proj.GetPublicId(),
+			Resource:     resource.Target,
+			Action:       action.ListResolvableAliases,
+			ResourceIds:  []string{tar.GetPublicId(), "ttcp_unknownid"},
+			OnlySelf:     false,
+			All:          false,
 		},
 	}
 	// Reverse since we read items in descending order (newest first)
@@ -76,16 +81,48 @@ func TestService_ListResolvableAliases(t *testing.T) {
 	}
 	byScopePerms := []perms.Permission{
 		{
-			ScopeId:  proj2.GetPublicId(),
-			Resource: resource.Target,
-			Action:   action.ListResolvableAliases,
-			OnlySelf: false,
-			All:      true,
+			GrantScopeId: proj2.GetPublicId(),
+			Resource:     resource.Target,
+			Action:       action.ListResolvableAliases,
+			OnlySelf:     false,
+			All:          true,
 		},
 	}
 	// Reverse since we read items in descending order (newest first)
 	slices.Reverse(byScopeResources)
 
+	org3, proj3 := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+	tar3 := tcp.TestTarget(ctx, t, conn, proj3.GetPublicId(), "target3")
+	var byChildrenResources []*target.Alias
+	for i := 0; i < 5; i++ {
+		r := target.TestAlias(t, rw, fmt.Sprintf("test%d.alias.by-children", i), target.WithDestinationId(tar3.GetPublicId()))
+		byChildrenResources = append(byChildrenResources, r)
+	}
+	byChildrenPerms := []perms.Permission{
+		{
+			RoleScopeId:       org3.GetPublicId(),
+			RoleParentScopeId: scope.Global.String(),
+			GrantScopeId:      globals.GrantScopeChildren,
+			Resource:          resource.Target,
+			Action:            action.ListResolvableAliases,
+			OnlySelf:          false,
+			All:               true,
+		},
+	}
+	// Reverse since we read items in descending order (newest first)
+	slices.Reverse(byChildrenResources)
+
+	byDescendantsPerms := []perms.Permission{
+		{
+			RoleScopeId:  scope.Global.String(),
+			GrantScopeId: globals.GrantScopeDescendants,
+			Resource:     resource.Target,
+			Action:       action.ListResolvableAliases,
+			OnlySelf:     false,
+			All:          true,
+		},
+	}
+
 	repo, repoErr := target.NewRepository(ctx, rw, rw, kmsCache)
 	require.NoError(t, repoErr)
 
@@ -296,21 +333,41 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		})
 	})
 
+	// Build the descendants resources for the first tests
+	byDescendantsResources := append([]*target.Alias{}, byChildrenResources...)
+	byDescendantsResources = append(byDescendantsResources, byScopeResources...)
+	byDescendantsResources = append(byDescendantsResources, byIdResources...)
+
 	t.Run("simple pagination", func(t *testing.T) {
 		cases := []struct {
 			name          string
 			perms         []perms.Permission
 			resourceSlice []*target.Alias
+			lastPageSize  int
 		}{
 			{
 				name:          "by-id",
 				perms:         byIdPerms,
 				resourceSlice: byIdResources,
+				lastPageSize:  1,
 			},
 			{
 				name:          "by-scope",
 				perms:         byScopePerms,
 				resourceSlice: byScopeResources,
+				lastPageSize:  1,
+			},
+			{
+				name:          "by-children",
+				perms:         byChildrenPerms,
+				resourceSlice: byChildrenResources,
+				lastPageSize:  1,
+			},
+			{
+				name:          "by-descendants",
+				perms:         byDescendantsPerms,
+				resourceSlice: byDescendantsResources,
+				lastPageSize:  11,
 			},
 		}
 		for _, tc := range cases {
@@ -320,7 +377,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NotNil(t, resp.ListToken)
 				require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
 				require.False(t, resp.CompleteListing)
-				require.Equal(t, resp.EstimatedItemCount, 10)
+				require.Equal(t, 15, resp.EstimatedItemCount)
 				require.Empty(t, resp.DeletedIds)
 				require.Len(t, resp.Items, 1)
 				require.Empty(t, cmp.Diff(resp.Items[0], tc.resourceSlice[0], cmpIgnoreUnexportedOpts), "resources did not match", tc.resourceSlice, "resp", resp.Items)
@@ -329,7 +386,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
 				require.False(t, resp2.CompleteListing)
-				require.Equal(t, resp2.EstimatedItemCount, 10)
+				require.Equal(t, 15, resp2.EstimatedItemCount)
 				require.Empty(t, resp2.DeletedIds)
 				require.Len(t, resp2.Items, 1)
 				require.Empty(t, cmp.Diff(resp2.Items[0], tc.resourceSlice[1], cmpIgnoreUnexportedOpts))
@@ -338,7 +395,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
 				require.False(t, resp3.CompleteListing)
-				require.Equal(t, resp3.EstimatedItemCount, 10)
+				require.Equal(t, 15, resp3.EstimatedItemCount)
 				require.Empty(t, resp3.DeletedIds)
 				require.Len(t, resp3.Items, 1)
 				require.Empty(t, cmp.Diff(resp3.Items[0], tc.resourceSlice[2], cmpIgnoreUnexportedOpts))
@@ -347,18 +404,18 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash"))
 				require.False(t, resp4.CompleteListing)
-				require.Equal(t, resp4.EstimatedItemCount, 10)
+				require.Equal(t, 15, resp4.EstimatedItemCount)
 				require.Empty(t, resp4.DeletedIds)
 				require.Len(t, resp4.Items, 1)
 				require.Empty(t, cmp.Diff(resp4.Items[0], tc.resourceSlice[3], cmpIgnoreUnexportedOpts))
 
-				resp5, err := target.ListResolvableAliasesPage(ctx, []byte("some hash"), 1, resp4.ListToken, repo, tc.perms)
+				resp5, err := target.ListResolvableAliasesPage(ctx, []byte("some hash"), tc.lastPageSize, resp4.ListToken, repo, tc.perms)
 				require.NoError(t, err)
 				require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash"))
 				require.True(t, resp5.CompleteListing)
-				require.Equal(t, resp5.EstimatedItemCount, 10)
+				require.Equal(t, 15, resp5.EstimatedItemCount)
 				require.Empty(t, resp5.DeletedIds)
-				require.Len(t, resp5.Items, 1)
+				require.Len(t, resp5.Items, tc.lastPageSize)
 				require.Empty(t, cmp.Diff(resp5.Items[0], tc.resourceSlice[4], cmpIgnoreUnexportedOpts))
 
 				// Finished initial pagination phase, request refresh
@@ -367,7 +424,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp6.ListToken.GrantsHash, []byte("some hash"))
 				require.True(t, resp6.CompleteListing)
-				require.Equal(t, resp6.EstimatedItemCount, 10)
+				require.Equal(t, 15, resp6.EstimatedItemCount)
 				require.Empty(t, resp6.DeletedIds)
 				require.Empty(t, resp6.Items)
 
@@ -392,7 +449,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp7.ListToken.GrantsHash, []byte("some hash"))
 				require.False(t, resp7.CompleteListing)
-				require.Equal(t, resp7.EstimatedItemCount, 12)
+				require.Equal(t, 17, resp7.EstimatedItemCount)
 				require.Empty(t, resp7.DeletedIds)
 				require.Len(t, resp7.Items, 1)
 				require.Empty(t, cmp.Diff(resp7.Items[0], newR2, cmpIgnoreUnexportedOpts))
@@ -402,7 +459,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp8.ListToken.GrantsHash, []byte("some hash"))
 				require.True(t, resp8.CompleteListing)
-				require.Equal(t, resp8.EstimatedItemCount, 12)
+				require.Equal(t, 17, resp8.EstimatedItemCount)
 				require.Empty(t, resp8.DeletedIds)
 				require.Len(t, resp8.Items, 1)
 				require.Empty(t, cmp.Diff(resp8.Items[0], newR1, cmpIgnoreUnexportedOpts))
@@ -412,14 +469,14 @@ func TestService_ListResolvableAliases(t *testing.T) {
 				require.NoError(t, err)
 				require.Equal(t, resp9.ListToken.GrantsHash, []byte("some hash"))
 				require.True(t, resp9.CompleteListing)
-				require.Equal(t, resp9.EstimatedItemCount, 12)
+				require.Equal(t, 17, resp9.EstimatedItemCount)
 				require.Empty(t, resp9.DeletedIds)
 				require.Empty(t, resp9.Items)
 			})
 		}
 	})
 
-	t.Run("simple pagination with destination id changes", func(t *testing.T) {
+	t.Run("simple pagination with destination id changes - id", func(t *testing.T) {
 		firstUpdatedA := byScopeResources[0]
 		// this no longer has the destination id that has permissions
 		firstUpdatedA.DestinationId = tar.GetPublicId()
@@ -442,7 +499,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		require.NotNil(t, resp.ListToken)
 		require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
 		require.False(t, resp.CompleteListing)
-		require.Equal(t, resp.EstimatedItemCount, 10)
+		require.Equal(t, resp.EstimatedItemCount, 15)
 		require.Empty(t, resp.DeletedIds)
 		require.Len(t, resp.Items, 1)
 		require.Empty(t, cmp.Diff(resp.Items[0], byScopeResources[0], cmpIgnoreUnexportedOpts))
@@ -452,7 +509,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
 		require.True(t, resp2.CompleteListing)
-		require.Equal(t, resp2.EstimatedItemCount, 10)
+		require.Equal(t, resp2.EstimatedItemCount, 15)
 		require.Empty(t, resp2.DeletedIds)
 		require.Len(t, resp2.Items, 3)
 		require.Empty(t, cmp.Diff(resp2.Items, byScopeResources[1:], cmpIgnoreUnexportedOpts))
@@ -479,12 +536,99 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
 		require.True(t, resp3.CompleteListing)
-		require.Equal(t, resp3.EstimatedItemCount, 10)
+		require.Equal(t, resp3.EstimatedItemCount, 15)
 		require.Contains(t, resp3.DeletedIds, secondA.GetPublicId())
 		require.Empty(t, resp3.Items)
 	})
 
-	t.Run("simple pagination with deletion", func(t *testing.T) {
+	t.Run("simple pagination with destination id changes - children", func(t *testing.T) {
+		firstUpdatedA := byChildrenResources[0]
+		// this no longer has the destination id that has permissions
+		firstUpdatedA.DestinationId = tar.GetPublicId()
+		firstUpdatedA, _, err := repo.UpdateAlias(ctx, firstUpdatedA, firstUpdatedA.GetVersion(), []string{"DestinationId"})
+		require.NoError(t, err)
+		byChildrenResources = byChildrenResources[1:]
+		t.Cleanup(func() {
+			firstUpdatedA.DestinationId = tar3.GetPublicId()
+			firstUpdatedA, _, err := repo.UpdateAlias(ctx, firstUpdatedA, firstUpdatedA.GetVersion(), []string{"DestinationId"})
+			require.NoError(t, err)
+			byChildrenResources = append([]*target.Alias{firstUpdatedA}, byChildrenResources...)
+		})
+
+		// Run analyze to update count estimate
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		resp, err := target.ListResolvableAliases(ctx, []byte("some hash"), 1, repo, byChildrenPerms)
+		require.NoError(t, err)
+		require.NotNil(t, resp.ListToken)
+		require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
+		require.False(t, resp.CompleteListing)
+		require.Equal(t, resp.EstimatedItemCount, 15)
+		require.Empty(t, resp.DeletedIds)
+		require.Len(t, resp.Items, 1)
+		require.Empty(t, cmp.Diff(resp.Items[0], byChildrenResources[0], cmpIgnoreUnexportedOpts))
+
+		// request remaining results
+		resp2, err := target.ListResolvableAliasesPage(ctx, []byte("some hash"), 3, resp.ListToken, repo, byChildrenPerms)
+		require.NoError(t, err)
+		require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
+		require.True(t, resp2.CompleteListing)
+		require.Equal(t, resp2.EstimatedItemCount, 15)
+		require.Empty(t, resp2.DeletedIds)
+		require.Len(t, resp2.Items, 3)
+		require.Empty(t, cmp.Diff(resp2.Items, byChildrenResources[1:], cmpIgnoreUnexportedOpts))
+	})
+
+	// We have to re-build the expected set of resources as the original slices
+	// have been updated with updated values
+	byDescendantsResources = append([]*target.Alias{}, byChildrenResources...)
+	byDescendantsResources = append(byDescendantsResources, byScopeResources...)
+	byDescendantsResources = append(byDescendantsResources, byIdResources...)
+
+	t.Run("simple pagination with destination id changes - descendants", func(t *testing.T) {
+		firstUpdatedA := byDescendantsResources[0]
+		// this no longer has the destination id that has permissions
+		firstUpdatedA.DestinationId = tar.GetPublicId()
+		firstUpdatedA, _, err := repo.UpdateAlias(ctx, firstUpdatedA, firstUpdatedA.GetVersion(), []string{"DestinationId"})
+		require.NoError(t, err)
+		// Descendants will keep permissions for everything so we don't elide
+		// one here, but we need to increment the version number to match
+		byDescendantsResources[0] = firstUpdatedA
+		t.Cleanup(func() {
+			firstUpdatedA.DestinationId = tar3.GetPublicId()
+			firstUpdatedA, _, err = repo.UpdateAlias(ctx, firstUpdatedA, firstUpdatedA.GetVersion(), []string{"DestinationId"})
+			require.NoError(t, err)
+			byDescendantsResources[0] = firstUpdatedA
+		})
+
+		// Run analyze to update count estimate
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		resp, err := target.ListResolvableAliases(ctx, []byte("some hash"), 1, repo, byDescendantsPerms)
+		require.NoError(t, err)
+		require.NotNil(t, resp.ListToken)
+		require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
+		require.False(t, resp.CompleteListing)
+		require.Equal(t, resp.EstimatedItemCount, 15)
+		require.Empty(t, resp.DeletedIds)
+		require.Len(t, resp.Items, 1)
+		require.Empty(t, cmp.Diff(resp.Items[0], byDescendantsResources[0], cmpIgnoreUnexportedOpts))
+
+		// request remaining results -- we should see all, because descendants
+		// is going to maintain permissions
+		resp2, err := target.ListResolvableAliasesPage(ctx, []byte("some hash"), 14, resp.ListToken, repo, byDescendantsPerms)
+		require.NoError(t, err)
+		require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
+		require.True(t, resp2.CompleteListing)
+		require.Equal(t, resp2.EstimatedItemCount, 15)
+		require.Empty(t, resp2.DeletedIds)
+		require.Len(t, resp2.Items, 14)
+		require.Empty(t, cmp.Diff(resp2.Items, byDescendantsResources[1:], cmpIgnoreUnexportedOpts))
+	})
+
+	t.Run("simple pagination with deletion - id", func(t *testing.T) {
 		deletedAliasId := byIdResources[0].GetPublicId()
 		_, err := repo.DeleteAlias(ctx, deletedAliasId)
 		require.NoError(t, err)
@@ -499,7 +643,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		require.NotNil(t, resp.ListToken)
 		require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
 		require.False(t, resp.CompleteListing)
-		require.Equal(t, resp.EstimatedItemCount, 9)
+		require.Equal(t, 14, resp.EstimatedItemCount)
 		require.Empty(t, resp.DeletedIds)
 		require.Len(t, resp.Items, 1)
 		require.Empty(t, cmp.Diff(resp.Items[0], byIdResources[0], cmpIgnoreUnexportedOpts))
@@ -509,7 +653,7 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
 		require.True(t, resp2.CompleteListing)
-		require.Equal(t, resp2.EstimatedItemCount, 9)
+		require.Equal(t, 14, resp2.EstimatedItemCount)
 		require.Empty(t, resp2.DeletedIds)
 		require.Len(t, resp2.Items, 3)
 		require.Empty(t, cmp.Diff(resp2.Items, byIdResources[1:], cmpIgnoreUnexportedOpts))
@@ -528,7 +672,111 @@ func TestService_ListResolvableAliases(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
 		require.True(t, resp3.CompleteListing)
-		require.Equal(t, resp3.EstimatedItemCount, 8)
+		require.Equal(t, 13, resp3.EstimatedItemCount)
+		require.Contains(t, resp3.DeletedIds, deletedAliasId)
+		require.Empty(t, resp3.Items)
+	})
+
+	t.Run("simple pagination with deletion - children", func(t *testing.T) {
+		deletedAliasId := byChildrenResources[0].GetPublicId()
+		_, err := repo.DeleteAlias(ctx, deletedAliasId)
+		require.NoError(t, err)
+		byChildrenResources = byChildrenResources[1:]
+
+		// Run analyze to update count estimate
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		resp, err := target.ListResolvableAliases(ctx, []byte("some hash"), 1, repo, byChildrenPerms)
+		require.NoError(t, err)
+		require.NotNil(t, resp.ListToken)
+		require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
+		require.False(t, resp.CompleteListing)
+		require.Equal(t, 12, resp.EstimatedItemCount)
+		require.Empty(t, resp.DeletedIds)
+		require.Len(t, resp.Items, 1)
+		require.Empty(t, cmp.Diff(resp.Items[0], byChildrenResources[0], cmpIgnoreUnexportedOpts))
+
+		// request remaining results
+		resp2, err := target.ListResolvableAliasesPage(ctx, []byte("some hash"), 8, resp.ListToken, repo, byChildrenPerms)
+		require.NoError(t, err)
+		require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
+		require.True(t, resp2.CompleteListing)
+		require.Equal(t, 12, resp2.EstimatedItemCount)
+		require.Empty(t, resp2.DeletedIds)
+		require.Len(t, resp2.Items, 3)
+		require.Empty(t, cmp.Diff(resp2.Items, byChildrenResources[1:], cmpIgnoreUnexportedOpts))
+
+		deletedAliasId = byChildrenResources[0].GetPublicId()
+		_, err = repo.DeleteAlias(ctx, deletedAliasId)
+		require.NoError(t, err)
+		byChildrenResources = byChildrenResources[1:]
+
+		// Run analyze to update count estimate
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		// request a refresh, nothing should be returned except the deleted id
+		resp3, err := target.ListResolvableAliasesRefresh(ctx, []byte("some hash"), 1, resp2.ListToken, repo, byChildrenPerms)
+		require.NoError(t, err)
+		require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
+		require.True(t, resp3.CompleteListing)
+		require.Equal(t, 11, resp3.EstimatedItemCount)
+		require.Contains(t, resp3.DeletedIds, deletedAliasId)
+		require.Empty(t, resp3.Items)
+	})
+
+	// We have to re-build the expected set of resources as the original slices
+	// have been updated with updated values again
+	byDescendantsResources = append([]*target.Alias{}, byChildrenResources...)
+	byDescendantsResources = append(byDescendantsResources, byScopeResources...)
+	byDescendantsResources = append(byDescendantsResources, byIdResources...)
+
+	t.Run("simple pagination with deletion - descendants", func(t *testing.T) {
+		deletedAliasId := byDescendantsResources[0].GetPublicId()
+		_, err := repo.DeleteAlias(ctx, deletedAliasId)
+		require.NoError(t, err)
+		byDescendantsResources = byDescendantsResources[1:]
+
+		// Run analyze to update count estimate
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		resp, err := target.ListResolvableAliases(ctx, []byte("some hash"), 1, repo, byDescendantsPerms)
+		require.NoError(t, err)
+		require.NotNil(t, resp.ListToken)
+		require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
+		require.False(t, resp.CompleteListing)
+		require.Equal(t, 10, resp.EstimatedItemCount)
+		require.Empty(t, resp.DeletedIds)
+		require.Len(t, resp.Items, 1)
+		require.Empty(t, cmp.Diff(resp.Items[0], byDescendantsResources[0], cmpIgnoreUnexportedOpts))
+
+		// request remaining results
+		resp2, err := target.ListResolvableAliasesPage(ctx, []byte("some hash"), 13, resp.ListToken, repo, byDescendantsPerms)
+		require.NoError(t, err)
+		require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
+		require.True(t, resp2.CompleteListing)
+		require.Equal(t, 10, resp2.EstimatedItemCount)
+		require.Empty(t, resp2.DeletedIds)
+		require.Len(t, resp2.Items, 9)
+		require.Empty(t, cmp.Diff(resp2.Items, byDescendantsResources[1:], cmpIgnoreUnexportedOpts))
+
+		deletedAliasId = byDescendantsResources[0].GetPublicId()
+		_, err = repo.DeleteAlias(ctx, deletedAliasId)
+		require.NoError(t, err)
+		byDescendantsResources = byDescendantsResources[1:]
+
+		// Run analyze to update count estimate
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		// request a refresh, nothing should be returned except the deleted id
+		resp3, err := target.ListResolvableAliasesRefresh(ctx, []byte("some hash"), 1, resp2.ListToken, repo, byDescendantsPerms)
+		require.NoError(t, err)
+		require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
+		require.True(t, resp3.CompleteListing)
+		require.Equal(t, 9, resp3.EstimatedItemCount)
 		require.Contains(t, resp3.DeletedIds, deletedAliasId)
 		require.Empty(t, resp3.Items)
 	})

internal/api/genapi/input.go

@@ -142,6 +142,10 @@ type structInfo struct {
 	// fields
 	fieldFilter []string
 
+	// nonPaginatedListing indicates a collection that does not support
+	// pagination
+	nonPaginatedListing bool
+
 	allowEmpty bool
 }
 
@@ -1426,5 +1430,6 @@ var inputStructs = []*structInfo{
 		createResponseTypes: []string{CreateResponseType, ReadResponseType, UpdateResponseType, DeleteResponseType, ListResponseType},
 		recursiveListing:    true,
 		versionEnabled:      true,
+		nonPaginatedListing: true,
 	},
 }

internal/api/genapi/templates.go

@@ -49,6 +49,7 @@ type templateInput struct {
 	SliceSubtypes         map[string]sliceSubtypeInfo
 	ExtraFields           []fieldInfo
 	VersionEnabled        bool
+	NonPaginatedListing   bool
 	CreateResponseTypes   []string
 	SkipListFiltering     bool
 	RecursiveListing      bool
@@ -69,6 +70,7 @@ func fillTemplates() {
 			ParentTypeName:      in.parentTypeName,
 			ExtraFields:         in.extraFields,
 			VersionEnabled:      in.versionEnabled,
+			NonPaginatedListing: in.nonPaginatedListing,
 			CreateResponseTypes: in.createResponseTypes,
 			SkipListFiltering:   in.skipListFiltering,
 			RecursiveListing:    in.recursiveListing,
@@ -153,7 +155,7 @@ func fillTemplates() {
 			optionsMap[input.Package] = optionMap
 		}
 		// Override some defined options
-		if len(in.fieldOverrides) > 0 && optionsMap != nil {
+		if len(in.fieldOverrides) > 0 {
 			for _, override := range in.fieldOverrides {
 				inOpts := optionsMap[input.Package]
 				if inOpts != nil {
@@ -241,7 +243,12 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
 	opts, apiOpts := getOpts(opt...)
 	opts.queryMap["{{ snakeCase .CollectionFunctionArg }}"] = {{ .CollectionFunctionArg }}
 
-	req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...)
+	requestPath := "{{ .CollectionPath }}"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating List request: %w", err)
 	}
@@ -268,96 +275,96 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
 		return nil, apiErr
 	}
 	target.Response = resp
+{{ if .NonPaginatedListing }}
+	return target, nil
+{{ end }}
+{{ if ( not ( .NonPaginatedListing ) ) }}
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+{{ if .RecursiveListing }} 
+	target.recursive = opts.withRecursive
+{{ end }}
+	target.pageSize = opts.withPageSize
+	target.{{ .CollectionFunctionArg }} = {{ .CollectionFunctionArg }}
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*{{ .Name }}, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new({{ .Name }}ListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any {{ .Name }} that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a {{ .Name }} has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *{{ .Name }}) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *{{ .Name }}) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -371,7 +378,98 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+{{ end  }}
 }
+
+{{ if ( not ( .NonPaginatedListing ) ) }}
+func (c *Client) ListNextPage(ctx context.Context, currentPage *{{ .Name }}ListResult, opt ...Option) (*{{ .Name }}ListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.{{ .CollectionFunctionArg }} == "" {
+		return nil, fmt.Errorf("empty {{ .CollectionFunctionArg }} value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["{{ snakeCase .CollectionFunctionArg }}"] = currentPage.{{ .CollectionFunctionArg }}
+
+{{ if .RecursiveListing }} 
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+{{ end }}
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	requestPath := "{{ .CollectionPath }}"
+	if opts.withResourcePathOverride != "" {
+		requestPath = opts.withResourcePathOverride
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", requestPath, nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new({{ .Name }}ListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.{{ .CollectionFunctionArg }} = currentPage.{{ .CollectionFunctionArg }}
+{{ if .RecursiveListing }}
+	nextPage.recursive = currentPage.recursive
+{{ end }} 
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
+}
+{{ end }}
 `))
 
 var readTemplate = template.Must(template.New("").Parse(`
@@ -764,6 +862,14 @@ type {{ .Name }}ListResult struct {
 	ListToken string            `, "`json:\"list_token,omitempty\"`", `
 	ResponseType string         `, "`json:\"response_type,omitempty\"`", `
 	Response *api.Response
+
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive bool
+	pageSize uint32
+	{{ .CollectionFunctionArg }} string
+	allRemovedIds []string
 }
 
 func (n {{ .Name }}ListResult) GetItems() []*{{ .Name }} {
@@ -845,6 +951,9 @@ type options struct {
 	withSkipCurlOutput bool
 	withFilter string
 	withListToken string
+	withClientDirectedPagination bool
+	withPageSize uint32
+    withResourcePathOverride string
 	{{ if .RecursiveListing }} withRecursive bool {{ end }}
 }
 
@@ -875,6 +984,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	} {{ end }}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -916,6 +1028,27 @@ func WithFilter(filter string) Option {
 	}
 }
 {{ end }}
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+			o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
+// WithResourcePathOverride tells the API to use the provided resource path
+func WithResourcePathOverride(path string) Option {
+	return func(o *options) {
+		o.withResourcePathOverride = path
+	}
+}
 {{ if .RecursiveListing }}
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource

internal/clientcache/cmd/cache/start.go

@@ -19,7 +19,6 @@ import (
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/mitchellh/cli"
-	"github.com/mitchellh/go-homedir"
 	"github.com/posener/complete"
 	"gopkg.in/natefinch/lumberjack.v2"
 )
@@ -51,6 +50,7 @@ type StartCommand struct {
 	flagLogFormat               string
 	flagStoreDebug              bool
 	flagBackground              bool
+	flagForceResetSchema        bool
 }
 
 func (c *StartCommand) Synopsis() string {
@@ -134,6 +134,13 @@ func (c *StartCommand) Flags() *base.FlagSets {
 		Default: false,
 		Usage:   `Run the cache daemon in the background`,
 	})
+	f.BoolVar(&base.BoolVar{
+		Name:    "force-reset-schema",
+		Target:  &c.flagForceResetSchema,
+		Default: false,
+		Usage:   `Force resetting the cache schema and all contained data`,
+		Hidden:  true,
+	})
 
 	return set
 }
@@ -208,6 +215,7 @@ func (c *StartCommand) Run(args []string) int {
 		LogFileName:             logFileName,
 		DotDirectory:            dotDir,
 		RunningInBackground:     os.Getenv(backgroundEnvName) == backgroundEnvVal,
+		ForceResetSchema:        c.flagForceResetSchema,
 	}
 
 	srv, err := daemon.New(ctx, cfg)
@@ -243,7 +251,7 @@ func (c *StartCommand) Run(args []string) int {
 // DefaultDotDirectory returns the default path to the boundary dot directory.
 func DefaultDotDirectory(ctx context.Context) (string, error) {
 	const op = "cache.DefaultDotDirectory"
-	homeDir, err := homedir.Dir()
+	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		return "", errors.Wrap(ctx, err, op)
 	}

internal/clientcache/cmd/search/search.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
+	"math"
 	"net/url"
 	"os"
 	"strings"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/aliases"
+	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/sessions"
 	"github.com/hashicorp/boundary/api/targets"
 	cachecmd "github.com/hashicorp/boundary/internal/clientcache/cmd/cache"
@@ -33,6 +35,7 @@ var (
 		"resolvable-aliases",
 		"targets",
 		"sessions",
+		"implicit-scopes",
 	}
 
 	errCacheNotRunning = stderrors.New("The cache process is not running.")
@@ -40,9 +43,10 @@ var (
 
 type SearchCommand struct {
 	*base.Command
-	flagQuery        string
-	flagResource     string
-	flagForceRefresh bool
+	flagQuery            string
+	flagResource         string
+	flagForceRefresh     bool
+	flagMaxResultSetSize int64
 }
 
 func (c *SearchCommand) Synopsis() string {
@@ -112,6 +116,12 @@ func (c *SearchCommand) Flags() *base.FlagSets {
 		Usage:      `Specifies the resource type to search over`,
 		Completion: complete.PredictSet(supportedResourceTypes...),
 	})
+	f.Int64Var(&base.Int64Var{
+		Name:       "max-result-set-size",
+		Target:     &c.flagMaxResultSetSize,
+		Usage:      `Specifies an override to the default maximum result set size. Set to -1 to disable the limit. 0 will use the default.`,
+		Completion: complete.PredictNothing,
+	})
 	f.BoolVar(&base.BoolVar{
 		Name:   "force-refresh",
 		Target: &c.flagForceRefresh,
@@ -148,6 +158,15 @@ func (c *SearchCommand) Run(args []string) int {
 		return base.CommandUserError
 	}
 
+	switch {
+	case c.flagMaxResultSetSize < -1:
+		c.PrintCliError(stderrors.New("Max result set size must be greater than or equal to -1"))
+		return base.CommandUserError
+	case c.flagMaxResultSetSize > math.MaxInt:
+		c.PrintCliError(stderrors.New(fmt.Sprintf("Max result set size must be less than or equal to the %v", math.MaxInt)))
+		return base.CommandUserError
+	}
+
 	resp, result, apiErr, err := c.Search(ctx)
 	if err != nil {
 		c.PrintCliError(err)
@@ -171,9 +190,17 @@ func (c *SearchCommand) Run(args []string) int {
 			c.UI.Output(printTargetListTable(result.Targets))
 		case len(result.Sessions) > 0:
 			c.UI.Output(printSessionListTable(result.Sessions))
+		case len(result.ImplicitScopes) > 0:
+			c.UI.Output(printImplicitScopesListTable(result.ImplicitScopes))
 		default:
 			c.UI.Output("No items found")
 		}
+
+		// Put this at the end or people may not see it as they may not scroll
+		// all the way up.
+		if result.Incomplete {
+			c.UI.Warn("The maximum result set size was reached and the search results are incomplete. Please narrow your search or adjust the -max-result-set-size parameter.")
+		}
 	}
 	return base.CommandSuccess
 }
@@ -199,6 +226,9 @@ func (c *SearchCommand) Search(ctx context.Context) (*api.Response, *daemon.Sear
 		authTokenId:  strings.Join(tSlice[:2], "_"),
 		forceRefresh: c.flagForceRefresh,
 	}
+	if c.flagMaxResultSetSize != 0 {
+		tf.maxResultSetSize = int(c.flagMaxResultSetSize)
+	}
 	var opts []client.Option
 	if c.FlagOutputCurlString {
 		opts = append(opts, client.WithOutputCurlString())
@@ -230,6 +260,9 @@ func search(ctx context.Context, daemonPath string, fb filterBy, opt ...client.O
 	if fb.forceRefresh {
 		q.Add("force_refresh", "true")
 	}
+	if fb.maxResultSetSize != 0 {
+		q.Add("max_result_set_size", fmt.Sprintf("%d", fb.maxResultSetSize))
+	}
 	resp, err := c.Get(ctx, "/v1/search", q, opt...)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("Error when sending request to the cache: %w.", err)
@@ -423,10 +456,38 @@ func printSessionListTable(items []*sessions.Session) string {
 	return base.WrapForHelpText(output)
 }
 
-type filterBy struct {
-	flagFilter   string
-	flagQuery    string
-	authTokenId  string
-	resource     string
-	forceRefresh bool
+func printImplicitScopesListTable(items []*scopes.Scope) string {
+	if len(items) == 0 {
+		return "No implicit scopes found"
+	}
+	var output []string
+	output = []string{
+		"",
+		"Scope information:",
+	}
+	for i, item := range items {
+		if i > 0 {
+			output = append(output, "")
+		}
+		if item.Id != "" {
+			output = append(output,
+				fmt.Sprintf("  ID:                    %s", item.Id),
+			)
+		} else {
+			output = append(output,
+				fmt.Sprintf("  ID:                    %s", "(not available)"),
+			)
+		}
+	}
+
+	return base.WrapForHelpText(output)
+}
+
+type filterBy struct {
+	flagFilter       string
+	flagQuery        string
+	authTokenId      string
+	resource         string
+	forceRefresh     bool
+	maxResultSetSize int
 }

internal/clientcache/cmd/search/search_test.go

@@ -140,7 +140,9 @@ func TestSearch(t *testing.T) {
 		assert.Nil(t, apiErr)
 		assert.NotNil(t, resp)
 		assert.NotNil(t, r)
-		assert.EqualValues(t, r, &daemon.SearchResult{})
+		assert.EqualValues(t, &daemon.SearchResult{
+			RefreshStatus: daemon.NotRefreshing,
+		}, r)
 	})
 
 	t.Run("empty response from query", func(t *testing.T) {
@@ -154,7 +156,9 @@ func TestSearch(t *testing.T) {
 		assert.Nil(t, apiErr)
 		assert.NotNil(t, resp)
 		assert.NotNil(t, r)
-		assert.EqualValues(t, r, &daemon.SearchResult{})
+		assert.EqualValues(t, r, &daemon.SearchResult{
+			RefreshStatus: daemon.NotRefreshing,
+		})
 	})
 
 	t.Run("unsupported boundary instance", func(t *testing.T) {

internal/clientcache/internal/cache/consts.go

@@ -0,0 +1,17 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+/*
+Package cache contains the domain logic for the client cache.
+*/
+package cache
+
+const (
+	// defaultLimitedResultSetSize is the default number of results to
+	// return when limiting
+	defaultLimitedResultSetSize = 250
+
+	// unlimitedMaxResultSetSize is the value to use when we want to return all
+	// results
+	unlimitedMaxResultSetSize = -1
+)

internal/clientcache/internal/cache/options.go

@@ -4,9 +4,16 @@
 package cache
 
 import (
+	stderrors "errors"
+
 	"github.com/hashicorp/go-dbw"
 )
 
+type testRefreshWaitChs struct {
+	firstSempahore  chan struct{}
+	secondSemaphore chan struct{}
+}
+
 type options struct {
 	withUpdateLastAccessedTime       bool
 	withDbType                       dbw.DbType
@@ -16,6 +23,9 @@ type options struct {
 	withTargetRetrievalFunc          TargetRetrievalFunc
 	withSessionRetrievalFunc         SessionRetrievalFunc
 	withIgnoreSearchStaleness        bool
+	withMaxResultSetSize             int
+	withTestRefreshWaitChs           *testRefreshWaitChs
+	withUseNonPagedListing           bool
 }
 
 // Option - how options are passed as args
@@ -23,7 +33,8 @@ type Option func(*options) error
 
 func getDefaultOptions() options {
 	return options{
-		withDbType: dbw.Sqlite,
+		withDbType:           dbw.Sqlite,
+		withMaxResultSetSize: defaultLimitedResultSetSize,
 	}
 }
 
@@ -94,3 +105,37 @@ func WithIgnoreSearchStaleness(b bool) Option {
 		return nil
 	}
 }
+
+// WithMaxResultSetSize provides an option for limiting the result set, e.g.
+// when no filter is provided on a list. A 0 does nothing (keeps the default).
+func WithMaxResultSetSize(with int) Option {
+	return func(o *options) error {
+		switch {
+		case with == 0:
+			return nil
+		case with < -1:
+			return stderrors.New("max result set size must be -1 or greater")
+		}
+		o.withMaxResultSetSize = with
+		return nil
+	}
+}
+
+// WithTestRefreshWaitChs provides an option for specifying channels to wait on
+// before proceeding. This allows testing the logic that ensures only one is
+// running at a time.
+func WithTestRefreshWaitChs(with *testRefreshWaitChs) Option {
+	return func(o *options) error {
+		o.withTestRefreshWaitChs = with
+		return nil
+	}
+}
+
+// WithUseNonPagedListing provides an option for ignoring the resource
+// staleness when performing a search.
+func WithUseNonPagedListing(b bool) Option {
+	return func(o *options) error {
+		o.withUseNonPagedListing = b
+		return nil
+	}
+}

internal/clientcache/internal/cache/options_test.go

@@ -22,7 +22,8 @@ func Test_GetOpts(t *testing.T) {
 		opts, err := getOpts()
 		require.NoError(t, err)
 		testOpts := options{
-			withDbType: dbw.Sqlite,
+			withDbType:           dbw.Sqlite,
+			withMaxResultSetSize: defaultLimitedResultSetSize,
 		}
 		assert.Equal(t, opts, testOpts)
 	})
@@ -48,8 +49,8 @@ func Test_GetOpts(t *testing.T) {
 		assert.Equal(t, opts, testOpts)
 	})
 	t.Run("WithTargetRetrievalFunc", func(t *testing.T) {
-		var f TargetRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
-			return nil, nil, "", nil
+		var f TargetRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (*targets.TargetListResult, RefreshTokenValue, error) {
+			return nil, "", nil
 		}
 		opts, err := getOpts(WithTargetRetrievalFunc(f))
 		require.NoError(t, err)
@@ -61,8 +62,8 @@ func Test_GetOpts(t *testing.T) {
 		assert.Equal(t, opts, testOpts)
 	})
 	t.Run("WithSessionRetrievalFunc", func(t *testing.T) {
-		var f SessionRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error) {
-			return nil, nil, "", nil
+		var f SessionRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *sessions.SessionListResult, opt ...Option) (*sessions.SessionListResult, RefreshTokenValue, error) {
+			return nil, "", nil
 		}
 		opts, err := getOpts(WithSessionRetrievalFunc(f))
 		require.NoError(t, err)
@@ -74,8 +75,8 @@ func Test_GetOpts(t *testing.T) {
 		assert.Equal(t, opts, testOpts)
 	})
 	t.Run("WithAliasRetrievalFunc", func(t *testing.T) {
-		var f ResolvableAliasRetrievalFunc = func(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue) ([]*aliases.Alias, []string, RefreshTokenValue, error) {
-			return nil, nil, "", nil
+		var f ResolvableAliasRetrievalFunc = func(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue, inPage *aliases.AliasListResult, opt ...Option) (*aliases.AliasListResult, RefreshTokenValue, error) {
+			return nil, "", nil
 		}
 		opts, err := getOpts(WithAliasRetrievalFunc(f))
 		require.NoError(t, err)
@@ -93,4 +94,36 @@ func Test_GetOpts(t *testing.T) {
 		testOpts.withIgnoreSearchStaleness = true
 		assert.Equal(t, opts, testOpts)
 	})
+	t.Run("withMaxResultSetSize", func(t *testing.T) {
+		opts, err := getOpts(WithMaxResultSetSize(defaultLimitedResultSetSize))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		testOpts.withMaxResultSetSize = defaultLimitedResultSetSize
+		assert.Equal(t, opts, testOpts)
+		opts, err = getOpts(WithMaxResultSetSize(0))
+		require.Nil(t, err)
+		assert.Equal(t, opts, testOpts)
+		_, err = getOpts(WithMaxResultSetSize(-2))
+		require.Error(t, err)
+	})
+	t.Run("withTestRefreshWaitChs", func(t *testing.T) {
+		waitCh := &testRefreshWaitChs{
+			firstSempahore:  make(chan struct{}),
+			secondSemaphore: make(chan struct{}),
+		}
+		opts, err := getOpts(WithTestRefreshWaitChs(waitCh))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.Empty(t, testOpts.withTestRefreshWaitChs)
+		testOpts.withTestRefreshWaitChs = waitCh
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithUseNonPagedListing", func(t *testing.T) {
+		opts, err := getOpts(WithUseNonPagedListing(true))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.False(t, testOpts.withUseNonPagedListing)
+		testOpts.withUseNonPagedListing = true
+		assert.Equal(t, opts, testOpts)
+	})
 }

internal/clientcache/internal/cache/refresh.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -17,9 +19,20 @@ import (
 	"github.com/hashicorp/go-hclog"
 )
 
+// ErrRefreshInProgress is used as an internal error to indicate that a refresh
+// is already in progress, as a signal that the caller may want to handle it a
+// different way.
+var ErrRefreshInProgress error = stderrors.New("cache refresh in progress")
+
 type RefreshService struct {
 	repo *Repository
 
+	// syncSempaphores is used to prevent multiple refreshes from happening at
+	// the same time for a given user ID and resource type. The key is a string
+	// (the user ID + resource type) and the value is an atomic bool we can CAS
+	// with.
+	syncSemaphores sync.Map
+
 	logger hclog.Logger
 	// the amount of time that should have passed since the last refresh for a
 	// call to RefreshForSearch will cause a refresh request to be sent to a
@@ -135,21 +148,25 @@ func (r *RefreshService) cacheSupportedUsers(ctx context.Context, in []*user) ([
 	return ret, nil
 }
 
-// RefreshForSearch refreshes a specific resource type owned by the user associated
-// with the provided auth token id, if it hasn't been refreshed in a certain
-// amount of time. If the resources has been updated recently, or if it is known
-// that the user's boundary instance doesn't support partial refreshing, this
-// method returns without any requests to the boundary controller.
-// While the criteria of whether the user will refresh or not is almost the same
-// as the criteria used in Refresh, RefreshForSearch will not refresh any
-// data if there is not a refresh token stored for the resource. It
-// might make sense to change this in the future, but the reasoning used is
-// that we should not be making an initial load of all resources while blocking
-// a search query, in case we have not yet even attempted to load the resources
-// for this user yet.
-// Note: Currently, if the context timesout we stop refreshing completely and
-// return to the caller.  A possible enhancement in the future would be to return
-// when the context is Done, but allow the refresh to proceed in the background.
+// RefreshForSearch refreshes a specific resource type owned by the user
+// associated with the provided auth token id, if it hasn't been refreshed in a
+// certain amount of time. If the resources has been updated recently, or if it
+// is known that the user's boundary instance doesn't support partial
+// refreshing, this method returns without any requests to the boundary
+// controller. While the criteria of whether the user will refresh or not is
+// almost the same as the criteria used in Refresh, RefreshForSearch will not
+// refresh any data if there is not a refresh token stored for the resource. It
+// might make sense to change this in the future, but the reasoning used is that
+// we should not be making an initial load of all resources while blocking a
+// search query, in case we have not yet even attempted to load the resources
+// for this user yet. Note: Currently, if the context timesout we stop
+// refreshing completely and return to the caller.  A possible enhancement in
+// the future would be to return when the context is Done, but allow the refresh
+// to proceed in the background.
+//
+// If a refresh is already running for that user and resource type we will
+// return ErrRefreshInProgress. If so we will not start another one and we can
+// make it clear in the response that cache filling is ongoing.
 func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid string, resourceType SearchableResource, opt ...Option) error {
 	const op = "cache.(RefreshService).RefreshForSearch"
 	if r.maxSearchRefreshTimeout > 0 {
@@ -191,6 +208,8 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 		return errors.Wrap(ctx, err, op, errors.WithoutEvent())
 	}
 
+	cacheKey := fmt.Sprintf("%s-%s", u.Id, resourceType)
+
 	switch resourceType {
 	case ResolvableAliases:
 		rtv, err := r.repo.lookupRefreshToken(ctx, u, resolvableAliasResourceType)
@@ -198,6 +217,11 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 			return errors.Wrap(ctx, err, op)
 		}
 		if opts.withIgnoreSearchStaleness || rtv != nil && time.Since(rtv.UpdateTime) > r.maxSearchStaleness {
+			semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+			if !semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+				return ErrRefreshInProgress
+			}
+			defer semaphore.(*atomic.Bool).Store(false)
 			tokens, err := r.cleanAndPickAuthTokens(ctx, u)
 			if err != nil {
 				return errors.Wrap(ctx, err, op, errors.WithoutEvent())
@@ -206,6 +230,12 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 			if rtv != nil {
 				args = append(args, "alias staleness", time.Since(rtv.UpdateTime))
 			}
+
+			if opts.withTestRefreshWaitChs != nil {
+				close(opts.withTestRefreshWaitChs.firstSempahore)
+				<-opts.withTestRefreshWaitChs.secondSemaphore
+			}
+
 			r.logger.Debug("refreshing aliases before performing search", args...)
 			if err := r.repo.refreshResolvableAliases(ctx, u, tokens, opt...); err != nil {
 				return errors.Wrap(ctx, err, op)
@@ -217,6 +247,12 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 			return errors.Wrap(ctx, err, op, errors.WithoutEvent())
 		}
 		if opts.withIgnoreSearchStaleness || rtv != nil && time.Since(rtv.UpdateTime) > r.maxSearchStaleness {
+			semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+			if !semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+				return ErrRefreshInProgress
+			}
+			defer semaphore.(*atomic.Bool).Store(false)
+
 			tokens, err := r.cleanAndPickAuthTokens(ctx, u)
 			if err != nil {
 				return errors.Wrap(ctx, err, op, errors.WithoutEvent())
@@ -225,6 +261,12 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 			if rtv != nil {
 				args = append(args, "target staleness", time.Since(rtv.UpdateTime))
 			}
+
+			if opts.withTestRefreshWaitChs != nil {
+				close(opts.withTestRefreshWaitChs.firstSempahore)
+				<-opts.withTestRefreshWaitChs.secondSemaphore
+			}
+
 			r.logger.Debug("refreshing targets before performing search", args...)
 			if err := r.repo.refreshTargets(ctx, u, tokens, opt...); err != nil {
 				return errors.Wrap(ctx, err, op, errors.WithoutEvent())
@@ -236,6 +278,12 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 			return errors.Wrap(ctx, err, op)
 		}
 		if opts.withIgnoreSearchStaleness || rtv != nil && time.Since(rtv.UpdateTime) > r.maxSearchStaleness {
+			semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+			if !semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+				return ErrRefreshInProgress
+			}
+			defer semaphore.(*atomic.Bool).Store(false)
+
 			tokens, err := r.cleanAndPickAuthTokens(ctx, u)
 			if err != nil {
 				return errors.Wrap(ctx, err, op, errors.WithoutEvent())
@@ -244,6 +292,12 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 			if rtv != nil {
 				args = append(args, "session staleness", time.Since(rtv.UpdateTime))
 			}
+
+			if opts.withTestRefreshWaitChs != nil {
+				close(opts.withTestRefreshWaitChs.firstSempahore)
+				<-opts.withTestRefreshWaitChs.secondSemaphore
+			}
+
 			r.logger.Debug("refreshing sessions before performing search", args...)
 			if err := r.repo.refreshSessions(ctx, u, tokens, opt...); err != nil {
 				return errors.Wrap(ctx, err, op, errors.WithoutEvent())
@@ -262,6 +316,11 @@ func (r *RefreshService) RefreshForSearch(ctx context.Context, authTokenid strin
 // cache with the values retrieved there. Refresh accepts the options
 // WithTargetRetrievalFunc and WithSessionRetrievalFunc which overwrites the
 // default functions used to retrieve those resources from boundary.
+//
+// This shares the map of sync semaphores with RefreshForSearch, so if a refresh
+// is already happening via either method, it will be skipped by the other. In
+// the case of this function it won't return ErrRefreshInProgress, but it will
+// log that the refresh is already in progress.
 func (r *RefreshService) Refresh(ctx context.Context, opt ...Option) error {
 	const op = "cache.(RefreshService).Refresh"
 	if err := r.repo.cleanExpiredOrOrphanedAuthTokens(ctx); err != nil {
@@ -290,16 +349,32 @@ func (r *RefreshService) Refresh(ctx context.Context, opt ...Option) error {
 			continue
 		}
 
-		if err := r.repo.refreshResolvableAliases(ctx, u, tokens, opt...); err != nil {
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
-		}
-		if err := r.repo.refreshTargets(ctx, u, tokens, opt...); err != nil {
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
-		}
-		if err := r.repo.refreshSessions(ctx, u, tokens, opt...); err != nil {
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+		cacheKey := fmt.Sprintf("%s-%s", u.Id, ResolvableAliases)
+		semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.refreshResolvableAliases(ctx, u, tokens, opt...); err != nil {
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
 		}
 
+		cacheKey = fmt.Sprintf("%s-%s", u.Id, Targets)
+		semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.refreshTargets(ctx, u, tokens, opt...); err != nil {
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
+		}
+
+		cacheKey = fmt.Sprintf("%s-%s", u.Id, Sessions)
+		semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.refreshSessions(ctx, u, tokens, opt...); err != nil {
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
+		}
 	}
 	return retErr
 }
@@ -346,28 +421,47 @@ func (r *RefreshService) RecheckCachingSupport(ctx context.Context, opt ...Optio
 			continue
 		}
 
-		if err := r.repo.checkCachingTargets(ctx, u, tokens, opt...); err != nil {
-			if err == ErrRefreshNotSupported {
-				// This is expected so no need to propagate the error up
-				continue
+		cacheKey := fmt.Sprintf("%s-%s", u.Id, Targets)
+		semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.checkCachingTargets(ctx, u, tokens, opt...); err != nil {
+				if err == ErrRefreshNotSupported {
+					semaphore.(*atomic.Bool).Store(false)
+					// This is expected so no need to propagate the error up
+					continue
+				}
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
 			}
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
-		}
-		if err := r.repo.checkCachingSessions(ctx, u, tokens, opt...); err != nil {
-			if err == ErrRefreshNotSupported {
-				// This is expected so no need to propagate the error up
-				continue
-			}
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
-		}
-		if err := r.repo.checkCachingResolvableAliases(ctx, u, tokens, opt...); err != nil {
-			if err == ErrRefreshNotSupported {
-				// This is expected so no need to propagate the error up
-				continue
-			}
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			semaphore.(*atomic.Bool).Store(false)
 		}
 
+		cacheKey = fmt.Sprintf("%s-%s", u.Id, ResolvableAliases)
+		semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.checkCachingResolvableAliases(ctx, u, tokens, opt...); err != nil {
+				if err == ErrRefreshNotSupported {
+					// This is expected so no need to propagate the error up
+					semaphore.(*atomic.Bool).Store(false)
+					continue
+				}
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
+		}
+
+		cacheKey = fmt.Sprintf("%s-%s", u.Id, Sessions)
+		semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.checkCachingSessions(ctx, u, tokens, opt...); err != nil {
+				if err == ErrRefreshNotSupported {
+					semaphore.(*atomic.Bool).Store(false)
+					// This is expected so no need to propagate the error up
+					continue
+				}
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
+		}
 	}
 	return retErr
 }

internal/clientcache/internal/cache/refresh_test.go

@@ -54,6 +54,54 @@ func testStaticResourceRetrievalFunc[T any](t *testing.T, ret [][]T, removed [][
 	}
 }
 
+func testTargetStaticResourceRetrievalFunc(inFunc func(ctx context.Context, s1, s2 string, refToken RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error)) TargetRetrievalFunc {
+	return func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (ret *targets.TargetListResult, refreshToken RefreshTokenValue, err error) {
+		retTargets, removed, refreshToken, err := inFunc(ctx, addr, authTok, refreshTok)
+		if err != nil {
+			return nil, "", err
+		}
+
+		ret = &targets.TargetListResult{
+			Items:        retTargets,
+			RemovedIds:   removed,
+			ResponseType: "complete",
+		}
+		return ret, refreshToken, nil
+	}
+}
+
+func testSessionStaticResourceRetrievalFunc(inFunc func(ctx context.Context, s1, s2 string, refToken RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error)) SessionRetrievalFunc {
+	return func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *sessions.SessionListResult, opt ...Option) (ret *sessions.SessionListResult, refreshToken RefreshTokenValue, err error) {
+		retSessions, removed, refreshToken, err := inFunc(ctx, addr, authTok, refreshTok)
+		if err != nil {
+			return nil, "", err
+		}
+
+		ret = &sessions.SessionListResult{
+			Items:        retSessions,
+			RemovedIds:   removed,
+			ResponseType: "complete",
+		}
+		return ret, refreshToken, nil
+	}
+}
+
+func testResolvableAliasStaticResourceRetrievalFunc(inFunc func(ctx context.Context, s1, s2, s3 string, refToken RefreshTokenValue) ([]*aliases.Alias, []string, RefreshTokenValue, error)) ResolvableAliasRetrievalFunc {
+	return func(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue, inPage *aliases.AliasListResult, opt ...Option) (ret *aliases.AliasListResult, refreshToken RefreshTokenValue, err error) {
+		retSessions, removed, refreshToken, err := inFunc(ctx, addr, authTok, userId, refreshTok)
+		if err != nil {
+			return nil, "", err
+		}
+
+		ret = &aliases.AliasListResult{
+			Items:        retSessions,
+			RemovedIds:   removed,
+			ResponseType: "complete",
+		}
+		return ret, refreshToken, nil
+	}
+}
+
 // testNoRefreshRetrievalFunc simulates a controller that doesn't support refresh
 // since it does not return any refresh token.
 func testNoRefreshRetrievalFunc[T any](t *testing.T) func(context.Context, string, string, RefreshTokenValue) ([]T, []string, RefreshTokenValue, error) {
@@ -424,9 +472,9 @@ func TestRefreshForSearch(t *testing.T) {
 			target("4"),
 		}
 		opts := []Option{
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -435,20 +483,23 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...))
 
 		cachedTargets, err := r.ListTargets(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, cachedTargets)
+		require.NoError(t, err)
+		assert.Empty(t, cachedTargets.Targets)
+		assert.Empty(t, cachedTargets.ResolvableAliases)
+		assert.Empty(t, cachedTargets.Sessions)
+		assert.False(t, cachedTargets.Incomplete)
 
 		// Now load up a few resources and a token, and trying again should
 		// see the RefreshForSearch update more fields.
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedTargets, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[:3], cachedTargets)
+		assert.ElementsMatch(t, retTargets[:3], cachedTargets.Targets)
 
 		// Let 2 milliseconds pass so the items are stale enough
 		time.Sleep(2 * time.Millisecond)
@@ -456,7 +507,7 @@ func TestRefreshForSearch(t *testing.T) {
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...))
 		cachedTargets, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[2:], cachedTargets)
+		assert.ElementsMatch(t, retTargets[2:], cachedTargets.Targets)
 	})
 
 	t.Run("targets forced refreshed for searching", func(t *testing.T) {
@@ -476,9 +527,9 @@ func TestRefreshForSearch(t *testing.T) {
 			target("4"),
 		}
 		opts := []Option{
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -487,32 +538,35 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...))
 
 		cachedTargets, err := r.ListTargets(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, cachedTargets)
+		require.NoError(t, err)
+		assert.Empty(t, cachedTargets.Targets)
+		assert.Empty(t, cachedTargets.ResolvableAliases)
+		assert.Empty(t, cachedTargets.Sessions)
+		assert.False(t, cachedTargets.Incomplete)
 
 		// Now load up a few resources and a token, and trying again should
 		// see the RefreshForSearch update more fields.
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedTargets, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[:3], cachedTargets)
+		assert.ElementsMatch(t, retTargets[:3], cachedTargets.Targets)
 
 		// No refresh happened because it is not considered stale
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...))
 		cachedTargets, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[:3], cachedTargets)
+		assert.ElementsMatch(t, retTargets[:3], cachedTargets.Targets)
 
 		// Now force refresh
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, append(opts, WithIgnoreSearchStaleness(true))...))
 		cachedTargets, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[2:], cachedTargets)
+		assert.ElementsMatch(t, retTargets[2:], cachedTargets.Targets)
 	})
 
 	t.Run("no refresh token no refresh for search", func(t *testing.T) {
@@ -531,45 +585,52 @@ func TestRefreshForSearch(t *testing.T) {
 
 		// Get the first set of resources, but no refresh tokens
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 		assert.ErrorContains(t, err, ErrRefreshNotSupported.Error())
 
 		got, err := r.ListTargets(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		// Now that we know that this user doesn't support refresh tokens, they
 		// wont be refreshed any more, and we wont see the error when refreshing
 		// any more.
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		assert.Nil(t, err)
 
 		err = rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		assert.Nil(t, err)
 
 		got, err = r.ListTargets(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		// Now simulate the controller updating to support refresh tokens and
 		// the resources starting to be cached.
 		err = rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{retTargets}, [][]string{{}})))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{retTargets}, [][]string{{}}))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+		)
 		assert.Nil(t, err, err)
 
 		got, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.Len(t, got, 2)
+		assert.Len(t, got.Targets, 2)
 	})
 
 	t.Run("sessions refreshed for searching", func(t *testing.T) {
@@ -588,9 +649,9 @@ func TestRefreshForSearch(t *testing.T) {
 			session("4"),
 		}
 		opts := []Option{
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
 					retSess[3:],
@@ -599,25 +660,28 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retSess[0].Id, retSess[1].Id},
 				},
-			)),
+			))),
 		}
 
 		// First call doesn't sync anything because no sessions were already synced yet
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Sessions, opts...))
 		cachedSessions, err := r.ListSessions(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, cachedSessions)
+		require.NoError(t, err)
+		assert.Empty(t, cachedSessions.Targets)
+		assert.Empty(t, cachedSessions.ResolvableAliases)
+		assert.Empty(t, cachedSessions.Sessions)
+		assert.False(t, cachedSessions.Incomplete)
 
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedSessions, err = r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[:3], cachedSessions)
+		assert.ElementsMatch(t, retSess[:3], cachedSessions.Sessions)
 
 		// Second call removes the first 2 resources from the cache and adds the last
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Sessions, opts...))
 		cachedSessions, err = r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[2:], cachedSessions)
+		assert.ElementsMatch(t, retSess[2:], cachedSessions.Sessions)
 	})
 
 	t.Run("sessions forced refreshed for searching", func(t *testing.T) {
@@ -637,9 +701,9 @@ func TestRefreshForSearch(t *testing.T) {
 			session("4"),
 		}
 		opts := []Option{
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
 					retSess[3:],
@@ -648,31 +712,34 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retSess[0].Id, retSess[1].Id},
 				},
-			)),
+			))),
 		}
 
 		// First call doesn't sync anything because no sessions were already synced yet
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Sessions, opts...))
 		cachedSessions, err := r.ListSessions(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, cachedSessions)
+		require.NoError(t, err)
+		assert.Empty(t, cachedSessions.Targets)
+		assert.Empty(t, cachedSessions.ResolvableAliases)
+		assert.Empty(t, cachedSessions.Sessions)
+		assert.False(t, cachedSessions.Incomplete)
 
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedSessions, err = r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[:3], cachedSessions)
+		assert.ElementsMatch(t, retSess[:3], cachedSessions.Sessions)
 
 		// Refresh for search doesn't refresh anything because it isn't stale
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Sessions, opts...))
 		cachedSessions, err = r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[:3], cachedSessions)
+		assert.ElementsMatch(t, retSess[:3], cachedSessions.Sessions)
 
 		// Now force the refresh and see things get updated
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Sessions, append(opts, WithIgnoreSearchStaleness(true))...))
 		cachedSessions, err = r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[2:], cachedSessions)
+		assert.ElementsMatch(t, retSess[2:], cachedSessions.Sessions)
 	})
 
 	t.Run("aliases refreshed for searching", func(t *testing.T) {
@@ -691,9 +758,9 @@ func TestRefreshForSearch(t *testing.T) {
 			alias("4"),
 		}
 		opts := []Option{
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAl[:3],
 					retAl[3:],
@@ -702,25 +769,28 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retAl[0].Id, retAl[1].Id},
 				},
-			)),
+			))),
 		}
 
 		// First call doesn't sync anything because no aliases were already synced yet
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, opts...))
 		cachedAliases, err := r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, cachedAliases)
+		require.NoError(t, err)
+		assert.Empty(t, cachedAliases.Targets)
+		assert.Empty(t, cachedAliases.ResolvableAliases)
+		assert.Empty(t, cachedAliases.Sessions)
+		assert.False(t, cachedAliases.Incomplete)
 
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedAliases, err = r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAl[:3], cachedAliases)
+		require.NoError(t, err)
+		assert.ElementsMatch(t, retAl[:3], cachedAliases.ResolvableAliases)
 
 		// Second call removes the first 2 resources from the cache and adds the last
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, opts...))
 		cachedAliases, err = r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAl[2:], cachedAliases)
+		require.NoError(t, err)
+		assert.ElementsMatch(t, retAl[2:], cachedAliases.ResolvableAliases)
 	})
 
 	t.Run("aliases forced refreshed for searching", func(t *testing.T) {
@@ -740,9 +810,9 @@ func TestRefreshForSearch(t *testing.T) {
 			alias("4"),
 		}
 		opts := []Option{
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAls[:3],
 					retAls[3:],
@@ -751,31 +821,237 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retAls[0].Id, retAls[1].Id},
 				},
-			)),
+			))),
 		}
 
 		// First call doesn't sync anything because no aliases were already synced yet
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, opts...))
 		cachedAliases, err := r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, cachedAliases)
+		require.NoError(t, err)
+		assert.Empty(t, cachedAliases.Targets)
+		assert.Empty(t, cachedAliases.ResolvableAliases)
+		assert.Empty(t, cachedAliases.Sessions)
+		assert.False(t, cachedAliases.Incomplete)
 
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedAliases, err = r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAls[:3], cachedAliases)
+		require.NoError(t, err)
+		assert.ElementsMatch(t, retAls[:3], cachedAliases.ResolvableAliases)
 
 		// Refresh for search doesn't refresh anything because it isn't stale
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, opts...))
 		cachedAliases, err = r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAls[:3], cachedAliases)
+		require.NoError(t, err)
+		assert.ElementsMatch(t, retAls[:3], cachedAliases.ResolvableAliases)
 
 		// Now force the refresh and see things get updated
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, append(opts, WithIgnoreSearchStaleness(true))...))
 		cachedAliases, err = r.ListResolvableAliases(ctx, at.Id)
+		require.NoError(t, err)
+		assert.ElementsMatch(t, retAls[2:], cachedAliases.ResolvableAliases)
+	})
+}
+
+func TestRefreshNonBlocking(t *testing.T) {
+	ctx := context.Background()
+
+	boundaryAddr := "address"
+	u := &user{Id: "u1", Address: boundaryAddr}
+	at := &authtokens.AuthToken{
+		Id:             "at_1",
+		Token:          "at_1_token",
+		UserId:         u.Id,
+		ExpirationTime: time.Now().Add(time.Minute),
+	}
+
+	boundaryAuthTokens := []*authtokens.AuthToken{at}
+	atMap := make(map[ringToken]*authtokens.AuthToken)
+
+	atMap[ringToken{"k", "t"}] = at
+
+	t.Run("targets refreshed for searching", func(t *testing.T) {
+		t.Parallel()
+		s, err := db.Open(ctx)
+		require.NoError(t, err)
+		r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(boundaryAuthTokens))
+		require.NoError(t, err)
+		rs, err := NewRefreshService(ctx, r, hclog.NewNullLogger(), time.Millisecond, 0)
+		require.NoError(t, err)
+		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
+
+		retTargets := []*targets.Target{
+			target("1"),
+			target("2"),
+			target("3"),
+			target("4"),
+		}
+		opts := []Option{
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+				[][]*targets.Target{
+					retTargets[:3],
+					retTargets[3:],
+				},
+				[][]string{
+					nil,
+					{retTargets[0].Id, retTargets[1].Id},
+				},
+			))),
+		}
+
+		refreshWaitChs := &testRefreshWaitChs{
+			firstSempahore:  make(chan struct{}),
+			secondSemaphore: make(chan struct{}),
+		}
+		wg := new(sync.WaitGroup)
+		wg.Add(2)
+		extraOpts := []Option{WithTestRefreshWaitChs(refreshWaitChs), WithIgnoreSearchStaleness(true)}
+		go func() {
+			defer wg.Done()
+			blockingRefreshError := rs.RefreshForSearch(ctx, at.Id, Targets, append(opts, extraOpts...)...)
+			assert.NoError(t, blockingRefreshError)
+		}()
+		go func() {
+			defer wg.Done()
+			// Sleep here to ensure ordering of the calls since both goroutines
+			// are spawned at the same time
+			<-refreshWaitChs.firstSempahore
+			nonblockingRefreshError := rs.RefreshForSearch(ctx, at.Id, Targets, append(opts, extraOpts...)...)
+			close(refreshWaitChs.secondSemaphore)
+			assert.ErrorIs(t, nonblockingRefreshError, ErrRefreshInProgress)
+		}()
+		wg.Wait()
+
+		// Unlike in the TestRefreshForSearch test, since we did a force
+		// refresh we do expect to see values
+		cachedTargets, err := r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAls[2:], cachedAliases)
+		assert.ElementsMatch(t, retTargets[:3], cachedTargets.Targets)
+	})
+
+	t.Run("sessions refreshed for searching", func(t *testing.T) {
+		t.Parallel()
+		s, err := db.Open(ctx)
+		require.NoError(t, err)
+		r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(boundaryAuthTokens))
+		require.NoError(t, err)
+		rs, err := NewRefreshService(ctx, r, hclog.NewNullLogger(), 0, 0)
+		require.NoError(t, err)
+		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
+
+		retSess := []*sessions.Session{
+			session("1"),
+			session("2"),
+			session("3"),
+			session("4"),
+		}
+		opts := []Option{
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
+				[][]*sessions.Session{
+					retSess[:3],
+					retSess[3:],
+				},
+				[][]string{
+					nil,
+					{retSess[0].Id, retSess[1].Id},
+				},
+			))),
+		}
+
+		refreshWaitChs := &testRefreshWaitChs{
+			firstSempahore:  make(chan struct{}),
+			secondSemaphore: make(chan struct{}),
+		}
+		wg := new(sync.WaitGroup)
+		wg.Add(2)
+		extraOpts := []Option{WithTestRefreshWaitChs(refreshWaitChs), WithIgnoreSearchStaleness(true)}
+		go func() {
+			defer wg.Done()
+			blockingRefreshError := rs.RefreshForSearch(ctx, at.Id, Sessions, append(opts, extraOpts...)...)
+			assert.NoError(t, blockingRefreshError)
+		}()
+		go func() {
+			defer wg.Done()
+			// Sleep here to ensure ordering of the calls since both goroutines
+			// are spawned at the same time
+			<-refreshWaitChs.firstSempahore
+			nonblockingRefreshError := rs.RefreshForSearch(ctx, at.Id, Sessions, append(opts, extraOpts...)...)
+			close(refreshWaitChs.secondSemaphore)
+			assert.ErrorIs(t, nonblockingRefreshError, ErrRefreshInProgress)
+		}()
+
+		wg.Wait()
+
+		// Unlike in the TestRefreshForSearch test, since we are did a force
+		// refresh we do expect to see values
+		cachedSessions, err := r.ListSessions(ctx, at.Id)
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, retSess[:3], cachedSessions.Sessions)
+	})
+
+	t.Run("aliases refreshed for searching", func(t *testing.T) {
+		t.Parallel()
+		s, err := db.Open(ctx)
+		require.NoError(t, err)
+		r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(boundaryAuthTokens))
+		require.NoError(t, err)
+		rs, err := NewRefreshService(ctx, r, hclog.Default(), 0, 0)
+		require.NoError(t, err)
+		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
+
+		retAl := []*aliases.Alias{
+			alias("1"),
+			alias("2"),
+			alias("3"),
+			alias("4"),
+		}
+		opts := []Option{
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
+				[][]*aliases.Alias{
+					retAl[:3],
+					retAl[3:],
+				},
+				[][]string{
+					nil,
+					{retAl[0].Id, retAl[1].Id},
+				},
+			))),
+		}
+
+		refreshWaitChs := &testRefreshWaitChs{
+			firstSempahore:  make(chan struct{}),
+			secondSemaphore: make(chan struct{}),
+		}
+		wg := new(sync.WaitGroup)
+		wg.Add(2)
+		extraOpts := []Option{WithTestRefreshWaitChs(refreshWaitChs), WithIgnoreSearchStaleness(true)}
+		go func() {
+			defer wg.Done()
+			blockingRefreshError := rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, append(opts, extraOpts...)...)
+			assert.NoError(t, blockingRefreshError)
+		}()
+		go func() {
+			defer wg.Done()
+			// Sleep here to ensure ordering of the calls since both goroutines
+			// are spawned at the same time
+			<-refreshWaitChs.firstSempahore
+			nonblockingRefreshError := rs.RefreshForSearch(ctx, at.Id, ResolvableAliases, append(opts, extraOpts...)...)
+			close(refreshWaitChs.secondSemaphore)
+			assert.ErrorIs(t, nonblockingRefreshError, ErrRefreshInProgress)
+		}()
+
+		wg.Wait()
+
+		// Unlike in the TestRefreshForSearch test, since we are did a force
+		// refresh we do expect to see values
+		cachedAliases, err := r.ListResolvableAliases(ctx, at.Id)
+		require.NoError(t, err)
+		assert.ElementsMatch(t, retAl[:3], cachedAliases.ResolvableAliases)
 	})
 }
 
@@ -812,9 +1088,9 @@ func TestRefresh(t *testing.T) {
 			target("4"),
 		}
 		opts := []Option{
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -823,19 +1099,19 @@ func TestRefresh(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 
 		cachedTargets, err := r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[:3], cachedTargets)
+		assert.ElementsMatch(t, retTargets[:3], cachedTargets.Targets)
 
 		// Second call removes the first 2 resources from the cache and adds the last
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedTargets, err = r.ListTargets(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retTargets[2:], cachedTargets)
+		assert.ElementsMatch(t, retTargets[2:], cachedTargets.Targets)
 	})
 
 	t.Run("set sessions", func(t *testing.T) {
@@ -854,9 +1130,9 @@ func TestRefresh(t *testing.T) {
 			session("4"),
 		}
 		opts := []Option{
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
 					retSess[3:],
@@ -865,18 +1141,18 @@ func TestRefresh(t *testing.T) {
 					nil,
 					{retSess[0].Id, retSess[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedSessions, err := r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[:3], cachedSessions)
+		assert.ElementsMatch(t, retSess[:3], cachedSessions.Sessions)
 
 		// Second call removes the first 2 resources from the cache and adds the last
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedSessions, err = r.ListSessions(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retSess[2:], cachedSessions)
+		assert.ElementsMatch(t, retSess[2:], cachedSessions.Sessions)
 	})
 
 	t.Run("set aliases", func(t *testing.T) {
@@ -895,9 +1171,9 @@ func TestRefresh(t *testing.T) {
 			alias("4"),
 		}
 		opts := []Option{
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAls[:3],
 					retAls[3:],
@@ -906,18 +1182,18 @@ func TestRefresh(t *testing.T) {
 					nil,
 					{retAls[0].Id, retAls[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedAliases, err := r.ListResolvableAliases(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAls[:3], cachedAliases)
+		assert.ElementsMatch(t, retAls[:3], cachedAliases.ResolvableAliases)
 
 		// Second call removes the first 2 resources from the cache and adds the last
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 		cachedAliases, err = r.ListResolvableAliases(ctx, at.Id)
 		assert.NoError(t, err)
-		assert.ElementsMatch(t, retAls[2:], cachedAliases)
+		assert.ElementsMatch(t, retAls[2:], cachedAliases.ResolvableAliases)
 	})
 
 	t.Run("error propagates up", func(t *testing.T) {
@@ -931,22 +1207,22 @@ func TestRefresh(t *testing.T) {
 
 		innerErr := errors.New("test error")
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
-			WithSessionRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error) {
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 	})
 
@@ -973,9 +1249,9 @@ func TestRefresh(t *testing.T) {
 		assert.Len(t, us, 1)
 
 		require.NoError(t, rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)))))
 
 		ps, err = r.listTokens(ctx, u)
 		require.NoError(t, err)
@@ -1017,29 +1293,35 @@ func TestRecheckCachingSupport(t *testing.T) {
 		// Since this user doesn't have any resources, the user's data will still
 		// only get updated with a call to Refresh.
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))))
 
 		got, err := r.ListTargets(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
 		got, err = r.ListTargets(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		// now a full fetch will work since the user has resources and no refresh token
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))))
 	})
 
 	t.Run("sessions", func(t *testing.T) {
@@ -1052,31 +1334,40 @@ func TestRecheckCachingSupport(t *testing.T) {
 		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))))
 
 		got, err := r.ListSessions(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
 		got, err = r.ListSessions(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))))
 		got, err = r.ListSessions(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 	})
 
 	t.Run("aliases", func(t *testing.T) {
@@ -1089,31 +1380,40 @@ func TestRecheckCachingSupport(t *testing.T) {
 		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))))
 
 		got, err := r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
 		got, err = r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))))
 		got, err = r.ListResolvableAliases(ctx, at.Id)
-		assert.NoError(t, err)
-		assert.Empty(t, got)
+		require.NoError(t, err)
+		assert.Empty(t, got.Targets)
+		assert.Empty(t, got.ResolvableAliases)
+		assert.Empty(t, got.Sessions)
+		assert.False(t, got.Incomplete)
 	})
 
 	t.Run("error propagates up", func(t *testing.T) {
@@ -1126,30 +1426,30 @@ func TestRecheckCachingSupport(t *testing.T) {
 		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
 
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
 		innerErr := errors.New("test error")
 		err = rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 
 		err = rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 	})
 
@@ -1163,9 +1463,9 @@ func TestRecheckCachingSupport(t *testing.T) {
 		require.NoError(t, r.AddKeyringToken(ctx, boundaryAddr, KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}))
 
 		err = rs.Refresh(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
 		// Remove the token from the keyring, see that we can still see the
@@ -1182,9 +1482,9 @@ func TestRecheckCachingSupport(t *testing.T) {
 		assert.Len(t, us, 1)
 
 		err = rs.RecheckCachingSupport(ctx,
-			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))),
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 		assert.NoError(t, err)
 
 		ps, err = r.listTokens(ctx, u)

internal/clientcache/internal/cache/repository_implicit_scopes.go

@@ -0,0 +1,114 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package cache
+
+import (
+	"context"
+	"fmt"
+	"slices"
+
+	"github.com/hashicorp/boundary/api/scopes"
+	"github.com/hashicorp/boundary/internal/errors"
+)
+
+func (r *Repository) ListImplicitScopes(ctx context.Context, authTokenId string, opt ...Option) (*SearchResult, error) {
+	const op = "cache.(Repository).ListImplicitScopes"
+	switch {
+	case authTokenId == "":
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "auth token id is missing")
+	}
+	ret, err := r.searchImplicitScopes(ctx, "true", nil, append(opt, withAuthTokenId(authTokenId))...)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+	return ret, nil
+}
+
+// QueryImplicitScopes is not supported currently so we return an error message
+func (r *Repository) QueryImplicitScopes(ctx context.Context, authTokenId, query string, opt ...Option) (*SearchResult, error) {
+	const op = "cache.(Repository).QueryImplicitScopes"
+
+	// Internal is used as we have checks at the handler level to ensure this
+	// can't be used so it's an internal error if we actually call this
+	// function.
+	return nil, errors.New(ctx, errors.Internal, op, "querying implicit scopes is not supported")
+}
+
+func (r *Repository) searchImplicitScopes(ctx context.Context, condition string, searchArgs []any, opt ...Option) (*SearchResult, error) {
+	const op = "cache.(Repository).searchImplicitScopes"
+	switch {
+	case condition == "":
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "condition is missing")
+	}
+
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+	switch {
+	case opts.withAuthTokenId != "" && opts.withUserId != "":
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "both user id and auth token id were provided")
+	case opts.withAuthTokenId == "" && opts.withUserId == "":
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "neither user id nor auth token id were provided")
+
+	// In these cases we append twice because we're doing a union of two tables
+	case opts.withAuthTokenId != "":
+		condition = "where fk_user_id in (select user_id from auth_token where id = ?)"
+		searchArgs = append(searchArgs, opts.withAuthTokenId, opts.withAuthTokenId)
+	case opts.withUserId != "":
+		condition = "where fk_user_id = ?"
+		searchArgs = append(searchArgs, opts.withUserId, opts.withUserId)
+	}
+
+	const unionQueryBase = `
+		select distinct fk_user_id, scope_id from session
+			%s
+	union
+		select distinct fk_user_id, scope_id from target
+			%s
+`
+	unionQuery := fmt.Sprintf(unionQueryBase, condition, condition)
+
+	rows, err := r.rw.Query(ctx, unionQuery, searchArgs)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+	defer rows.Close()
+
+	type ScopeIdsResult struct {
+		FkUserId string `gorm:"primaryKey"`
+		ScopeId  string `gorm:"default:null"`
+	}
+
+	var scopeIdsResults []ScopeIdsResult
+	for rows.Next() {
+		var res ScopeIdsResult
+		if err := r.rw.ScanRows(ctx, rows, &res); err != nil {
+			return nil, errors.Wrap(ctx, err, op)
+		}
+		scopeIdsResults = append(scopeIdsResults, res)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+
+	dedupMap := make(map[string]struct{}, len(scopeIdsResults))
+	for _, res := range scopeIdsResults {
+		dedupMap[res.ScopeId] = struct{}{}
+	}
+	scopeIds := make([]string, 0, len(dedupMap))
+	for k := range dedupMap {
+		scopeIds = append(scopeIds, k)
+	}
+	slices.Sort(scopeIds)
+
+	sr := &SearchResult{
+		ImplicitScopes: make([]*scopes.Scope, 0, len(dedupMap)),
+	}
+	for _, scopeId := range scopeIds {
+		sr.ImplicitScopes = append(sr.ImplicitScopes, &scopes.Scope{Id: scopeId})
+	}
+
+	return sr, nil
+}

internal/clientcache/internal/cache/repository_implicit_scopes_test.go

@@ -0,0 +1,124 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package cache
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/hashicorp/boundary/api/authtokens"
+	"github.com/hashicorp/boundary/api/scopes"
+	"github.com/hashicorp/boundary/api/sessions"
+	"github.com/hashicorp/boundary/api/targets"
+	cachedb "github.com/hashicorp/boundary/internal/clientcache/internal/db"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
+)
+
+func TestRepository_ImplicitScopes(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u1 := &user{
+		Id:      "u1",
+		Address: addr,
+	}
+	at1 := &authtokens.AuthToken{
+		Id:     "at_1",
+		Token:  "at_1_token",
+		UserId: u1.Id,
+	}
+	kt1 := KeyringToken{KeyringType: "k1", TokenName: "t1", AuthTokenId: at1.Id}
+
+	u2 := &user{
+		Id:      "u2",
+		Address: addr,
+	}
+	at2 := &authtokens.AuthToken{
+		Id:     "at_2",
+		Token:  "at_2_token",
+		UserId: u2.Id,
+	}
+	kt2 := KeyringToken{KeyringType: "k2", TokenName: "t2", AuthTokenId: at2.Id}
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k1", "t1"}: at1,
+		{"k2", "t2"}: at2,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt1))
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt2))
+
+	var expectedScopes []*scopes.Scope
+
+	ts := []*targets.Target{
+		target("1"),
+		target("2"),
+		target("3"),
+	}
+	require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
+
+	for _, t := range ts {
+		expectedScopes = append(expectedScopes, &scopes.Scope{
+			Id: t.ScopeId,
+		})
+	}
+
+	ss := []*sessions.Session{
+		{
+			Id:       "ttcp_1",
+			Status:   "status1",
+			Endpoint: "address1",
+			ScopeId:  "p_123",
+			TargetId: "ttcp_123",
+			UserId:   "u_123",
+			Type:     "tcp",
+		},
+		{
+			Id:       "ttcp_2",
+			Status:   "status2",
+			Endpoint: "address2",
+			ScopeId:  "p_123",
+			TargetId: "ttcp_123",
+			UserId:   "u_123",
+			Type:     "tcp",
+		},
+		{
+			Id:       "ttcp_3",
+			Status:   "status3",
+			Endpoint: "address3",
+			ScopeId:  "p_123",
+			TargetId: "ttcp_123",
+			UserId:   "u_123",
+			Type:     "tcp",
+		},
+	}
+	require.NoError(t, r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ss}, [][]string{nil})))))
+
+	expectedScopes = append(expectedScopes, &scopes.Scope{
+		Id: ss[0].ScopeId,
+	})
+
+	t.Run("wrong user gets no implicit scopes", func(t *testing.T) {
+		l, err := r.ListImplicitScopes(ctx, kt2.AuthTokenId)
+		require.NoError(t, err)
+		assert.Empty(t, l.ImplicitScopes)
+	})
+	t.Run("correct token gets implicit scopes from listing", func(t *testing.T) {
+		l, err := r.ListImplicitScopes(ctx, kt1.AuthTokenId)
+		require.NoError(t, err)
+		assert.Len(t, l.ImplicitScopes, len(expectedScopes))
+		assert.ElementsMatch(t, l.ImplicitScopes, expectedScopes)
+	})
+	t.Run("querying returns error", func(t *testing.T) {
+		_, err := r.QueryImplicitScopes(ctx, kt1.AuthTokenId, "anything")
+		require.Error(t, err)
+	})
+}

internal/clientcache/internal/cache/repository_resolvable_aliases.go

@@ -22,35 +22,45 @@ import (
 
 // ResolvableAliasRetrievalFunc is a function that retrieves aliases
 // from the provided boundary addr using the provided token.
-type ResolvableAliasRetrievalFunc func(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue) (ret []*aliases.Alias, removedIds []string, refreshToken RefreshTokenValue, err error)
+type ResolvableAliasRetrievalFunc func(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue, inPage *aliases.AliasListResult, opt ...Option) (ret *aliases.AliasListResult, refreshToken RefreshTokenValue, err error)
 
-func defaultResolvableAliasFunc(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue) ([]*aliases.Alias, []string, RefreshTokenValue, error) {
+func defaultResolvableAliasFunc(ctx context.Context, addr, authTok, userId string, refreshTok RefreshTokenValue, inPage *aliases.AliasListResult, opt ...Option) (*aliases.AliasListResult, RefreshTokenValue, error) {
 	const op = "cache.defaultResolvableAliasFunc"
 	conf, err := api.DefaultConfig()
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
+	}
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	conf.Addr = addr
 	conf.Token = authTok
 	client, err := api.NewClient(conf)
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	aClient := users.NewClient(client)
-	l, err := aClient.ListResolvableAliases(ctx, userId, users.WithListToken(string(refreshTok)))
+	var l *aliases.AliasListResult
+	switch inPage {
+	case nil:
+		l, err = aClient.ListResolvableAliases(ctx, userId, users.WithRecursive(true), users.WithListToken(string(refreshTok)), users.WithClientDirectedPagination(!opts.withUseNonPagedListing))
+	default:
+		l, err = aClient.ListResolvableAliasesNextPage(ctx, userId, inPage, users.WithListToken(string(refreshTok)))
+	}
 	if err != nil {
 		if api.ErrInvalidListToken.Is(err) {
-			return nil, nil, "", err
+			return nil, "", err
 		}
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	if l.ResponseType == "" {
-		return nil, nil, "", ErrRefreshNotSupported
+		return nil, "", ErrRefreshNotSupported
 	}
-	return l.Items, l.RemovedIds, RefreshTokenValue(l.ListToken), nil
+	return l, RefreshTokenValue(l.ListToken), nil
 }
 
-// refreshResolvableAliases attempts to refresh the resolvabl aliases for the
+// refreshResolvableAliases attempts to refresh the resolvable aliases for the
 // provided user using the provided tokens. If available, it uses the refresh
 // tokens in storage to retrieve and apply only the delta.
 func (r *Repository) refreshResolvableAliases(ctx context.Context, u *user, tokens map[AuthToken]string, opt ...Option) error {
@@ -83,13 +93,13 @@ func (r *Repository) refreshResolvableAliases(ctx context.Context, u *user, toke
 
 	// Find and use a token for retrieving aliases
 	var gotResponse bool
-	var resp []*aliases.Alias
+	var currentPage *aliases.AliasListResult
 	var newRefreshToken RefreshTokenValue
+	var foundAuthToken string
 	var unsupportedCacheRequest bool
-	var removedIds []string
 	var retErr error
 	for at, t := range tokens {
-		resp, removedIds, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, t, u.Id, oldRefreshTokenVal)
+		currentPage, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, t, u.Id, oldRefreshTokenVal, currentPage)
 		if api.ErrInvalidListToken.Is(err) {
 			event.WriteSysEvent(ctx, op, "old list token is no longer valid, starting new initial fetch", "user_id", u.Id)
 			if err := r.deleteRefreshToken(ctx, u, resourceType); err != nil {
@@ -97,7 +107,7 @@ func (r *Repository) refreshResolvableAliases(ctx context.Context, u *user, toke
 			}
 			// try again without the refresh token
 			oldRefreshToken = nil
-			resp, removedIds, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, t, u.Id, "")
+			currentPage, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, t, u.Id, "", currentPage)
 		}
 		if err != nil {
 			if err == ErrRefreshNotSupported {
@@ -107,6 +117,7 @@ func (r *Repository) refreshResolvableAliases(ctx context.Context, u *user, toke
 				continue
 			}
 		}
+		foundAuthToken = t
 		gotResponse = true
 		break
 	}
@@ -121,44 +132,57 @@ func (r *Repository) refreshResolvableAliases(ctx context.Context, u *user, toke
 	}
 
 	var numDeleted int
-	_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
-		var err error
-		switch {
-		case oldRefreshToken == nil:
-			if numDeleted, err = w.Exec(ctx, "delete from resolvable_alias where fk_user_id = @fk_user_id",
-				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
-				return err
+	var numUpserted int
+	var clearPerformed bool
+	for {
+		_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
+			var err error
+			if (oldRefreshToken == nil || unsupportedCacheRequest) && !clearPerformed {
+				if numDeleted, err = w.Exec(ctx, "delete from resolvable_alias where fk_user_id = @fk_user_id",
+					[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
+					return err
+				}
+				clearPerformed = true
 			}
-		case len(removedIds) > 0:
-			if numDeleted, err = w.Exec(ctx, "delete from resolvable_alias where id in @ids",
-				[]any{sql.Named("ids", removedIds)}); err != nil {
-				return err
+			switch {
+			case unsupportedCacheRequest:
+				if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
+					return err
+				}
+			case newRefreshToken != "":
+				numUpserted += len(currentPage.Items)
+				if err := upsertResolvableAliases(ctx, w, u, currentPage.Items); err != nil {
+					return err
+				}
+				if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
+					return err
+				}
+			default:
+				// controller supports caching, but doesn't have any resources
 			}
+			if !unsupportedCacheRequest && len(currentPage.RemovedIds) > 0 {
+				if numDeleted, err = w.Exec(ctx, "delete from resolvable_alias where id in @ids",
+					[]any{sql.Named("ids", currentPage.RemovedIds)}); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		if unsupportedCacheRequest || currentPage.ResponseType == "" || currentPage.ResponseType == "complete" {
+			break
 		}
-		switch {
-		case unsupportedCacheRequest:
-			if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
-				return err
-			}
-		case newRefreshToken != "":
-			if err := upsertResolvableAliases(ctx, w, u, resp); err != nil {
-				return err
-			}
-			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
-				return err
-			}
-		default:
-			// controller supports caching, but doesn't have any resources
+		currentPage, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, foundAuthToken, u.Id, newRefreshToken, currentPage)
+		if err != nil {
+			break
 		}
-		return nil
-	})
+	}
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "resolvable-aliases updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "resolvable-aliases updated", "deleted", numDeleted, "upserted", numUpserted, "user_id", u.Id)
 	return nil
 }
 
@@ -187,12 +211,12 @@ func (r *Repository) checkCachingResolvableAliases(ctx context.Context, u *user,
 
 	// Find and use a token for retrieving aliases
 	var gotResponse bool
-	var resp []*aliases.Alias
+	var resp *aliases.AliasListResult
 	var newRefreshToken RefreshTokenValue
 	var unsupportedCacheRequest bool
 	var retErr error
 	for at, t := range tokens {
-		resp, _, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, t, u.Id, "")
+		resp, newRefreshToken, err = opts.withResolvableAliasRetrievalFunc(ctx, u.Address, t, u.Id, "", nil, WithUseNonPagedListing(true))
 		if err != nil {
 			if err == ErrRefreshNotSupported {
 				unsupportedCacheRequest = true
@@ -227,7 +251,7 @@ func (r *Repository) checkCachingResolvableAliases(ctx context.Context, u *user,
 				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
 				return err
 			}
-			if err := upsertResolvableAliases(ctx, w, u, resp); err != nil {
+			if err := upsertResolvableAliases(ctx, w, u, resp.Items); err != nil {
 				return err
 			}
 			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
@@ -248,7 +272,7 @@ func (r *Repository) checkCachingResolvableAliases(ctx context.Context, u *user,
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "resolvable-aliases updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "resolvable-aliases updated", "deleted", numDeleted, "upserted", len(resp.Items), "user_id", u.Id)
 	return nil
 }
 
@@ -288,20 +312,20 @@ func upsertResolvableAliases(ctx context.Context, w db.Writer, u *user, in []*al
 	return nil
 }
 
-func (r *Repository) ListResolvableAliases(ctx context.Context, authTokenId string) ([]*aliases.Alias, error) {
+func (r *Repository) ListResolvableAliases(ctx context.Context, authTokenId string, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).ListResolvableAliases"
 	switch {
 	case authTokenId == "":
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "auth token id is missing")
 	}
-	ret, err := r.searchResolvableAliases(ctx, "true", nil, withAuthTokenId(authTokenId))
+	ret, err := r.searchResolvableAliases(ctx, "true", nil, append(opt, withAuthTokenId(authTokenId))...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return ret, nil
 }
 
-func (r *Repository) QueryResolvableAliases(ctx context.Context, authTokenId, query string) ([]*aliases.Alias, error) {
+func (r *Repository) QueryResolvableAliases(ctx context.Context, authTokenId, query string, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).QueryResolvableAliases"
 	switch {
 	case authTokenId == "":
@@ -314,14 +338,14 @@ func (r *Repository) QueryResolvableAliases(ctx context.Context, authTokenId, qu
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter))
 	}
-	ret, err := r.searchResolvableAliases(ctx, w.Condition, w.Args, withAuthTokenId(authTokenId))
+	ret, err := r.searchResolvableAliases(ctx, w.Condition, w.Args, append(opt, withAuthTokenId(authTokenId))...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return ret, nil
 }
 
-func (r *Repository) searchResolvableAliases(ctx context.Context, condition string, searchArgs []any, opt ...Option) ([]*aliases.Alias, error) {
+func (r *Repository) searchResolvableAliases(ctx context.Context, condition string, searchArgs []any, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).searchResolvableAliases"
 	switch {
 	case condition == "":
@@ -346,7 +370,7 @@ func (r *Repository) searchResolvableAliases(ctx context.Context, condition stri
 	}
 
 	var cachedResolvableAliases []*ResolvableAlias
-	if err := r.rw.SearchWhere(ctx, &cachedResolvableAliases, condition, searchArgs, db.WithLimit(-1)); err != nil {
+	if err := r.rw.SearchWhere(ctx, &cachedResolvableAliases, condition, searchArgs, db.WithLimit(opts.withMaxResultSetSize+1)); err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
@@ -358,7 +382,15 @@ func (r *Repository) searchResolvableAliases(ctx context.Context, condition stri
 		}
 		retAliases = append(retAliases, &a)
 	}
-	return retAliases, nil
+
+	sr := &SearchResult{
+		ResolvableAliases: retAliases,
+	}
+	if opts.withMaxResultSetSize > 0 && len(sr.ResolvableAliases) > opts.withMaxResultSetSize {
+		sr.ResolvableAliases = sr.ResolvableAliases[:opts.withMaxResultSetSize]
+		sr.Incomplete = true
+	}
+	return sr, nil
 }
 
 type ResolvableAlias struct {

internal/clientcache/internal/cache/repository_resolvable_aliases_test.go

@@ -6,6 +6,7 @@ package cache
 import (
 	"context"
 	"encoding/json"
+	"strconv"
 	"sync"
 	"testing"
 
@@ -138,7 +139,7 @@ func TestRepository_refreshAliases(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			err := r.refreshResolvableAliases(ctx, tc.u, map[AuthToken]string{{Id: "id"}: "something"},
-				WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{tc.al}, [][]string{nil})))
+				WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{tc.al}, [][]string{nil}))))
 			if tc.errorContains == "" {
 				assert.NoError(t, err)
 				rw := db.New(s)
@@ -217,41 +218,41 @@ func TestRepository_RefreshAliases_withRefreshTokens(t *testing.T) {
 	}
 
 	err = r.refreshResolvableAliases(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, ss, [][]string{nil, nil})))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, ss, [][]string{nil, nil}))))
 	assert.NoError(t, err)
 
 	got, err := r.ListResolvableAliases(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 2)
+	assert.Len(t, got.ResolvableAliases, 2)
 
 	// Refreshing again uses the refresh token and get additional aliases, appending
 	// them to the response
 	err = r.refreshResolvableAliases(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, ss, [][]string{nil, nil})))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, ss, [][]string{nil, nil}))))
 	assert.NoError(t, err)
 
 	got, err = r.ListResolvableAliases(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 3)
+	assert.Len(t, got.ResolvableAliases, 3)
 
 	// Refreshing again wont return any more resources, but also none should be
 	// removed
 	require.NoError(t, r.refreshResolvableAliases(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, ss, [][]string{nil, nil}))))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, ss, [][]string{nil, nil})))))
 	assert.NoError(t, err)
 
 	got, err = r.ListResolvableAliases(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 3)
+	assert.Len(t, got.ResolvableAliases, 3)
 
 	// Refresh again with the refresh token being reported as invalid.
 	require.NoError(t, r.refreshResolvableAliases(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testErroringForRefreshTokenRetrievalFuncForId(t, ss[0]))))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testErroringForRefreshTokenRetrievalFuncForId(t, ss[0])))))
 	assert.NoError(t, err)
 
 	got, err = r.ListResolvableAliases(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 2)
+	assert.Len(t, got.ResolvableAliases, 2)
 }
 
 func TestRepository_ListAliases(t *testing.T) {
@@ -327,18 +328,78 @@ func TestRepository_ListAliases(t *testing.T) {
 		},
 	}
 	require.NoError(t, r.refreshResolvableAliases(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{ss}, [][]string{nil}))))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{ss}, [][]string{nil})))))
 
 	t.Run("wrong user gets no aliases", func(t *testing.T) {
 		l, err := r.ListResolvableAliases(ctx, kt2.AuthTokenId)
 		assert.NoError(t, err)
-		assert.Empty(t, l)
+		assert.Empty(t, l.ResolvableAliases)
 	})
 	t.Run("correct token gets aliases", func(t *testing.T) {
 		l, err := r.ListResolvableAliases(ctx, kt1.AuthTokenId)
 		assert.NoError(t, err)
-		assert.Len(t, l, len(ss))
-		assert.ElementsMatch(t, l, ss)
+		assert.Len(t, l.ResolvableAliases, len(ss))
+		assert.ElementsMatch(t, l.ResolvableAliases, ss)
+	})
+}
+
+func TestRepository_ListAliasesLimiting(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u := &user{
+		Id:      "u",
+		Address: addr,
+	}
+	at := &authtokens.AuthToken{
+		Id:     "at",
+		Token:  "at_token",
+		UserId: u.Id,
+	}
+	kt := KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}
+
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k", "t"}: at,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt))
+
+	var ts []*aliases.Alias
+	for i := 0; i < defaultLimitedResultSetSize*2; i++ {
+		ts = append(ts, alias("s"+strconv.Itoa(i)))
+	}
+	require.NoError(t, r.refreshResolvableAliases(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{ts}, [][]string{nil})))))
+
+	searchService, err := NewSearchService(ctx, r)
+	require.NoError(t, err)
+	params := SearchParams{
+		Resource:    ResolvableAliases,
+		AuthTokenId: kt.AuthTokenId,
+	}
+
+	t.Run("default limit", func(t *testing.T) {
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.ResolvableAliases, defaultLimitedResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("custom limit", func(t *testing.T) {
+		params.MaxResultSetSize = 20
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.ResolvableAliases, params.MaxResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("no limit", func(t *testing.T) {
+		params.MaxResultSetSize = -1
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.ResolvableAliases, defaultLimitedResultSetSize*2)
+		assert.False(t, searchResult.Incomplete)
 	})
 }
 
@@ -437,18 +498,79 @@ func TestRepository_QueryAliases(t *testing.T) {
 		},
 	}
 	require.NoError(t, r.refreshResolvableAliases(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{ss}, [][]string{nil}))))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{ss}, [][]string{nil})))))
 
 	t.Run("wrong token gets no aliases", func(t *testing.T) {
 		l, err := r.QueryResolvableAliases(ctx, kt2.AuthTokenId, query)
 		assert.NoError(t, err)
-		assert.Empty(t, l)
+		assert.Empty(t, l.ResolvableAliases)
 	})
 	t.Run("correct token gets aliases", func(t *testing.T) {
 		l, err := r.QueryResolvableAliases(ctx, kt1.AuthTokenId, query)
 		assert.NoError(t, err)
-		assert.Len(t, l, 2)
-		assert.ElementsMatch(t, l, ss[0:2])
+		assert.Len(t, l.ResolvableAliases, 2)
+		assert.ElementsMatch(t, l.ResolvableAliases, ss[0:2])
+	})
+}
+
+func TestRepository_QueryResolvableAliasesLimiting(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u := &user{
+		Id:      "u",
+		Address: addr,
+	}
+	at := &authtokens.AuthToken{
+		Id:     "at",
+		Token:  "at_token",
+		UserId: u.Id,
+	}
+	kt := KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}
+
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k", "t"}: at,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt))
+
+	var ts []*aliases.Alias
+	for i := 0; i < defaultLimitedResultSetSize*2; i++ {
+		ts = append(ts, alias("s"+strconv.Itoa(i)))
+	}
+	require.NoError(t, r.refreshResolvableAliases(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{ts}, [][]string{nil})))))
+
+	searchService, err := NewSearchService(ctx, r)
+	require.NoError(t, err)
+	params := SearchParams{
+		Resource:    ResolvableAliases,
+		AuthTokenId: kt.AuthTokenId,
+		Query:       `(type % 'target')`,
+	}
+
+	t.Run("default limit", func(t *testing.T) {
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.ResolvableAliases, defaultLimitedResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("custom limit", func(t *testing.T) {
+		params.MaxResultSetSize = 20
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.ResolvableAliases, params.MaxResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("no limit", func(t *testing.T) {
+		params.MaxResultSetSize = -1
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.ResolvableAliases, defaultLimitedResultSetSize*2)
+		assert.False(t, searchResult.Incomplete)
 	})
 }
 
@@ -471,15 +593,15 @@ func TestDefaultAliasRetrievalFunc(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, tar1)
 
-	got, removed, refTok, err := defaultResolvableAliasFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, tc.Token().UserId, "")
+	got, refTok, err := defaultResolvableAliasFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, tc.Token().UserId, "", nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok)
-	assert.Empty(t, removed)
-	assert.Len(t, got, 1)
+	assert.Empty(t, got.RemovedIds)
+	assert.Len(t, got.Items, 1)
 
-	got2, removed2, refTok2, err := defaultResolvableAliasFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, tc.Token().UserId, refTok)
+	got2, refTok2, err := defaultResolvableAliasFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, tc.Token().UserId, refTok, nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok2)
-	assert.Empty(t, removed2)
-	assert.Empty(t, got2)
+	assert.Empty(t, got2.RemovedIds)
+	assert.Empty(t, got2.Items)
 }

internal/clientcache/internal/cache/repository_sessions.go

@@ -21,32 +21,42 @@ import (
 
 // SessionRetrievalFunc is a function that retrieves sessions
 // from the provided boundary addr using the provided token.
-type SessionRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) (ret []*sessions.Session, removedIds []string, refreshToken RefreshTokenValue, err error)
+type SessionRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *sessions.SessionListResult, opt ...Option) (ret *sessions.SessionListResult, refreshToken RefreshTokenValue, err error)
 
-func defaultSessionFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error) {
+func defaultSessionFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *sessions.SessionListResult, opt ...Option) (ret *sessions.SessionListResult, refreshToken RefreshTokenValue, err error) {
 	const op = "cache.defaultSessionFunc"
 	conf, err := api.DefaultConfig()
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
+	}
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	conf.Addr = addr
 	conf.Token = authTok
 	client, err := api.NewClient(conf)
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	sClient := sessions.NewClient(client)
-	l, err := sClient.List(ctx, "global", sessions.WithIncludeTerminated(true), sessions.WithRecursive(true), sessions.WithListToken(string(refreshTok)))
+	var l *sessions.SessionListResult
+	switch inPage {
+	case nil:
+		l, err = sClient.List(ctx, "global", sessions.WithIncludeTerminated(true), sessions.WithRecursive(true), sessions.WithListToken(string(refreshTok)), sessions.WithClientDirectedPagination(!opts.withUseNonPagedListing))
+	default:
+		l, err = sClient.ListNextPage(ctx, inPage, sessions.WithListToken(string(refreshTok)))
+	}
 	if err != nil {
 		if api.ErrInvalidListToken.Is(err) {
-			return nil, nil, "", err
+			return nil, "", err
 		}
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	if l.ResponseType == "" {
-		return nil, nil, "", ErrRefreshNotSupported
+		return nil, "", ErrRefreshNotSupported
 	}
-	return l.Items, l.RemovedIds, RefreshTokenValue(l.ListToken), nil
+	return l, RefreshTokenValue(l.ListToken), nil
 }
 
 // refreshSessions uses attempts to refresh the sessions for the provided user
@@ -59,8 +69,6 @@ func (r *Repository) refreshSessions(ctx context.Context, u *user, tokens map[Au
 		return errors.New(ctx, errors.InvalidParameter, op, "user is nil")
 	case u.Id == "":
 		return errors.New(ctx, errors.InvalidParameter, op, "user id is missing")
-	case u.Address == "":
-		return errors.New(ctx, errors.InvalidParameter, op, "user boundary address is missing")
 	}
 	const resourceType = sessionResourceType
 
@@ -82,13 +90,13 @@ func (r *Repository) refreshSessions(ctx context.Context, u *user, tokens map[Au
 
 	// Find and use a token for retrieving sessions
 	var gotResponse bool
-	var resp []*sessions.Session
+	var currentPage *sessions.SessionListResult
 	var newRefreshToken RefreshTokenValue
+	var foundAuthToken string
 	var unsupportedCacheRequest bool
-	var removedIds []string
 	var retErr error
 	for at, t := range tokens {
-		resp, removedIds, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal)
+		currentPage, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal, currentPage)
 		if api.ErrInvalidListToken.Is(err) {
 			event.WriteSysEvent(ctx, op, "old list token is no longer valid, starting new initial fetch", "user_id", u.Id)
 			if err := r.deleteRefreshToken(ctx, u, resourceType); err != nil {
@@ -96,7 +104,7 @@ func (r *Repository) refreshSessions(ctx context.Context, u *user, tokens map[Au
 			}
 			// try again without the refresh token
 			oldRefreshToken = nil
-			resp, removedIds, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, t, "")
+			currentPage, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, t, "", currentPage)
 		}
 		if err != nil {
 			if err == ErrRefreshNotSupported {
@@ -106,6 +114,7 @@ func (r *Repository) refreshSessions(ctx context.Context, u *user, tokens map[Au
 				continue
 			}
 		}
+		foundAuthToken = t
 		gotResponse = true
 		break
 	}
@@ -120,44 +129,56 @@ func (r *Repository) refreshSessions(ctx context.Context, u *user, tokens map[Au
 	}
 
 	var numDeleted int
-	_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
-		var err error
-		switch {
-		case oldRefreshToken == nil:
-			if numDeleted, err = w.Exec(ctx, "delete from session where fk_user_id = @fk_user_id",
-				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
-				return err
+	var numUpserted int
+	var clearPerformed bool
+	for {
+		_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
+			var err error
+			if (oldRefreshToken == nil || unsupportedCacheRequest) && !clearPerformed {
+				if numDeleted, err = w.Exec(ctx, "delete from session where fk_user_id = @fk_user_id",
+					[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
+					return err
+				}
 			}
-		case len(removedIds) > 0:
-			if numDeleted, err = w.Exec(ctx, "delete from session where id in @ids",
-				[]any{sql.Named("ids", removedIds)}); err != nil {
-				return err
+			switch {
+			case unsupportedCacheRequest:
+				if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
+					return err
+				}
+			case newRefreshToken != "":
+				numUpserted += len(currentPage.Items)
+				if err := upsertSessions(ctx, w, u, currentPage.Items); err != nil {
+					return err
+				}
+				if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
+					return err
+				}
+			default:
+				// controller supports caching, but doesn't have any resources
 			}
+			if !unsupportedCacheRequest && len(currentPage.RemovedIds) > 0 {
+				if numDeleted, err = w.Exec(ctx, "delete from session where id in @ids",
+					[]any{sql.Named("ids", currentPage.RemovedIds)}); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		if unsupportedCacheRequest || currentPage.ResponseType == "" || currentPage.ResponseType == "complete" {
+			break
 		}
-		switch {
-		case unsupportedCacheRequest:
-			if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
-				return err
-			}
-		case newRefreshToken != "":
-			if err := upsertSessions(ctx, w, u, resp); err != nil {
-				return err
-			}
-			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
-				return err
-			}
-		default:
-			// controller supports caching, but doesn't have any resources
+		currentPage, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, foundAuthToken, newRefreshToken, currentPage)
+		if err != nil {
+			break
 		}
-		return nil
-	})
+	}
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "sessions updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "sessions updated", "deleted", numDeleted, "upserted", numUpserted, "user id", u.Id)
 	return nil
 }
 
@@ -186,12 +207,12 @@ func (r *Repository) checkCachingSessions(ctx context.Context, u *user, tokens m
 
 	// Find and use a token for retrieving sessions
 	var gotResponse bool
-	var resp []*sessions.Session
+	var resp *sessions.SessionListResult
 	var newRefreshToken RefreshTokenValue
 	var unsupportedCacheRequest bool
 	var retErr error
 	for at, t := range tokens {
-		resp, _, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, t, "")
+		resp, newRefreshToken, err = opts.withSessionRetrievalFunc(ctx, u.Address, t, "", nil, WithUseNonPagedListing(true))
 		if err != nil {
 			if err == ErrRefreshNotSupported {
 				unsupportedCacheRequest = true
@@ -217,24 +238,29 @@ func (r *Repository) checkCachingSessions(ctx context.Context, u *user, tokens m
 	_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(reader db.Reader, w db.Writer) error {
 		switch {
 		case unsupportedCacheRequest:
+			// Since we know the controller doesn't support caching, we mark the
+			// user as unable to cache the data.
 			if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
 				return err
 			}
 		case newRefreshToken != "":
+			// Now that there is a refresh token, the data can be cached, so
+			// cache it and store the refresh token for future refreshes. First
+			// remove any values, then add the new ones
 			var err error
 			if numDeleted, err = w.Exec(ctx, "delete from session where fk_user_id = @fk_user_id",
 				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
 				return err
 			}
-			if err := upsertSessions(ctx, w, u, resp); err != nil {
+			if err := upsertSessions(ctx, w, u, resp.Items); err != nil {
 				return err
 			}
 			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
 				return err
 			}
 		default:
-			// This is no longer flagged as not supported, but we dont have a
-			// refresh token so clear out any refresh token we have stored.
+			// We know the controller supports caching, but doesn't have a
+			// refresh token so clear out any refresh token we have for this resource.
 			if err := deleteRefreshToken(ctx, w, u, resourceType); err != nil {
 				return err
 			}
@@ -247,7 +273,7 @@ func (r *Repository) checkCachingSessions(ctx context.Context, u *user, tokens m
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "sessions updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "sessions updated", "deleted", numDeleted, "upserted", len(resp.Items), "user_id", u.Id)
 	return nil
 }
 
@@ -290,20 +316,20 @@ func upsertSessions(ctx context.Context, w db.Writer, u *user, in []*sessions.Se
 	return nil
 }
 
-func (r *Repository) ListSessions(ctx context.Context, authTokenId string) ([]*sessions.Session, error) {
+func (r *Repository) ListSessions(ctx context.Context, authTokenId string, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).ListSessions"
 	switch {
 	case authTokenId == "":
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "auth token id is missing")
 	}
-	ret, err := r.searchSessions(ctx, "true", nil, withAuthTokenId(authTokenId))
+	ret, err := r.searchSessions(ctx, "true", nil, append(opt, withAuthTokenId(authTokenId))...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return ret, nil
 }
 
-func (r *Repository) QuerySessions(ctx context.Context, authTokenId, query string) ([]*sessions.Session, error) {
+func (r *Repository) QuerySessions(ctx context.Context, authTokenId, query string, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).QuerySessions"
 	switch {
 	case authTokenId == "":
@@ -316,14 +342,14 @@ func (r *Repository) QuerySessions(ctx context.Context, authTokenId, query strin
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter))
 	}
-	ret, err := r.searchSessions(ctx, w.Condition, w.Args, withAuthTokenId(authTokenId))
+	ret, err := r.searchSessions(ctx, w.Condition, w.Args, append(opt, withAuthTokenId(authTokenId))...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return ret, nil
 }
 
-func (r *Repository) searchSessions(ctx context.Context, condition string, searchArgs []any, opt ...Option) ([]*sessions.Session, error) {
+func (r *Repository) searchSessions(ctx context.Context, condition string, searchArgs []any, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).searchSessions"
 	switch {
 	case condition == "":
@@ -348,7 +374,7 @@ func (r *Repository) searchSessions(ctx context.Context, condition string, searc
 	}
 
 	var cachedSessions []*Session
-	if err := r.rw.SearchWhere(ctx, &cachedSessions, condition, searchArgs, db.WithLimit(-1)); err != nil {
+	if err := r.rw.SearchWhere(ctx, &cachedSessions, condition, searchArgs, db.WithLimit(opts.withMaxResultSetSize+1)); err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
@@ -360,7 +386,15 @@ func (r *Repository) searchSessions(ctx context.Context, condition string, searc
 		}
 		retSessions = append(retSessions, &sess)
 	}
-	return retSessions, nil
+
+	sr := &SearchResult{
+		Sessions: retSessions,
+	}
+	if opts.withMaxResultSetSize > 0 && len(sr.Sessions) > opts.withMaxResultSetSize {
+		sr.Sessions = sr.Sessions[:opts.withMaxResultSetSize]
+		sr.Incomplete = true
+	}
+	return sr, nil
 }
 
 type Session struct {

internal/clientcache/internal/cache/repository_sessions_test.go

@@ -6,6 +6,7 @@ package cache
 import (
 	"context"
 	"encoding/json"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
@@ -149,7 +150,7 @@ func TestRepository_refreshSessions(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			err := r.refreshSessions(ctx, tc.u, map[AuthToken]string{{Id: "id"}: "something"},
-				WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{tc.sess}, [][]string{nil})))
+				WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{tc.sess}, [][]string{nil}))))
 			if tc.errorContains == "" {
 				assert.NoError(t, err)
 				rw := db.New(s)
@@ -234,41 +235,41 @@ func TestRepository_RefreshSessions_withRefreshTokens(t *testing.T) {
 	}
 
 	err = r.refreshSessions(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, ss, [][]string{nil, nil})))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ss, [][]string{nil, nil}))))
 	assert.NoError(t, err)
 
 	got, err := r.ListSessions(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 2)
+	assert.Len(t, got.Sessions, 2)
 
 	// Refreshing again uses the refresh token and get additional sessions, appending
 	// them to the response
 	err = r.refreshSessions(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, ss, [][]string{nil, nil})))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ss, [][]string{nil, nil}))))
 	assert.NoError(t, err)
 
 	got, err = r.ListSessions(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 3)
+	assert.Len(t, got.Sessions, 3)
 
 	// Refreshing again wont return any more resources, but also none should be
 	// removed
 	require.NoError(t, r.refreshSessions(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, ss, [][]string{nil, nil}))))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ss, [][]string{nil, nil})))))
 	assert.NoError(t, err)
 
 	got, err = r.ListSessions(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 3)
+	assert.Len(t, got.Sessions, 3)
 
 	// Refresh again with the refresh token being reported as invalid.
 	require.NoError(t, r.refreshSessions(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ss[0]))))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ss[0])))))
 	assert.NoError(t, err)
 
 	got, err = r.ListSessions(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 2)
+	assert.Len(t, got.Sessions, 2)
 }
 
 func TestRepository_ListSessions(t *testing.T) {
@@ -350,18 +351,78 @@ func TestRepository_ListSessions(t *testing.T) {
 		},
 	}
 	require.NoError(t, r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ss}, [][]string{nil}))))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ss}, [][]string{nil})))))
 
 	t.Run("wrong user gets no sessions", func(t *testing.T) {
 		l, err := r.ListSessions(ctx, kt2.AuthTokenId)
 		assert.NoError(t, err)
-		assert.Empty(t, l)
+		assert.Empty(t, l.Sessions)
 	})
 	t.Run("correct token gets sessions", func(t *testing.T) {
 		l, err := r.ListSessions(ctx, kt1.AuthTokenId)
 		assert.NoError(t, err)
-		assert.Len(t, l, len(ss))
-		assert.ElementsMatch(t, l, ss)
+		assert.Len(t, l.Sessions, len(ss))
+		assert.ElementsMatch(t, l.Sessions, ss)
+	})
+}
+
+func TestRepository_ListSessionsLimiting(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u := &user{
+		Id:      "u",
+		Address: addr,
+	}
+	at := &authtokens.AuthToken{
+		Id:     "at",
+		Token:  "at_token",
+		UserId: u.Id,
+	}
+	kt := KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}
+
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k", "t"}: at,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt))
+
+	var ts []*sessions.Session
+	for i := 0; i < defaultLimitedResultSetSize*2; i++ {
+		ts = append(ts, session("s"+strconv.Itoa(i)))
+	}
+	require.NoError(t, r.refreshSessions(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ts}, [][]string{nil})))))
+
+	searchService, err := NewSearchService(ctx, r)
+	require.NoError(t, err)
+	params := SearchParams{
+		Resource:    Sessions,
+		AuthTokenId: kt.AuthTokenId,
+	}
+
+	t.Run("default limit", func(t *testing.T) {
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Sessions, defaultLimitedResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("custom limit", func(t *testing.T) {
+		params.MaxResultSetSize = 20
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Sessions, params.MaxResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("no limit", func(t *testing.T) {
+		params.MaxResultSetSize = -1
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Sessions, defaultLimitedResultSetSize*2)
+		assert.False(t, searchResult.Incomplete)
 	})
 }
 
@@ -466,18 +527,79 @@ func TestRepository_QuerySessions(t *testing.T) {
 		},
 	}
 	require.NoError(t, r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ss}, [][]string{nil}))))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ss}, [][]string{nil})))))
 
 	t.Run("wrong token gets no sessions", func(t *testing.T) {
 		l, err := r.QuerySessions(ctx, kt2.AuthTokenId, query)
 		assert.NoError(t, err)
-		assert.Empty(t, l)
+		assert.Empty(t, l.Sessions)
 	})
 	t.Run("correct token gets sessions", func(t *testing.T) {
 		l, err := r.QuerySessions(ctx, kt1.AuthTokenId, query)
 		assert.NoError(t, err)
-		assert.Len(t, l, 2)
-		assert.ElementsMatch(t, l, ss[0:2])
+		assert.Len(t, l.Sessions, 2)
+		assert.ElementsMatch(t, l.Sessions, ss[0:2])
+	})
+}
+
+func TestRepository_QuerySessionsLimiting(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u := &user{
+		Id:      "u",
+		Address: addr,
+	}
+	at := &authtokens.AuthToken{
+		Id:     "at",
+		Token:  "at_token",
+		UserId: u.Id,
+	}
+	kt := KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}
+
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k", "t"}: at,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt))
+
+	var ts []*sessions.Session
+	for i := 0; i < defaultLimitedResultSetSize*2; i++ {
+		ts = append(ts, session("t"+strconv.Itoa(i)))
+	}
+	require.NoError(t, r.refreshSessions(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{ts}, [][]string{nil})))))
+
+	searchService, err := NewSearchService(ctx, r)
+	require.NoError(t, err)
+	params := SearchParams{
+		Resource:    Sessions,
+		AuthTokenId: kt.AuthTokenId,
+		Query:       `(id % 'session')`,
+	}
+
+	t.Run("default limit", func(t *testing.T) {
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Sessions, defaultLimitedResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("custom limit", func(t *testing.T) {
+		params.MaxResultSetSize = 20
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Sessions, params.MaxResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("no limit", func(t *testing.T) {
+		params.MaxResultSetSize = -1
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Sessions, defaultLimitedResultSetSize*2)
+		assert.False(t, searchResult.Incomplete)
 	})
 }
 
@@ -515,15 +637,15 @@ func TestDefaultSessionRetrievalFunc(t *testing.T) {
 		}
 	}
 
-	got, removed, refTok, err := defaultSessionFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "")
+	got, refTok, err := defaultSessionFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "", nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok)
-	assert.Empty(t, removed)
-	assert.Len(t, got, 1)
+	assert.Empty(t, got.RemovedIds)
+	assert.Len(t, got.Items, 1)
 
-	got2, removed2, refTok2, err := defaultSessionFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok)
+	got2, refTok2, err := defaultSessionFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok, nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok2)
-	assert.Empty(t, removed2)
-	assert.Empty(t, got2)
+	assert.Empty(t, got2.RemovedIds)
+	assert.Empty(t, got2.Items)
 }

internal/clientcache/internal/cache/repository_targets.go

@@ -21,32 +21,42 @@ import (
 
 // TargetRetrievalFunc is a function that retrieves targets
 // from the provided boundary addr using the provided token.
-type TargetRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) (ret []*targets.Target, removedIds []string, refreshToken RefreshTokenValue, err error)
+type TargetRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (ret *targets.TargetListResult, refreshToken RefreshTokenValue, err error)
 
-func defaultTargetFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+func defaultTargetFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (*targets.TargetListResult, RefreshTokenValue, error) {
 	const op = "cache.defaultTargetFunc"
 	conf, err := api.DefaultConfig()
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
+	}
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	conf.Addr = addr
 	conf.Token = authTok
 	client, err := api.NewClient(conf)
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	tarClient := targets.NewClient(client)
-	l, err := tarClient.List(ctx, "global", targets.WithRecursive(true), targets.WithListToken(string(refreshTok)))
+	var l *targets.TargetListResult
+	switch inPage {
+	case nil:
+		l, err = tarClient.List(ctx, "global", targets.WithRecursive(true), targets.WithListToken(string(refreshTok)), targets.WithClientDirectedPagination(!opts.withUseNonPagedListing))
+	default:
+		l, err = tarClient.ListNextPage(ctx, inPage, targets.WithListToken(string(refreshTok)))
+	}
 	if err != nil {
 		if api.ErrInvalidListToken.Is(err) {
-			return nil, nil, "", err
+			return nil, "", err
 		}
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	if l.ResponseType == "" {
-		return nil, nil, "", ErrRefreshNotSupported
+		return nil, "", ErrRefreshNotSupported
 	}
-	return l.Items, l.RemovedIds, RefreshTokenValue(l.ListToken), nil
+	return l, RefreshTokenValue(l.ListToken), nil
 }
 
 // refreshTargets uses attempts to refresh the targets for the provided user
@@ -81,13 +91,13 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 
 	// Find and use a token for retrieving targets
 	var gotResponse bool
-	var resp []*targets.Target
-	var removedIds []string
+	var currentPage *targets.TargetListResult
 	var newRefreshToken RefreshTokenValue
+	var foundAuthToken string
 	var unsupportedCacheRequest bool
 	var retErr error
 	for at, t := range tokens {
-		resp, removedIds, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal)
+		currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal, currentPage)
 		if api.ErrInvalidListToken.Is(err) {
 			event.WriteSysEvent(ctx, op, "old list token is no longer valid, starting new initial fetch", "user_id", u.Id)
 			if err := r.deleteRefreshToken(ctx, u, resourceType); err != nil {
@@ -95,7 +105,7 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 			}
 			// try again without the refresh token
 			oldRefreshToken = nil
-			resp, removedIds, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "")
+			currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "", currentPage)
 		}
 		if err != nil {
 			if err == ErrRefreshNotSupported {
@@ -105,6 +115,7 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 				continue
 			}
 		}
+		foundAuthToken = t
 		gotResponse = true
 		break
 	}
@@ -118,44 +129,57 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 	}
 
 	var numDeleted int
-	_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
-		var err error
-		switch {
-		case oldRefreshToken == nil || unsupportedCacheRequest:
-			if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id",
-				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
-				return err
+	var numUpserted int
+	var clearPerformed bool
+	for {
+		_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
+			var err error
+			if (oldRefreshToken == nil || unsupportedCacheRequest) && !clearPerformed {
+				if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id",
+					[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
+					return err
+				}
+				clearPerformed = true
 			}
-		case len(removedIds) > 0:
-			if numDeleted, err = w.Exec(ctx, "delete from target where id in @ids",
-				[]any{sql.Named("ids", removedIds)}); err != nil {
-				return err
+			switch {
+			case unsupportedCacheRequest:
+				if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
+					return err
+				}
+			case newRefreshToken != "":
+				if err := upsertTargets(ctx, w, u, currentPage.Items); err != nil {
+					return err
+				}
+				numUpserted += len(currentPage.Items)
+				if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
+					return err
+				}
+			default:
+				// controller supports caching, but doesn't have any resources
 			}
+			if !unsupportedCacheRequest && len(currentPage.RemovedIds) > 0 {
+				if numDeleted, err = w.Exec(ctx, "delete from target where id in @ids",
+					[]any{sql.Named("ids", currentPage.RemovedIds)}); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		if unsupportedCacheRequest || currentPage.ResponseType == "" || currentPage.ResponseType == "complete" {
+			break
 		}
-		switch {
-		case unsupportedCacheRequest:
-			if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
-				return err
-			}
-		case newRefreshToken != "":
-			if err := upsertTargets(ctx, w, u, resp); err != nil {
-				return err
-			}
-			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
-				return err
-			}
-		default:
-			// controller supports caching, but doesn't have any resources
+		currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, foundAuthToken, newRefreshToken, currentPage)
+		if err != nil {
+			break
 		}
-		return nil
-	})
+	}
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", numUpserted, "user_id", u.Id)
 	return nil
 }
 
@@ -185,12 +209,12 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma
 
 	// Find and use a token for retrieving targets
 	var gotResponse bool
-	var resp []*targets.Target
+	var resp *targets.TargetListResult
 	var newRefreshToken RefreshTokenValue
 	var unsupportedCacheRequest bool
 	var retErr error
 	for at, t := range tokens {
-		resp, _, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "")
+		resp, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "", nil, WithUseNonPagedListing(true))
 		if err != nil {
 			if err == ErrRefreshNotSupported {
 				unsupportedCacheRequest = true
@@ -221,14 +245,15 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma
 				return err
 			}
 		case newRefreshToken != "":
-			var err error
 			// Now that there is a refresh token, the data can be cached, so
-			// cache it and store the refresh token for future refreshes.
+			// cache it and store the refresh token for future refreshes. First
+			// remove any values, then add the new ones
+			var err error
 			if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id",
 				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
 				return err
 			}
-			if err := upsertTargets(ctx, w, u, resp); err != nil {
+			if err := upsertTargets(ctx, w, u, resp.Items); err != nil {
 				return err
 			}
 			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
@@ -249,7 +274,7 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp.Items), "user_id", u.Id)
 	return nil
 }
 
@@ -291,20 +316,20 @@ func upsertTargets(ctx context.Context, w db.Writer, u *user, in []*targets.Targ
 	return nil
 }
 
-func (r *Repository) ListTargets(ctx context.Context, authTokenId string) ([]*targets.Target, error) {
+func (r *Repository) ListTargets(ctx context.Context, authTokenId string, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).ListTargets"
 	switch {
 	case authTokenId == "":
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "auth token id is missing")
 	}
-	ret, err := r.searchTargets(ctx, "true", nil, withAuthTokenId(authTokenId))
+	ret, err := r.searchTargets(ctx, "true", nil, append(opt, withAuthTokenId(authTokenId))...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return ret, nil
 }
 
-func (r *Repository) QueryTargets(ctx context.Context, authTokenId, query string) ([]*targets.Target, error) {
+func (r *Repository) QueryTargets(ctx context.Context, authTokenId, query string, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).QueryTargets"
 	switch {
 	case authTokenId == "":
@@ -317,14 +342,14 @@ func (r *Repository) QueryTargets(ctx context.Context, authTokenId, query string
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter))
 	}
-	ret, err := r.searchTargets(ctx, w.Condition, w.Args, withAuthTokenId(authTokenId))
+	ret, err := r.searchTargets(ctx, w.Condition, w.Args, append(opt, withAuthTokenId(authTokenId))...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return ret, nil
 }
 
-func (r *Repository) searchTargets(ctx context.Context, condition string, searchArgs []any, opt ...Option) ([]*targets.Target, error) {
+func (r *Repository) searchTargets(ctx context.Context, condition string, searchArgs []any, opt ...Option) (*SearchResult, error) {
 	const op = "cache.(Repository).searchTargets"
 	switch {
 	case condition == "":
@@ -349,7 +374,7 @@ func (r *Repository) searchTargets(ctx context.Context, condition string, search
 	}
 
 	var cachedTargets []*Target
-	if err := r.rw.SearchWhere(ctx, &cachedTargets, condition, searchArgs, db.WithLimit(-1)); err != nil {
+	if err := r.rw.SearchWhere(ctx, &cachedTargets, condition, searchArgs, db.WithLimit(opts.withMaxResultSetSize+1)); err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
@@ -361,7 +386,15 @@ func (r *Repository) searchTargets(ctx context.Context, condition string, search
 		}
 		retTargets = append(retTargets, &tar)
 	}
-	return retTargets, nil
+
+	sr := &SearchResult{
+		Targets: retTargets,
+	}
+	if opts.withMaxResultSetSize > 0 && len(sr.Targets) > opts.withMaxResultSetSize {
+		sr.Targets = sr.Targets[:opts.withMaxResultSetSize]
+		sr.Incomplete = true
+	}
+	return sr, nil
 }
 
 type Target struct {

internal/clientcache/internal/cache/repository_targets_test.go

@@ -6,6 +6,7 @@ package cache
 import (
 	"context"
 	"encoding/json"
+	"strconv"
 	"sync"
 	"testing"
 
@@ -140,7 +141,7 @@ func TestRepository_refreshTargets(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			err := r.refreshTargets(ctx, tc.u, map[AuthToken]string{{Id: "id"}: "something"},
-				WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{tc.targets}, [][]string{nil})))
+				WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{tc.targets}, [][]string{nil}))))
 			if tc.errorContains == "" {
 				assert.NoError(t, err)
 				rw := db.New(s)
@@ -212,12 +213,12 @@ func TestRepository_RefreshTargets_InvalidListTokenError(t *testing.T) {
 	}
 
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(invalidAuthTokenFunc)))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(invalidAuthTokenFunc))))
 
 	// This time an invalid auth token should be returned, and refreshTargets should fall back
 	// to requesting without one.
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(invalidAuthTokenFunc)))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(invalidAuthTokenFunc))))
 
 	assert.Equal(t, 1, withRefreshToken)
 	assert.Equal(t, 2, withoutRefreshToken)
@@ -260,40 +261,40 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) {
 	}
 
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))))
 
 	got, err := r.ListTargets(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 2)
+	assert.Len(t, got.Targets, 2)
 
 	// Refreshing again uses the refresh token and get additional sessions, appending
 	// them to the response
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))))
 	assert.NoError(t, err)
 
 	got, err = r.ListTargets(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 3)
+	assert.Len(t, got.Targets, 3)
 
 	// Refreshing again wont return any more resources, but also none should be
 	// removed
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))))
 	assert.NoError(t, err)
 
 	got, err = r.ListTargets(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 3)
+	assert.Len(t, got.Targets, 3)
 
 	// Refresh again with the refresh token being reported as invalid.
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ts[0]))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ts[0])))))
 	assert.NoError(t, err)
 
 	got, err = r.ListTargets(ctx, at.Id)
 	require.NoError(t, err)
-	assert.Len(t, got, 2)
+	assert.Len(t, got.Targets, 2)
 }
 
 func TestRepository_ListTargets(t *testing.T) {
@@ -344,18 +345,78 @@ func TestRepository_ListTargets(t *testing.T) {
 		target("3"),
 	}
 	require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	t.Run("wrong user gets no targets", func(t *testing.T) {
 		l, err := r.ListTargets(ctx, kt2.AuthTokenId)
 		assert.NoError(t, err)
-		assert.Empty(t, l)
+		assert.Empty(t, l.Targets)
 	})
 	t.Run("correct token gets targets", func(t *testing.T) {
 		l, err := r.ListTargets(ctx, kt1.AuthTokenId)
 		assert.NoError(t, err)
-		assert.Len(t, l, len(ts))
-		assert.ElementsMatch(t, l, ts)
+		assert.Len(t, l.Targets, len(ts))
+		assert.ElementsMatch(t, l.Targets, ts)
+	})
+}
+
+func TestRepository_ListTargetsLimiting(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u := &user{
+		Id:      "u",
+		Address: addr,
+	}
+	at := &authtokens.AuthToken{
+		Id:     "at",
+		Token:  "at_token",
+		UserId: u.Id,
+	}
+	kt := KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}
+
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k", "t"}: at,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt))
+
+	var ts []*targets.Target
+	for i := 0; i < defaultLimitedResultSetSize*2; i++ {
+		ts = append(ts, target("t"+strconv.Itoa(i)))
+	}
+	require.NoError(t, r.refreshTargets(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
+
+	searchService, err := NewSearchService(ctx, r)
+	require.NoError(t, err)
+	params := SearchParams{
+		Resource:    Targets,
+		AuthTokenId: kt.AuthTokenId,
+	}
+
+	t.Run("default limit", func(t *testing.T) {
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Targets, defaultLimitedResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("custom limit", func(t *testing.T) {
+		params.MaxResultSetSize = 20
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Targets, params.MaxResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("no limit", func(t *testing.T) {
+		params.MaxResultSetSize = -1
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Targets, defaultLimitedResultSetSize*2)
+		assert.False(t, searchResult.Incomplete)
 	})
 }
 
@@ -451,18 +512,79 @@ func TestRepository_QueryTargets(t *testing.T) {
 		},
 	}
 	require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	t.Run("wrong token gets no targets", func(t *testing.T) {
 		l, err := r.QueryTargets(ctx, kt2.AuthTokenId, query)
 		assert.NoError(t, err)
-		assert.Empty(t, l)
+		assert.Empty(t, l.Targets)
 	})
 	t.Run("correct token gets targets", func(t *testing.T) {
 		l, err := r.QueryTargets(ctx, kt1.AuthTokenId, query)
 		assert.NoError(t, err)
-		assert.Len(t, l, 2)
-		assert.ElementsMatch(t, l, ts[0:2])
+		assert.Len(t, l.Targets, 2)
+		assert.ElementsMatch(t, l.Targets, ts[0:2])
+	})
+}
+
+func TestRepository_QueryTargetsLimiting(t *testing.T) {
+	ctx := context.Background()
+	s, err := cachedb.Open(ctx)
+	require.NoError(t, err)
+
+	addr := "address"
+	u := &user{
+		Id:      "u",
+		Address: addr,
+	}
+	at := &authtokens.AuthToken{
+		Id:     "at",
+		Token:  "at_token",
+		UserId: u.Id,
+	}
+	kt := KeyringToken{KeyringType: "k", TokenName: "t", AuthTokenId: at.Id}
+
+	atMap := map[ringToken]*authtokens.AuthToken{
+		{"k", "t"}: at,
+	}
+	r, err := NewRepository(ctx, s, &sync.Map{}, mapBasedAuthTokenKeyringLookup(atMap), sliceBasedAuthTokenBoundaryReader(maps.Values(atMap)))
+	require.NoError(t, err)
+	require.NoError(t, r.AddKeyringToken(ctx, addr, kt))
+
+	var ts []*targets.Target
+	for i := 0; i < defaultLimitedResultSetSize*2; i++ {
+		ts = append(ts, target("t"+strconv.Itoa(i)))
+	}
+	require.NoError(t, r.refreshTargets(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
+
+	searchService, err := NewSearchService(ctx, r)
+	require.NoError(t, err)
+	params := SearchParams{
+		Resource:    Targets,
+		AuthTokenId: kt.AuthTokenId,
+		Query:       `(name % 'name')`,
+	}
+
+	t.Run("default limit", func(t *testing.T) {
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Targets, defaultLimitedResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("custom limit", func(t *testing.T) {
+		params.MaxResultSetSize = 20
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Targets, params.MaxResultSetSize)
+		assert.True(t, searchResult.Incomplete)
+	})
+	t.Run("no limit", func(t *testing.T) {
+		params.MaxResultSetSize = -1
+		searchResult, err := searchService.Search(ctx, params)
+		require.NoError(t, err)
+		assert.Len(t, searchResult.Targets, defaultLimitedResultSetSize*2)
+		assert.False(t, searchResult.Incomplete)
 	})
 }
 
@@ -484,13 +606,13 @@ func TestDefaultTargetRetrievalFunc(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, tar2)
 
-	got, removed, refTok, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "")
+	got, refTok, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "", nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok)
-	assert.Empty(t, removed)
+	assert.Empty(t, got.RemovedIds)
 	found1 := false
 	found2 := false
-	for _, t := range got {
+	for _, t := range got.Items {
 		if t.Id == tar1.Item.Id {
 			found1 = true
 		}
@@ -501,10 +623,10 @@ func TestDefaultTargetRetrievalFunc(t *testing.T) {
 	assert.True(t, found1, "expected to find target %s in list", tar1.Item.Id)
 	assert.True(t, found2, "expected to find target %s in list", tar2.Item.Id)
 
-	got2, removed2, refTok2, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok)
+	got2, refTok2, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok, nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok2)
 	assert.NotEqual(t, refTok2, refTok)
-	assert.Empty(t, removed2)
-	assert.Empty(t, got2)
+	assert.Empty(t, got.RemovedIds)
+	assert.Empty(t, got2.Items)
 }

internal/clientcache/internal/cache/search.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/boundary/api/aliases"
+	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/sessions"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/internal/errors"
@@ -23,11 +24,12 @@ const (
 	ResolvableAliases SearchableResource = "resolvable-aliases"
 	Targets           SearchableResource = "targets"
 	Sessions          SearchableResource = "sessions"
+	ImplicitScopes    SearchableResource = "implicit-scopes"
 )
 
 func (r SearchableResource) Valid() bool {
 	switch r {
-	case ResolvableAliases, Targets, Sessions:
+	case ResolvableAliases, Targets, Sessions, ImplicitScopes:
 		return true
 	}
 	return false
@@ -41,6 +43,8 @@ func ToSearchableResource(s string) SearchableResource {
 		return Targets
 	case strings.EqualFold(s, string(Sessions)):
 		return Sessions
+	case strings.EqualFold(s, string(ImplicitScopes)):
+		return ImplicitScopes
 	}
 	return Unknown
 }
@@ -55,6 +59,8 @@ type SearchParams struct {
 	Query string
 	// the optional bexpr filter string that all results will be filtered by
 	Filter string
+	// Max result set size is an override to the default max result set size
+	MaxResultSetSize int
 }
 
 // SearchResult returns the results from searching the cache.
@@ -62,6 +68,11 @@ type SearchResult struct {
 	ResolvableAliases []*aliases.Alias    `json:"resolvable_aliases,omitempty"`
 	Targets           []*targets.Target   `json:"targets,omitempty"`
 	Sessions          []*sessions.Session `json:"sessions,omitempty"`
+	ImplicitScopes    []*scopes.Scope     `json:"implicit_scopes,omitempty"`
+
+	// Incomplete is true if the search results are incomplete, that is, we are
+	// returning only a subset based on the max result set size
+	Incomplete bool `json:"incomplete,omitempty"`
 }
 
 // SearchService is a domain service that can search across all resources in the
@@ -83,22 +94,53 @@ func NewSearchService(ctx context.Context, repo *Repository) (*SearchService, er
 			ResolvableAliases: &resourceSearchFns[*aliases.Alias]{
 				list:  repo.ListResolvableAliases,
 				query: repo.QueryResolvableAliases,
-				searchResult: func(a []*aliases.Alias) *SearchResult {
-					return &SearchResult{ResolvableAliases: a}
+				filter: func(in *SearchResult, e *bexpr.Evaluator) {
+					finalResults := make([]*aliases.Alias, 0, len(in.ResolvableAliases))
+					for _, item := range in.ResolvableAliases {
+						if m, err := e.Evaluate(filterItem{item}); err == nil && m {
+							finalResults = append(finalResults, item)
+						}
+					}
+					in.ResolvableAliases = finalResults
 				},
 			},
 			Targets: &resourceSearchFns[*targets.Target]{
 				list:  repo.ListTargets,
 				query: repo.QueryTargets,
-				searchResult: func(t []*targets.Target) *SearchResult {
-					return &SearchResult{Targets: t}
+				filter: func(in *SearchResult, e *bexpr.Evaluator) {
+					finalResults := make([]*targets.Target, 0, len(in.Targets))
+					for _, item := range in.Targets {
+						if m, err := e.Evaluate(filterItem{item}); err == nil && m {
+							finalResults = append(finalResults, item)
+						}
+					}
+					in.Targets = finalResults
 				},
 			},
 			Sessions: &resourceSearchFns[*sessions.Session]{
 				list:  repo.ListSessions,
 				query: repo.QuerySessions,
-				searchResult: func(s []*sessions.Session) *SearchResult {
-					return &SearchResult{Sessions: s}
+				filter: func(in *SearchResult, e *bexpr.Evaluator) {
+					finalResults := make([]*sessions.Session, 0, len(in.Sessions))
+					for _, item := range in.Sessions {
+						if m, err := e.Evaluate(filterItem{item}); err == nil && m {
+							finalResults = append(finalResults, item)
+						}
+					}
+					in.Sessions = finalResults
+				},
+			},
+			ImplicitScopes: &resourceSearchFns[*scopes.Scope]{
+				list:  repo.ListImplicitScopes,
+				query: repo.QueryImplicitScopes,
+				filter: func(in *SearchResult, e *bexpr.Evaluator) {
+					finalResults := make([]*scopes.Scope, 0, len(in.ImplicitScopes))
+					for _, item := range in.ImplicitScopes {
+						if m, err := e.Evaluate(filterItem{item}); err == nil && m {
+							finalResults = append(finalResults, item)
+						}
+					}
+					in.ImplicitScopes = finalResults
 				},
 			},
 		},
@@ -160,19 +202,15 @@ type resourceSearchFns[T any] struct {
 	// list takes a context and an auth token and returns all resources for the
 	// user of that auth token. If the provided auth token is not in the cache
 	// an empty slice and no error is returned.
-	list func(context.Context, string) ([]T, error)
+	list func(context.Context, string, ...Option) (*SearchResult, error)
 	// query takes a context, an auth token, and a query string and returns all
 	// resources for that auth token that matches the provided query parameter.
 	// If the provided auth token is not in the cache an empty slice and no
 	// error is returned.
-	query func(context.Context, string, string) ([]T, error)
-	// searchResult is a function which provides a SearchResult based on the
-	// type of T. SearchResult contains different fields for the different
-	// resource types returned, so for example if T is *targets.Target the
-	// returned SearchResult will have it's "Targets" field populated so the
-	// searchResult should take the passed in paramater and assign it to the
-	// appropriate field in the SearchResult.
-	searchResult func([]T) *SearchResult
+	query func(context.Context, string, string, ...Option) (*SearchResult, error)
+	// filter takes results and a ready-to-use evaluator and filters the items
+	// in the result
+	filter func(*SearchResult, *bexpr.Evaluator)
 }
 
 // resourceSearcher is an interface that only resourceSearchFns[T] is expected
@@ -191,33 +229,29 @@ type resourceSearcher interface {
 func (l *resourceSearchFns[T]) search(ctx context.Context, p SearchParams) (*SearchResult, error) {
 	const op = "cache.(resourceSearchFns).search"
 
-	var found []T
+	var found *SearchResult
 	var err error
 	switch p.Query {
 	case "":
-		found, err = l.list(ctx, p.AuthTokenId)
+		found, err = l.list(ctx, p.AuthTokenId, WithMaxResultSetSize(p.MaxResultSetSize))
 	default:
-		found, err = l.query(ctx, p.AuthTokenId, p.Query)
+		found, err = l.query(ctx, p.AuthTokenId, p.Query, WithMaxResultSetSize(p.MaxResultSetSize))
 	}
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
 	if p.Filter == "" {
-		return l.searchResult(found), nil
+		return found, nil
 	}
 
 	e, err := bexpr.CreateEvaluator(p.Filter, bexpr.WithTagName("json"))
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("couldn't build filter"), errors.WithCode(errors.InvalidParameter))
 	}
-	finalResults := make([]T, 0, len(found))
-	for _, item := range found {
-		if m, err := e.Evaluate(filterItem{item}); err == nil && m {
-			finalResults = append(finalResults, item)
-		}
-	}
-	return l.searchResult(finalResults), nil
+
+	l.filter(found, e)
+	return found, nil
 }
 
 type filterItem struct {

internal/clientcache/internal/cache/status_test.go

@@ -190,11 +190,11 @@ func TestStatus(t *testing.T) {
 			target("4"),
 		}
 		err = r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
 		require.NoError(t, err)
 
 		err = r.refreshTargets(ctx, u2, map[AuthToken]string{{Id: "id"}: "something"},
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts[:2]}, [][]string{nil})))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts[:2]}, [][]string{nil}))))
 		require.NoError(t, err)
 
 		sess := []*sessions.Session{
@@ -203,7 +203,7 @@ func TestStatus(t *testing.T) {
 			session("3"),
 		}
 		err := r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{sess}, [][]string{nil})))
+			WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*sessions.Session{sess}, [][]string{nil}))))
 		require.NoError(t, err)
 
 		als := []*aliases.Alias{
@@ -212,7 +212,7 @@ func TestStatus(t *testing.T) {
 			alias("3"),
 		}
 		err = r.refreshResolvableAliases(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{als}, [][]string{nil})))
+			WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testStaticResourceRetrievalFuncForId(t, [][]*aliases.Alias{als}, [][]string{nil}))))
 		require.NoError(t, err)
 
 		got, err := ss.Status(ctx)
@@ -308,15 +308,15 @@ func TestStatus_unsupported(t *testing.T) {
 	}))
 
 	err = r.refreshResolvableAliases(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)))
+		WithAliasRetrievalFunc(testResolvableAliasStaticResourceRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t))))
 	require.ErrorIs(t, err, ErrRefreshNotSupported)
 
 	err = r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 	require.ErrorIs(t, err, ErrRefreshNotSupported)
 
 	err = r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
+		WithSessionRetrievalFunc(testSessionStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 	require.ErrorIs(t, err, ErrRefreshNotSupported)
 
 	got, err := ss.Status(ctx)

internal/clientcache/internal/daemon/404_handler.go

@@ -9,7 +9,7 @@ import (
 )
 
 // new404Func creates a handler that returns a custom 404 error message.
-func new404Func(ctx context.Context) http.HandlerFunc {
+func new404Func(_ context.Context) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		writeError(w, "Not found", http.StatusNotFound)
 	}

internal/clientcache/internal/daemon/options.go

@@ -20,8 +20,10 @@ type options struct {
 	WithReadyToServeNotificationCh         chan struct{}
 	withBoundaryTokenReaderFunc            cache.BoundaryTokenReaderFn
 
-	withUrl    string
-	withLogger hclog.Logger
+	withUrl              string
+	withLogger           hclog.Logger
+	withHomeDir          string
+	withForceResetSchema bool
 }
 
 // Option - how options are passed as args
@@ -42,6 +44,14 @@ func getOpts(opt ...Option) (options, error) {
 	return opts, nil
 }
 
+// WithHomeDir provides an optional home directory to use.
+func WithHomeDir(_ context.Context, dir string) Option {
+	return func(o *options) error {
+		o.withHomeDir = dir
+		return nil
+	}
+}
+
 // withRefreshInterval provides an optional refresh interval.
 func withRefreshInterval(_ context.Context, d time.Duration) Option {
 	return func(o *options) error {
@@ -101,6 +111,15 @@ func WithBoundaryTokenReaderFunc(_ context.Context, fn cache.BoundaryTokenReader
 	}
 }
 
+// WithForceResetSchema provides an optional way to force resetting the schema,
+// e.g. wiping the cache
+func WithForceResetSchema(_ context.Context, force bool) Option {
+	return func(o *options) error {
+		o.withForceResetSchema = force
+		return nil
+	}
+}
+
 // WithReadyToServeNotificationCh provides an optional channel to notify when
 // the server is ready to serve; mainly used for test timing but exported for
 // availability. The channel will be closed just before the HTTP server is

internal/clientcache/internal/daemon/options_test.go

@@ -101,6 +101,21 @@ func Test_GetOpts(t *testing.T) {
 		testOpts := getDefaultOptions()
 		assert.Equal(t, opts, testOpts)
 	})
+	t.Run("WithHomeDir", func(t *testing.T) {
+		opts, err := getOpts(WithHomeDir(ctx, "/tmp"))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		testOpts.withHomeDir = "/tmp"
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithForceResetSchema", func(t *testing.T) {
+		opts, err := getOpts(WithForceResetSchema(ctx, true))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.False(t, testOpts.withForceResetSchema)
+		testOpts.withForceResetSchema = true
+		assert.Equal(t, opts, testOpts)
+	})
 	t.Run("WithReadyToServeNotificationCh", func(t *testing.T) {
 		ch := make(chan struct{})
 		opts, err := getOpts(WithReadyToServeNotificationCh(ctx, ch))
@@ -109,6 +124,5 @@ func Test_GetOpts(t *testing.T) {
 		testOpts := getDefaultOptions()
 		assert.Nil(t, testOpts.WithReadyToServeNotificationCh)
 		testOpts.WithReadyToServeNotificationCh = ch
-		assert.Equal(t, opts, testOpts)
 	})
 }

internal/clientcache/internal/daemon/search_handler.go

@@ -6,12 +6,14 @@ package daemon
 import (
 	"context"
 	"encoding/json"
+	stderrors "errors"
 	"fmt"
 	"net/http"
 	"strconv"
 
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/aliases"
+	"github.com/hashicorp/boundary/api/scopes"
 	"github.com/hashicorp/boundary/api/sessions"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/internal/clientcache/internal/cache"
@@ -21,19 +23,40 @@ import (
 	"github.com/hashicorp/go-hclog"
 )
 
+type RefreshStatus string
+
+const (
+	// Not refreshing means the result is complete, that is, the cache is not in
+	// the process of being built or refreshed
+	NotRefreshing RefreshStatus = "not-refreshing"
+	// Refreshing means that the cache is in the process of being refreshed, so
+	// the result may not be complete and the caller should try again later for
+	// more complete results
+	Refreshing RefreshStatus = "refreshing"
+	// RefreshError means that there was an error refreshing the cache. It says
+	// nothing about the completeness of the result, only that when attempting
+	// to refresh the cache in-line with the search an error was encountered.
+	RefreshError RefreshStatus = "refresh-error"
+)
+
 // SearchResult is the struct returned to search requests.
 type SearchResult struct {
 	ResolvableAliases []*aliases.Alias    `json:"resolvable_aliases,omitempty"`
 	Targets           []*targets.Target   `json:"targets,omitempty"`
 	Sessions          []*sessions.Session `json:"sessions,omitempty"`
+	ImplicitScopes    []*scopes.Scope     `json:"implicit_scopes,omitempty"`
+	Incomplete        bool                `json:"incomplete,omitempty"`
+	RefreshStatus     RefreshStatus       `json:"refresh_status,omitempty"`
+	RefreshError      string              `json:"refresh_error,omitempty"`
 }
 
 const (
-	filterKey       = "filter"
-	queryKey        = "query"
-	resourceKey     = "resource"
-	forceRefreshKey = "force_refresh"
-	authTokenIdKey  = "auth_token_id"
+	filterKey           = "filter"
+	queryKey            = "query"
+	resourceKey         = "resource"
+	forceRefreshKey     = "force_refresh"
+	authTokenIdKey      = "auth_token_id"
+	maxResultSetSizeKey = "max_result_set_size"
 )
 
 func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshService *cache.RefreshService, logger hclog.Logger) (http.HandlerFunc, error) {
@@ -54,8 +77,13 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 
 	return func(w http.ResponseWriter, r *http.Request) {
 		reqCtx := r.Context()
-		resource := r.URL.Query().Get(resourceKey)
-		authTokenId := r.URL.Query().Get(authTokenIdKey)
+		q := r.URL.Query()
+		resource := q.Get(resourceKey)
+		authTokenId := q.Get(authTokenIdKey)
+		maxResultSetSizeStr := q.Get(maxResultSetSizeKey)
+		maxResultSetSizeInt, maxResultSetSizeIntErr := strconv.Atoi(maxResultSetSizeStr)
+		query := q.Get(queryKey)
+		filter := q.Get(filterKey)
 
 		searchableResource := cache.ToSearchableResource(resource)
 		switch {
@@ -71,6 +99,26 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 			event.WriteError(ctx, op, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("%s is a required field but was empty", authTokenIdKey)))
 			writeError(w, fmt.Sprintf("%s is a required field but was empty", authTokenIdKey), http.StatusBadRequest)
 			return
+		case maxResultSetSizeStr != "" && maxResultSetSizeIntErr != nil:
+			event.WriteError(ctx, op, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("%s is not able to be parsed as an integer", maxResultSetSizeStr)))
+			writeError(w, fmt.Sprintf("%s is not able to be parsed as an integer", maxResultSetSizeStr), http.StatusBadRequest)
+			return
+		case maxResultSetSizeInt < -1:
+			event.WriteError(ctx, op, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("%s must be greater than or equal to -1", maxResultSetSizeStr)))
+			writeError(w, fmt.Sprintf("%s must be greater than or equal to -1", maxResultSetSizeStr), http.StatusBadRequest)
+			return
+		case searchableResource == cache.ImplicitScopes && maxResultSetSizeStr != "":
+			event.WriteError(ctx, op, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("max result set size is not supported for resource %q", resource)))
+			writeError(w, fmt.Sprintf("max result set size is not supported for resource %q", resource), http.StatusBadRequest)
+			return
+		case searchableResource == cache.ImplicitScopes && query != "":
+			event.WriteError(ctx, op, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("query is not supported for resource %q", resource)))
+			writeError(w, fmt.Sprintf("query is not supported for resource %q", resource), http.StatusBadRequest)
+			return
+		case searchableResource == cache.ImplicitScopes && filter != "":
+			event.WriteError(ctx, op, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("filter is not supported for resource %q", resource)))
+			writeError(w, fmt.Sprintf("filter is not supported for resource %q", resource), http.StatusBadRequest)
+			return
 		}
 
 		t, err := repo.LookupToken(reqCtx, authTokenId, cache.WithUpdateLastAccessedTime(true))
@@ -102,22 +150,31 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 			opts = append(opts, cache.WithIgnoreSearchStaleness(true))
 		}
 
+		var refreshError error
 		// Refresh the resources for the provided user, if possible. This is best
 		// effort, so if there is any problem refreshing, we just log the error
 		// and move on to handling the search request.
-		if err := refreshService.RefreshForSearch(reqCtx, authTokenId, searchableResource, opts...); err != nil {
-			// we don't stop the search, we just log that the inline refresh failed
-			event.WriteError(ctx, op, err, event.WithInfoMsg("when refreshing the resources inline for search", "auth_token_id", authTokenId, "resource", searchableResource))
+		switch searchableResource {
+		case cache.ImplicitScopes:
+			// This is not able to be refreshed, so continue on
+		default:
+			refreshError = refreshService.RefreshForSearch(reqCtx, authTokenId, searchableResource, opts...)
+			switch {
+			case refreshError == nil,
+				stderrors.Is(refreshError, cache.ErrRefreshInProgress):
+				// Don't event in these cases
+			default:
+				// we don't stop the search, we just log that the inline refresh failed
+				event.WriteError(ctx, op, err, event.WithInfoMsg("when refreshing the resources inline for search", "auth_token_id", authTokenId, "resource", searchableResource))
+			}
 		}
 
-		query := r.URL.Query().Get(queryKey)
-		filter := r.URL.Query().Get(filterKey)
-
 		res, err := s.Search(reqCtx, cache.SearchParams{
-			AuthTokenId: authTokenId,
-			Resource:    searchableResource,
-			Query:       query,
-			Filter:      filter,
+			AuthTokenId:      authTokenId,
+			Resource:         searchableResource,
+			Query:            query,
+			Filter:           filter,
+			MaxResultSetSize: maxResultSetSizeInt,
 		})
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("when performing search", "auth_token_id", authTokenId, "resource", searchableResource, "query", query, "filter", filter))
@@ -135,6 +192,16 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 		}
 
 		apiRes := toApiResult(res)
+		switch {
+		case refreshError == nil:
+			apiRes.RefreshStatus = NotRefreshing
+		case stderrors.Is(refreshError, cache.ErrRefreshInProgress):
+			apiRes.RefreshStatus = Refreshing
+		default:
+			apiRes.RefreshStatus = RefreshError
+			apiRes.RefreshError = refreshError.Error()
+		}
+
 		j, err := json.Marshal(apiRes)
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("when marshaling search result to JSON", "auth_token_id", authTokenId, "resource", searchableResource, "query", query, "filter", filter))
@@ -152,6 +219,8 @@ func toApiResult(sr *cache.SearchResult) *SearchResult {
 		ResolvableAliases: sr.ResolvableAliases,
 		Targets:           sr.Targets,
 		Sessions:          sr.Sessions,
+		ImplicitScopes:    sr.ImplicitScopes,
+		Incomplete:        sr.Incomplete,
 	}
 }
 

internal/clientcache/internal/daemon/server.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -89,6 +90,8 @@ type Config struct {
 	// The maximum amount of time a refresh should block a search request from
 	// completing before it times out.
 	MaxSearchRefreshTimeout time.Duration
+	// Force resetting the schema, that is, drop all existing data
+	ForceResetSchema bool
 }
 
 func (sc *Config) validate(ctx context.Context) error {
@@ -228,7 +231,9 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 	var store *db.DB
 	store, err = openStore(ctx,
 		WithUrl(ctx, s.conf.DatabaseUrl),
-		WithLogger(ctx, s.logger))
+		WithLogger(ctx, s.logger),
+		WithForceResetSchema(ctx, s.conf.ForceResetSchema),
+	)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
@@ -499,6 +504,10 @@ func setupEventing(ctx context.Context, logger hclog.Logger, serializationLock *
 	return nil
 }
 
+// openStore will open the underlying store for the db. If no options are
+// provided, it will default to an on disk store using the user's home dir +
+// ".boundary/cache.db". If a url is provided, it will use that as the store.
+// Supported options: WithUrl, WithLogger, WithHomeDir
 func openStore(ctx context.Context, opt ...Option) (*db.DB, error) {
 	const op = "daemon.openStore"
 	opts, err := getOpts(opt...)
@@ -514,13 +523,69 @@ func openStore(ctx context.Context, opt ...Option) (*db.DB, error) {
 			return nil, errors.Wrap(ctx, err, op)
 		}
 		dbOpts = append(dbOpts, cachedb.WithUrl(url))
+	default:
+		url, err := defaultDbUrl(ctx, opt...)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op)
+		}
+		dbOpts = append(dbOpts, cachedb.WithUrl(url))
 	}
 	if !util.IsNil(opts.withLogger) {
-		dbOpts = append(dbOpts, cachedb.WithGormFormatter(opts.withLogger))
+		opts.withLogger.Log(hclog.Debug, "Store GormFormatter", "LogLevel", opts.withLogger.GetLevel())
+		switch {
+		case opts.withLogger.IsDebug():
+			dbOpts = append(dbOpts, cachedb.WithGormFormatter(gormDebugLogger{Logger: opts.withLogger}))
+			dbOpts = append(dbOpts, cachedb.WithDebug(true))
+		default:
+			dbOpts = append(dbOpts, cachedb.WithGormFormatter(opts.withLogger))
+		}
 	}
+	dbOpts = append(dbOpts, cachedb.WithForceResetSchema(opts.withForceResetSchema))
 	store, err := cachedb.Open(ctx, dbOpts...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
 	return store, nil
 }
+
+// defaultDbUrl returns the default db name including the path. It will ensure
+// the directory exists by creating it if it doesn't.
+func defaultDbUrl(ctx context.Context, opt ...Option) (string, error) {
+	const op = "daemon.DefaultDotDirectory"
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return "", errors.Wrap(ctx, err, op)
+	}
+	if opts.withHomeDir == "" {
+		opts.withHomeDir, err = os.UserHomeDir()
+		if err != nil {
+			return "", errors.Wrap(ctx, err, op)
+		}
+	}
+	dotDir := filepath.Join(opts.withHomeDir, dotDirname)
+	if err := os.MkdirAll(dotDir, 0o700); err != nil {
+		return "", errors.Wrap(ctx, err, op)
+	}
+	fileName := filepath.Join(dotDir, dbFileName)
+	return fmt.Sprintf("%s%s", fileName, fkPragma), nil
+}
+
+const (
+	dotDirname = ".boundary"
+	dbFileName = "cache.db"
+	fkPragma   = "?_pragma=foreign_keys(1)"
+)
+
+type gormDebugLogger struct {
+	hclog.Logger
+}
+
+func (g gormDebugLogger) Printf(msg string, values ...any) {
+	b := new(strings.Builder)
+	fmt.Fprintf(b, msg, values...)
+	g.Debug(b.String())
+}
+
+func getGormLogger(log hclog.Logger) gormDebugLogger {
+	return gormDebugLogger{Logger: log}
+}

internal/clientcache/internal/daemon/server_test.go

@@ -5,16 +5,68 @@ package daemon
 
 import (
 	"context"
+	"strings"
+	"sync"
 	"testing"
 
 	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/roles"
 	"github.com/hashicorp/boundary/internal/cmd/base"
 	"github.com/hashicorp/boundary/internal/daemon/controller"
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func Test_openStore(t *testing.T) {
+	ctx := context.Background()
+	t.Run("success", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		store, err := openStore(ctx, WithUrl(ctx, tmpDir+"/test.db"+fkPragma))
+		require.NoError(t, err)
+		require.NotNil(t, store)
+		assert.FileExists(t, tmpDir+"/test.db")
+		rw := db.New(store)
+		rows, err := rw.Query(ctx, "select * from target", nil)
+		require.NoError(t, err)
+		rows.Close()
+	})
+	t.Run("homedir", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		db, err := openStore(ctx, WithHomeDir(ctx, tmpDir))
+		require.NoError(t, err)
+		require.NotNil(t, db)
+		assert.FileExists(t, tmpDir+"/"+dotDirname+"/"+dbFileName)
+	})
+	t.Run("log-level-debug", func(t *testing.T) {
+		buf := new(strings.Builder)
+		testLock := &sync.Mutex{}
+		testLogger := hclog.New(&hclog.LoggerOptions{
+			Mutex:      testLock,
+			Name:       "test",
+			JSONFormat: true,
+			Output:     buf,
+			Level:      hclog.Debug,
+		})
+		tmpDir := t.TempDir()
+		store, err := openStore(ctx,
+			WithUrl(ctx, tmpDir+"/test.db"+fkPragma),
+			WithLogger(ctx, testLogger),
+		)
+		require.NoError(t, err)
+		require.NotNil(t, store)
+		assert.FileExists(t, tmpDir+"/test.db")
+		rw := db.New(store)
+
+		rows, err := rw.Query(ctx, "select * from target", nil)
+		require.NoError(t, err)
+		defer rows.Close()
+		assert.Contains(t, buf.String(), "select * from target")
+		t.Log(buf.String())
+	})
+}
+
 // Note: the name of this test must remain short because the temp dir created
 // includes the name of the test and there is a 108 character limit in allowed
 // unix socket path names.

internal/clientcache/internal/daemon/testing.go

@@ -37,6 +37,9 @@ func NewTestServer(t *testing.T, cmd Commander, opt ...Option) *TestServer {
 		RecheckSupportInterval: DefaultRecheckSupportInterval,
 		LogWriter:              io.Discard,
 		DotDirectory:           dotDir,
+		// we need to provide this, otherwise it will open a store in the user's
+		// home dir. See db.Open(...)
+		DatabaseUrl: dotDir + "cache.db?_pragma=foreign_keys(1)",
 	}
 
 	s, err := New(ctx, cfg)
@@ -88,23 +91,29 @@ func (s *TestServer) AddResources(t *testing.T, p *authtokens.AuthToken, alts []
 	r, err := cache.NewRepository(ctx, s.CacheServer.store.Load(), &sync.Map{}, s.cmd.ReadTokenFromKeyring, atReadFn)
 	require.NoError(t, err)
 
-	altFn := func(ctx context.Context, _, tok, _ string, _ cache.RefreshTokenValue) ([]*aliases.Alias, []string, cache.RefreshTokenValue, error) {
+	altFn := func(ctx context.Context, _ string, tok, _ string, _ cache.RefreshTokenValue, inPage *aliases.AliasListResult, opt ...cache.Option) (*aliases.AliasListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return nil, "", nil
 		}
-		return alts, nil, "addedaliases", nil
+		return &aliases.AliasListResult{
+			Items: alts,
+		}, "addedaliases", nil
 	}
-	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) {
+	tarFn := func(ctx context.Context, _ string, tok string, _ cache.RefreshTokenValue, inPage *targets.TargetListResult, opt ...cache.Option) (*targets.TargetListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return nil, "", nil
 		}
-		return tars, nil, "addedtargets", nil
+		return &targets.TargetListResult{
+			Items: tars,
+		}, "addedtargets", nil
 	}
-	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) {
+	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue, inPage *sessions.SessionListResult, opt ...cache.Option) (*sessions.SessionListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return nil, "", nil
 		}
-		return sess, nil, "addedsessions", nil
+		return &sessions.SessionListResult{
+			Items: sess,
+		}, "addedsessions", nil
 	}
 	rs, err := cache.NewRefreshService(ctx, r, hclog.NewNullLogger(), 0, 0)
 	require.NoError(t, err)
@@ -121,19 +130,25 @@ func (s *TestServer) AddUnsupportedCachingData(t *testing.T, p *authtokens.AuthT
 	r, err := cache.NewRepository(ctx, s.CacheServer.store.Load(), &sync.Map{}, s.cmd.ReadTokenFromKeyring, atReadFn)
 	require.NoError(t, err)
 
-	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) {
+	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue, inPage *targets.TargetListResult, opt ...cache.Option) (*targets.TargetListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return &targets.TargetListResult{}, "", nil
 		}
-		return []*targets.Target{
-			{Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"},
-		}, nil, "", cache.ErrRefreshNotSupported
+		return &targets.TargetListResult{
+			Items: []*targets.Target{
+				{Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"},
+			},
+		}, "", cache.ErrRefreshNotSupported
 	}
-	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) {
+	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue, inPage *sessions.SessionListResult, opt ...cache.Option) (*sessions.SessionListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return &sessions.SessionListResult{}, "", nil
 		}
-		return []*sessions.Session{}, nil, "", cache.ErrRefreshNotSupported
+		return &sessions.SessionListResult{
+			Items: []*sessions.Session{
+				{Id: "s_unsupported"},
+			},
+		}, "", cache.ErrRefreshNotSupported
 	}
 	rs, err := cache.NewRefreshService(ctx, r, hclog.NewNullLogger(), 0, 0)
 	require.NoError(t, err)

internal/clientcache/internal/db/db.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	_ "embed"
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/errors"
@@ -17,11 +19,16 @@ import (
 //go:embed schema.sql
 var cacheSchema string
 
+//go:embed schema_reset.sql
+var cacheSchemaReset string
+
 // DefaultStoreUrl uses a temp in-memory sqlite database see: https://www.sqlite.org/inmemorydb.html
 const DefaultStoreUrl = "file::memory:?_pragma=foreign_keys(1)"
 
 // Open creates a database connection. WithUrl is supported, but by default it
 // uses an in memory sqlite table. Sqlite is the only supported dbtype.
+// Supported options: WithUrl, WithGormFormatter, WithDebug,
+// WithTestValidSchemaVersion (for testing purposes)
 func Open(ctx context.Context, opt ...Option) (*db.DB, error) {
 	const op = "db.Open"
 	opts, err := getOpts(opt...)
@@ -47,19 +54,44 @@ func Open(ctx context.Context, opt ...Option) (*db.DB, error) {
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}
-	conn.Debug(opts.withDebug)
-
+	defer func() {
+		// let's not capture the output of resetSchema and createTables, so
+		// we'll defer turning on debug
+		conn.Debug(opts.withDebug)
+	}()
 	switch {
-	case opts.withDbType == dbw.Sqlite:
+	case opts.withDbType == dbw.Sqlite && url == DefaultStoreUrl:
 		if err := createTables(ctx, conn); err != nil {
 			return nil, errors.Wrap(ctx, err, op)
 		}
+	case opts.withDbType == dbw.Sqlite && url != DefaultStoreUrl:
+		ok, err := validSchema(ctx, conn, opt...)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op)
+		}
+		if !ok || opts.withForceResetSchema {
+			if err := resetSchema(ctx, conn); err != nil {
+				return nil, errors.Wrap(ctx, err, op)
+			}
+			if err := createTables(ctx, conn); err != nil {
+				return nil, errors.Wrap(ctx, err, op)
+			}
+		}
 	default:
 		return nil, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("%q is not a supported cache store type", opts.withDbType))
 	}
 	return conn, nil
 }
 
+func resetSchema(ctx context.Context, conn *db.DB) error {
+	const op = "db.resetSchema"
+	rw := db.New(conn)
+	if _, err := rw.Exec(ctx, cacheSchemaReset, nil); err != nil {
+		return errors.Wrap(ctx, err, op)
+	}
+	return nil
+}
+
 func createTables(ctx context.Context, conn *db.DB) error {
 	const op = "db.createTables"
 	rw := db.New(conn)
@@ -68,3 +100,56 @@ func createTables(ctx context.Context, conn *db.DB) error {
 	}
 	return nil
 }
+
+// validSchema checks of the schema is valid based on its version. Options
+// supported: withTestValidSchemaVersion (for testing purposes)
+func validSchema(ctx context.Context, conn *db.DB, opt ...Option) (bool, error) {
+	const op = "validateSchema"
+	switch {
+	case conn == nil:
+		return false, errors.New(ctx, errors.InvalidParameter, op, "conn is missing")
+	}
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return false, errors.Wrap(ctx, err, op)
+	}
+	if opts.withSchemaVersion == "" {
+		opts.withSchemaVersion = schemaCurrentVersion
+	}
+
+	rw := db.New(conn)
+	s := schema{}
+	err = rw.LookupWhere(ctx, &s, "1=1", nil)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "no such table: schema_version"):
+		return false, nil
+	case err != nil:
+		// not sure if we should return the error or just return false so the
+		// schema is recreated... for now return the error.
+		return false, fmt.Errorf("%s: unable to get version: %w", op, err)
+	case s.Version != opts.withSchemaVersion:
+		return false, nil
+	default:
+		return true, nil
+	}
+}
+
+// schema represents the current schema in the database
+type schema struct {
+	// Version of the schema
+	Version string
+	// UpdateTime is the last update of the version
+	UpdateTime time.Time
+	// CreateTime is the create time of the initial version
+	CreateTime time.Time
+}
+
+const (
+	schemaTableName      = "schema_version"
+	schemaCurrentVersion = "v0.0.2"
+)
+
+// TableName returns the table name
+func (s *schema) TableName() string {
+	return schemaTableName
+}

internal/clientcache/internal/db/db_test.go

@@ -0,0 +1,66 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package db
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOpen(t *testing.T) {
+	ctx := context.Background()
+	t.Run("success-file-url-with-reopening", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		db, err := Open(ctx, WithUrl(tmpDir+"/test.db"+fkPragma))
+		require.NoError(t, err)
+		require.NotNil(t, db)
+		assert.FileExists(t, tmpDir+"/test.db")
+
+		info, err := os.Stat(tmpDir + "/test.db")
+		require.NoError(t, err)
+		origCreatedAt := info.ModTime()
+
+		// Reopen the db and make sure the file is not recreated
+		db, err = Open(ctx, WithUrl(tmpDir+"/test.db"+fkPragma))
+		require.NoError(t, err)
+		require.NotNil(t, db)
+		info, err = os.Stat(tmpDir + "/test.db")
+		require.NoError(t, err)
+		assert.Equal(t, origCreatedAt, info.ModTime())
+	})
+	t.Run("success-mem-default-url", func(t *testing.T) {
+		db, err := Open(ctx)
+		require.NoError(t, err)
+		require.NotNil(t, db)
+	})
+	t.Run("recreate-on-version-mismatch", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		db, err := Open(ctx, WithUrl(tmpDir+"/test.db"+fkPragma))
+		require.NoError(t, err)
+		require.NotNil(t, db)
+		assert.FileExists(t, tmpDir+"/test.db")
+		info, err := os.Stat(tmpDir + "/test.db")
+		require.NoError(t, err)
+		origCreatedAt := info.ModTime()
+
+		// Reopen the db with a different schema version: forcing the db to be recreated
+		db, err = Open(ctx, WithUrl(tmpDir+"/test.db"+fkPragma), withTestValidSchemaVersion("2"))
+		require.NoError(t, err)
+		require.NotNil(t, db)
+		info, err = os.Stat(tmpDir + "/test.db")
+		require.NoError(t, err)
+		// The file should have been recreated with a new timestamp
+		assert.NotEqual(t, origCreatedAt, info.ModTime())
+	})
+}
+
+const (
+	dotDirname = ".boundary"
+	dbFileName = "cache.db"
+	fkPragma   = "?_pragma=foreign_keys(1)"
+)

internal/clientcache/internal/db/options.go

@@ -9,10 +9,12 @@ import (
 )
 
 type options struct {
-	withDebug         bool
-	withUrl           string
-	withDbType        dbw.DbType
-	withGormFormatter hclog.Logger
+	withSchemaVersion    string
+	withDebug            bool
+	withUrl              string
+	withDbType           dbw.DbType
+	withGormFormatter    hclog.Logger
+	withForceResetSchema bool
 }
 
 // Option - how options are passed as args
@@ -42,6 +44,15 @@ func WithGormFormatter(logger hclog.Logger) Option {
 	}
 }
 
+// withTestValidSchemaVersion provides optional valid schema version for testing
+// purposes. This is used to simulate a schema version that is valid/invalid.
+func withTestValidSchemaVersion(useVersion string) Option {
+	return func(o *options) error {
+		o.withSchemaVersion = useVersion
+		return nil
+	}
+}
+
 // WithUrls provides optional url
 func WithUrl(url string) Option {
 	return func(o *options) error {
@@ -57,3 +68,11 @@ func WithDebug(debug bool) Option {
 		return nil
 	}
 }
+
+// WithForceResetSchema provides an optional way to force resetting the cache
+func WithForceResetSchema(debug bool) Option {
+	return func(o *options) error {
+		o.withForceResetSchema = debug
+		return nil
+	}
+}

internal/clientcache/internal/db/options_test.go

@@ -37,4 +37,20 @@ func Test_GetOpts(t *testing.T) {
 		testOpts.withDebug = true
 		assert.Equal(t, opts, testOpts)
 	})
+	t.Run("withTestValidSchemaVersion", func(t *testing.T) {
+		version := "v1"
+		opts, err := getOpts(withTestValidSchemaVersion(version))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		testOpts.withSchemaVersion = version
+		assert.Equal(t, opts, testOpts)
+	})
+	t.Run("WithForceResetSchema", func(t *testing.T) {
+		opts, err := getOpts(WithForceResetSchema(true))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.False(t, testOpts.withForceResetSchema)
+		testOpts.withForceResetSchema = true
+		assert.Equal(t, opts, testOpts)
+	})
 }

internal/clientcache/internal/db/schema.sql

@@ -2,6 +2,40 @@
 -- SPDX-License-Identifier: BUSL-1.1
 
 begin;
+
+-- schema_version is a one row table to keep the version
+create table if not exists schema_version (
+    version text not null,
+    create_time timestamp not null default current_timestamp,
+    update_time timestamp not null default current_timestamp
+);
+
+-- ensure that it's only ever one row
+create unique index schema_version_one_row
+ON schema_version((version is not null));
+
+create trigger immutable_columns_schema_version
+before update on schema_version
+for each row 
+  when 
+    new.create_time <> old.create_time 
+	begin
+	  select raise(abort, 'immutable column');
+	end;
+
+
+create trigger update_time_column_schema_version
+before update on schema_version
+for each row 
+when 
+  new.version <> old.version 
+  begin
+    update schema_version set update_time = datetime('now','localtime') where rowid == new.rowid;
+  end;
+
+
+insert into schema_version(version) values('v0.0.2');
+
 -- user contains the boundary user information for the boundary user that owns
 -- the information in the cache.
 create table if not exists user (
@@ -129,6 +163,9 @@ create table if not exists target (
   primary key (fk_user_id, id)
 );
 
+-- index for implicit scope search
+create index target_scope_id_ix on target(scope_id);
+
 -- session contains cached boundary session resource for a specific user and
 -- with specific fields extracted to facilitate searching over those fields
 create table if not exists session (
@@ -156,6 +193,9 @@ create table if not exists session (
   primary key (fk_user_id, id)
 );
 
+-- implicit scope search
+create index session_scope_id_ix on session(scope_id);
+
 -- alias contains cached boundary alias resource for a specific user and
 -- with specific fields extracted to facilitate searching over those fields
 create table if not exists resolvable_alias (
@@ -177,6 +217,9 @@ create table if not exists resolvable_alias (
   primary key (fk_user_id, id)
 );
 
+-- optimize query for destination_id
+create index destination_id_resolvable_alias_ix on resolvable_alias(destination_id);
+
 -- contains errors from the last attempt to sync data from boundary for a
 -- specific resource type
 create table if not exists api_error (

internal/clientcache/internal/db/schema_reset.sql

@@ -0,0 +1,11 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+-- cannot vacuum from within a transaction, so we're not using a transaction 
+-- when running these statements
+PRAGMA writable_schema = 1;
+DELETE FROM sqlite_master;
+PRAGMA writable_schema = 0;
+VACUUM;
+PRAGMA integrity_check;
+

internal/cmd/commands/rolescmd/funcs.go

@@ -259,7 +259,7 @@ func extraFlagsHandlingFuncImpl(c *Command, _ *base.FlagSets, opts *[]roles.Opti
 
 	if len(c.flagGrants) > 0 {
 		for _, grant := range c.flagGrants {
-			parsed, err := perms.Parse(c.Context, scope.Global.String(), grant)
+			parsed, err := perms.Parse(c.Context, perms.GrantTuple{RoleScopeId: scope.Global.String(), GrantScopeId: scope.Global.String(), Grant: grant})
 			if err != nil {
 				c.UI.Error(fmt.Errorf("Grant %q could not be parsed successfully: %w", grant, err).Error())
 				return false

internal/daemon/controller/auth/auth.go

@@ -273,6 +273,7 @@ func Verify(ctx context.Context, opt ...Option) (ret VerifyResults) {
 		Id:      opts.withId,
 		Pin:     opts.withPin,
 		Type:    opts.withType,
+		// Parent Scope ID will be filled in via performAuthCheck
 	}
 	// Global scope has no parent ID; account for this
 	if opts.withId == scope.Global.String() && opts.withType == resource.Scope {
@@ -336,7 +337,7 @@ func Verify(ctx context.Context, opt ...Option) (ret VerifyResults) {
 		grants = append(grants, event.Grant{
 			Grant:   g.Grant,
 			RoleId:  g.RoleId,
-			ScopeId: g.ScopeId,
+			ScopeId: g.GrantScopeId,
 		})
 	}
 	ea.UserInfo = &event.UserInfo{
@@ -636,6 +637,7 @@ func (v verifier) performAuthCheck(ctx context.Context) (
 			ParentScopeId: scp.GetParentId(),
 		}
 	}
+	v.res.ParentScopeId = scopeInfo.ParentScopeId
 
 	// At this point we don't need to look up grants since it's automatically allowed
 	if v.requestInfo.TokenFormat == uint32(AuthTokenTypeRecoveryKms) {
@@ -658,7 +660,7 @@ func (v verifier) performAuthCheck(ctx context.Context) (
 	// Note: Below, we always skip validation so that we don't error on formats
 	// that we've since restricted, e.g. "ids=foo;actions=create,read". These
 	// will simply not have an effect.
-	for _, pair := range grantTuples {
+	for _, tuple := range grantTuples {
 		permsOpts := []perms.Option{
 			perms.WithUserId(*userData.User.Id),
 			perms.WithSkipFinalValidation(true),
@@ -668,11 +670,10 @@ func (v verifier) performAuthCheck(ctx context.Context) (
 		}
 		parsed, err := perms.Parse(
 			ctx,
-			pair.ScopeId,
-			pair.Grant,
+			tuple,
 			permsOpts...)
 		if err != nil {
-			retErr = errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("failed to parse grant %#v", pair.Grant)))
+			retErr = errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("failed to parse grant %#v", tuple.Grant)))
 			return
 		}
 		parsedGrants = append(parsedGrants, parsed)
@@ -867,7 +868,7 @@ func (r *VerifyResults) ScopesAuthorizedForList(ctx context.Context, rootScopeId
 		aSet := r.FetchActionSetForType(ctx,
 			resource.Unknown, // This is overridden by `WithResource` option.
 			action.NewActionSet(action.List),
-			WithResource(&perms.Resource{Type: resourceType, ScopeId: scpId}),
+			WithResource(&perms.Resource{Type: resourceType, ScopeId: scpId, ParentScopeId: scp.GetParentId()}),
 		)
 
 		// We only expect the action set to be nothing, or list. In case

internal/daemon/controller/auth/authorized_actions.go

@@ -9,6 +9,7 @@ import (
 	"github.com/hashicorp/boundary/internal/perms"
 	"github.com/hashicorp/boundary/internal/types/action"
 	"github.com/hashicorp/boundary/internal/types/resource"
+	"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/scopes"
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 	"google.golang.org/protobuf/types/known/structpb"
 )
@@ -22,11 +23,12 @@ import (
 func CalculateAuthorizedCollectionActions(ctx context.Context,
 	authResults VerifyResults,
 	mapToRange map[resource.Type]action.ActionSet,
-	scopeId, pin string,
+	scopeInfo *scopes.ScopeInfo, pin string,
 ) (map[string]*structpb.ListValue, error) {
 	res := &perms.Resource{
-		ScopeId: scopeId,
-		Pin:     pin,
+		ScopeId:       scopeInfo.GetId(),
+		Pin:           pin,
+		ParentScopeId: scopeInfo.GetParentScopeId(),
 	}
 	// Range over the defined collections and check permissions against those
 	// collections.

internal/daemon/controller/handlers/accounts/account_service_test.go

@@ -4385,3 +4385,118 @@ func TestChangePassword(t *testing.T) {
 		})
 	}
 }
+
+// The purpose of this test is mainly to ensure that we are properly fetching
+// membership information in GrantsForUser across managed group types
+func TestGrantsAcrossManagedGroups(t *testing.T) {
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrap := db.TestWrapper(t)
+	kmsCache := kms.TestKms(t, conn, wrap)
+
+	org, _ := iam.TestScopes(t, iam.TestRepo(t, conn, wrap))
+
+	databaseWrapper, err := kmsCache.GetWrapper(ctx, org.PublicId, kms.KeyPurposeDatabase)
+	require.NoError(t, err)
+	oidcAm := oidc.TestAuthMethod(
+		t, conn, databaseWrapper, org.PublicId, oidc.ActivePrivateState,
+		"alice-rp", "fido",
+		oidc.WithIssuer(oidc.TestConvertToUrls(t, "https://www.alice.com")[0]),
+		oidc.WithSigningAlgs(oidc.RS256),
+		oidc.WithApiUrl(oidc.TestConvertToUrls(t, "https://www.alice.com/callback")[0]),
+	)
+	oidcAcct := oidc.TestAccount(t, conn, oidcAm, "test-subject")
+	// Create a managed group that will always match, so we can test that it is
+	// returned in results
+	oidcMg := oidc.TestManagedGroup(t, conn, oidcAm, `"/token/sub" matches ".*"`)
+	oidc.TestManagedGroupMember(t, conn, oidcMg.GetPublicId(), oidcAcct.GetPublicId())
+
+	ldapAm := ldap.TestAuthMethod(t, conn, databaseWrapper, org.PublicId, []string{"ldaps://ldap1"})
+	ldapAcct := ldap.TestAccount(t, conn, ldapAm, "test-acct",
+		ldap.WithMemberOfGroups(ctx, "admin"),
+		ldap.WithFullName(ctx, "test-name"),
+		ldap.WithEmail(ctx, "test-email"),
+		ldap.WithDn(ctx, "test-dn"),
+	)
+	ldapMg := ldap.TestManagedGroup(t, conn, ldapAm, []string{"admin"})
+
+	iamRepoFn := func() (*iam.Repository, error) {
+		return iam.NewRepository(ctx, rw, rw, kmsCache)
+	}
+	iamRepo, err := iamRepoFn()
+	require.NoError(t, err)
+
+	user := iam.TestUser(t, iamRepo, org.PublicId, iam.WithAccountIds(oidcAcct.GetPublicId(), ldapAcct.GetPublicId()))
+
+	// Create two roles, each containing a single managed group, and add a
+	// unique grant to each
+	oidcRole := iam.TestRole(t, conn, org.GetPublicId(), iam.WithGrantScopeIds([]string{globals.GrantScopeChildren}))
+	iam.TestManagedGroupRole(t, conn, oidcRole.GetPublicId(), oidcMg.GetPublicId())
+	iam.TestRoleGrant(t, conn, oidcRole.GetPublicId(), "ids=ttcp_oidc;actions=read")
+	ldapRole := iam.TestRole(t, conn, org.GetPublicId(), iam.WithGrantScopeIds([]string{globals.GrantScopeChildren}))
+	iam.TestManagedGroupRole(t, conn, ldapRole.GetPublicId(), ldapMg.GetPublicId())
+	iam.TestRoleGrant(t, conn, ldapRole.GetPublicId(), "ids=ttcp_ldap;actions=read")
+
+	grants, err := iamRepo.GrantsForUser(ctx, user.GetPublicId())
+	require.NoError(t, err)
+
+	// Verify we see both grants
+	var foundOidc, foundLdap bool
+	for _, grant := range grants {
+		if grant.Grant == "ids=ttcp_oidc;actions=read" {
+			foundOidc = true
+		}
+		if grant.Grant == "ids=ttcp_ldap;actions=read" {
+			foundLdap = true
+		}
+	}
+	assert.True(t, foundOidc)
+	assert.True(t, foundLdap)
+
+	// Delete the ldap managed group
+	ldapRepo, err := ldap.NewRepository(ctx, rw, rw, kmsCache)
+	require.NoError(t, err)
+	numDeleted, err := ldapRepo.DeleteManagedGroup(ctx, org.GetPublicId(), ldapMg.GetPublicId())
+	require.NoError(t, err)
+	assert.Equal(t, 1, numDeleted)
+
+	// Verify we don't see the ldap grant anymore
+	grants, err = iamRepo.GrantsForUser(ctx, user.GetPublicId())
+	require.NoError(t, err)
+	foundOidc = false
+	foundLdap = false
+	for _, grant := range grants {
+		if grant.Grant == "ids=ttcp_oidc;actions=read" {
+			foundOidc = true
+		}
+		if grant.Grant == "ids=ttcp_ldap;actions=read" {
+			foundLdap = true
+		}
+	}
+	assert.True(t, foundOidc)
+	assert.False(t, foundLdap)
+
+	// Delete the oidc managed group
+	oidcRepo, err := oidc.NewRepository(ctx, rw, rw, kmsCache)
+	require.NoError(t, err)
+	numDeleted, err = oidcRepo.DeleteManagedGroup(ctx, org.GetPublicId(), oidcMg.GetPublicId())
+	require.NoError(t, err)
+	assert.Equal(t, 1, numDeleted)
+
+	// Verify we don't see the oidc grant anymore
+	grants, err = iamRepo.GrantsForUser(ctx, user.GetPublicId())
+	require.NoError(t, err)
+	foundOidc = false
+	foundLdap = false
+	for _, grant := range grants {
+		if grant.Grant == "ids=ttcp_oidc;actions=read" {
+			foundOidc = true
+		}
+		if grant.Grant == "ids=ttcp_ldap;actions=read" {
+			foundLdap = true
+		}
+	}
+	assert.False(t, foundOidc)
+	assert.False(t, foundLdap)
+}

internal/daemon/controller/handlers/authmethods/authmethod_service.go

@@ -343,7 +343,7 @@ func (s Service) GetAuthMethod(ctx context.Context, req *pbs.GetAuthMethodReques
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, am.GetPublicId(), IdActions[globals.ResourceInfoFromPrefix(am.GetPublicId()).Subtype]).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, am.GetPublicId())
+		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope, am.GetPublicId())
 		if err != nil {
 			return nil, err
 		}
@@ -388,7 +388,7 @@ func (s Service) CreateAuthMethod(ctx context.Context, req *pbs.CreateAuthMethod
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, am.GetPublicId(), IdActions[globals.ResourceInfoFromPrefix(am.GetPublicId()).Subtype]).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, am.GetPublicId())
+		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope, am.GetPublicId())
 		if err != nil {
 			return nil, err
 		}
@@ -438,7 +438,7 @@ func (s Service) UpdateAuthMethod(ctx context.Context, req *pbs.UpdateAuthMethod
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, am.GetPublicId(), IdActions[globals.ResourceInfoFromPrefix(am.GetPublicId()).Subtype]).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, am.GetPublicId())
+		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope, am.GetPublicId())
 		if err != nil {
 			return nil, err
 		}
@@ -492,7 +492,7 @@ func (s Service) ChangeState(ctx context.Context, req *pbs.ChangeStateRequest) (
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, am.GetPublicId(), IdActions[globals.ResourceInfoFromPrefix(am.GetPublicId()).Subtype]).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, am.GetPublicId())
+		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope, am.GetPublicId())
 		if err != nil {
 			return nil, err
 		}
@@ -1442,6 +1442,10 @@ func (s Service) convertToAuthenticateResponse(ctx context.Context, req *pbs.Aut
 		ScopeId: authResults.Scope.Id,
 		Type:    resource.AuthToken,
 	}
+	// Auth methods are only at global or org, so we can figure out the parent
+	if strings.HasPrefix(res.ScopeId, scope.Org.Prefix()) {
+		res.ParentScopeId = scope.Global.String()
+	}
 	tokenType := req.GetType()
 	if tokenType == "" {
 		// Fall back to deprecated field if type is not set
@@ -1588,7 +1592,7 @@ func newOutputOpts(ctx context.Context, item auth.AuthMethod, scopeInfoMap map[s
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope.Id, item.GetPublicId())
+		collectionActions, err := requestauth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap, authResults.Scope, item.GetPublicId())
 		if err != nil {
 			return nil, false, err
 		}

internal/daemon/controller/handlers/credentialstores/credentialstore_service.go

@@ -996,10 +996,10 @@ func calculateAuthorizedCollectionActions(ctx context.Context, authResults auth.
 	var err error
 	switch globals.ResourceInfoFromPrefix(id).Subtype {
 	case vault.Subtype:
-		collectionActions, err = auth.CalculateAuthorizedCollectionActions(ctx, authResults, vaultCollectionTypeMap, authResults.Scope.Id, id)
+		collectionActions, err = auth.CalculateAuthorizedCollectionActions(ctx, authResults, vaultCollectionTypeMap, authResults.Scope, id)
 
 	case static.Subtype:
-		collectionActions, err = auth.CalculateAuthorizedCollectionActions(ctx, authResults, staticCollectionTypeMap, authResults.Scope.Id, id)
+		collectionActions, err = auth.CalculateAuthorizedCollectionActions(ctx, authResults, staticCollectionTypeMap, authResults.Scope, id)
 	}
 	if err != nil {
 		return nil, err

internal/daemon/controller/handlers/host_catalogs/host_catalog_service.go

@@ -342,7 +342,7 @@ func (s Service) GetHostCatalog(ctx context.Context, req *pbs.GetHostCatalogRequ
 			subtype = hostplugin.Subtype
 		}
 		if subtype != "" {
-			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope.Id, hc.GetPublicId())
+			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope, hc.GetPublicId())
 			if err != nil {
 				return nil, err
 			}
@@ -399,7 +399,7 @@ func (s Service) CreateHostCatalog(ctx context.Context, req *pbs.CreateHostCatal
 			subtype = hostplugin.Subtype
 		}
 		if subtype != "" {
-			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope.Id, hc.GetPublicId())
+			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope, hc.GetPublicId())
 			if err != nil {
 				return nil, err
 			}
@@ -462,7 +462,7 @@ func (s Service) UpdateHostCatalog(ctx context.Context, req *pbs.UpdateHostCatal
 			subtype = hostplugin.Subtype
 		}
 		if subtype != "" {
-			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope.Id, hc.GetPublicId())
+			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope, hc.GetPublicId())
 			if err != nil {
 				return nil, err
 			}
@@ -795,7 +795,7 @@ func newOutputOpts(
 			subtype = hostplugin.Subtype
 		}
 		if subtype != "" {
-			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope.Id, item.GetPublicId())
+			collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, collectionTypeMap[subtype], authResults.Scope, item.GetPublicId())
 			if err != nil {
 				return nil, false, err
 			}

internal/daemon/controller/handlers/roles/role_service.go

@@ -1123,7 +1123,7 @@ func toProto(ctx context.Context, in *iam.Role, principals []*iam.PrincipalRole,
 	}
 	if outputFields.Has(globals.GrantsField) {
 		for _, g := range grants {
-			parsed, err := perms.Parse(ctx, in.GetScopeId(), g.GetRawGrant())
+			parsed, err := perms.Parse(ctx, perms.GrantTuple{RoleScopeId: in.GetPublicId(), GrantScopeId: in.GetScopeId(), Grant: g.GetRawGrant()})
 			if err != nil {
 				// This should never happen as we validate on the way in, but let's
 				// return what we can since we are still returning the raw grant
@@ -1319,7 +1319,7 @@ func validateAddRoleGrantsRequest(ctx context.Context, req *pbs.AddRoleGrantsReq
 			badFields["grant_strings"] = "Grant strings must not be empty."
 			break
 		}
-		grant, err := perms.Parse(ctx, "p_anything", v)
+		grant, err := perms.Parse(ctx, perms.GrantTuple{RoleScopeId: req.GetId(), GrantScopeId: "p_anything", Grant: v})
 		if err != nil {
 			badFields["grant_strings"] = fmt.Sprintf("Improperly formatted grant %q.", v)
 			break
@@ -1356,9 +1356,9 @@ func validateSetRoleGrantsRequest(ctx context.Context, req *pbs.SetRoleGrantsReq
 			badFields["grant_strings"] = "Grant strings must not be empty."
 			break
 		}
-		grant, err := perms.Parse(ctx, "p_anything", v)
+		grant, err := perms.Parse(ctx, perms.GrantTuple{RoleScopeId: req.GetId(), GrantScopeId: "p_anything", Grant: v})
 		if err != nil {
-			badFields["grant_strings"] = fmt.Sprintf("Improperly formatted grant %q.", v)
+			badFields["grant_strings"] = fmt.Sprintf("Improperly formatted grant %q: %s.", v, err.Error())
 			break
 		}
 		_, actStrs := grant.Actions()
@@ -1396,7 +1396,7 @@ func validateRemoveRoleGrantsRequest(ctx context.Context, req *pbs.RemoveRoleGra
 			badFields["grant_strings"] = "Grant strings must not be empty."
 			break
 		}
-		if _, err := perms.Parse(ctx, "p_anything", v); err != nil {
+		if _, err := perms.Parse(ctx, perms.GrantTuple{RoleScopeId: req.GetId(), GrantScopeId: "p_anything", Grant: v}); err != nil {
 			badFields["grant_strings"] = fmt.Sprintf("Improperly formatted grant %q.", v)
 			break
 		}

internal/daemon/controller/handlers/roles/role_service_test.go

@@ -1026,7 +1026,7 @@ func TestCreate(t *testing.T) {
 func TestUpdate(t *testing.T) {
 	ctx := context.Background()
 	grantString := "ids=*;type=*;actions=*"
-	g, err := perms.Parse(context.Background(), "global", grantString)
+	g, err := perms.Parse(context.Background(), perms.GrantTuple{RoleScopeId: "global", GrantScopeId: "global", Grant: grantString})
 	require.NoError(t, err)
 	_, actions := g.Actions()
 	grant := &pb.Grant{
@@ -2127,7 +2127,7 @@ func checkEqualGrants(t *testing.T, expected []string, got *pb.Role) {
 		return got.GrantStrings[i] < got.GrantStrings[j]
 	})
 	for i, v := range expected {
-		parsed, err := perms.Parse(context.Background(), "o_abc123", v)
+		parsed, err := perms.Parse(context.Background(), perms.GrantTuple{RoleScopeId: "o_abc123", GrantScopeId: "o_abc123", Grant: v})
 		require.NoError(err)
 		assert.Equal(expected[i], got.GrantStrings[i])
 		assert.Equal(expected[i], got.Grants[i].GetRaw())

internal/daemon/controller/handlers/scopes/scope_service.go

@@ -337,7 +337,14 @@ func (s *Service) GetScope(ctx context.Context, req *pbs.GetScopeRequest) (*pbs.
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, p.GetPublicId(), idActionsById(p.GetPublicId())).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[p.Type], p.GetPublicId(), "")
+		scopeInfo := &pb.ScopeInfo{
+			Id:            p.GetPublicId(),
+			Type:          p.Type,
+			Name:          p.GetName(),
+			Description:   p.GetDescription(),
+			ParentScopeId: p.GetParentId(),
+		}
+		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[p.Type], scopeInfo, "")
 		if err != nil {
 			return nil, err
 		}
@@ -382,7 +389,14 @@ func (s *Service) CreateScope(ctx context.Context, req *pbs.CreateScopeRequest)
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, p.GetPublicId(), idActionsById(p.GetPublicId())).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[p.Type], p.GetPublicId(), "")
+		scopeInfo := &pb.ScopeInfo{
+			Id:            p.GetPublicId(),
+			Type:          p.Type,
+			Name:          p.GetName(),
+			Description:   p.GetDescription(),
+			ParentScopeId: p.GetParentId(),
+		}
+		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[p.Type], scopeInfo, "")
 		if err != nil {
 			return nil, err
 		}
@@ -427,7 +441,14 @@ func (s *Service) UpdateScope(ctx context.Context, req *pbs.UpdateScopeRequest)
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authResults.FetchActionSetForId(ctx, p.GetPublicId(), idActionsById(p.GetPublicId())).Strings()))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[p.Type], p.GetPublicId(), "")
+		scopeInfo := &pb.ScopeInfo{
+			Id:            p.GetPublicId(),
+			Type:          p.Type,
+			Name:          p.GetName(),
+			Description:   p.GetDescription(),
+			ParentScopeId: p.GetParentId(),
+		}
+		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[p.Type], scopeInfo, "")
 		if err != nil {
 			return nil, err
 		}
@@ -1137,7 +1158,14 @@ func newOutputOpts(ctx context.Context, item *iam.Scope, authResults auth.Verify
 		outputOpts = append(outputOpts, handlers.WithAuthorizedActions(authorizedActions))
 	}
 	if outputFields.Has(globals.AuthorizedCollectionActionsField) {
-		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[item.Type], item.GetPublicId(), "")
+		scopeInfo := &pb.ScopeInfo{
+			Id:            item.GetPublicId(),
+			Type:          item.Type,
+			Name:          item.GetName(),
+			Description:   item.GetDescription(),
+			ParentScopeId: item.GetParentId(),
+		}
+		collectionActions, err := auth.CalculateAuthorizedCollectionActions(ctx, authResults, scopeCollectionTypeMapMap[item.Type], scopeInfo, "")
 		if err != nil {
 			return nil, false, err
 		}

internal/daemon/controller/handlers/sessions/session_service.go

@@ -575,9 +575,10 @@ func validateCancelRequest(req *pbs.CancelSessionRequest) error {
 
 func newOutputOpts(ctx context.Context, item *session.Session, scopeIds map[string]*scopes.ScopeInfo, authResults auth.VerifyResults) ([]handlers.Option, bool) {
 	res := perms.Resource{
-		Type:    resource.Session,
-		Id:      item.GetPublicId(),
-		ScopeId: item.GetProjectId(),
+		Type:          resource.Session,
+		Id:            item.GetPublicId(),
+		ScopeId:       item.GetProjectId(),
+		ParentScopeId: scopeIds[item.ProjectId].ParentScopeId,
 	}
 	authorizedActions := authResults.FetchActionSetForId(ctx, item.GetPublicId(), IdActions, auth.WithResource(&res)).Strings()
 	if len(authorizedActions) == 0 {

internal/daemon/controller/handlers/targets/target_service.go

@@ -2006,7 +2006,7 @@ func validateListRequest(ctx context.Context, req *pbs.ListTargetsRequest) error
 }
 
 func newOutputOpts(ctx context.Context, item target.Target, authResults auth.VerifyResults, authzScopes map[string]*scopes.ScopeInfo) []handlers.Option {
-	pr := perms.Resource{Id: item.GetPublicId(), ScopeId: item.GetProjectId(), Type: resource.Target}
+	pr := perms.Resource{Id: item.GetPublicId(), ScopeId: item.GetProjectId(), Type: resource.Target, ParentScopeId: authzScopes[item.GetProjectId()].GetParentScopeId()}
 	outputFields := authResults.FetchOutputFields(pr, action.List).SelfOrDefaults(authResults.UserId)
 
 	outputOpts := make([]handlers.Option, 0, 3)

internal/daemon/controller/handlers/targets/tcp/target_service_test.go

@@ -489,13 +489,8 @@ func TestList(t *testing.T) {
 	}
 }
 
-func TestListPagination(t *testing.T) {
-	// Set database read timeout to avoid duplicates in response
-	oldReadTimeout := globals.RefreshReadLookbackDuration
-	globals.RefreshReadLookbackDuration = 0
-	t.Cleanup(func() {
-		globals.RefreshReadLookbackDuration = oldReadTimeout
-	})
+func TestListGrantScopes(t *testing.T) {
+	t.Parallel()
 	ctx := context.Background()
 	conn, _ := db.TestSetup(t, "postgres")
 	sqlDB, err := conn.SqlDB(ctx)
@@ -515,27 +510,395 @@ func TestListPagination(t *testing.T) {
 	serversRepoFn := func() (*server.Repository, error) {
 		return server.NewRepository(ctx, rw, rw, kms)
 	}
-	repo, err := target.NewRepository(ctx, rw, rw, kms)
+
+	at := authtoken.TestAuthToken(t, conn, kms, scope.Global.String())
+
+	var projects []*iam.Scope
+	org1, proj1 := iam.TestScopes(t, iamRepo)
+	projects = append(projects, proj1)
+	org2, proj2 := iam.TestScopes(t, iamRepo)
+	projects = append(projects, proj2)
+
+	var totalTars []*pb.Target
+	for i, proj := range projects {
+		for j := 0; j < 5; j++ {
+			name := fmt.Sprintf("tar-%d-%d", i, j)
+			tar := tcp.TestTarget(ctx, t, conn, proj.GetPublicId(), name, target.WithAddress(fmt.Sprintf("1.1.%d.%d", i, j)))
+			totalTars = append(totalTars, &pb.Target{
+				Id:                     tar.GetPublicId(),
+				ScopeId:                proj.GetPublicId(),
+				Name:                   wrapperspb.String(name),
+				Scope:                  &scopes.ScopeInfo{Id: proj.GetPublicId(), Type: scope.Project.String(), ParentScopeId: proj.ParentId},
+				CreatedTime:            tar.GetCreateTime().GetTimestamp(),
+				UpdatedTime:            tar.GetUpdateTime().GetTimestamp(),
+				Version:                tar.GetVersion(),
+				Type:                   tcp.Subtype.String(),
+				Attrs:                  &pb.Target_TcpTargetAttributes{},
+				SessionMaxSeconds:      wrapperspb.UInt32(28800),
+				SessionConnectionLimit: wrapperspb.Int32(-1),
+				AuthorizedActions:      testAuthorizedActions,
+				Address:                &wrapperspb.StringValue{Value: fmt.Sprintf("1.1.%d.%d", i, j)},
+			})
+		}
+	}
+
+	// Run analyze to update postgres estimates
+	_, err = sqlDB.ExecContext(ctx, "analyze")
 	require.NoError(t, err)
 
-	org, proj := iam.TestScopes(t, iamRepo)
-	at := authtoken.TestAuthToken(t, conn, kms, org.GetPublicId())
-	r := iam.TestRole(t, conn, proj.GetPublicId())
-	_ = iam.TestUserRole(t, conn, r.GetPublicId(), at.GetIamUserId())
-	_ = iam.TestRoleGrant(t, conn, r.GetPublicId(), "ids=*;type=*;actions=*")
-	hc := static.TestCatalogs(t, conn, proj.GetPublicId(), 1)[0]
-	hss := static.TestSets(t, conn, hc.GetPublicId(), 2)
-	s, err := testService(t, context.Background(), conn, kms, wrapper)
-	require.NoError(t, err)
+	_ = org1
+	_ = org2
 
-	var allTargets []*pb.Target
-	for i := 0; i < 10; i++ {
-		tar := tcp.TestTarget(ctx, t, conn, proj.GetPublicId(), fmt.Sprintf("tar%d", i), target.WithHostSources([]string{hss[0].GetPublicId(), hss[1].GetPublicId()}))
-		allTargets = append(allTargets, &pb.Target{
+	cases := []struct {
+		name      string
+		pageSize  uint32
+		setupFunc func(t *testing.T)
+		res       *pbs.ListTargetsResponse
+		err       error
+	}{
+		{
+			name: "global-with-direct-grants-wildcard",
+			setupFunc: func(t *testing.T) {
+				globalRole := iam.TestRole(t, conn, scope.Global.String(), iam.WithGrantScopeIds([]string{proj1.GetPublicId(), proj2.GetPublicId()}))
+				_ = iam.TestUserRole(t, conn, globalRole.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, globalRole.GetPublicId(), "ids=*;type=*;actions=*")
+			},
+			res: &pbs.ListTargetsResponse{
+				Items:        totalTars,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				EstItemCount: 10,
+			},
+		},
+		{
+			name: "global-with-direct-grants-non-wildcard",
+			setupFunc: func(t *testing.T) {
+				globalRole := iam.TestRole(t, conn, scope.Global.String(), iam.WithGrantScopeIds([]string{proj1.GetPublicId(), proj2.GetPublicId()}))
+				_ = iam.TestUserRole(t, conn, globalRole.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, globalRole.GetPublicId(), "ids=*;type=target;actions=list")
+				_ = iam.TestRoleGrant(t, conn, globalRole.GetPublicId(), fmt.Sprintf("ids=%s,%s;actions=*", totalTars[0].Id, totalTars[1].Id))
+			},
+			res: &pbs.ListTargetsResponse{
+				Items:        totalTars[0:2],
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				EstItemCount: 2,
+			},
+		},
+		{
+			name: "global-with-descendants-wildcard",
+			setupFunc: func(t *testing.T) {
+				globalRole := iam.TestRole(t, conn, scope.Global.String(), iam.WithGrantScopeIds([]string{globals.GrantScopeDescendants}))
+				_ = iam.TestUserRole(t, conn, globalRole.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, globalRole.GetPublicId(), "ids=*;type=*;actions=*")
+			},
+			res: &pbs.ListTargetsResponse{
+				Items:        totalTars,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				EstItemCount: 10,
+			},
+		},
+		{
+			name: "org-with-direct-grants-wildcard",
+			setupFunc: func(t *testing.T) {
+				org1Role := iam.TestRole(t, conn, org1.GetPublicId(), iam.WithGrantScopeIds([]string{proj1.GetPublicId()}))
+				_ = iam.TestUserRole(t, conn, org1Role.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, org1Role.GetPublicId(), "ids=*;type=*;actions=*")
+				org2Role := iam.TestRole(t, conn, org2.GetPublicId(), iam.WithGrantScopeIds([]string{proj2.GetPublicId()}))
+				_ = iam.TestUserRole(t, conn, org2Role.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, org2Role.GetPublicId(), "ids=*;type=*;actions=*")
+			},
+			res: &pbs.ListTargetsResponse{
+				Items:        totalTars,
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				EstItemCount: 10,
+			},
+		},
+		{
+			name: "org-with-direct-grants-non-wildcard",
+			setupFunc: func(t *testing.T) {
+				org1Role := iam.TestRole(t, conn, org1.GetPublicId(), iam.WithGrantScopeIds([]string{proj1.GetPublicId()}))
+				_ = iam.TestUserRole(t, conn, org1Role.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, org1Role.GetPublicId(), "ids=*;type=target;actions=list")
+				_ = iam.TestRoleGrant(t, conn, org1Role.GetPublicId(), fmt.Sprintf("ids=%s,%s;actions=*", totalTars[0].Id, totalTars[1].Id))
+				org2Role := iam.TestRole(t, conn, org2.GetPublicId(), iam.WithGrantScopeIds([]string{proj2.GetPublicId()}))
+				_ = iam.TestUserRole(t, conn, org2Role.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, org2Role.GetPublicId(), "ids=*;type=target;actions=list")
+				_ = iam.TestRoleGrant(t, conn, org2Role.GetPublicId(), fmt.Sprintf("ids=%s,%s;actions=*", totalTars[5].Id, totalTars[6].Id))
+			},
+			res: &pbs.ListTargetsResponse{
+				Items:        append([]*pb.Target{}, append(append([]*pb.Target{}, totalTars[0:2]...), totalTars[5:7]...)...),
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				EstItemCount: 4,
+			},
+		},
+		{
+			name: "org-with-children-wildcard",
+			setupFunc: func(t *testing.T) {
+				org1Role := iam.TestRole(t, conn, org1.GetPublicId(), iam.WithGrantScopeIds([]string{globals.GrantScopeChildren}))
+				_ = iam.TestUserRole(t, conn, org1Role.GetPublicId(), at.GetIamUserId())
+				_ = iam.TestRoleGrant(t, conn, org1Role.GetPublicId(), "ids=*;type=*;actions=*")
+			},
+			res: &pbs.ListTargetsResponse{
+				Items:        totalTars[0:5],
+				ResponseType: "complete",
+				SortBy:       "created_time",
+				SortDir:      "desc",
+				EstItemCount: 5,
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert, require := assert.New(t), require.New(t)
+			_, err := sqlDB.Exec("delete from iam_role")
+			require.NoError(err)
+			tc.setupFunc(t)
+
+			s, err := testService(t, context.Background(), conn, kms, wrapper)
+			require.NoError(err, "Couldn't create new target service.")
+
+			requestInfo := authpb.RequestInfo{
+				TokenFormat: uint32(auth.AuthTokenTypeBearer),
+				PublicId:    at.GetPublicId(),
+				Token:       at.GetToken(),
+			}
+			requestContext := context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
+			ctx := auth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
+			got, gErr := s.ListTargets(ctx, &pbs.ListTargetsRequest{
+				ScopeId:   scope.Global.String(),
+				Recursive: true,
+				PageSize:  tc.pageSize,
+			})
+			if tc.err != nil {
+				require.Error(gErr)
+				assert.True(errors.Is(gErr, tc.err), "got error %v, wanted %v", gErr, tc.err)
+				return
+			}
+			require.NoError(gErr)
+			assert.Equal(len(tc.res.Items), len(got.Items))
+			wantById := make(map[string]*pb.Target, len(tc.res.Items))
+			for _, t := range tc.res.Items {
+				wantById[t.Id] = t
+			}
+			for _, t := range got.Items {
+				want, ok := wantById[t.Id]
+				assert.True(ok, "Got unexpected target with id: %s", t.Id)
+				assert.Empty(cmp.Diff(
+					t,
+					want,
+					protocmp.Transform(),
+					cmpopts.SortSlices(func(a, b string) bool {
+						return a < b
+					}),
+				), "got %v, wanted %v", t, want)
+			}
+		})
+	}
+}
+
+func TestListPagination(t *testing.T) {
+	testListPagination := func(t *testing.T, useDescendants bool) {
+		// Set database read timeout to avoid duplicates in response
+		oldReadTimeout := globals.RefreshReadLookbackDuration
+		globals.RefreshReadLookbackDuration = 0
+		t.Cleanup(func() {
+			globals.RefreshReadLookbackDuration = oldReadTimeout
+		})
+		ctx := context.Background()
+		conn, _ := db.TestSetup(t, "postgres")
+		sqlDB, err := conn.SqlDB(ctx)
+		require.NoError(t, err)
+		wrapper := db.TestWrapper(t)
+		kms := kms.TestKms(t, conn, wrapper)
+
+		rw := db.New(conn)
+
+		iamRepo := iam.TestRepo(t, conn, wrapper)
+		iamRepoFn := func() (*iam.Repository, error) {
+			return iamRepo, nil
+		}
+		tokenRepoFn := func() (*authtoken.Repository, error) {
+			return authtoken.NewRepository(ctx, rw, rw, kms)
+		}
+		serversRepoFn := func() (*server.Repository, error) {
+			return server.NewRepository(ctx, rw, rw, kms)
+		}
+		repo, err := target.NewRepository(ctx, rw, rw, kms)
+		require.NoError(t, err)
+
+		// We're going to run the same test in two projects; one with
+		// descendants and one with direct grants in one project and a child
+		// grant from org in another project
+		org1, proj1 := iam.TestScopes(t, iamRepo)
+		org2, proj2 := iam.TestScopes(t, iamRepo)
+		at := authtoken.TestAuthToken(t, conn, kms, scope.Global.String())
+		if useDescendants {
+			r := iam.TestRole(t, conn, scope.Global.String(), iam.WithGrantScopeIds([]string{globals.GrantScopeDescendants}))
+			_ = iam.TestUserRole(t, conn, r.GetPublicId(), at.GetIamUserId())
+			_ = iam.TestRoleGrant(t, conn, r.GetPublicId(), "ids=*;type=*;actions=*")
+		} else {
+			r1 := iam.TestRole(t, conn, proj1.GetPublicId())
+			_ = iam.TestUserRole(t, conn, r1.GetPublicId(), at.GetIamUserId())
+			_ = iam.TestRoleGrant(t, conn, r1.GetPublicId(), "ids=*;type=*;actions=*")
+			r2 := iam.TestRole(t, conn, org2.GetPublicId(), iam.WithGrantScopeIds([]string{globals.GrantScopeChildren}))
+			_ = iam.TestUserRole(t, conn, r2.GetPublicId(), at.GetIamUserId())
+			_ = iam.TestRoleGrant(t, conn, r2.GetPublicId(), "ids=*;type=*;actions=*")
+		}
+		hc := static.TestCatalogs(t, conn, proj1.GetPublicId(), 1)[0]
+		hss := static.TestSets(t, conn, hc.GetPublicId(), 2)
+		s, err := testService(t, context.Background(), conn, kms, wrapper)
+		require.NoError(t, err)
+
+		var allTargets []*pb.Target
+		for i := 0; i < 10; i++ {
+			tar := tcp.TestTarget(ctx, t, conn, proj1.GetPublicId(), fmt.Sprintf("tar-1-%d", i), target.WithHostSources([]string{hss[0].GetPublicId(), hss[1].GetPublicId()}))
+			allTargets = append(allTargets, &pb.Target{
+				Id:                     tar.GetPublicId(),
+				ScopeId:                proj1.GetPublicId(),
+				Name:                   wrapperspb.String(tar.GetName()),
+				Scope:                  &scopes.ScopeInfo{Id: proj1.GetPublicId(), Type: scope.Project.String(), ParentScopeId: org1.GetPublicId()},
+				CreatedTime:            tar.GetCreateTime().GetTimestamp(),
+				UpdatedTime:            tar.GetUpdateTime().GetTimestamp(),
+				Version:                tar.GetVersion(),
+				Type:                   tcp.Subtype.String(),
+				Attrs:                  &pb.Target_TcpTargetAttributes{},
+				SessionMaxSeconds:      wrapperspb.UInt32(28800),
+				SessionConnectionLimit: wrapperspb.Int32(-1),
+				AuthorizedActions:      testAuthorizedActions,
+				Address:                &wrapperspb.StringValue{},
+			})
+		}
+		for i := 0; i < 10; i++ {
+			tar := tcp.TestTarget(ctx, t, conn, proj2.GetPublicId(), fmt.Sprintf("tar-2-%d", i), target.WithAddress(fmt.Sprintf("127.0.0.%d", i)))
+			allTargets = append(allTargets, &pb.Target{
+				Id:                     tar.GetPublicId(),
+				ScopeId:                proj2.GetPublicId(),
+				Name:                   wrapperspb.String(tar.GetName()),
+				Scope:                  &scopes.ScopeInfo{Id: proj2.GetPublicId(), Type: scope.Project.String(), ParentScopeId: org2.GetPublicId()},
+				CreatedTime:            tar.GetCreateTime().GetTimestamp(),
+				UpdatedTime:            tar.GetUpdateTime().GetTimestamp(),
+				Version:                tar.GetVersion(),
+				Type:                   tcp.Subtype.String(),
+				Attrs:                  &pb.Target_TcpTargetAttributes{},
+				SessionMaxSeconds:      wrapperspb.UInt32(28800),
+				SessionConnectionLimit: wrapperspb.Int32(-1),
+				AuthorizedActions:      testAuthorizedActions,
+				Address:                &wrapperspb.StringValue{Value: fmt.Sprintf("127.0.0.%d", i)},
+			})
+		}
+		// Reverse since we read items in descending order (newest first)
+		slices.Reverse(allTargets)
+
+		// Run analyze to update postgres estimates
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		requestInfo := authpb.RequestInfo{
+			TokenFormat: uint32(auth.AuthTokenTypeBearer),
+			PublicId:    at.GetPublicId(),
+			Token:       at.GetToken(),
+		}
+		requestContext := context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
+		ctx = auth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
+
+		// Start paginating, recursively
+		req := &pbs.ListTargetsRequest{
+			ScopeId:   "global",
+			Recursive: true,
+			Filter:    "",
+			ListToken: "",
+			PageSize:  2,
+		}
+		got, err := s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 2)
+		// Compare without comparing the list token
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        allTargets[0:2],
+					ResponseType: "delta",
+					SortBy:       "created_time",
+					SortDir:      "desc",
+					RemovedIds:   nil,
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+
+		// Request second page
+		req.ListToken = got.ListToken
+		got, err = s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 2)
+		// Compare without comparing the list token
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        allTargets[2:4],
+					ResponseType: "delta",
+					SortBy:       "created_time",
+					SortDir:      "desc",
+					RemovedIds:   nil,
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+
+		// Request rest of results
+		req.ListToken = got.ListToken
+		req.PageSize = 20
+		got, err = s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 16)
+		// Compare without comparing the list token
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        allTargets[4:],
+					ResponseType: "complete",
+					SortBy:       "created_time",
+					SortDir:      "desc",
+					RemovedIds:   nil,
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+
+		// Create another target
+		tar := tcp.TestTarget(ctx, t, conn, proj1.GetPublicId(), "test-target-1", target.WithHostSources([]string{hss[0].GetPublicId(), hss[1].GetPublicId()}))
+		newTarget := &pb.Target{
 			Id:                     tar.GetPublicId(),
-			ScopeId:                proj.GetPublicId(),
+			ScopeId:                proj1.GetPublicId(),
 			Name:                   wrapperspb.String(tar.GetName()),
-			Scope:                  &scopes.ScopeInfo{Id: proj.GetPublicId(), Type: scope.Project.String(), ParentScopeId: org.GetPublicId()},
+			Scope:                  &scopes.ScopeInfo{Id: proj1.GetPublicId(), Type: scope.Project.String(), ParentScopeId: org1.GetPublicId()},
 			CreatedTime:            tar.GetCreateTime().GetTimestamp(),
 			UpdatedTime:            tar.GetUpdateTime().GetTimestamp(),
 			Version:                tar.GetVersion(),
@@ -545,285 +908,230 @@ func TestListPagination(t *testing.T) {
 			SessionConnectionLimit: wrapperspb.Int32(-1),
 			AuthorizedActions:      testAuthorizedActions,
 			Address:                &wrapperspb.StringValue{},
+		}
+		// Add to the front since it's most recently updated
+		allTargets = append([]*pb.Target{newTarget}, allTargets...)
+		tar = tcp.TestTarget(ctx, t, conn, proj2.GetPublicId(), "test-target-2", target.WithAddress(fmt.Sprintf("127.0.0.11")))
+		newTarget = &pb.Target{
+			Id:                     tar.GetPublicId(),
+			ScopeId:                proj2.GetPublicId(),
+			Name:                   wrapperspb.String(tar.GetName()),
+			Scope:                  &scopes.ScopeInfo{Id: proj2.GetPublicId(), Type: scope.Project.String(), ParentScopeId: org2.GetPublicId()},
+			CreatedTime:            tar.GetCreateTime().GetTimestamp(),
+			UpdatedTime:            tar.GetUpdateTime().GetTimestamp(),
+			Version:                tar.GetVersion(),
+			Type:                   tcp.Subtype.String(),
+			Attrs:                  &pb.Target_TcpTargetAttributes{},
+			SessionMaxSeconds:      wrapperspb.UInt32(28800),
+			SessionConnectionLimit: wrapperspb.Int32(-1),
+			AuthorizedActions:      testAuthorizedActions,
+			Address:                &wrapperspb.StringValue{Value: fmt.Sprintf("127.0.0.11")},
+		}
+		allTargets = append([]*pb.Target{newTarget}, allTargets...)
+
+		// Leaving this function here as it is very useful if test objects change
+		/*
+			printNames := func(step string, tars []*pb.Target) {
+				names := make([]string, len(tars))
+				for i, t := range tars {
+					names[i] = t.GetName().GetValue()
+				}
+				log.Println(step, pretty.Sprint(strings.Join(names, ", ")))
+			}
+		*/
+
+		// printNames("before delete      ", allTargets)
+
+		// Delete one of the other targets in each project
+		_, err = repo.DeleteTarget(ctx, allTargets[len(allTargets)-11].Id)
+		require.NoError(t, err)
+		deletedTarget1 := allTargets[len(allTargets)-11]
+		allTargets = append(allTargets[:len(allTargets)-11], allTargets[len(allTargets)-11+1:]...)
+		// printNames("after first delete ", allTargets)
+
+		_, err = repo.DeleteTarget(ctx, allTargets[len(allTargets)-1].Id)
+		require.NoError(t, err)
+		deletedTarget2 := allTargets[len(allTargets)-1]
+		allTargets = allTargets[:len(allTargets)-1]
+		// printNames("after second delete", allTargets)
+
+		// Update two of the other targets
+		allTargets[2].Name = wrapperspb.String("new-name-1")
+		allTargets[2].Version = 2
+		updatedTarget := &tcp.Target{
+			Target: &store.Target{
+				PublicId:  allTargets[2].Id,
+				Name:      allTargets[2].Name.GetValue(),
+				ProjectId: allTargets[2].ScopeId,
+			},
+		}
+		tg, _, err := repo.UpdateTarget(ctx, updatedTarget, 1, []string{"name"})
+		require.NoError(t, err)
+		allTargets[2].UpdatedTime = tg.GetUpdateTime().GetTimestamp()
+		allTargets[2].Version = tg.GetVersion()
+		// Add to the front since it's most recently updated
+		newAllTargets := append([]*pb.Target{allTargets[2]}, allTargets[0:2]...)
+		newAllTargets = append(newAllTargets, allTargets[3:]...)
+		allTargets = newAllTargets
+		// printNames("after first update ", allTargets)
+		allTargets[11].Name = wrapperspb.String("new-name-11")
+		allTargets[11].Version = 2
+		updatedTarget = &tcp.Target{
+			Target: &store.Target{
+				PublicId:  allTargets[11].Id,
+				Name:      allTargets[11].Name.GetValue(),
+				ProjectId: allTargets[11].ScopeId,
+			},
+		}
+		tg, _, err = repo.UpdateTarget(ctx, updatedTarget, 1, []string{"name"})
+		require.NoError(t, err)
+		allTargets[11].UpdatedTime = tg.GetUpdateTime().GetTimestamp()
+		allTargets[11].Version = tg.GetVersion()
+		// Add to the front since it's most recently updated
+		newAllTargets = append([]*pb.Target{allTargets[11]}, allTargets[0:11]...)
+		newAllTargets = append(newAllTargets, allTargets[12:]...)
+		allTargets = newAllTargets
+		// printNames("after second update", allTargets)
+
+		// Run analyze to update postgres estimates
+		_, err = sqlDB.ExecContext(ctx, "analyze")
+		require.NoError(t, err)
+
+		// Request updated results
+		req.ListToken = got.ListToken
+		req.PageSize = 2
+		got, err = s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 2)
+		// Compare without comparing the list token
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        []*pb.Target{allTargets[0], allTargets[1]},
+					ResponseType: "delta",
+					SortBy:       "updated_time",
+					SortDir:      "desc",
+					// Should contain the deleted target
+					RemovedIds:   []string{deletedTarget1.Id, deletedTarget2.Id},
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+
+		// Get next page
+		req.ListToken = got.ListToken
+		got, err = s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 2)
+		// Compare without comparing the list token
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        []*pb.Target{allTargets[2], allTargets[3]},
+					ResponseType: "complete",
+					SortBy:       "updated_time",
+					SortDir:      "desc",
+					RemovedIds:   nil,
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+
+		// Request new page with filter requiring looping
+		// to fill the page.
+		req.ListToken = ""
+		req.PageSize = 1
+		req.Filter = fmt.Sprintf(`"/item/id"==%q or "/item/id"==%q`, allTargets[len(allTargets)-2].Id, allTargets[len(allTargets)-1].Id)
+		got, err = s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 1)
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        []*pb.Target{allTargets[len(allTargets)-2]},
+					ResponseType: "delta",
+					SortBy:       "created_time",
+					SortDir:      "desc",
+					// Should be empty again
+					RemovedIds:   nil,
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+		req.ListToken = got.ListToken
+		// Get the second page
+		got, err = s.ListTargets(ctx, req)
+		require.NoError(t, err)
+		require.Len(t, got.GetItems(), 1)
+		assert.Empty(t,
+			cmp.Diff(
+				got,
+				&pbs.ListTargetsResponse{
+					Items:        []*pb.Target{allTargets[len(allTargets)-1]},
+					ResponseType: "complete",
+					SortBy:       "created_time",
+					SortDir:      "desc",
+					RemovedIds:   nil,
+					EstItemCount: 20,
+				},
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+				protocmp.Transform(),
+				protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
+			),
+		)
+
+		// Create unauthenticated user
+		unauthAt := authtoken.TestAuthToken(t, conn, kms, org1.GetPublicId())
+		unauthR := iam.TestRole(t, conn, proj1.GetPublicId())
+		_ = iam.TestUserRole(t, conn, unauthR.GetPublicId(), unauthAt.GetIamUserId())
+
+		// Make a request with the unauthenticated user,
+		// ensure the response contains the pagination parameters.
+		requestInfo = authpb.RequestInfo{
+			TokenFormat: uint32(auth.AuthTokenTypeBearer),
+			PublicId:    unauthAt.GetPublicId(),
+			Token:       unauthAt.GetToken(),
+		}
+		requestContext = context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
+		ctx = auth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
+
+		got, err = s.ListTargets(ctx, &pbs.ListTargetsRequest{
+			ScopeId:   "global",
+			Recursive: true,
 		})
+		require.NoError(t, err)
+		assert.Empty(t, got.Items)
+		assert.Equal(t, "created_time", got.SortBy)
+		assert.Equal(t, "desc", got.SortDir)
+		assert.Equal(t, "complete", got.ResponseType)
 	}
-	// Reverse since we read items in descending order (newest first)
-	slices.Reverse(allTargets)
 
-	// Run analyze to update postgres estimates
-	_, err = sqlDB.ExecContext(ctx, "analyze")
-	require.NoError(t, err)
-
-	requestInfo := authpb.RequestInfo{
-		TokenFormat: uint32(auth.AuthTokenTypeBearer),
-		PublicId:    at.GetPublicId(),
-		Token:       at.GetToken(),
-	}
-	requestContext := context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
-	ctx = auth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
-
-	// Start paginating, recursively
-	req := &pbs.ListTargetsRequest{
-		ScopeId:   "global",
-		Recursive: true,
-		Filter:    "",
-		ListToken: "",
-		PageSize:  2,
-	}
-	got, err := s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 2)
-	// Compare without comparing the list token
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        allTargets[0:2],
-				ResponseType: "delta",
-				SortBy:       "created_time",
-				SortDir:      "desc",
-				RemovedIds:   nil,
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-
-	// Request second page
-	req.ListToken = got.ListToken
-	got, err = s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 2)
-	// Compare without comparing the list token
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        allTargets[2:4],
-				ResponseType: "delta",
-				SortBy:       "created_time",
-				SortDir:      "desc",
-				RemovedIds:   nil,
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-
-	// Request rest of results
-	req.ListToken = got.ListToken
-	req.PageSize = 10
-	got, err = s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 6)
-	// Compare without comparing the list token
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        allTargets[4:],
-				ResponseType: "complete",
-				SortBy:       "created_time",
-				SortDir:      "desc",
-				RemovedIds:   nil,
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-
-	// Create another target
-	tar := tcp.TestTarget(ctx, t, conn, proj.GetPublicId(), "test-target", target.WithHostSources([]string{hss[0].GetPublicId(), hss[1].GetPublicId()}))
-	newTarget := &pb.Target{
-		Id:                     tar.GetPublicId(),
-		ScopeId:                proj.GetPublicId(),
-		Name:                   wrapperspb.String(tar.GetName()),
-		Scope:                  &scopes.ScopeInfo{Id: proj.GetPublicId(), Type: scope.Project.String(), ParentScopeId: org.GetPublicId()},
-		CreatedTime:            tar.GetCreateTime().GetTimestamp(),
-		UpdatedTime:            tar.GetUpdateTime().GetTimestamp(),
-		Version:                tar.GetVersion(),
-		Type:                   tcp.Subtype.String(),
-		Attrs:                  &pb.Target_TcpTargetAttributes{},
-		SessionMaxSeconds:      wrapperspb.UInt32(28800),
-		SessionConnectionLimit: wrapperspb.Int32(-1),
-		AuthorizedActions:      testAuthorizedActions,
-		Address:                &wrapperspb.StringValue{},
-	}
-	// Add to the front since it's most recently updated
-	allTargets = append([]*pb.Target{newTarget}, allTargets...)
-
-	// Delete one of the other targets
-	_, err = repo.DeleteTarget(ctx, allTargets[len(allTargets)-1].Id)
-	require.NoError(t, err)
-	deletedTarget := allTargets[len(allTargets)-1]
-	allTargets = allTargets[:len(allTargets)-1]
-
-	// Update one of the other targets
-	allTargets[1].Name = wrapperspb.String("new-name")
-	allTargets[1].Version = 2
-	updatedTarget := &tcp.Target{
-		Target: &store.Target{
-			PublicId:  allTargets[1].Id,
-			Name:      allTargets[1].Name.GetValue(),
-			ProjectId: allTargets[1].ScopeId,
-		},
-	}
-	tg, _, err := repo.UpdateTarget(ctx, updatedTarget, 1, []string{"name"})
-	require.NoError(t, err)
-	allTargets[1].UpdatedTime = tg.GetUpdateTime().GetTimestamp()
-	allTargets[1].Version = tg.GetVersion()
-	// Add to the front since it's most recently updated
-	allTargets = append(
-		[]*pb.Target{allTargets[1]},
-		append(
-			[]*pb.Target{allTargets[0]},
-			allTargets[2:]...,
-		)...,
-	)
-
-	// Run analyze to update postgres estimates
-	_, err = sqlDB.ExecContext(ctx, "analyze")
-	require.NoError(t, err)
-
-	// Request updated results
-	req.ListToken = got.ListToken
-	req.PageSize = 1
-	got, err = s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 1)
-	// Compare without comparing the list token
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        []*pb.Target{allTargets[0]},
-				ResponseType: "delta",
-				SortBy:       "updated_time",
-				SortDir:      "desc",
-				// Should contain the deleted target
-				RemovedIds:   []string{deletedTarget.Id},
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-
-	// Get next page
-	req.ListToken = got.ListToken
-	got, err = s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 1)
-	// Compare without comparing the list token
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        []*pb.Target{allTargets[1]},
-				ResponseType: "complete",
-				SortBy:       "updated_time",
-				SortDir:      "desc",
-				RemovedIds:   nil,
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-
-	// Request new page with filter requiring looping
-	// to fill the page.
-	req.ListToken = ""
-	req.PageSize = 1
-	req.Filter = fmt.Sprintf(`"/item/id"==%q or "/item/id"==%q`, allTargets[len(allTargets)-2].Id, allTargets[len(allTargets)-1].Id)
-	got, err = s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 1)
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        []*pb.Target{allTargets[len(allTargets)-2]},
-				ResponseType: "delta",
-				SortBy:       "created_time",
-				SortDir:      "desc",
-				// Should be empty again
-				RemovedIds:   nil,
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-	req.ListToken = got.ListToken
-	// Get the second page
-	got, err = s.ListTargets(ctx, req)
-	require.NoError(t, err)
-	require.Len(t, got.GetItems(), 1)
-	assert.Empty(t,
-		cmp.Diff(
-			got,
-			&pbs.ListTargetsResponse{
-				Items:        []*pb.Target{allTargets[len(allTargets)-1]},
-				ResponseType: "complete",
-				SortBy:       "created_time",
-				SortDir:      "desc",
-				RemovedIds:   nil,
-				EstItemCount: 10,
-			},
-			cmpopts.SortSlices(func(a, b string) bool {
-				return a < b
-			}),
-			protocmp.Transform(),
-			protocmp.IgnoreFields(&pbs.ListTargetsResponse{}, "list_token"),
-		),
-	)
-
-	// Create unauthenticated user
-	unauthAt := authtoken.TestAuthToken(t, conn, kms, org.GetPublicId())
-	unauthR := iam.TestRole(t, conn, proj.GetPublicId())
-	_ = iam.TestUserRole(t, conn, unauthR.GetPublicId(), unauthAt.GetIamUserId())
-
-	// Make a request with the unauthenticated user,
-	// ensure the response contains the pagination parameters.
-	requestInfo = authpb.RequestInfo{
-		TokenFormat: uint32(auth.AuthTokenTypeBearer),
-		PublicId:    unauthAt.GetPublicId(),
-		Token:       unauthAt.GetToken(),
-	}
-	requestContext = context.WithValue(context.Background(), requests.ContextRequestInformationKey, &requests.RequestContext{})
-	ctx = auth.NewVerifierContext(requestContext, iamRepoFn, tokenRepoFn, serversRepoFn, kms, &requestInfo)
-
-	got, err = s.ListTargets(ctx, &pbs.ListTargetsRequest{
-		ScopeId:   "global",
-		Recursive: true,
+	t.Run("with-descendants", func(t *testing.T) {
+		testListPagination(t, true)
+	})
+	t.Run("without-descendants", func(t *testing.T) {
+		testListPagination(t, false)
 	})
-	require.NoError(t, err)
-	assert.Empty(t, got.Items)
-	assert.Equal(t, "created_time", got.SortBy)
-	assert.Equal(t, "desc", got.SortDir)
-	assert.Equal(t, "complete", got.ResponseType)
 }
 
 func TestDelete(t *testing.T) {

internal/daemon/controller/handlers/users/user_service.go

@@ -506,7 +506,7 @@ func (s Service) ListResolvableAliases(ctx context.Context, req *pbs.ListResolva
 		}
 	}
 
-	permissions := acl.ListResolvablePermissions(resource.Target, targets.IdActions)
+	permissions := acl.ListResolvableAliasesPermissions(resource.Target, targets.IdActions)
 
 	if len(permissions) == 0 {
 		// if there are no permitted targets then there will be no aliases that
@@ -615,15 +615,14 @@ func (s Service) aclAndGrantHashForUser(ctx context.Context, userId string) (per
 	// Note: Below, we always skip validation so that we don't error on formats
 	// that we've since restricted, e.g. "ids=foo;actions=create,read". These
 	// will simply not have an effect.
-	for _, pair := range grantTuples {
+	for _, tuple := range grantTuples {
 		permsOpts := []perms.Option{
 			perms.WithUserId(userId),
 			perms.WithSkipFinalValidation(true),
 		}
 		parsed, err := perms.Parse(
 			ctx,
-			pair.ScopeId,
-			pair.Grant,
+			tuple,
 			permsOpts...)
 		if err != nil {
 			return perms.ACL{}, nil, errors.Wrap(ctx, err, op)

internal/db/schema/migrations/oss/postgres/81/01_deleted_tables_and_triggers.up.sql

@@ -308,6 +308,7 @@ begin;
     'affected by the trigger and the current timestamp. It is used to populate rows '
     'of the deleted tables.';
 
+  -- Removed in 91/05_deletion_tables_view and replaced with a view.
   create function get_deletion_tables() returns setof name
   as $$
     select c.relname

internal/db/schema/migrations/oss/postgres/91/02_indexes_fk_delete.up.sql

@@ -0,0 +1,46 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+  -- Index to help when setting target_id to null on a session
+  -- when the corresponding target is deleted.
+  drop index if exists session_target_id_ix;
+  create index session_target_id_ix
+            on session (target_id);
+
+  -- Index to help when setting auth_token_id to null on a session
+  -- when the corresponding auth_token is deleted.
+  drop index if exists session_auth_token_id_ix;
+  create index session_auth_token_id_ix
+            on session (auth_token_id);
+
+  -- Index to help when setting session_id to null on a credential_vault_credential
+  -- when the corresponding session is deleted.
+  drop index if exists credential_vault_credential_session_id_ix;
+  create index credential_vault_credential_session_id_ix
+            on credential_vault_credential (session_id);
+
+  -- Index to help delete cascade of session_worker_protocol
+  -- when the corresponding session is deleted.
+  drop index if exists session_worker_protocol_session_id_ix;
+  create index session_worker_protocol_session_id_ix
+            on session_worker_protocol (session_id);
+
+  -- Index to help when setting session_id to null on recording_connection
+  -- when the corresponding session is deleted.
+  drop index if exists recording_connection_session_id_ix;
+  create index recording_connection_session_id_ix
+            on recording_connection (session_id);
+
+  -- Index to help delete of terminated sessions.
+  drop index if exists session_state_state_start_time_ix;
+  create index session_state_state_terminated_start_time_ix
+            on session_state (state, start_time)
+         where state = 'terminated';
+
+  analyze session,
+          credential_vault_credential,
+          session_worker_protocol,
+          recording_connection,
+          session_state;
+commit;

internal/db/schema/migrations/oss/postgres/91/03_indexes_grants_query.up.sql

@@ -0,0 +1,39 @@
+-- Copyright (c) HashiCorp, Inc.
+-- SPDX-License-Identifier: BUSL-1.1
+
+begin;
+  -- For each of these tables, swap the ordering of the
+  -- columns in the index for the primary key.
+  -- This helps the grants query that contains
+  -- several where clauses on what is currently the second
+  -- column in these indexes. By swapping the order, this
+  -- will make it more likely that the query planner will
+  -- choose to use the index.
+  -- See: https://www.postgresql.org/docs/current/indexes-multicolumn.html
+
+      alter table auth_oidc_managed_group_member_account
+  drop constraint auth_oidc_managed_group_member_account_pkey,
+  add primary key (member_id, managed_group_id);
+
+      alter table iam_managed_group_role
+  drop constraint iam_managed_group_role_pkey,
+  add primary key (principal_id, role_id);
+
+      alter table iam_group_member_user
+  drop constraint iam_group_member_user_pkey,
+  add primary key (member_id, group_id);
+
+      alter table iam_group_role
+  drop constraint iam_group_role_pkey,
+  add primary key (principal_id, role_id);
+
+      alter table iam_user_role
+  drop constraint iam_user_role_pkey,
+  add primary key (principal_id, role_id);
+
+  analyze auth_oidc_managed_group_member_account,
+          iam_managed_group_role,
+          iam_group_member_user,
+          iam_group_role,
+          iam_user_role;
+commit;