Paginate targets in cache (#5101)

* Update Go API to support pagination
* Implement pagination of targets in cache

(cherry picked from commit cd25fad42f325fc050b7ea3bec3336780d2b5e16)

api/accounts/account.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type AccountListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	authMethodId  string
+	allRemovedIds []string
 }
 
 func (n AccountListResult) GetItems() []*Account {
@@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.authMethodId = authMethodId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Account, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AccountListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Account that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Account has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Account) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Account) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AccountListResult, opt ...Option) (*AccountListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.authMethodId == "" {
+		return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["auth_method_id"] = currentPage.authMethodId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AccountListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.authMethodId = currentPage.authMethodId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/accounts/option.gen.go

@@ -5,6 +5,7 @@
 package accounts
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/aliases/alias.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type AliasListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AliasListResult) GetItems() []*Alias {
@@ -346,96 +354,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Alias, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AliasListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Alias that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Alias has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Alias) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Alias) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +454,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AliasListResult, opt ...Option) (*AliasListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AliasListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/aliases/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/authmethods/authmethods.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type AuthMethodListResult struct {
 	ListToken    string        `json:"list_token,omitempty"`
 	ResponseType string        `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AuthMethodListResult) GetItems() []*AuthMethod {
@@ -346,96 +354,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*AuthMethod, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AuthMethodListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any AuthMethod that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a AuthMethod has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *AuthMethod) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *AuthMethod) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +454,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthMethodListResult, opt ...Option) (*AuthMethodListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AuthMethodListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/authmethods/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/authtokens/authtokens.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -66,6 +67,13 @@ type AuthTokenListResult struct {
 	ListToken    string       `json:"list_token,omitempty"`
 	ResponseType string       `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AuthTokenListResult) GetItems() []*AuthToken {
@@ -231,96 +239,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*AuthToken, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AuthTokenListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any AuthToken that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a AuthToken has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *AuthToken) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *AuthToken) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -334,4 +339,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthTokenListResult, opt ...Option) (*AuthTokenListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AuthTokenListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/authtokens/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -85,6 +90,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/billing/option.gen.go

@@ -6,6 +6,7 @@ package billing
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -21,12 +22,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -53,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -91,6 +97,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithEndTime(inEndTime string) Option {
 	return func(o *options) {
 		o.queryMap["end_time"] = fmt.Sprintf("%v", inEndTime)

api/credentiallibraries/credential_library.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type CredentialLibraryListResult struct {
 	ListToken    string               `json:"list_token,omitempty"`
 	ResponseType string               `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive         bool
+	pageSize          uint32
+	credentialStoreId string
+	allRemovedIds     []string
 }
 
 func (n CredentialLibraryListResult) GetItems() []*CredentialLibrary {
@@ -346,96 +354,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.credentialStoreId = credentialStoreId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*CredentialLibrary, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialLibraryListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any CredentialLibrary that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a CredentialLibrary has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *CredentialLibrary) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *CredentialLibrary) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +452,80 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialLibraryListResult, opt ...Option) (*CredentialLibraryListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.credentialStoreId == "" {
+		return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["credential_store_id"] = currentPage.credentialStoreId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialLibraryListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.credentialStoreId = currentPage.credentialStoreId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentiallibraries/option.gen.go

@@ -5,6 +5,7 @@
 package credentiallibraries
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithVaultSSHCertificateCredentialLibraryAdditionalValidPrincipals(inAdditionalValidPrincipals []string) Option {
 	return func(o *options) {
 		raw, ok := o.postMap["attributes"]

api/credentials/credential.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type CredentialListResult struct {
 	ListToken    string        `json:"list_token,omitempty"`
 	ResponseType string        `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive         bool
+	pageSize          uint32
+	credentialStoreId string
+	allRemovedIds     []string
 }
 
 func (n CredentialListResult) GetItems() []*Credential {
@@ -344,96 +352,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.credentialStoreId = credentialStoreId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Credential, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Credential that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Credential has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Credential) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Credential) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,4 +450,80 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialListResult, opt ...Option) (*CredentialListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.credentialStoreId == "" {
+		return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["credential_store_id"] = currentPage.credentialStoreId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.credentialStoreId = currentPage.credentialStoreId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentials/option.gen.go

@@ -5,6 +5,7 @@
 package credentials
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/credentialstores/credential_store.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type CredentialStoreListResult struct {
 	ListToken    string             `json:"list_token,omitempty"`
 	ResponseType string             `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n CredentialStoreListResult) GetItems() []*CredentialStore {
@@ -345,96 +353,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*CredentialStore, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialStoreListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any CredentialStore that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a CredentialStore has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *CredentialStore) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *CredentialStore) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -448,4 +453,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialStoreListResult, opt ...Option) (*CredentialStoreListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialStoreListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentialstores/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/groups/group.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type GroupListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n GroupListResult) GetItems() []*Group {
@@ -339,96 +347,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Group, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(GroupListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Group that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Group has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Group) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Group) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -442,6 +447,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *GroupListResult, opt ...Option) (*GroupListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(GroupListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddMembers(ctx context.Context, id string, version uint32, memberIds []string, opt ...Option) (*GroupUpdateResult, error) {

api/groups/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/hostcatalogs/host_catalog.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -73,6 +74,13 @@ type HostCatalogListResult struct {
 	ListToken    string         `json:"list_token,omitempty"`
 	ResponseType string         `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n HostCatalogListResult) GetItems() []*HostCatalog {
@@ -350,96 +358,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*HostCatalog, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostCatalogListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any HostCatalog that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a HostCatalog has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *HostCatalog) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *HostCatalog) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -453,4 +458,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostCatalogListResult, opt ...Option) (*HostCatalogListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostCatalogListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/hostcatalogs/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/hosts/host.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -74,6 +75,13 @@ type HostListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	hostCatalogId string
+	allRemovedIds []string
 }
 
 func (n HostListResult) GetItems() []*Host {
@@ -346,96 +354,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.hostCatalogId = hostCatalogId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Host, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Host that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Host has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Host) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Host) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +452,80 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostListResult, opt ...Option) (*HostListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.hostCatalogId == "" {
+		return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.hostCatalogId = currentPage.hostCatalogId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/hosts/option.gen.go

@@ -5,6 +5,7 @@
 package hosts
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithStaticHostAddress(inAddress string) Option {
 	return func(o *options) {
 		raw, ok := o.postMap["attributes"]

api/hostsets/host_set.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -72,6 +73,13 @@ type HostSetListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	hostCatalogId string
+	allRemovedIds []string
 }
 
 func (n HostSetListResult) GetItems() []*HostSet {
@@ -344,96 +352,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.hostCatalogId = hostCatalogId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*HostSet, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostSetListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any HostSet that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a HostSet has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *HostSet) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *HostSet) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,6 +450,82 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostSetListResult, opt ...Option) (*HostSetListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.hostCatalogId == "" {
+		return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostSetListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.hostCatalogId = currentPage.hostCatalogId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddHosts(ctx context.Context, id string, version uint32, hostIds []string, opt ...Option) (*HostSetUpdateResult, error) {

api/hostsets/option.gen.go

@@ -5,6 +5,7 @@
 package hostsets
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/managedgroups/managedgroups.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ManagedGroupListResult struct {
 	ListToken    string          `json:"list_token,omitempty"`
 	ResponseType string          `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	authMethodId  string
+	allRemovedIds []string
 }
 
 func (n ManagedGroupListResult) GetItems() []*ManagedGroup {
@@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.authMethodId = authMethodId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*ManagedGroup, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(ManagedGroupListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any ManagedGroup that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a ManagedGroup has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *ManagedGroup) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *ManagedGroup) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *ManagedGroupListResult, opt ...Option) (*ManagedGroupListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.authMethodId == "" {
+		return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["auth_method_id"] = currentPage.authMethodId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(ManagedGroupListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.authMethodId = currentPage.authMethodId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/managedgroups/option.gen.go

@@ -5,6 +5,7 @@
 package managedgroups
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/policies/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/policies/policy.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type PolicyListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n PolicyListResult) GetItems() []*Policy {
@@ -344,96 +352,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Policy, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(PolicyListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Policy that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Policy has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Policy) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Policy) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,4 +452,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *PolicyListResult, opt ...Option) (*PolicyListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(PolicyListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/roles/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/roles/role.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -70,6 +71,13 @@ type RoleListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n RoleListResult) GetItems() []*Role {
@@ -342,96 +350,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Role, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(RoleListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Role that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Role has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Role) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Role) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -445,6 +450,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *RoleListResult, opt ...Option) (*RoleListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(RoleListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddGrantScopes(ctx context.Context, id string, version uint32, grantScopeIds []string, opt ...Option) (*RoleUpdateResult, error) {

api/scopes/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/scopes/scope.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ScopeListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n ScopeListResult) GetItems() []*Scope {
@@ -340,96 +348,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Scope, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(ScopeListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Scope that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Scope has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Scope) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Scope) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +448,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *ScopeListResult, opt ...Option) (*ScopeListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(ScopeListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/sessionrecordings/option.gen.go

@@ -20,13 +20,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -56,6 +58,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -75,6 +80,21 @@ func WithListToken(listToken string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/sessionrecordings/session_recording.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -73,6 +74,13 @@ type SessionRecordingListResult struct {
 	ListToken    string              `json:"list_token,omitempty"`
 	ResponseType string              `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n SessionRecordingListResult) GetItems() []*SessionRecording {
@@ -238,96 +246,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*SessionRecording, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(SessionRecordingListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any SessionRecording that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a SessionRecording has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *SessionRecording) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *SessionRecording) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -341,4 +346,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionRecordingListResult, opt ...Option) (*SessionRecordingListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(SessionRecordingListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/sessions/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/sessions/session.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -75,6 +76,13 @@ type SessionListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n SessionListResult) GetItems() []*Session {
@@ -198,96 +206,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Session, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(SessionListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Session that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Session has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Session) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Session) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -301,4 +306,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionListResult, opt ...Option) (*SessionListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(SessionListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/storagebuckets/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/storagebuckets/storage_bucket.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -76,6 +77,13 @@ type StorageBucketListResult struct {
 	ListToken    string           `json:"list_token,omitempty"`
 	ResponseType string           `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n StorageBucketListResult) GetItems() []*StorageBucket {
@@ -348,96 +356,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*StorageBucket, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(StorageBucketListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any StorageBucket that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a StorageBucket has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *StorageBucket) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *StorageBucket) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -451,4 +456,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *StorageBucketListResult, opt ...Option) (*StorageBucketListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(StorageBucketListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/targets/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/targets/target.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -81,6 +82,13 @@ type TargetListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n TargetListResult) GetItems() []*Target {
@@ -358,96 +366,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Target, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(TargetListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Target that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Target has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Target) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Target) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -461,6 +466,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *TargetListResult, opt ...Option) (*TargetListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(TargetListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddCredentialSources(ctx context.Context, id string, version uint32, opt ...Option) (*TargetUpdateResult, error) {

api/users/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/users/user.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -71,6 +72,13 @@ type UserListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n UserListResult) GetItems() []*User {
@@ -343,96 +351,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*User, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(UserListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any User that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a User has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *User) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *User) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -446,6 +451,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *UserListResult, opt ...Option) (*UserListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(UserListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddAccounts(ctx context.Context, id string, version uint32, accountIds []string, opt ...Option) (*UserUpdateResult, error) {

api/workers/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/workers/worker.gen.go

@@ -6,11 +6,9 @@ package workers
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
-	"slices"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -78,6 +76,13 @@ type WorkerListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n WorkerListResult) GetItems() []*Worker {
@@ -399,109 +404,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work
 		return nil, apiErr
 	}
 	target.Response = resp
-	if target.ResponseType == "complete" || target.ResponseType == "" {
-		return target, nil
-	}
-	// If there are more results, automatically fetch the rest of the results.
-	// idToIndex keeps a map from the ID of an item to its index in target.Items.
-	// This is used to update updated items in-place and remove deleted items
-	// from the result after pagination is done.
-	idToIndex := map[string]int{}
-	for i, item := range target.Items {
-		idToIndex[item.Id] = i
-	}
-	for {
-		req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...)
-		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
-		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(WorkerListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
-			if i, ok := idToIndex[item.Id]; ok {
-				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
-			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
-			}
-		}
-		// RemovedIds contain any Worker that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
-			break
-		}
-	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
-	// Remove items that were deleted since the end of the last iteration.
-	// If a Worker has been updated and subsequently removed, we don't want
-	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
-		if i, ok := idToIndex[removedId]; ok {
-			// Remove the item at index i without preserving order
-			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
-			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
-		}
-	}
-	// Since we paginated to the end, we can avoid confusion
-	// for the user by setting the estimated item count to the
-	// length of the items slice. If we don't set this here, it
-	// will equal the value returned in the last response, which is
-	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Worker) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
-	// Finally, since we made at least 2 requests to the server to fulfill this
-	// function call, resp.Body and resp.Map will only contain the most recent response.
-	// Overwrite them with the true response.
-	target.Response.Body.Reset()
-	if err := json.NewEncoder(target.Response.Body).Encode(target); err != nil {
-		return nil, fmt.Errorf("error encoding final JSON list response: %w", err)
-	}
-	if err := json.Unmarshal(target.Response.Body.Bytes(), &target.Response.Map); err != nil {
-		return nil, fmt.Errorf("error encoding final map list response: %w", err)
-	}
-	// Note: the HTTP response body is consumed by resp.Decode in the loop,
-	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
 }
 
 func (c *Client) AddWorkerTags(ctx context.Context, id string, version uint32, apiTags map[string][]string, opt ...Option) (*WorkerUpdateResult, error) {

internal/api/genapi/input.go

@@ -142,6 +142,10 @@ type structInfo struct {
 	// fields
 	fieldFilter []string
 
+	// nonPaginatedListing indicates a collection that does not support
+	// pagination
+	nonPaginatedListing bool
+
 	allowEmpty bool
 }
 
@@ -1426,5 +1430,6 @@ var inputStructs = []*structInfo{
 		createResponseTypes: []string{CreateResponseType, ReadResponseType, UpdateResponseType, DeleteResponseType, ListResponseType},
 		recursiveListing:    true,
 		versionEnabled:      true,
+		nonPaginatedListing: true,
 	},
 }

internal/api/genapi/templates.go

@@ -49,6 +49,7 @@ type templateInput struct {
 	SliceSubtypes         map[string]sliceSubtypeInfo
 	ExtraFields           []fieldInfo
 	VersionEnabled        bool
+	NonPaginatedListing   bool
 	CreateResponseTypes   []string
 	SkipListFiltering     bool
 	RecursiveListing      bool
@@ -69,6 +70,7 @@ func fillTemplates() {
 			ParentTypeName:      in.parentTypeName,
 			ExtraFields:         in.extraFields,
 			VersionEnabled:      in.versionEnabled,
+			NonPaginatedListing: in.nonPaginatedListing,
 			CreateResponseTypes: in.createResponseTypes,
 			SkipListFiltering:   in.skipListFiltering,
 			RecursiveListing:    in.recursiveListing,
@@ -268,96 +270,96 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
 		return nil, apiErr
 	}
 	target.Response = resp
+{{ if .NonPaginatedListing }}
+	return target, nil
+{{ end }}
+{{ if ( not ( .NonPaginatedListing ) ) }}
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+{{ if .RecursiveListing }} 
+	target.recursive = opts.withRecursive
+{{ end }}
+	target.pageSize = opts.withPageSize
+	target.{{ .CollectionFunctionArg }} = {{ .CollectionFunctionArg }}
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*{{ .Name }}, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new({{ .Name }}ListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any {{ .Name }} that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a {{ .Name }} has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *{{ .Name }}) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *{{ .Name }}) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -371,7 +373,93 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+{{ end  }}
 }
+
+{{ if ( not ( .NonPaginatedListing ) ) }}
+func (c *Client) ListNextPage(ctx context.Context, currentPage *{{ .Name }}ListResult, opt ...Option) (*{{ .Name }}ListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.{{ .CollectionFunctionArg }} == "" {
+		return nil, fmt.Errorf("empty {{ .CollectionFunctionArg }} value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["{{ snakeCase .CollectionFunctionArg }}"] = currentPage.{{ .CollectionFunctionArg }}
+
+{{ if .RecursiveListing }} 
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+{{ end }}
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new({{ .Name }}ListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.{{ .CollectionFunctionArg }} = currentPage.{{ .CollectionFunctionArg }}
+{{ if .RecursiveListing }} 
+	nextPage.recursive = currentPage.recursive
+{{ end }} 
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
+}
+{{ end }}
 `))
 
 var readTemplate = template.Must(template.New("").Parse(`
@@ -764,6 +852,14 @@ type {{ .Name }}ListResult struct {
 	ListToken string            `, "`json:\"list_token,omitempty\"`", `
 	ResponseType string         `, "`json:\"response_type,omitempty\"`", `
 	Response *api.Response
+
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive bool
+	pageSize uint32
+	{{ .CollectionFunctionArg }} string
+	allRemovedIds []string
 }
 
 func (n {{ .Name }}ListResult) GetItems() []*{{ .Name }} {
@@ -845,6 +941,8 @@ type options struct {
 	withSkipCurlOutput bool
 	withFilter string
 	withListToken string
+	withClientDirectedPagination bool
+	withPageSize uint32
 	{{ if .RecursiveListing }} withRecursive bool {{ end }}
 }
 
@@ -875,6 +973,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	} {{ end }}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -916,6 +1017,20 @@ func WithFilter(filter string) Option {
 	}
 }
 {{ end }}
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+			o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
 {{ if .RecursiveListing }}
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource

internal/clientcache/cmd/cache/start.go

@@ -50,6 +50,7 @@ type StartCommand struct {
 	flagLogFormat               string
 	flagStoreDebug              bool
 	flagBackground              bool
+	flagForceResetSchema        bool
 }
 
 func (c *StartCommand) Synopsis() string {
@@ -133,6 +134,13 @@ func (c *StartCommand) Flags() *base.FlagSets {
 		Default: false,
 		Usage:   `Run the cache daemon in the background`,
 	})
+	f.BoolVar(&base.BoolVar{
+		Name:    "force-reset-schema",
+		Target:  &c.flagForceResetSchema,
+		Default: false,
+		Usage:   `Force resetting the cache schema and all contained data`,
+		Hidden:  true,
+	})
 
 	return set
 }
@@ -207,6 +215,7 @@ func (c *StartCommand) Run(args []string) int {
 		LogFileName:             logFileName,
 		DotDirectory:            dotDir,
 		RunningInBackground:     os.Getenv(backgroundEnvName) == backgroundEnvVal,
+		ForceResetSchema:        c.flagForceResetSchema,
 	}
 
 	srv, err := daemon.New(ctx, cfg)

internal/clientcache/internal/cache/options.go

@@ -25,6 +25,7 @@ type options struct {
 	withIgnoreSearchStaleness        bool
 	withMaxResultSetSize             int
 	withTestRefreshWaitChs           *testRefreshWaitChs
+	withUseNonPagedListing           bool
 }
 
 // Option - how options are passed as args
@@ -129,3 +130,12 @@ func WithTestRefreshWaitChs(with *testRefreshWaitChs) Option {
 		return nil
 	}
 }
+
+// WithUseNonPagedListing provides an option for ignoring the resource
+// staleness when performing a search.
+func WithUseNonPagedListing(b bool) Option {
+	return func(o *options) error {
+		o.withUseNonPagedListing = b
+		return nil
+	}
+}

internal/clientcache/internal/cache/options_test.go

@@ -49,8 +49,8 @@ func Test_GetOpts(t *testing.T) {
 		assert.Equal(t, opts, testOpts)
 	})
 	t.Run("WithTargetRetrievalFunc", func(t *testing.T) {
-		var f TargetRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
-			return nil, nil, "", nil
+		var f TargetRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (*targets.TargetListResult, RefreshTokenValue, error) {
+			return nil, "", nil
 		}
 		opts, err := getOpts(WithTargetRetrievalFunc(f))
 		require.NoError(t, err)
@@ -118,4 +118,12 @@ func Test_GetOpts(t *testing.T) {
 		testOpts.withTestRefreshWaitChs = waitCh
 		assert.Equal(t, opts, testOpts)
 	})
+	t.Run("WithUseNonPagedListing", func(t *testing.T) {
+		opts, err := getOpts(WithUseNonPagedListing(true))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.False(t, testOpts.withUseNonPagedListing)
+		testOpts.withUseNonPagedListing = true
+		assert.Equal(t, opts, testOpts)
+	})
 }

internal/clientcache/internal/cache/refresh.go

@@ -420,28 +420,47 @@ func (r *RefreshService) RecheckCachingSupport(ctx context.Context, opt ...Optio
 			continue
 		}
 
-		if err := r.repo.checkCachingTargets(ctx, u, tokens, opt...); err != nil {
-			if err == ErrRefreshNotSupported {
-				// This is expected so no need to propagate the error up
-				continue
+		cacheKey := fmt.Sprintf("%s-%s", u.Id, Targets)
+		semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.checkCachingTargets(ctx, u, tokens, opt...); err != nil {
+				if err == ErrRefreshNotSupported {
+					semaphore.(*atomic.Bool).Store(false)
+					// This is expected so no need to propagate the error up
+					continue
+				}
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
 			}
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
-		}
-		if err := r.repo.checkCachingSessions(ctx, u, tokens, opt...); err != nil {
-			if err == ErrRefreshNotSupported {
-				// This is expected so no need to propagate the error up
-				continue
-			}
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
-		}
-		if err := r.repo.checkCachingResolvableAliases(ctx, u, tokens, opt...); err != nil {
-			if err == ErrRefreshNotSupported {
-				// This is expected so no need to propagate the error up
-				continue
-			}
-			retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			semaphore.(*atomic.Bool).Store(false)
 		}
 
+		cacheKey = fmt.Sprintf("%s-%s", u.Id, ResolvableAliases)
+		semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.checkCachingResolvableAliases(ctx, u, tokens, opt...); err != nil {
+				if err == ErrRefreshNotSupported {
+					// This is expected so no need to propagate the error up
+					semaphore.(*atomic.Bool).Store(false)
+					continue
+				}
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
+		}
+
+		cacheKey = fmt.Sprintf("%s-%s", u.Id, Sessions)
+		semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool))
+		if semaphore.(*atomic.Bool).CompareAndSwap(false, true) {
+			if err := r.repo.checkCachingSessions(ctx, u, tokens, opt...); err != nil {
+				if err == ErrRefreshNotSupported {
+					semaphore.(*atomic.Bool).Store(false)
+					// This is expected so no need to propagate the error up
+					continue
+				}
+				retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id))))
+			}
+			semaphore.(*atomic.Bool).Store(false)
+		}
 	}
 	return retErr
 }

internal/clientcache/internal/cache/refresh_test.go

@@ -54,6 +54,22 @@ func testStaticResourceRetrievalFunc[T any](t *testing.T, ret [][]T, removed [][
 	}
 }
 
+func testTargetStaticResourceRetrievalFunc(inFunc func(ctx context.Context, s1, s2 string, refToken RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error)) TargetRetrievalFunc {
+	return func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (ret *targets.TargetListResult, refreshToken RefreshTokenValue, err error) {
+		retTargets, removed, refreshToken, err := inFunc(ctx, addr, authTok, refreshTok)
+		if err != nil {
+			return nil, "", err
+		}
+
+		ret = &targets.TargetListResult{
+			Items:        retTargets,
+			RemovedIds:   removed,
+			ResponseType: "complete",
+		}
+		return ret, refreshToken, nil
+	}
+}
+
 // testNoRefreshRetrievalFunc simulates a controller that doesn't support refresh
 // since it does not return any refresh token.
 func testNoRefreshRetrievalFunc[T any](t *testing.T) func(context.Context, string, string, RefreshTokenValue) ([]T, []string, RefreshTokenValue, error) {
@@ -426,7 +442,7 @@ func TestRefreshForSearch(t *testing.T) {
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -435,7 +451,7 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...))
 
@@ -481,7 +497,7 @@ func TestRefreshForSearch(t *testing.T) {
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -490,7 +506,7 @@ func TestRefreshForSearch(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...))
 
@@ -538,8 +554,8 @@ func TestRefreshForSearch(t *testing.T) {
 		// Get the first set of resources, but no refresh tokens
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.ErrorContains(t, err, ErrRefreshNotSupported.Error())
 
 		got, err := r.ListTargets(ctx, at.Id)
@@ -554,14 +570,14 @@ func TestRefreshForSearch(t *testing.T) {
 		// any more.
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.Nil(t, err)
 
 		err = rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
+			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.Nil(t, err)
 
 		got, err = r.ListTargets(ctx, at.Id)
@@ -575,8 +591,9 @@ func TestRefreshForSearch(t *testing.T) {
 		// the resources starting to be cached.
 		err = rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{retTargets}, [][]string{{}}))),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{retTargets}, [][]string{{}})))
+		)
 		assert.Nil(t, err, err)
 
 		got, err = r.ListTargets(ctx, at.Id)
@@ -601,7 +618,7 @@ func TestRefreshForSearch(t *testing.T) {
 		}
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
@@ -653,7 +670,7 @@ func TestRefreshForSearch(t *testing.T) {
 		}
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
@@ -710,7 +727,7 @@ func TestRefreshForSearch(t *testing.T) {
 		}
 		opts := []Option{
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAl[:3],
@@ -762,7 +779,7 @@ func TestRefreshForSearch(t *testing.T) {
 		}
 		opts := []Option{
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAls[:3],
@@ -839,7 +856,7 @@ func TestRefreshNonBlocking(t *testing.T) {
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -848,7 +865,7 @@ func TestRefreshNonBlocking(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 
 		refreshWaitChs := &testRefreshWaitChs{
@@ -899,7 +916,7 @@ func TestRefreshNonBlocking(t *testing.T) {
 		}
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
@@ -961,7 +978,7 @@ func TestRefreshNonBlocking(t *testing.T) {
 		}
 		opts := []Option{
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAl[:3],
@@ -1041,7 +1058,7 @@ func TestRefresh(t *testing.T) {
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t,
 				[][]*targets.Target{
 					retTargets[:3],
 					retTargets[3:],
@@ -1050,7 +1067,7 @@ func TestRefresh(t *testing.T) {
 					nil,
 					{retTargets[0].Id, retTargets[1].Id},
 				},
-			)),
+			))),
 		}
 		assert.NoError(t, rs.Refresh(ctx, opts...))
 
@@ -1082,7 +1099,7 @@ func TestRefresh(t *testing.T) {
 		}
 		opts := []Option{
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t,
 				[][]*sessions.Session{
 					retSess[:3],
@@ -1123,7 +1140,7 @@ func TestRefresh(t *testing.T) {
 		}
 		opts := []Option{
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t,
 				[][]*aliases.Alias{
 					retAls[:3],
@@ -1160,15 +1177,15 @@ func TestRefresh(t *testing.T) {
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))),
 			WithSessionRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
@@ -1202,7 +1219,7 @@ func TestRefresh(t *testing.T) {
 		require.NoError(t, rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)),
 			WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)),
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)))))
 
 		ps, err = r.listTokens(ctx, u)
 		require.NoError(t, err)
@@ -1246,7 +1263,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))))
 
 		got, err := r.ListTargets(ctx, at.Id)
 		require.NoError(t, err)
@@ -1258,7 +1275,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
 		got, err = r.ListTargets(ctx, at.Id)
@@ -1272,7 +1289,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))))
 	})
 
 	t.Run("sessions", func(t *testing.T) {
@@ -1286,7 +1303,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 
 		got, err := r.ListSessions(ctx, at.Id)
@@ -1298,7 +1315,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
@@ -1311,7 +1328,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		got, err = r.ListSessions(ctx, at.Id)
 		require.NoError(t, err)
@@ -1332,7 +1349,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 
 		got, err := r.ListResolvableAliases(ctx, at.Id)
@@ -1344,7 +1361,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
@@ -1357,7 +1374,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		assert.NoError(t, rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))))
 		got, err = r.ListResolvableAliases(ctx, at.Id)
 		require.NoError(t, err)
@@ -1378,7 +1395,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
@@ -1386,21 +1403,21 @@ func TestRecheckCachingSupport(t *testing.T) {
 		err = rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 
 		err = rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
 				require.Equal(t, boundaryAddr, addr)
 				require.Equal(t, at.Token, token)
 				return nil, nil, "", innerErr
-			}))
+			})))
 		assert.ErrorContains(t, err, innerErr.Error())
 	})
 
@@ -1415,7 +1432,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 
 		err = rs.Refresh(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)),
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))
 		assert.ErrorIs(t, err, ErrRefreshNotSupported)
 
@@ -1435,7 +1452,7 @@ func TestRecheckCachingSupport(t *testing.T) {
 		err = rs.RecheckCachingSupport(ctx,
 			WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)),
 			WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)),
-			WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 		assert.NoError(t, err)
 
 		ps, err = r.listTokens(ctx, u)

internal/clientcache/internal/cache/repository_implicit_scopes_test.go

@@ -62,7 +62,7 @@ func TestRepository_ImplicitScopes(t *testing.T) {
 		target("3"),
 	}
 	require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	for _, t := range ts {
 		expectedScopes = append(expectedScopes, &scopes.Scope{

internal/clientcache/internal/cache/repository_targets.go

@@ -21,32 +21,42 @@ import (
 
 // TargetRetrievalFunc is a function that retrieves targets
 // from the provided boundary addr using the provided token.
-type TargetRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) (ret []*targets.Target, removedIds []string, refreshToken RefreshTokenValue, err error)
+type TargetRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (ret *targets.TargetListResult, refreshToken RefreshTokenValue, err error)
 
-func defaultTargetFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) {
+func defaultTargetFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (*targets.TargetListResult, RefreshTokenValue, error) {
 	const op = "cache.defaultTargetFunc"
 	conf, err := api.DefaultConfig()
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
+	}
+	opts, err := getOpts(opt...)
+	if err != nil {
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	conf.Addr = addr
 	conf.Token = authTok
 	client, err := api.NewClient(conf)
 	if err != nil {
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	tarClient := targets.NewClient(client)
-	l, err := tarClient.List(ctx, "global", targets.WithRecursive(true), targets.WithListToken(string(refreshTok)))
+	var l *targets.TargetListResult
+	switch inPage {
+	case nil:
+		l, err = tarClient.List(ctx, "global", targets.WithRecursive(true), targets.WithListToken(string(refreshTok)), targets.WithClientDirectedPagination(!opts.withUseNonPagedListing))
+	default:
+		l, err = tarClient.ListNextPage(ctx, inPage, targets.WithListToken(string(refreshTok)))
+	}
 	if err != nil {
 		if api.ErrInvalidListToken.Is(err) {
-			return nil, nil, "", err
+			return nil, "", err
 		}
-		return nil, nil, "", errors.Wrap(ctx, err, op)
+		return nil, "", errors.Wrap(ctx, err, op)
 	}
 	if l.ResponseType == "" {
-		return nil, nil, "", ErrRefreshNotSupported
+		return nil, "", ErrRefreshNotSupported
 	}
-	return l.Items, l.RemovedIds, RefreshTokenValue(l.ListToken), nil
+	return l, RefreshTokenValue(l.ListToken), nil
 }
 
 // refreshTargets uses attempts to refresh the targets for the provided user
@@ -81,13 +91,13 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 
 	// Find and use a token for retrieving targets
 	var gotResponse bool
-	var resp []*targets.Target
-	var removedIds []string
+	var currentPage *targets.TargetListResult
 	var newRefreshToken RefreshTokenValue
+	var foundAuthToken string
 	var unsupportedCacheRequest bool
 	var retErr error
 	for at, t := range tokens {
-		resp, removedIds, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal)
+		currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal, currentPage)
 		if api.ErrInvalidListToken.Is(err) {
 			event.WriteSysEvent(ctx, op, "old list token is no longer valid, starting new initial fetch", "user_id", u.Id)
 			if err := r.deleteRefreshToken(ctx, u, resourceType); err != nil {
@@ -95,7 +105,7 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 			}
 			// try again without the refresh token
 			oldRefreshToken = nil
-			resp, removedIds, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "")
+			currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "", currentPage)
 		}
 		if err != nil {
 			if err == ErrRefreshNotSupported {
@@ -105,6 +115,7 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 				continue
 			}
 		}
+		foundAuthToken = t
 		gotResponse = true
 		break
 	}
@@ -118,44 +129,57 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut
 	}
 
 	var numDeleted int
-	_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
-		var err error
-		switch {
-		case oldRefreshToken == nil || unsupportedCacheRequest:
-			if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id",
-				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
-				return err
+	var numUpserted int
+	var clearPerformed bool
+	for {
+		_, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error {
+			var err error
+			if (oldRefreshToken == nil || unsupportedCacheRequest) && !clearPerformed {
+				if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id",
+					[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
+					return err
+				}
+				clearPerformed = true
 			}
-		case len(removedIds) > 0:
-			if numDeleted, err = w.Exec(ctx, "delete from target where id in @ids",
-				[]any{sql.Named("ids", removedIds)}); err != nil {
-				return err
+			switch {
+			case unsupportedCacheRequest:
+				if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
+					return err
+				}
+			case newRefreshToken != "":
+				if err := upsertTargets(ctx, w, u, currentPage.Items); err != nil {
+					return err
+				}
+				numUpserted += len(currentPage.Items)
+				if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
+					return err
+				}
+			default:
+				// controller supports caching, but doesn't have any resources
 			}
+			if !unsupportedCacheRequest && len(currentPage.RemovedIds) > 0 {
+				if numDeleted, err = w.Exec(ctx, "delete from target where id in @ids",
+					[]any{sql.Named("ids", currentPage.RemovedIds)}); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		if unsupportedCacheRequest || currentPage.ResponseType == "" || currentPage.ResponseType == "complete" {
+			break
 		}
-		switch {
-		case unsupportedCacheRequest:
-			if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil {
-				return err
-			}
-		case newRefreshToken != "":
-			if err := upsertTargets(ctx, w, u, resp); err != nil {
-				return err
-			}
-			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
-				return err
-			}
-		default:
-			// controller supports caching, but doesn't have any resources
+		currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, foundAuthToken, newRefreshToken, currentPage)
+		if err != nil {
+			break
 		}
-		return nil
-	})
+	}
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", numUpserted, "user_id", u.Id)
 	return nil
 }
 
@@ -185,12 +209,12 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma
 
 	// Find and use a token for retrieving targets
 	var gotResponse bool
-	var resp []*targets.Target
+	var resp *targets.TargetListResult
 	var newRefreshToken RefreshTokenValue
 	var unsupportedCacheRequest bool
 	var retErr error
 	for at, t := range tokens {
-		resp, _, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "")
+		resp, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "", nil, WithUseNonPagedListing(true))
 		if err != nil {
 			if err == ErrRefreshNotSupported {
 				unsupportedCacheRequest = true
@@ -221,14 +245,15 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma
 				return err
 			}
 		case newRefreshToken != "":
-			var err error
 			// Now that there is a refresh token, the data can be cached, so
-			// cache it and store the refresh token for future refreshes.
+			// cache it and store the refresh token for future refreshes. First
+			// remove any values, then add the new ones
+			var err error
 			if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id",
 				[]any{sql.Named("fk_user_id", u.Id)}); err != nil {
 				return err
 			}
-			if err := upsertTargets(ctx, w, u, resp); err != nil {
+			if err := upsertTargets(ctx, w, u, resp.Items); err != nil {
 				return err
 			}
 			if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil {
@@ -249,7 +274,7 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma
 	if unsupportedCacheRequest {
 		return ErrRefreshNotSupported
 	}
-	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id)
+	event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp.Items), "user_id", u.Id)
 	return nil
 }
 

internal/clientcache/internal/cache/repository_targets_test.go

@@ -141,7 +141,7 @@ func TestRepository_refreshTargets(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
 			err := r.refreshTargets(ctx, tc.u, map[AuthToken]string{{Id: "id"}: "something"},
-				WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{tc.targets}, [][]string{nil})))
+				WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{tc.targets}, [][]string{nil}))))
 			if tc.errorContains == "" {
 				assert.NoError(t, err)
 				rw := db.New(s)
@@ -213,12 +213,12 @@ func TestRepository_RefreshTargets_InvalidListTokenError(t *testing.T) {
 	}
 
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(invalidAuthTokenFunc)))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(invalidAuthTokenFunc))))
 
 	// This time an invalid auth token should be returned, and refreshTargets should fall back
 	// to requesting without one.
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(invalidAuthTokenFunc)))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(invalidAuthTokenFunc))))
 
 	assert.Equal(t, 1, withRefreshToken)
 	assert.Equal(t, 2, withoutRefreshToken)
@@ -261,7 +261,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) {
 	}
 
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))))
 
 	got, err := r.ListTargets(ctx, at.Id)
 	require.NoError(t, err)
@@ -270,7 +270,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) {
 	// Refreshing again uses the refresh token and get additional sessions, appending
 	// them to the response
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))))
 	assert.NoError(t, err)
 
 	got, err = r.ListTargets(ctx, at.Id)
@@ -280,7 +280,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) {
 	// Refreshing again wont return any more resources, but also none should be
 	// removed
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))))
 	assert.NoError(t, err)
 
 	got, err = r.ListTargets(ctx, at.Id)
@@ -289,7 +289,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) {
 
 	// Refresh again with the refresh token being reported as invalid.
 	require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ts[0]))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ts[0])))))
 	assert.NoError(t, err)
 
 	got, err = r.ListTargets(ctx, at.Id)
@@ -345,7 +345,7 @@ func TestRepository_ListTargets(t *testing.T) {
 		target("3"),
 	}
 	require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	t.Run("wrong user gets no targets", func(t *testing.T) {
 		l, err := r.ListTargets(ctx, kt2.AuthTokenId)
@@ -389,7 +389,7 @@ func TestRepository_ListTargetsLimiting(t *testing.T) {
 		ts = append(ts, target("t"+strconv.Itoa(i)))
 	}
 	require.NoError(t, r.refreshTargets(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	searchService, err := NewSearchService(ctx, r)
 	require.NoError(t, err)
@@ -512,7 +512,7 @@ func TestRepository_QueryTargets(t *testing.T) {
 		},
 	}
 	require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	t.Run("wrong token gets no targets", func(t *testing.T) {
 		l, err := r.QueryTargets(ctx, kt2.AuthTokenId, query)
@@ -556,7 +556,7 @@ func TestRepository_QueryTargetsLimiting(t *testing.T) {
 		ts = append(ts, target("t"+strconv.Itoa(i)))
 	}
 	require.NoError(t, r.refreshTargets(ctx, u, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))))
 
 	searchService, err := NewSearchService(ctx, r)
 	require.NoError(t, err)
@@ -606,13 +606,13 @@ func TestDefaultTargetRetrievalFunc(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, tar2)
 
-	got, removed, refTok, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "")
+	got, refTok, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "", nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok)
-	assert.Empty(t, removed)
+	assert.Empty(t, got.RemovedIds)
 	found1 := false
 	found2 := false
-	for _, t := range got {
+	for _, t := range got.Items {
 		if t.Id == tar1.Item.Id {
 			found1 = true
 		}
@@ -623,10 +623,10 @@ func TestDefaultTargetRetrievalFunc(t *testing.T) {
 	assert.True(t, found1, "expected to find target %s in list", tar1.Item.Id)
 	assert.True(t, found2, "expected to find target %s in list", tar2.Item.Id)
 
-	got2, removed2, refTok2, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok)
+	got2, refTok2, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok, nil)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, refTok2)
 	assert.NotEqual(t, refTok2, refTok)
-	assert.Empty(t, removed2)
-	assert.Empty(t, got2)
+	assert.Empty(t, got.RemovedIds)
+	assert.Empty(t, got2.Items)
 }

internal/clientcache/internal/cache/status_test.go

@@ -190,11 +190,11 @@ func TestStatus(t *testing.T) {
 			target("4"),
 		}
 		err = r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))
 		require.NoError(t, err)
 
 		err = r.refreshTargets(ctx, u2, map[AuthToken]string{{Id: "id"}: "something"},
-			WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts[:2]}, [][]string{nil})))
+			WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts[:2]}, [][]string{nil}))))
 		require.NoError(t, err)
 
 		sess := []*sessions.Session{
@@ -312,7 +312,7 @@ func TestStatus_unsupported(t *testing.T) {
 	require.ErrorIs(t, err, ErrRefreshNotSupported)
 
 	err = r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},
-		WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))
+		WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))
 	require.ErrorIs(t, err, ErrRefreshNotSupported)
 
 	err = r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"},

internal/clientcache/internal/daemon/options.go

@@ -20,9 +20,10 @@ type options struct {
 	WithReadyToServeNotificationCh         chan struct{}
 	withBoundaryTokenReaderFunc            cache.BoundaryTokenReaderFn
 
-	withUrl     string
-	withLogger  hclog.Logger
-	withHomeDir string
+	withUrl              string
+	withLogger           hclog.Logger
+	withHomeDir          string
+	withForceResetSchema bool
 }
 
 // Option - how options are passed as args
@@ -110,6 +111,15 @@ func WithBoundaryTokenReaderFunc(_ context.Context, fn cache.BoundaryTokenReader
 	}
 }
 
+// WithForceResetSchema provides an optional way to force resetting the schema,
+// e.g. wiping the cache
+func WithForceResetSchema(_ context.Context, force bool) Option {
+	return func(o *options) error {
+		o.withForceResetSchema = force
+		return nil
+	}
+}
+
 // WithReadyToServeNotificationCh provides an optional channel to notify when
 // the server is ready to serve; mainly used for test timing but exported for
 // availability. The channel will be closed just before the HTTP server is

internal/clientcache/internal/daemon/options_test.go

@@ -108,6 +108,14 @@ func Test_GetOpts(t *testing.T) {
 		testOpts.withHomeDir = "/tmp"
 		assert.Equal(t, opts, testOpts)
 	})
+	t.Run("WithForceResetSchema", func(t *testing.T) {
+		opts, err := getOpts(WithForceResetSchema(ctx, true))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.False(t, testOpts.withForceResetSchema)
+		testOpts.withForceResetSchema = true
+		assert.Equal(t, opts, testOpts)
+	})
 	t.Run("WithReadyToServeNotificationCh", func(t *testing.T) {
 		ch := make(chan struct{})
 		opts, err := getOpts(WithReadyToServeNotificationCh(ctx, ch))

internal/clientcache/internal/daemon/server.go

@@ -90,6 +90,8 @@ type Config struct {
 	// The maximum amount of time a refresh should block a search request from
 	// completing before it times out.
 	MaxSearchRefreshTimeout time.Duration
+	// Force resetting the schema, that is, drop all existing data
+	ForceResetSchema bool
 }
 
 func (sc *Config) validate(ctx context.Context) error {
@@ -230,6 +232,7 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 	store, err = openStore(ctx,
 		WithUrl(ctx, s.conf.DatabaseUrl),
 		WithLogger(ctx, s.logger),
+		WithForceResetSchema(ctx, s.conf.ForceResetSchema),
 	)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
@@ -537,6 +540,7 @@ func openStore(ctx context.Context, opt ...Option) (*db.DB, error) {
 			dbOpts = append(dbOpts, cachedb.WithGormFormatter(opts.withLogger))
 		}
 	}
+	dbOpts = append(dbOpts, cachedb.WithForceResetSchema(opts.withForceResetSchema))
 	store, err := cachedb.Open(ctx, dbOpts...)
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)

internal/clientcache/internal/daemon/testing.go

@@ -97,11 +97,13 @@ func (s *TestServer) AddResources(t *testing.T, p *authtokens.AuthToken, alts []
 		}
 		return alts, nil, "addedaliases", nil
 	}
-	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) {
+	tarFn := func(ctx context.Context, _ string, tok string, _ cache.RefreshTokenValue, inPage *targets.TargetListResult, opt ...cache.Option) (*targets.TargetListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return nil, "", nil
 		}
-		return tars, nil, "addedtargets", nil
+		return &targets.TargetListResult{
+			Items: tars,
+		}, "addedtargets", nil
 	}
 	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
@@ -124,13 +126,15 @@ func (s *TestServer) AddUnsupportedCachingData(t *testing.T, p *authtokens.AuthT
 	r, err := cache.NewRepository(ctx, s.CacheServer.store.Load(), &sync.Map{}, s.cmd.ReadTokenFromKeyring, atReadFn)
 	require.NoError(t, err)
 
-	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) {
+	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue, inPage *targets.TargetListResult, opt ...cache.Option) (*targets.TargetListResult, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
-			return nil, nil, "", nil
+			return &targets.TargetListResult{}, "", nil
 		}
-		return []*targets.Target{
-			{Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"},
-		}, nil, "", cache.ErrRefreshNotSupported
+		return &targets.TargetListResult{
+			Items: []*targets.Target{
+				{Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"},
+			},
+		}, "", cache.ErrRefreshNotSupported
 	}
 	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) {
 		if tok != p.Token {

internal/clientcache/internal/db/db.go

@@ -69,7 +69,7 @@ func Open(ctx context.Context, opt ...Option) (*db.DB, error) {
 		if err != nil {
 			return nil, errors.Wrap(ctx, err, op)
 		}
-		if !ok {
+		if !ok || opts.withForceResetSchema {
 			if err := resetSchema(ctx, conn); err != nil {
 				return nil, errors.Wrap(ctx, err, op)
 			}

internal/clientcache/internal/db/options.go

@@ -9,11 +9,12 @@ import (
 )
 
 type options struct {
-	withSchemaVersion string
-	withDebug         bool
-	withUrl           string
-	withDbType        dbw.DbType
-	withGormFormatter hclog.Logger
+	withSchemaVersion    string
+	withDebug            bool
+	withUrl              string
+	withDbType           dbw.DbType
+	withGormFormatter    hclog.Logger
+	withForceResetSchema bool
 }
 
 // Option - how options are passed as args
@@ -67,3 +68,11 @@ func WithDebug(debug bool) Option {
 		return nil
 	}
 }
+
+// WithForceResetSchema provides an optional way to force resetting the cache
+func WithForceResetSchema(debug bool) Option {
+	return func(o *options) error {
+		o.withForceResetSchema = debug
+		return nil
+	}
+}

internal/clientcache/internal/db/options_test.go

@@ -45,4 +45,12 @@ func Test_GetOpts(t *testing.T) {
 		testOpts.withSchemaVersion = version
 		assert.Equal(t, opts, testOpts)
 	})
+	t.Run("WithForceResetSchema", func(t *testing.T) {
+		opts, err := getOpts(WithForceResetSchema(true))
+		require.NoError(t, err)
+		testOpts := getDefaultOptions()
+		assert.False(t, testOpts.withForceResetSchema)
+		testOpts.withForceResetSchema = true
+		assert.Equal(t, opts, testOpts)
+	})
 }

internal/tests/api/targets/target_test.go

@@ -336,6 +336,39 @@ func TestListWithListToken(t *testing.T) {
 	require.Empty(res.Items)
 }
 
+func TestListWithPageSize(t *testing.T) {
+	// Set database read timeout to avoid duplicates in response
+	oldReadTimeout := globals.RefreshReadLookbackDuration
+	globals.RefreshReadLookbackDuration = 0
+	t.Cleanup(func() {
+		globals.RefreshReadLookbackDuration = oldReadTimeout
+	})
+	require := require.New(t)
+	tc := controller.NewTestController(t, nil)
+	defer tc.Shutdown()
+
+	client := tc.Client()
+	token := tc.Token()
+	client.SetToken(token.Token)
+	_, proj := iam.TestScopes(t, tc.IamRepo(), iam.WithUserId(token.UserId))
+
+	tarClient := targets.NewClient(client)
+	_, err := tarClient.Create(tc.Context(), "tcp", proj.GetPublicId(), targets.WithName("1"), targets.WithTcpTargetDefaultPort(2))
+	require.NoError(err)
+	_, err = tarClient.Create(tc.Context(), "tcp", proj.GetPublicId(), targets.WithName("2"), targets.WithTcpTargetDefaultPort(2))
+	require.NoError(err)
+
+	// Refresh tokens recursive listing over global scope
+	res, err := tarClient.List(tc.Context(), "global", targets.WithRecursive(true), targets.WithPageSize(2))
+	require.NoError(err)
+	require.Len(res.Items, 4, "expected the 2 targets created above and the 2 auto created for the test controller")
+	refTok := res.ListToken
+
+	res, err = tarClient.List(tc.Context(), "global", targets.WithRecursive(true), targets.WithListToken(refTok))
+	require.NoError(err)
+	require.Empty(res.Items)
+}
+
 func TestTarget_AddressMutualExclusiveRelationship(t *testing.T) {
 	tc := controller.NewTestController(t, nil)