Paginate targets in cache (#5101)

* Update Go API to support pagination
* Implement pagination of targets in cache

(cherry picked from commit cd25fad42f325fc050b7ea3bec3336780d2b5e16)

api/accounts/account.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type AccountListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	authMethodId  string
+	allRemovedIds []string
 }
 
 func (n AccountListResult) GetItems() []*Account {
@@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.authMethodId = authMethodId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Account, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AccountListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Account that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Account has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Account) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Account) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AccountListResult, opt ...Option) (*AccountListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.authMethodId == "" {
+		return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["auth_method_id"] = currentPage.authMethodId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AccountListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.authMethodId = currentPage.authMethodId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/accounts/option.gen.go

@@ -5,6 +5,7 @@
 package accounts
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/aliases/alias.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type AliasListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AliasListResult) GetItems() []*Alias {
@@ -346,96 +354,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Alias, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AliasListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Alias that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Alias has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Alias) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Alias) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +454,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AliasListResult, opt ...Option) (*AliasListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AliasListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/aliases/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/authmethods/authmethods.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type AuthMethodListResult struct {
 	ListToken    string        `json:"list_token,omitempty"`
 	ResponseType string        `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AuthMethodListResult) GetItems() []*AuthMethod {
@@ -346,96 +354,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*AuthMethod, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AuthMethodListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any AuthMethod that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a AuthMethod has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *AuthMethod) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *AuthMethod) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +454,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthMethodListResult, opt ...Option) (*AuthMethodListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AuthMethodListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/authmethods/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/authtokens/authtokens.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -66,6 +67,13 @@ type AuthTokenListResult struct {
 	ListToken    string       `json:"list_token,omitempty"`
 	ResponseType string       `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n AuthTokenListResult) GetItems() []*AuthToken {
@@ -231,96 +239,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*AuthToken, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(AuthTokenListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any AuthToken that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a AuthToken has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *AuthToken) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *AuthToken) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -334,4 +339,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthTokenListResult, opt ...Option) (*AuthTokenListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(AuthTokenListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/authtokens/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -85,6 +90,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/billing/option.gen.go

@@ -6,6 +6,7 @@ package billing
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -21,12 +22,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -53,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -91,6 +97,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithEndTime(inEndTime string) Option {
 	return func(o *options) {
 		o.queryMap["end_time"] = fmt.Sprintf("%v", inEndTime)

api/credentiallibraries/credential_library.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -69,6 +70,13 @@ type CredentialLibraryListResult struct {
 	ListToken    string               `json:"list_token,omitempty"`
 	ResponseType string               `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive         bool
+	pageSize          uint32
+	credentialStoreId string
+	allRemovedIds     []string
 }
 
 func (n CredentialLibraryListResult) GetItems() []*CredentialLibrary {
@@ -346,96 +354,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.credentialStoreId = credentialStoreId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*CredentialLibrary, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialLibraryListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any CredentialLibrary that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a CredentialLibrary has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *CredentialLibrary) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *CredentialLibrary) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +452,80 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialLibraryListResult, opt ...Option) (*CredentialLibraryListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.credentialStoreId == "" {
+		return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["credential_store_id"] = currentPage.credentialStoreId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialLibraryListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.credentialStoreId = currentPage.credentialStoreId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentiallibraries/option.gen.go

@@ -5,6 +5,7 @@
 package credentiallibraries
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithVaultSSHCertificateCredentialLibraryAdditionalValidPrincipals(inAdditionalValidPrincipals []string) Option {
 	return func(o *options) {
 		raw, ok := o.postMap["attributes"]

api/credentials/credential.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type CredentialListResult struct {
 	ListToken    string        `json:"list_token,omitempty"`
 	ResponseType string        `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive         bool
+	pageSize          uint32
+	credentialStoreId string
+	allRemovedIds     []string
 }
 
 func (n CredentialListResult) GetItems() []*Credential {
@@ -344,96 +352,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.credentialStoreId = credentialStoreId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Credential, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Credential that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Credential has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Credential) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Credential) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,4 +450,80 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialListResult, opt ...Option) (*CredentialListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.credentialStoreId == "" {
+		return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["credential_store_id"] = currentPage.credentialStoreId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.credentialStoreId = currentPage.credentialStoreId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentials/option.gen.go

@@ -5,6 +5,7 @@
 package credentials
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/credentialstores/credential_store.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type CredentialStoreListResult struct {
 	ListToken    string             `json:"list_token,omitempty"`
 	ResponseType string             `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n CredentialStoreListResult) GetItems() []*CredentialStore {
@@ -345,96 +353,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*CredentialStore, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(CredentialStoreListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any CredentialStore that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a CredentialStore has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *CredentialStore) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *CredentialStore) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -448,4 +453,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialStoreListResult, opt ...Option) (*CredentialStoreListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(CredentialStoreListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/credentialstores/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/groups/group.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type GroupListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n GroupListResult) GetItems() []*Group {
@@ -339,96 +347,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Group, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(GroupListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Group that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Group has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Group) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Group) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -442,6 +447,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *GroupListResult, opt ...Option) (*GroupListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(GroupListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddMembers(ctx context.Context, id string, version uint32, memberIds []string, opt ...Option) (*GroupUpdateResult, error) {

api/groups/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/hostcatalogs/host_catalog.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -73,6 +74,13 @@ type HostCatalogListResult struct {
 	ListToken    string         `json:"list_token,omitempty"`
 	ResponseType string         `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n HostCatalogListResult) GetItems() []*HostCatalog {
@@ -350,96 +358,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*HostCatalog, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostCatalogListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any HostCatalog that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a HostCatalog has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *HostCatalog) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *HostCatalog) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -453,4 +458,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostCatalogListResult, opt ...Option) (*HostCatalogListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostCatalogListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/hostcatalogs/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/hosts/host.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -74,6 +75,13 @@ type HostListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	hostCatalogId string
+	allRemovedIds []string
 }
 
 func (n HostListResult) GetItems() []*Host {
@@ -346,96 +354,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.hostCatalogId = hostCatalogId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Host, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Host that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Host has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Host) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Host) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -449,4 +452,80 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostListResult, opt ...Option) (*HostListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.hostCatalogId == "" {
+		return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.hostCatalogId = currentPage.hostCatalogId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/hosts/option.gen.go

@@ -5,6 +5,7 @@
 package hosts
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithStaticHostAddress(inAddress string) Option {
 	return func(o *options) {
 		raw, ok := o.postMap["attributes"]

api/hostsets/host_set.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -72,6 +73,13 @@ type HostSetListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	hostCatalogId string
+	allRemovedIds []string
 }
 
 func (n HostSetListResult) GetItems() []*HostSet {
@@ -344,96 +352,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.hostCatalogId = hostCatalogId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*HostSet, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(HostSetListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any HostSet that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a HostSet has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *HostSet) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *HostSet) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,6 +450,82 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option)
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *HostSetListResult, opt ...Option) (*HostSetListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.hostCatalogId == "" {
+		return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(HostSetListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.hostCatalogId = currentPage.hostCatalogId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddHosts(ctx context.Context, id string, version uint32, hostIds []string, opt ...Option) (*HostSetUpdateResult, error) {

api/hostsets/option.gen.go

@@ -5,6 +5,7 @@
 package hostsets
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/managedgroups/managedgroups.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ManagedGroupListResult struct {
 	ListToken    string          `json:"list_token,omitempty"`
 	ResponseType string          `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	authMethodId  string
+	allRemovedIds []string
 }
 
 func (n ManagedGroupListResult) GetItems() []*ManagedGroup {
@@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.pageSize = opts.withPageSize
+	target.authMethodId = authMethodId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*ManagedGroup, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(ManagedGroupListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any ManagedGroup that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a ManagedGroup has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *ManagedGroup) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *ManagedGroup) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *ManagedGroupListResult, opt ...Option) (*ManagedGroupListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.authMethodId == "" {
+		return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["auth_method_id"] = currentPage.authMethodId
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(ManagedGroupListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.authMethodId = currentPage.authMethodId
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/managedgroups/option.gen.go

@@ -5,6 +5,7 @@
 package managedgroups
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/hashicorp/boundary/api"
@@ -20,12 +21,14 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
 }
 
 func getDefaultOptions() options {
@@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withListToken != "" {
 		opts.queryMap["list_token"] = opts.withListToken
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -90,6 +96,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 func WithAttributes(inAttributes map[string]interface{}) Option {
 	return func(o *options) {
 		o.postMap["attributes"] = inAttributes

api/policies/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/policies/policy.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -67,6 +68,13 @@ type PolicyListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n PolicyListResult) GetItems() []*Policy {
@@ -344,96 +352,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Policy, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(PolicyListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Policy that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Policy has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Policy) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Policy) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -447,4 +452,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *PolicyListResult, opt ...Option) (*PolicyListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(PolicyListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/roles/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/roles/role.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -70,6 +71,13 @@ type RoleListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n RoleListResult) GetItems() []*Role {
@@ -342,96 +350,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Role, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(RoleListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Role that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Role has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Role) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Role) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -445,6 +450,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *RoleListResult, opt ...Option) (*RoleListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(RoleListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddGrantScopes(ctx context.Context, id string, version uint32, grantScopeIds []string, opt ...Option) (*RoleUpdateResult, error) {

api/scopes/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/scopes/scope.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ScopeListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n ScopeListResult) GetItems() []*Scope {
@@ -340,96 +348,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Scope, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(ScopeListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Scope that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Scope has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Scope) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Scope) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -443,4 +448,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *ScopeListResult, opt ...Option) (*ScopeListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(ScopeListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/sessionrecordings/option.gen.go

@@ -20,13 +20,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -56,6 +58,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -75,6 +80,21 @@ func WithListToken(listToken string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/sessionrecordings/session_recording.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -73,6 +74,13 @@ type SessionRecordingListResult struct {
 	ListToken    string              `json:"list_token,omitempty"`
 	ResponseType string              `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n SessionRecordingListResult) GetItems() []*SessionRecording {
@@ -238,96 +246,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*SessionRecording, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(SessionRecordingListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any SessionRecording that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a SessionRecording has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *SessionRecording) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *SessionRecording) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -341,4 +346,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionRecordingListResult, opt ...Option) (*SessionRecordingListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(SessionRecordingListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/sessions/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/sessions/session.gen.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -75,6 +76,13 @@ type SessionListResult struct {
 	ListToken    string     `json:"list_token,omitempty"`
 	ResponseType string     `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n SessionListResult) GetItems() []*Session {
@@ -198,96 +206,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Session, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(SessionListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Session that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Session has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Session) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Session) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -301,4 +306,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionListResult, opt ...Option) (*SessionListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(SessionListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/storagebuckets/option.gen.go

@@ -22,13 +22,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -96,6 +101,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/storagebuckets/storage_bucket.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -76,6 +77,13 @@ type StorageBucketListResult struct {
 	ListToken    string           `json:"list_token,omitempty"`
 	ResponseType string           `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n StorageBucketListResult) GetItems() []*StorageBucket {
@@ -348,96 +356,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*StorageBucket, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(StorageBucketListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any StorageBucket that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a StorageBucket has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *StorageBucket) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *StorageBucket) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -451,4 +456,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *StorageBucketListResult, opt ...Option) (*StorageBucketListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(StorageBucketListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }

api/targets/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/targets/target.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -81,6 +82,13 @@ type TargetListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n TargetListResult) GetItems() []*Target {
@@ -358,96 +366,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*Target, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(TargetListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any Target that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a Target has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *Target) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Target) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -461,6 +466,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *TargetListResult, opt ...Option) (*TargetListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(TargetListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddCredentialSources(ctx context.Context, id string, version uint32, opt ...Option) (*TargetUpdateResult, error) {

api/users/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/users/user.gen.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -71,6 +72,13 @@ type UserListResult struct {
 	ListToken    string   `json:"list_token,omitempty"`
 	ResponseType string   `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n UserListResult) GetItems() []*User {
@@ -343,96 +351,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 		return nil, apiErr
 	}
 	target.Response = resp
+
 	if target.ResponseType == "complete" || target.ResponseType == "" {
 		return target, nil
 	}
+
+	// In case we shortcut out due to client directed pagination, ensure these
+	// are set
+
+	target.recursive = opts.withRecursive
+
+	target.pageSize = opts.withPageSize
+	target.scopeId = scopeId
+	target.allRemovedIds = target.RemovedIds
+	if opts.withClientDirectedPagination {
+		return target, nil
+	}
+
+	allItems := make([]*User, 0, target.EstItemCount)
+	allItems = append(allItems, target.Items...)
+
 	// If there are more results, automatically fetch the rest of the results.
 	// idToIndex keeps a map from the ID of an item to its index in target.Items.
 	// This is used to update updated items in-place and remove deleted items
 	// from the result after pagination is done.
 	idToIndex := map[string]int{}
-	for i, item := range target.Items {
+	for i, item := range allItems {
 		idToIndex[item.Id] = i
 	}
+
+	// If we're here there are more pages and the client does not want to
+	// paginate on their own; fetch them as this call returns all values.
+	currentPage := target
 	for {
-		req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...)
+		nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
 		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
+			return nil, fmt.Errorf("error getting next page in List call: %w", err)
 		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(UserListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
+		for _, item := range nextPage.Items {
 			if i, ok := idToIndex[item.Id]; ok {
 				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
+				allItems[i] = item
 			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
+				allItems = append(allItems, item)
+				idToIndex[item.Id] = len(allItems) - 1
 			}
 		}
-		// RemovedIds contain any User that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
+
+		currentPage = nextPage
+
+		if currentPage.ResponseType == "complete" {
 			break
 		}
 	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
+
+	// The current page here is the final page of the results, that is, the
+	// response type is "complete"
+
 	// Remove items that were deleted since the end of the last iteration.
 	// If a User has been updated and subsequently removed, we don't want
 	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
+	for _, removedId := range currentPage.RemovedIds {
 		if i, ok := idToIndex[removedId]; ok {
 			// Remove the item at index i without preserving order
 			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
+			allItems[i] = allItems[len(allItems)-1]
+			allItems = allItems[:len(allItems)-1]
 			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
+			idToIndex[allItems[i].Id] = i
 		}
 	}
+	// Sort the results again since in-place updates and deletes
+	// may have shuffled items. We sort by created time descending
+	// (most recently created first), same as the API.
+	slices.SortFunc(allItems, func(i, j *User) int {
+		return j.CreatedTime.Compare(i.CreatedTime)
+	})
 	// Since we paginated to the end, we can avoid confusion
 	// for the user by setting the estimated item count to the
 	// length of the items slice. If we don't set this here, it
 	// will equal the value returned in the last response, which is
 	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *User) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
+	currentPage.EstItemCount = uint(len(allItems))
+	// Set items to the full list we have collected here
+	currentPage.Items = allItems
+	// Set the returned value to the last page with calculated values
+	target = currentPage
 	// Finally, since we made at least 2 requests to the server to fulfill this
 	// function call, resp.Body and resp.Map will only contain the most recent response.
 	// Overwrite them with the true response.
@@ -446,6 +451,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User
 	// Note: the HTTP response body is consumed by resp.Decode in the loop,
 	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
+}
+
+func (c *Client) ListNextPage(ctx context.Context, currentPage *UserListResult, opt ...Option) (*UserListResult, error) {
+	if currentPage == nil {
+		return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
+	}
+	if currentPage.scopeId == "" {
+		return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request")
+	}
+	if c.client == nil {
+		return nil, fmt.Errorf("nil client")
+	}
+	if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
+		return nil, fmt.Errorf("no more pages available in ListNextPage request")
+	}
+
+	opts, apiOpts := getOpts(opt...)
+	opts.queryMap["scope_id"] = currentPage.scopeId
+
+	// Don't require them to re-specify recursive
+	if currentPage.recursive {
+		opts.queryMap["recursive"] = "true"
+	}
+
+	if currentPage.pageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
+	}
+
+	req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating List request: %w", err)
+	}
+
+	opts.queryMap["list_token"] = currentPage.ListToken
+	if len(opts.queryMap) > 0 {
+		q := url.Values{}
+		for k, v := range opts.queryMap {
+			q.Add(k, v)
+		}
+		req.URL.RawQuery = q.Encode()
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
+	}
+
+	nextPage := new(UserListResult)
+	apiErr, err := resp.Decode(nextPage)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
+	}
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	// Ensure values are carried forward to the next call
+	nextPage.scopeId = currentPage.scopeId
+
+	nextPage.recursive = currentPage.recursive
+
+	nextPage.pageSize = currentPage.pageSize
+	// Cache the removed IDs from this page
+	nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
+	// Set the response body to the current response
+	nextPage.Response = resp
+	// If we're done iterating, pull the full set of removed IDs into the last
+	// response
+	if nextPage.ResponseType == "complete" {
+		// Collect up the last values
+		nextPage.RemovedIds = nextPage.allRemovedIds
+		// For now, removedIds will only be populated if this pagination cycle
+		// was the result of a "refresh" operation (i.e., the caller provided a
+		// list token option to this call).
+		//
+		// Sort to make response deterministic
+		slices.Sort(nextPage.RemovedIds)
+		// Remove any duplicates
+		nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
+	}
+
+	return nextPage, nil
 }
 
 func (c *Client) AddAccounts(ctx context.Context, id string, version uint32, accountIds []string, opt ...Option) (*UserUpdateResult, error) {

api/workers/option.gen.go

@@ -21,13 +21,15 @@ import (
 type Option func(*options)
 
 type options struct {
-	postMap                 map[string]any
-	queryMap                map[string]string
-	withAutomaticVersioning bool
-	withSkipCurlOutput      bool
-	withFilter              string
-	withListToken           string
-	withRecursive           bool
+	postMap                      map[string]any
+	queryMap                     map[string]string
+	withAutomaticVersioning      bool
+	withSkipCurlOutput           bool
+	withFilter                   string
+	withListToken                string
+	withClientDirectedPagination bool
+	withPageSize                 uint32
+	withRecursive                bool
 }
 
 func getDefaultOptions() options {
@@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
+	if opts.withPageSize != 0 {
+		opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
+	}
 	return opts, apiOpts
 }
 
@@ -95,6 +100,21 @@ func WithFilter(filter string) Option {
 	}
 }
 
+// WithClientDirectedPagination tells the List function to return only the first
+// page, if more pages are available
+func WithClientDirectedPagination(with bool) Option {
+	return func(o *options) {
+		o.withClientDirectedPagination = with
+	}
+}
+
+// WithPageSize controls the size of pages used during List
+func WithPageSize(with uint32) Option {
+	return func(o *options) {
+		o.withPageSize = with
+	}
+}
+
 // WithRecursive tells the API to use recursion for listing operations on this
 // resource
 func WithRecursive(recurse bool) Option {

api/workers/worker.gen.go

@@ -6,11 +6,9 @@ package workers
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
-	"slices"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -78,6 +76,13 @@ type WorkerListResult struct {
 	ListToken    string    `json:"list_token,omitempty"`
 	ResponseType string    `json:"response_type,omitempty"`
 	Response     *api.Response
+
+	// The following fields are used for cached information when client-directed
+	// pagination is used.
+	recursive     bool
+	pageSize      uint32
+	scopeId       string
+	allRemovedIds []string
 }
 
 func (n WorkerListResult) GetItems() []*Worker {
@@ -399,109 +404,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work
 		return nil, apiErr
 	}
 	target.Response = resp
-	if target.ResponseType == "complete" || target.ResponseType == "" {
-		return target, nil
-	}
-	// If there are more results, automatically fetch the rest of the results.
-	// idToIndex keeps a map from the ID of an item to its index in target.Items.
-	// This is used to update updated items in-place and remove deleted items
-	// from the result after pagination is done.
-	idToIndex := map[string]int{}
-	for i, item := range target.Items {
-		idToIndex[item.Id] = i
-	}
-	for {
-		req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...)
-		if err != nil {
-			return nil, fmt.Errorf("error creating List request: %w", err)
-		}
 
-		opts.queryMap["list_token"] = target.ListToken
-		if len(opts.queryMap) > 0 {
-			q := url.Values{}
-			for k, v := range opts.queryMap {
-				q.Add(k, v)
-			}
-			req.URL.RawQuery = q.Encode()
-		}
-
-		resp, err := c.client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("error performing client request during List call: %w", err)
-		}
-
-		page := new(WorkerListResult)
-		apiErr, err := resp.Decode(page)
-		if err != nil {
-			return nil, fmt.Errorf("error decoding List response: %w", err)
-		}
-		if apiErr != nil {
-			return nil, apiErr
-		}
-		for _, item := range page.Items {
-			if i, ok := idToIndex[item.Id]; ok {
-				// Item has already been seen at index i, update in-place
-				target.Items[i] = item
-			} else {
-				target.Items = append(target.Items, item)
-				idToIndex[item.Id] = len(target.Items) - 1
-			}
-		}
-		// RemovedIds contain any Worker that were deleted since the last response.
-		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
-		target.EstItemCount = page.EstItemCount
-		target.ListToken = page.ListToken
-		target.ResponseType = page.ResponseType
-		target.Response = resp
-		if target.ResponseType == "complete" {
-			break
-		}
-	}
-	// For now, removedIds will only be populated if this pagination cycle was the result of a
-	// "refresh" operation (i.e., the caller provided a list token option to this call).
-	//
-	// Sort to make response deterministic
-	slices.Sort(target.RemovedIds)
-	// Remove any duplicates
-	target.RemovedIds = slices.Compact(target.RemovedIds)
-	// Remove items that were deleted since the end of the last iteration.
-	// If a Worker has been updated and subsequently removed, we don't want
-	// it to appear both in the Items and RemovedIds, so we remove it from the Items.
-	for _, removedId := range target.RemovedIds {
-		if i, ok := idToIndex[removedId]; ok {
-			// Remove the item at index i without preserving order
-			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
-			target.Items[i] = target.Items[len(target.Items)-1]
-			target.Items = target.Items[:len(target.Items)-1]
-			// Update the index of the previously last element
-			idToIndex[target.Items[i].Id] = i
-		}
-	}
-	// Since we paginated to the end, we can avoid confusion
-	// for the user by setting the estimated item count to the
-	// length of the items slice. If we don't set this here, it
-	// will equal the value returned in the last response, which is
-	// often much smaller than the total number returned.
-	target.EstItemCount = uint(len(target.Items))
-	// Sort the results again since in-place updates and deletes
-	// may have shuffled items. We sort by created time descending
-	// (most recently created first), same as the API.
-	slices.SortFunc(target.Items, func(i, j *Worker) int {
-		return j.CreatedTime.Compare(i.CreatedTime)
-	})
-	// Finally, since we made at least 2 requests to the server to fulfill this
-	// function call, resp.Body and resp.Map will only contain the most recent response.
-	// Overwrite them with the true response.
-	target.Response.Body.Reset()
-	if err := json.NewEncoder(target.Response.Body).Encode(target); err != nil {
-		return nil, fmt.Errorf("error encoding final JSON list response: %w", err)
-	}
-	if err := json.Unmarshal(target.Response.Body.Bytes(), &target.Response.Map); err != nil {
-		return nil, fmt.Errorf("error encoding final map list response: %w", err)
-	}
-	// Note: the HTTP response body is consumed by resp.Decode in the loop,
-	// so it doesn't need to be updated (it will always be, and has always been, empty).
 	return target, nil
+
 }
 
 func (c *Client) AddWorkerTags(ctx context.Context, id string, version uint32, apiTags map[string][]string, opt ...Option) (*WorkerUpdateResult, error) {