api: add automatic list pagination

Updates all List endpoints to automatically paginate
through results to preserve backwards compatibility.
Adding support for user controllable pagination could
be considered at a later time.

api/workers/option.gen.go

@@ -26,6 +26,7 @@ type options struct {
 	withAutomaticVersioning bool
 	withSkipCurlOutput      bool
 	withFilter              string
+	withRefreshToken        string
 	withRecursive           bool
 }
 
@@ -50,6 +51,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
 	if opts.withFilter != "" {
 		opts.queryMap["filter"] = opts.withFilter
 	}
+	if opts.withRefreshToken != "" {
+		opts.queryMap["refresh_token"] = opts.withRefreshToken
+	}
 	if opts.withRecursive {
 		opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
 	}
@@ -74,6 +78,14 @@ func WithSkipCurlOutput(skip bool) Option {
 	}
 }
 
+// WithRefreshToken tells the API to use the provided refresh token
+// for listing operations on this resource.
+func WithRefreshToken(refreshToken string) Option {
+	return func(o *options) {
+		o.withRefreshToken = refreshToken
+	}
+}
+
 // WithFilter tells the API to filter the items returned using the provided
 // filter term.  The filter should be in a format supported by
 // hashicorp/go-bexpr.

api/workers/worker.gen.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"slices"
 	"time"
 
 	"github.com/hashicorp/boundary/api"
@@ -70,14 +71,34 @@ func (n WorkerDeleteResult) GetResponse() *api.Response {
 }
 
 type WorkerListResult struct {
-	Items    []*Worker
-	response *api.Response
+	Items        []*Worker `json:"items,omitempty"`
+	EstItemCount uint      `json:"est_item_count,omitempty"`
+	RemovedIds   []string  `json:"removed_ids,omitempty"`
+	RefreshToken string    `json:"refresh_token,omitempty"`
+	ResponseType string    `json:"response_type,omitempty"`
+	response     *api.Response
 }
 
 func (n WorkerListResult) GetItems() []*Worker {
 	return n.Items
 }
 
+func (n WorkerListResult) GetEstItemCount() uint {
+	return n.EstItemCount
+}
+
+func (n WorkerListResult) GetRemovedIds() []string {
+	return n.RemovedIds
+}
+
+func (n WorkerListResult) GetRefreshToken() string {
+	return n.RefreshToken
+}
+
+func (n WorkerListResult) GetResponseType() string {
+	return n.ResponseType
+}
+
 func (n WorkerListResult) GetResponse() *api.Response {
 	return n.response
 }
@@ -377,6 +398,78 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work
 		return nil, apiErr
 	}
 	target.response = resp
+	if target.ResponseType == "complete" || target.ResponseType == "" {
+		return target, nil
+	}
+	// If there are more results, automatically fetch the rest of the results.
+	// idToIndex keeps a map from the ID of an item to its index in target.Items.
+	// This is used to update updated items in-place and remove deleted items
+	// from the result after pagination is done.
+	idToIndex := map[string]int{}
+	for i, item := range target.Items {
+		idToIndex[item.Id] = i
+	}
+	for {
+		req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...)
+		if err != nil {
+			return nil, fmt.Errorf("error creating List request: %w", err)
+		}
+
+		opts.queryMap["refresh_token"] = target.RefreshToken
+		if len(opts.queryMap) > 0 {
+			q := url.Values{}
+			for k, v := range opts.queryMap {
+				q.Add(k, v)
+			}
+			req.URL.RawQuery = q.Encode()
+		}
+
+		resp, err := c.client.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("error performing client request during List call: %w", err)
+		}
+
+		page := new(WorkerListResult)
+		apiErr, err := resp.Decode(page)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding List response: %w", err)
+		}
+		if apiErr != nil {
+			return nil, apiErr
+		}
+		for _, item := range page.Items {
+			if i, ok := idToIndex[item.Id]; ok {
+				// Item has already been seen at index i, update in-place
+				target.Items[i] = item
+			} else {
+				target.Items = append(target.Items, item)
+				idToIndex[item.Id] = len(target.Items) - 1
+			}
+		}
+		target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
+		target.EstItemCount = page.EstItemCount
+		target.RefreshToken = page.RefreshToken
+		target.ResponseType = page.ResponseType
+		target.response = resp
+		if target.ResponseType == "complete" {
+			break
+		}
+	}
+	for _, removedId := range target.RemovedIds {
+		if i, ok := idToIndex[removedId]; ok {
+			// Remove the item at index i without preserving order
+			// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
+			target.Items[i] = target.Items[len(target.Items)-1]
+			target.Items = target.Items[:len(target.Items)-1]
+			// Update the index of the last element
+			idToIndex[target.Items[i].Id] = i
+		}
+	}
+	// Finally, sort the results again since in-place updates and deletes
+	// may have shuffled items.
+	slices.SortFunc(target.Items, func(i, j *Worker) int {
+		return i.UpdatedTime.Compare(j.UpdatedTime)
+	})
 	return target, nil
 }