From 3a4ee032d954c6ed01f3539d9667ad70882111b7 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 12 Sep 2024 15:56:33 -0400 Subject: [PATCH] Paginate targets in cache (#5101) * Update Go API to support pagination * Implement pagination of targets in cache (cherry picked from commit cd25fad42f325fc050b7ea3bec3336780d2b5e16) --- api/accounts/account.gen.go | 187 ++++++++++----- api/accounts/option.gen.go | 33 ++- api/aliases/alias.gen.go | 196 ++++++++++----- api/aliases/option.gen.go | 34 ++- api/authmethods/authmethods.gen.go | 196 ++++++++++----- api/authmethods/option.gen.go | 34 ++- api/authtokens/authtokens.gen.go | 196 ++++++++++----- api/authtokens/option.gen.go | 34 ++- api/billing/option.gen.go | 33 ++- .../credential_library.gen.go | 187 ++++++++++----- api/credentiallibraries/option.gen.go | 33 ++- api/credentials/credential.gen.go | 187 ++++++++++----- api/credentials/option.gen.go | 33 ++- api/credentialstores/credential_store.gen.go | 196 ++++++++++----- api/credentialstores/option.gen.go | 34 ++- api/groups/group.gen.go | 196 ++++++++++----- api/groups/option.gen.go | 34 ++- api/hostcatalogs/host_catalog.gen.go | 196 ++++++++++----- api/hostcatalogs/option.gen.go | 34 ++- api/hosts/host.gen.go | 187 ++++++++++----- api/hosts/option.gen.go | 33 ++- api/hostsets/host_set.gen.go | 187 ++++++++++----- api/hostsets/option.gen.go | 33 ++- api/managedgroups/managedgroups.gen.go | 187 ++++++++++----- api/managedgroups/option.gen.go | 33 ++- api/policies/option.gen.go | 34 ++- api/policies/policy.gen.go | 196 ++++++++++----- api/roles/option.gen.go | 34 ++- api/roles/role.gen.go | 196 ++++++++++----- api/scopes/option.gen.go | 34 ++- api/scopes/scope.gen.go | 196 ++++++++++----- api/sessionrecordings/option.gen.go | 34 ++- .../session_recording.gen.go | 196 ++++++++++----- api/sessions/option.gen.go | 34 ++- api/sessions/session.gen.go | 196 ++++++++++----- api/storagebuckets/option.gen.go | 34 ++- api/storagebuckets/storage_bucket.gen.go | 196 ++++++++++----- api/targets/option.gen.go | 34 ++- api/targets/target.gen.go | 196 ++++++++++----- api/users/option.gen.go | 34 ++- api/users/user.gen.go | 196 ++++++++++----- api/workers/option.gen.go | 34 ++- api/workers/worker.gen.go | 111 +-------- internal/api/genapi/input.go | 5 + internal/api/genapi/templates.go | 223 +++++++++++++----- internal/clientcache/cmd/cache/start.go | 9 + .../clientcache/internal/cache/options.go | 10 + .../internal/cache/options_test.go | 12 +- .../clientcache/internal/cache/refresh.go | 57 +++-- .../internal/cache/refresh_test.go | 103 ++++---- .../cache/repository_implicit_scopes_test.go | 2 +- .../internal/cache/repository_targets.go | 119 ++++++---- .../internal/cache/repository_targets_test.go | 34 +-- .../clientcache/internal/cache/status_test.go | 6 +- .../clientcache/internal/daemon/options.go | 16 +- .../internal/daemon/options_test.go | 8 + .../clientcache/internal/daemon/server.go | 4 + .../clientcache/internal/daemon/testing.go | 20 +- internal/clientcache/internal/db/db.go | 2 +- internal/clientcache/internal/db/options.go | 19 +- .../clientcache/internal/db/options_test.go | 8 + internal/tests/api/targets/target_test.go | 33 +++ 62 files changed, 3875 insertions(+), 1533 deletions(-) diff --git a/api/accounts/account.gen.go b/api/accounts/account.gen.go index 98245e699..c55e8b1cb 100644 --- a/api/accounts/account.gen.go +++ b/api/accounts/account.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -68,6 +69,13 @@ type AccountListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + authMethodId string + allRemovedIds []string } func (n AccountListResult) GetItems() []*Account { @@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.pageSize = opts.withPageSize + target.authMethodId = authMethodId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Account, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(AccountListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Account that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Account has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Account) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Account) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *AccountListResult, opt ...Option) (*AccountListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.authMethodId == "" { + return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["auth_method_id"] = currentPage.authMethodId + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(AccountListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.authMethodId = currentPage.authMethodId + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/accounts/option.gen.go b/api/accounts/option.gen.go index b70d6950c..384121c54 100644 --- a/api/accounts/option.gen.go +++ b/api/accounts/option.gen.go @@ -5,6 +5,7 @@ package accounts import ( + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -20,12 +21,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -90,6 +96,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithAttributes(inAttributes map[string]interface{}) Option { return func(o *options) { o.postMap["attributes"] = inAttributes diff --git a/api/aliases/alias.gen.go b/api/aliases/alias.gen.go index d4a25dff7..54b3803cf 100644 --- a/api/aliases/alias.gen.go +++ b/api/aliases/alias.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -69,6 +70,13 @@ type AliasListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n AliasListResult) GetItems() []*Alias { @@ -346,96 +354,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Alias, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(AliasListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Alias that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Alias has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Alias) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Alias) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -449,4 +454,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Alia // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *AliasListResult, opt ...Option) (*AliasListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "aliases", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(AliasListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/aliases/option.gen.go b/api/aliases/option.gen.go index 6e235a94e..9ec819435 100644 --- a/api/aliases/option.gen.go +++ b/api/aliases/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/authmethods/authmethods.gen.go b/api/authmethods/authmethods.gen.go index 719d43fae..afd51e5d6 100644 --- a/api/authmethods/authmethods.gen.go +++ b/api/authmethods/authmethods.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -69,6 +70,13 @@ type AuthMethodListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n AuthMethodListResult) GetItems() []*AuthMethod { @@ -346,96 +354,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*AuthMethod, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(AuthMethodListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any AuthMethod that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a AuthMethod has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *AuthMethod) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *AuthMethod) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -449,4 +454,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthMethodListResult, opt ...Option) (*AuthMethodListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(AuthMethodListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/authmethods/option.gen.go b/api/authmethods/option.gen.go index 9584d61ea..079c35a85 100644 --- a/api/authmethods/option.gen.go +++ b/api/authmethods/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/authtokens/authtokens.gen.go b/api/authtokens/authtokens.gen.go index 541456a18..ced3e321d 100644 --- a/api/authtokens/authtokens.gen.go +++ b/api/authtokens/authtokens.gen.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -66,6 +67,13 @@ type AuthTokenListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n AuthTokenListResult) GetItems() []*AuthToken { @@ -231,96 +239,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*AuthToken, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(AuthTokenListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any AuthToken that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a AuthToken has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *AuthToken) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *AuthToken) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -334,4 +339,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *AuthTokenListResult, opt ...Option) (*AuthTokenListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(AuthTokenListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/authtokens/option.gen.go b/api/authtokens/option.gen.go index e1ed767ed..9fa43031a 100644 --- a/api/authtokens/option.gen.go +++ b/api/authtokens/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -85,6 +90,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/billing/option.gen.go b/api/billing/option.gen.go index 88ddf32ed..76babe8e8 100644 --- a/api/billing/option.gen.go +++ b/api/billing/option.gen.go @@ -6,6 +6,7 @@ package billing import ( "fmt" + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -21,12 +22,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -53,6 +56,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -91,6 +97,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithEndTime(inEndTime string) Option { return func(o *options) { o.queryMap["end_time"] = fmt.Sprintf("%v", inEndTime) diff --git a/api/credentiallibraries/credential_library.gen.go b/api/credentiallibraries/credential_library.gen.go index a298196fb..7206eb892 100644 --- a/api/credentiallibraries/credential_library.gen.go +++ b/api/credentiallibraries/credential_library.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -69,6 +70,13 @@ type CredentialLibraryListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + credentialStoreId string + allRemovedIds []string } func (n CredentialLibraryListResult) GetItems() []*CredentialLibrary { @@ -346,96 +354,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.pageSize = opts.withPageSize + target.credentialStoreId = credentialStoreId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*CredentialLibrary, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(CredentialLibraryListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any CredentialLibrary that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a CredentialLibrary has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *CredentialLibrary) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *CredentialLibrary) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -449,4 +452,80 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialLibraryListResult, opt ...Option) (*CredentialLibraryListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.credentialStoreId == "" { + return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["credential_store_id"] = currentPage.credentialStoreId + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(CredentialLibraryListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.credentialStoreId = currentPage.credentialStoreId + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/credentiallibraries/option.gen.go b/api/credentiallibraries/option.gen.go index 09600aae6..a4dc80b1d 100644 --- a/api/credentiallibraries/option.gen.go +++ b/api/credentiallibraries/option.gen.go @@ -5,6 +5,7 @@ package credentiallibraries import ( + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -20,12 +21,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -90,6 +96,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithVaultSSHCertificateCredentialLibraryAdditionalValidPrincipals(inAdditionalValidPrincipals []string) Option { return func(o *options) { raw, ok := o.postMap["attributes"] diff --git a/api/credentials/credential.gen.go b/api/credentials/credential.gen.go index ef8f49640..d77e7d128 100644 --- a/api/credentials/credential.gen.go +++ b/api/credentials/credential.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -67,6 +68,13 @@ type CredentialListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + credentialStoreId string + allRemovedIds []string } func (n CredentialListResult) GetItems() []*Credential { @@ -344,96 +352,91 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.pageSize = opts.withPageSize + target.credentialStoreId = credentialStoreId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Credential, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(CredentialListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Credential that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Credential has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Credential) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Credential) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -447,4 +450,80 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialListResult, opt ...Option) (*CredentialListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.credentialStoreId == "" { + return nil, fmt.Errorf("empty credentialStoreId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["credential_store_id"] = currentPage.credentialStoreId + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(CredentialListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.credentialStoreId = currentPage.credentialStoreId + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/credentials/option.gen.go b/api/credentials/option.gen.go index 6d7d00260..7cf0fb75e 100644 --- a/api/credentials/option.gen.go +++ b/api/credentials/option.gen.go @@ -5,6 +5,7 @@ package credentials import ( + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -20,12 +21,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -90,6 +96,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithAttributes(inAttributes map[string]interface{}) Option { return func(o *options) { o.postMap["attributes"] = inAttributes diff --git a/api/credentialstores/credential_store.gen.go b/api/credentialstores/credential_store.gen.go index d0ec32269..055af8150 100644 --- a/api/credentialstores/credential_store.gen.go +++ b/api/credentialstores/credential_store.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -68,6 +69,13 @@ type CredentialStoreListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n CredentialStoreListResult) GetItems() []*CredentialStore { @@ -345,96 +353,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*CredentialStore, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(CredentialStoreListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any CredentialStore that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a CredentialStore has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *CredentialStore) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *CredentialStore) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -448,4 +453,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *CredentialStoreListResult, opt ...Option) (*CredentialStoreListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(CredentialStoreListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/credentialstores/option.gen.go b/api/credentialstores/option.gen.go index 1b9ec58bc..eb44bdecf 100644 --- a/api/credentialstores/option.gen.go +++ b/api/credentialstores/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/groups/group.gen.go b/api/groups/group.gen.go index 49d6441f4..ed0c31b65 100644 --- a/api/groups/group.gen.go +++ b/api/groups/group.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -67,6 +68,13 @@ type GroupListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n GroupListResult) GetItems() []*Group { @@ -339,96 +347,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Group, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(GroupListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Group that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Group has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Group) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Group) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -442,6 +447,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *GroupListResult, opt ...Option) (*GroupListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(GroupListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } func (c *Client) AddMembers(ctx context.Context, id string, version uint32, memberIds []string, opt ...Option) (*GroupUpdateResult, error) { diff --git a/api/groups/option.gen.go b/api/groups/option.gen.go index 13828df81..d2e7b5a3d 100644 --- a/api/groups/option.gen.go +++ b/api/groups/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/hostcatalogs/host_catalog.gen.go b/api/hostcatalogs/host_catalog.gen.go index 968282435..5e4feb45a 100644 --- a/api/hostcatalogs/host_catalog.gen.go +++ b/api/hostcatalogs/host_catalog.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -73,6 +74,13 @@ type HostCatalogListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n HostCatalogListResult) GetItems() []*HostCatalog { @@ -350,96 +358,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*HostCatalog, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(HostCatalogListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any HostCatalog that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a HostCatalog has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *HostCatalog) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *HostCatalog) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -453,4 +458,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *HostCatalogListResult, opt ...Option) (*HostCatalogListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(HostCatalogListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/hostcatalogs/option.gen.go b/api/hostcatalogs/option.gen.go index 466fd5036..fe5d26fd1 100644 --- a/api/hostcatalogs/option.gen.go +++ b/api/hostcatalogs/option.gen.go @@ -22,13 +22,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -96,6 +101,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/hosts/host.gen.go b/api/hosts/host.gen.go index be36382a1..016869f44 100644 --- a/api/hosts/host.gen.go +++ b/api/hosts/host.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -74,6 +75,13 @@ type HostListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + hostCatalogId string + allRemovedIds []string } func (n HostListResult) GetItems() []*Host { @@ -346,96 +354,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.pageSize = opts.withPageSize + target.hostCatalogId = hostCatalogId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Host, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(HostListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Host that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Host has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Host) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Host) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -449,4 +452,80 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *HostListResult, opt ...Option) (*HostListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.hostCatalogId == "" { + return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(HostListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.hostCatalogId = currentPage.hostCatalogId + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/hosts/option.gen.go b/api/hosts/option.gen.go index 9868141e5..32809d61c 100644 --- a/api/hosts/option.gen.go +++ b/api/hosts/option.gen.go @@ -5,6 +5,7 @@ package hosts import ( + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -20,12 +21,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -90,6 +96,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithStaticHostAddress(inAddress string) Option { return func(o *options) { raw, ok := o.postMap["attributes"] diff --git a/api/hostsets/host_set.gen.go b/api/hostsets/host_set.gen.go index 1b6642eda..7ce7e6640 100644 --- a/api/hostsets/host_set.gen.go +++ b/api/hostsets/host_set.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -72,6 +73,13 @@ type HostSetListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + hostCatalogId string + allRemovedIds []string } func (n HostSetListResult) GetItems() []*HostSet { @@ -344,96 +352,91 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.pageSize = opts.withPageSize + target.hostCatalogId = hostCatalogId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*HostSet, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(HostSetListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any HostSet that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a HostSet has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *HostSet) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *HostSet) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -447,6 +450,82 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *HostSetListResult, opt ...Option) (*HostSetListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.hostCatalogId == "" { + return nil, fmt.Errorf("empty hostCatalogId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["host_catalog_id"] = currentPage.hostCatalogId + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(HostSetListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.hostCatalogId = currentPage.hostCatalogId + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } func (c *Client) AddHosts(ctx context.Context, id string, version uint32, hostIds []string, opt ...Option) (*HostSetUpdateResult, error) { diff --git a/api/hostsets/option.gen.go b/api/hostsets/option.gen.go index f8649cf47..fa81eddb7 100644 --- a/api/hostsets/option.gen.go +++ b/api/hostsets/option.gen.go @@ -5,6 +5,7 @@ package hostsets import ( + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -20,12 +21,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -90,6 +96,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithAttributes(inAttributes map[string]interface{}) Option { return func(o *options) { o.postMap["attributes"] = inAttributes diff --git a/api/managedgroups/managedgroups.gen.go b/api/managedgroups/managedgroups.gen.go index 9c7497fe4..02cc9d5be 100644 --- a/api/managedgroups/managedgroups.gen.go +++ b/api/managedgroups/managedgroups.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -68,6 +69,13 @@ type ManagedGroupListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + authMethodId string + allRemovedIds []string } func (n ManagedGroupListResult) GetItems() []*ManagedGroup { @@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.pageSize = opts.withPageSize + target.authMethodId = authMethodId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*ManagedGroup, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(ManagedGroupListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any ManagedGroup that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a ManagedGroup has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *ManagedGroup) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *ManagedGroup) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *ManagedGroupListResult, opt ...Option) (*ManagedGroupListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.authMethodId == "" { + return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["auth_method_id"] = currentPage.authMethodId + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(ManagedGroupListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.authMethodId = currentPage.authMethodId + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/managedgroups/option.gen.go b/api/managedgroups/option.gen.go index 15ca4a69f..7da2f0803 100644 --- a/api/managedgroups/option.gen.go +++ b/api/managedgroups/option.gen.go @@ -5,6 +5,7 @@ package managedgroups import ( + "strconv" "strings" "github.com/hashicorp/boundary/api" @@ -20,12 +21,14 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 } func getDefaultOptions() options { @@ -52,6 +55,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withListToken != "" { opts.queryMap["list_token"] = opts.withListToken } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -90,6 +96,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + func WithAttributes(inAttributes map[string]interface{}) Option { return func(o *options) { o.postMap["attributes"] = inAttributes diff --git a/api/policies/option.gen.go b/api/policies/option.gen.go index 59a9674b2..376e19c66 100644 --- a/api/policies/option.gen.go +++ b/api/policies/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/policies/policy.gen.go b/api/policies/policy.gen.go index 4229172ec..bf105a2ba 100644 --- a/api/policies/policy.gen.go +++ b/api/policies/policy.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -67,6 +68,13 @@ type PolicyListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n PolicyListResult) GetItems() []*Policy { @@ -344,96 +352,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Policy, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(PolicyListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Policy that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Policy has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Policy) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Policy) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -447,4 +452,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Poli // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *PolicyListResult, opt ...Option) (*PolicyListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "policies", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(PolicyListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/roles/option.gen.go b/api/roles/option.gen.go index fd552e1f4..06cda67ea 100644 --- a/api/roles/option.gen.go +++ b/api/roles/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/roles/role.gen.go b/api/roles/role.gen.go index 7fae3fd18..7b6b55071 100644 --- a/api/roles/role.gen.go +++ b/api/roles/role.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -70,6 +71,13 @@ type RoleListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n RoleListResult) GetItems() []*Role { @@ -342,96 +350,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Role, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(RoleListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Role that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Role has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Role) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Role) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -445,6 +450,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *RoleListResult, opt ...Option) (*RoleListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(RoleListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } func (c *Client) AddGrantScopes(ctx context.Context, id string, version uint32, grantScopeIds []string, opt ...Option) (*RoleUpdateResult, error) { diff --git a/api/scopes/option.gen.go b/api/scopes/option.gen.go index f8d59d6c1..d20936ba0 100644 --- a/api/scopes/option.gen.go +++ b/api/scopes/option.gen.go @@ -22,13 +22,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -96,6 +101,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/scopes/scope.gen.go b/api/scopes/scope.gen.go index eeb91f42c..93119cd5e 100644 --- a/api/scopes/scope.gen.go +++ b/api/scopes/scope.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -68,6 +69,13 @@ type ScopeListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n ScopeListResult) GetItems() []*Scope { @@ -340,96 +348,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Scope, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(ScopeListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Scope that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Scope has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Scope) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Scope) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -443,4 +448,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *ScopeListResult, opt ...Option) (*ScopeListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(ScopeListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/sessionrecordings/option.gen.go b/api/sessionrecordings/option.gen.go index fb53594df..beae7de2a 100644 --- a/api/sessionrecordings/option.gen.go +++ b/api/sessionrecordings/option.gen.go @@ -20,13 +20,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -56,6 +58,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -75,6 +80,21 @@ func WithListToken(listToken string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/sessionrecordings/session_recording.gen.go b/api/sessionrecordings/session_recording.gen.go index e217ea6fa..164880cf9 100644 --- a/api/sessionrecordings/session_recording.gen.go +++ b/api/sessionrecordings/session_recording.gen.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -73,6 +74,13 @@ type SessionRecordingListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n SessionRecordingListResult) GetItems() []*SessionRecording { @@ -238,96 +246,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*SessionRecording, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(SessionRecordingListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any SessionRecording that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a SessionRecording has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *SessionRecording) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *SessionRecording) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -341,4 +346,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionRecordingListResult, opt ...Option) (*SessionRecordingListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(SessionRecordingListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/sessions/option.gen.go b/api/sessions/option.gen.go index 15310f74d..3887657af 100644 --- a/api/sessions/option.gen.go +++ b/api/sessions/option.gen.go @@ -22,13 +22,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -96,6 +101,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/sessions/session.gen.go b/api/sessions/session.gen.go index 16b2693e3..5ffaa9f6f 100644 --- a/api/sessions/session.gen.go +++ b/api/sessions/session.gen.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -75,6 +76,13 @@ type SessionListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n SessionListResult) GetItems() []*Session { @@ -198,96 +206,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Session, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(SessionListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Session that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Session has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Session) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Session) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -301,4 +306,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *SessionListResult, opt ...Option) (*SessionListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(SessionListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/storagebuckets/option.gen.go b/api/storagebuckets/option.gen.go index ad2b9f623..fc97a4586 100644 --- a/api/storagebuckets/option.gen.go +++ b/api/storagebuckets/option.gen.go @@ -22,13 +22,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -58,6 +60,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -96,6 +101,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/storagebuckets/storage_bucket.gen.go b/api/storagebuckets/storage_bucket.gen.go index ee75c9a41..b232f5a2d 100644 --- a/api/storagebuckets/storage_bucket.gen.go +++ b/api/storagebuckets/storage_bucket.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -76,6 +77,13 @@ type StorageBucketListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n StorageBucketListResult) GetItems() []*StorageBucket { @@ -348,96 +356,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*StorageBucket, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(StorageBucketListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any StorageBucket that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a StorageBucket has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *StorageBucket) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *StorageBucket) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -451,4 +456,87 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *StorageBucketListResult, opt ...Option) (*StorageBucketListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(StorageBucketListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } diff --git a/api/targets/option.gen.go b/api/targets/option.gen.go index 73b3e85e0..4d71168be 100644 --- a/api/targets/option.gen.go +++ b/api/targets/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/targets/target.gen.go b/api/targets/target.gen.go index 36d28555e..6d7ad4331 100644 --- a/api/targets/target.gen.go +++ b/api/targets/target.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -81,6 +82,13 @@ type TargetListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n TargetListResult) GetItems() []*Target { @@ -358,96 +366,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*Target, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(TargetListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any Target that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a Target has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *Target) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Target) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -461,6 +466,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *TargetListResult, opt ...Option) (*TargetListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(TargetListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } func (c *Client) AddCredentialSources(ctx context.Context, id string, version uint32, opt ...Option) (*TargetUpdateResult, error) { diff --git a/api/users/option.gen.go b/api/users/option.gen.go index 7c0c5ed80..a6349cde1 100644 --- a/api/users/option.gen.go +++ b/api/users/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/users/user.gen.go b/api/users/user.gen.go index c1499f836..51065721f 100644 --- a/api/users/user.gen.go +++ b/api/users/user.gen.go @@ -11,6 +11,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "time" "github.com/hashicorp/boundary/api" @@ -71,6 +72,13 @@ type UserListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n UserListResult) GetItems() []*User { @@ -343,96 +351,93 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User return nil, apiErr } target.Response = resp + if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set + + target.recursive = opts.withRecursive + + target.pageSize = opts.withPageSize + target.scopeId = scopeId + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*User, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(UserListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any User that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a User has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *User) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *User) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -446,6 +451,89 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + +} + +func (c *Client) ListNextPage(ctx context.Context, currentPage *UserListResult, opt ...Option) (*UserListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.scopeId == "" { + return nil, fmt.Errorf("empty scopeId value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["scope_id"] = currentPage.scopeId + + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } + + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new(UserListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.scopeId = currentPage.scopeId + + nextPage.recursive = currentPage.recursive + + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil } func (c *Client) AddAccounts(ctx context.Context, id string, version uint32, accountIds []string, opt ...Option) (*UserUpdateResult, error) { diff --git a/api/workers/option.gen.go b/api/workers/option.gen.go index c9f51af41..1ca3ada82 100644 --- a/api/workers/option.gen.go +++ b/api/workers/option.gen.go @@ -21,13 +21,15 @@ import ( type Option func(*options) type options struct { - postMap map[string]any - queryMap map[string]string - withAutomaticVersioning bool - withSkipCurlOutput bool - withFilter string - withListToken string - withRecursive bool + postMap map[string]any + queryMap map[string]string + withAutomaticVersioning bool + withSkipCurlOutput bool + withFilter string + withListToken string + withClientDirectedPagination bool + withPageSize uint32 + withRecursive bool } func getDefaultOptions() options { @@ -57,6 +59,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -95,6 +100,21 @@ func WithFilter(filter string) Option { } } +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} + // WithRecursive tells the API to use recursion for listing operations on this // resource func WithRecursive(recurse bool) Option { diff --git a/api/workers/worker.gen.go b/api/workers/worker.gen.go index 140dc8e92..efe18d859 100644 --- a/api/workers/worker.gen.go +++ b/api/workers/worker.gen.go @@ -6,11 +6,9 @@ package workers import ( "context" - "encoding/json" "errors" "fmt" "net/url" - "slices" "time" "github.com/hashicorp/boundary/api" @@ -78,6 +76,13 @@ type WorkerListResult struct { ListToken string `json:"list_token,omitempty"` ResponseType string `json:"response_type,omitempty"` Response *api.Response + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + scopeId string + allRemovedIds []string } func (n WorkerListResult) GetItems() []*Worker { @@ -399,109 +404,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work return nil, apiErr } target.Response = resp - if target.ResponseType == "complete" || target.ResponseType == "" { - return target, nil - } - // If there are more results, automatically fetch the rest of the results. - // idToIndex keeps a map from the ID of an item to its index in target.Items. - // This is used to update updated items in-place and remove deleted items - // from the result after pagination is done. - idToIndex := map[string]int{} - for i, item := range target.Items { - idToIndex[item.Id] = i - } - for { - req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...) - if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) - } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new(WorkerListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { - if i, ok := idToIndex[item.Id]; ok { - // Item has already been seen at index i, update in-place - target.Items[i] = item - } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 - } - } - // RemovedIds contain any Worker that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { - break - } - } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) - // Remove items that were deleted since the end of the last iteration. - // If a Worker has been updated and subsequently removed, we don't want - // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { - if i, ok := idToIndex[removedId]; ok { - // Remove the item at index i without preserving order - // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] - // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i - } - } - // Since we paginated to the end, we can avoid confusion - // for the user by setting the estimated item count to the - // length of the items slice. If we don't set this here, it - // will equal the value returned in the last response, which is - // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *Worker) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) - // Finally, since we made at least 2 requests to the server to fulfill this - // function call, resp.Body and resp.Map will only contain the most recent response. - // Overwrite them with the true response. - target.Response.Body.Reset() - if err := json.NewEncoder(target.Response.Body).Encode(target); err != nil { - return nil, fmt.Errorf("error encoding final JSON list response: %w", err) - } - if err := json.Unmarshal(target.Response.Body.Bytes(), &target.Response.Map); err != nil { - return nil, fmt.Errorf("error encoding final map list response: %w", err) - } - // Note: the HTTP response body is consumed by resp.Decode in the loop, - // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil + } func (c *Client) AddWorkerTags(ctx context.Context, id string, version uint32, apiTags map[string][]string, opt ...Option) (*WorkerUpdateResult, error) { diff --git a/internal/api/genapi/input.go b/internal/api/genapi/input.go index 5e9291f32..0f497ec2b 100644 --- a/internal/api/genapi/input.go +++ b/internal/api/genapi/input.go @@ -142,6 +142,10 @@ type structInfo struct { // fields fieldFilter []string + // nonPaginatedListing indicates a collection that does not support + // pagination + nonPaginatedListing bool + allowEmpty bool } @@ -1426,5 +1430,6 @@ var inputStructs = []*structInfo{ createResponseTypes: []string{CreateResponseType, ReadResponseType, UpdateResponseType, DeleteResponseType, ListResponseType}, recursiveListing: true, versionEnabled: true, + nonPaginatedListing: true, }, } diff --git a/internal/api/genapi/templates.go b/internal/api/genapi/templates.go index 8e769e77a..aa37311e5 100644 --- a/internal/api/genapi/templates.go +++ b/internal/api/genapi/templates.go @@ -49,6 +49,7 @@ type templateInput struct { SliceSubtypes map[string]sliceSubtypeInfo ExtraFields []fieldInfo VersionEnabled bool + NonPaginatedListing bool CreateResponseTypes []string SkipListFiltering bool RecursiveListing bool @@ -69,6 +70,7 @@ func fillTemplates() { ParentTypeName: in.parentTypeName, ExtraFields: in.extraFields, VersionEnabled: in.versionEnabled, + NonPaginatedListing: in.nonPaginatedListing, CreateResponseTypes: in.createResponseTypes, SkipListFiltering: in.skipListFiltering, RecursiveListing: in.recursiveListing, @@ -268,96 +270,96 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string, return nil, apiErr } target.Response = resp +{{ if .NonPaginatedListing }} + return target, nil +{{ end }} +{{ if ( not ( .NonPaginatedListing ) ) }} if target.ResponseType == "complete" || target.ResponseType == "" { return target, nil } + + // In case we shortcut out due to client directed pagination, ensure these + // are set +{{ if .RecursiveListing }} + target.recursive = opts.withRecursive +{{ end }} + target.pageSize = opts.withPageSize + target.{{ .CollectionFunctionArg }} = {{ .CollectionFunctionArg }} + target.allRemovedIds = target.RemovedIds + if opts.withClientDirectedPagination { + return target, nil + } + + allItems := make([]*{{ .Name }}, 0, target.EstItemCount) + allItems = append(allItems, target.Items...) + // If there are more results, automatically fetch the rest of the results. // idToIndex keeps a map from the ID of an item to its index in target.Items. // This is used to update updated items in-place and remove deleted items // from the result after pagination is done. idToIndex := map[string]int{} - for i, item := range target.Items { + for i, item := range allItems { idToIndex[item.Id] = i } + + // If we're here there are more pages and the client does not want to + // paginate on their own; fetch them as this call returns all values. + currentPage := target for { - req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...) + nextPage, err := c.ListNextPage(ctx, currentPage, opt...) if err != nil { - return nil, fmt.Errorf("error creating List request: %w", err) + return nil, fmt.Errorf("error getting next page in List call: %w", err) } - opts.queryMap["list_token"] = target.ListToken - if len(opts.queryMap) > 0 { - q := url.Values{} - for k, v := range opts.queryMap { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("error performing client request during List call: %w", err) - } - - page := new({{ .Name }}ListResult) - apiErr, err := resp.Decode(page) - if err != nil { - return nil, fmt.Errorf("error decoding List response: %w", err) - } - if apiErr != nil { - return nil, apiErr - } - for _, item := range page.Items { + for _, item := range nextPage.Items { if i, ok := idToIndex[item.Id]; ok { // Item has already been seen at index i, update in-place - target.Items[i] = item + allItems[i] = item } else { - target.Items = append(target.Items, item) - idToIndex[item.Id] = len(target.Items) - 1 + allItems = append(allItems, item) + idToIndex[item.Id] = len(allItems) - 1 } } - // RemovedIds contain any {{ .Name }} that were deleted since the last response. - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) - target.EstItemCount = page.EstItemCount - target.ListToken = page.ListToken - target.ResponseType = page.ResponseType - target.Response = resp - if target.ResponseType == "complete" { + + currentPage = nextPage + + if currentPage.ResponseType == "complete" { break } } - // For now, removedIds will only be populated if this pagination cycle was the result of a - // "refresh" operation (i.e., the caller provided a list token option to this call). - // - // Sort to make response deterministic - slices.Sort(target.RemovedIds) - // Remove any duplicates - target.RemovedIds = slices.Compact(target.RemovedIds) + + // The current page here is the final page of the results, that is, the + // response type is "complete" + // Remove items that were deleted since the end of the last iteration. // If a {{ .Name }} has been updated and subsequently removed, we don't want // it to appear both in the Items and RemovedIds, so we remove it from the Items. - for _, removedId := range target.RemovedIds { + for _, removedId := range currentPage.RemovedIds { if i, ok := idToIndex[removedId]; ok { // Remove the item at index i without preserving order // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - target.Items[i] = target.Items[len(target.Items)-1] - target.Items = target.Items[:len(target.Items)-1] + allItems[i] = allItems[len(allItems)-1] + allItems = allItems[:len(allItems)-1] // Update the index of the previously last element - idToIndex[target.Items[i].Id] = i + idToIndex[allItems[i].Id] = i } } + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. + slices.SortFunc(allItems, func(i, j *{{ .Name }}) int { + return j.CreatedTime.Compare(i.CreatedTime) + }) // Since we paginated to the end, we can avoid confusion // for the user by setting the estimated item count to the // length of the items slice. If we don't set this here, it // will equal the value returned in the last response, which is // often much smaller than the total number returned. - target.EstItemCount = uint(len(target.Items)) - // Sort the results again since in-place updates and deletes - // may have shuffled items. We sort by created time descending - // (most recently created first), same as the API. - slices.SortFunc(target.Items, func(i, j *{{ .Name }}) int { - return j.CreatedTime.Compare(i.CreatedTime) - }) + currentPage.EstItemCount = uint(len(allItems)) + // Set items to the full list we have collected here + currentPage.Items = allItems + // Set the returned value to the last page with calculated values + target = currentPage // Finally, since we made at least 2 requests to the server to fulfill this // function call, resp.Body and resp.Map will only contain the most recent response. // Overwrite them with the true response. @@ -371,7 +373,93 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string, // Note: the HTTP response body is consumed by resp.Decode in the loop, // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil +{{ end }} } + +{{ if ( not ( .NonPaginatedListing ) ) }} +func (c *Client) ListNextPage(ctx context.Context, currentPage *{{ .Name }}ListResult, opt ...Option) (*{{ .Name }}ListResult, error) { + if currentPage == nil { + return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request") + } + if currentPage.{{ .CollectionFunctionArg }} == "" { + return nil, fmt.Errorf("empty {{ .CollectionFunctionArg }} value in currentPage passed into ListNextPage request") + } + if c.client == nil { + return nil, fmt.Errorf("nil client") + } + if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" { + return nil, fmt.Errorf("no more pages available in ListNextPage request") + } + + opts, apiOpts := getOpts(opt...) + opts.queryMap["{{ snakeCase .CollectionFunctionArg }}"] = currentPage.{{ .CollectionFunctionArg }} + +{{ if .RecursiveListing }} + // Don't require them to re-specify recursive + if currentPage.recursive { + opts.queryMap["recursive"] = "true" + } +{{ end }} + if currentPage.pageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10) + } + + req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...) + if err != nil { + return nil, fmt.Errorf("error creating List request: %w", err) + } + + opts.queryMap["list_token"] = currentPage.ListToken + if len(opts.queryMap) > 0 { + q := url.Values{} + for k, v := range opts.queryMap { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err) + } + + nextPage := new({{ .Name }}ListResult) + apiErr, err := resp.Decode(nextPage) + if err != nil { + return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err) + } + if apiErr != nil { + return nil, apiErr + } + + // Ensure values are carried forward to the next call + nextPage.{{ .CollectionFunctionArg }} = currentPage.{{ .CollectionFunctionArg }} +{{ if .RecursiveListing }} + nextPage.recursive = currentPage.recursive +{{ end }} + nextPage.pageSize = currentPage.pageSize + // Cache the removed IDs from this page + nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...) + // Set the response body to the current response + nextPage.Response = resp + // If we're done iterating, pull the full set of removed IDs into the last + // response + if nextPage.ResponseType == "complete" { + // Collect up the last values + nextPage.RemovedIds = nextPage.allRemovedIds + // For now, removedIds will only be populated if this pagination cycle + // was the result of a "refresh" operation (i.e., the caller provided a + // list token option to this call). + // + // Sort to make response deterministic + slices.Sort(nextPage.RemovedIds) + // Remove any duplicates + nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds) + } + + return nextPage, nil +} +{{ end }} `)) var readTemplate = template.Must(template.New("").Parse(` @@ -764,6 +852,14 @@ type {{ .Name }}ListResult struct { ListToken string `, "`json:\"list_token,omitempty\"`", ` ResponseType string `, "`json:\"response_type,omitempty\"`", ` Response *api.Response + + + // The following fields are used for cached information when client-directed + // pagination is used. + recursive bool + pageSize uint32 + {{ .CollectionFunctionArg }} string + allRemovedIds []string } func (n {{ .Name }}ListResult) GetItems() []*{{ .Name }} { @@ -845,6 +941,8 @@ type options struct { withSkipCurlOutput bool withFilter string withListToken string + withClientDirectedPagination bool + withPageSize uint32 {{ if .RecursiveListing }} withRecursive bool {{ end }} } @@ -875,6 +973,9 @@ func getOpts(opt ...Option) (options, []api.Option) { if opts.withRecursive { opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive) } {{ end }} + if opts.withPageSize != 0 { + opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10) + } return opts, apiOpts } @@ -916,6 +1017,20 @@ func WithFilter(filter string) Option { } } {{ end }} +// WithClientDirectedPagination tells the List function to return only the first +// page, if more pages are available +func WithClientDirectedPagination(with bool) Option { + return func(o *options) { + o.withClientDirectedPagination = with + } +} + +// WithPageSize controls the size of pages used during List +func WithPageSize(with uint32) Option { + return func(o *options) { + o.withPageSize = with + } +} {{ if .RecursiveListing }} // WithRecursive tells the API to use recursion for listing operations on this // resource diff --git a/internal/clientcache/cmd/cache/start.go b/internal/clientcache/cmd/cache/start.go index c94d507e0..e9d038597 100644 --- a/internal/clientcache/cmd/cache/start.go +++ b/internal/clientcache/cmd/cache/start.go @@ -50,6 +50,7 @@ type StartCommand struct { flagLogFormat string flagStoreDebug bool flagBackground bool + flagForceResetSchema bool } func (c *StartCommand) Synopsis() string { @@ -133,6 +134,13 @@ func (c *StartCommand) Flags() *base.FlagSets { Default: false, Usage: `Run the cache daemon in the background`, }) + f.BoolVar(&base.BoolVar{ + Name: "force-reset-schema", + Target: &c.flagForceResetSchema, + Default: false, + Usage: `Force resetting the cache schema and all contained data`, + Hidden: true, + }) return set } @@ -207,6 +215,7 @@ func (c *StartCommand) Run(args []string) int { LogFileName: logFileName, DotDirectory: dotDir, RunningInBackground: os.Getenv(backgroundEnvName) == backgroundEnvVal, + ForceResetSchema: c.flagForceResetSchema, } srv, err := daemon.New(ctx, cfg) diff --git a/internal/clientcache/internal/cache/options.go b/internal/clientcache/internal/cache/options.go index 36907f883..f57ab7180 100644 --- a/internal/clientcache/internal/cache/options.go +++ b/internal/clientcache/internal/cache/options.go @@ -25,6 +25,7 @@ type options struct { withIgnoreSearchStaleness bool withMaxResultSetSize int withTestRefreshWaitChs *testRefreshWaitChs + withUseNonPagedListing bool } // Option - how options are passed as args @@ -129,3 +130,12 @@ func WithTestRefreshWaitChs(with *testRefreshWaitChs) Option { return nil } } + +// WithUseNonPagedListing provides an option for ignoring the resource +// staleness when performing a search. +func WithUseNonPagedListing(b bool) Option { + return func(o *options) error { + o.withUseNonPagedListing = b + return nil + } +} diff --git a/internal/clientcache/internal/cache/options_test.go b/internal/clientcache/internal/cache/options_test.go index 44074fdbe..9f25faebe 100644 --- a/internal/clientcache/internal/cache/options_test.go +++ b/internal/clientcache/internal/cache/options_test.go @@ -49,8 +49,8 @@ func Test_GetOpts(t *testing.T) { assert.Equal(t, opts, testOpts) }) t.Run("WithTargetRetrievalFunc", func(t *testing.T) { - var f TargetRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { - return nil, nil, "", nil + var f TargetRetrievalFunc = func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (*targets.TargetListResult, RefreshTokenValue, error) { + return nil, "", nil } opts, err := getOpts(WithTargetRetrievalFunc(f)) require.NoError(t, err) @@ -118,4 +118,12 @@ func Test_GetOpts(t *testing.T) { testOpts.withTestRefreshWaitChs = waitCh assert.Equal(t, opts, testOpts) }) + t.Run("WithUseNonPagedListing", func(t *testing.T) { + opts, err := getOpts(WithUseNonPagedListing(true)) + require.NoError(t, err) + testOpts := getDefaultOptions() + assert.False(t, testOpts.withUseNonPagedListing) + testOpts.withUseNonPagedListing = true + assert.Equal(t, opts, testOpts) + }) } diff --git a/internal/clientcache/internal/cache/refresh.go b/internal/clientcache/internal/cache/refresh.go index 6cc294be0..015ca4480 100644 --- a/internal/clientcache/internal/cache/refresh.go +++ b/internal/clientcache/internal/cache/refresh.go @@ -420,28 +420,47 @@ func (r *RefreshService) RecheckCachingSupport(ctx context.Context, opt ...Optio continue } - if err := r.repo.checkCachingTargets(ctx, u, tokens, opt...); err != nil { - if err == ErrRefreshNotSupported { - // This is expected so no need to propagate the error up - continue + cacheKey := fmt.Sprintf("%s-%s", u.Id, Targets) + semaphore, _ := r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool)) + if semaphore.(*atomic.Bool).CompareAndSwap(false, true) { + if err := r.repo.checkCachingTargets(ctx, u, tokens, opt...); err != nil { + if err == ErrRefreshNotSupported { + semaphore.(*atomic.Bool).Store(false) + // This is expected so no need to propagate the error up + continue + } + retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id)))) } - retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id)))) - } - if err := r.repo.checkCachingSessions(ctx, u, tokens, opt...); err != nil { - if err == ErrRefreshNotSupported { - // This is expected so no need to propagate the error up - continue - } - retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id)))) - } - if err := r.repo.checkCachingResolvableAliases(ctx, u, tokens, opt...); err != nil { - if err == ErrRefreshNotSupported { - // This is expected so no need to propagate the error up - continue - } - retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id)))) + semaphore.(*atomic.Bool).Store(false) } + cacheKey = fmt.Sprintf("%s-%s", u.Id, ResolvableAliases) + semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool)) + if semaphore.(*atomic.Bool).CompareAndSwap(false, true) { + if err := r.repo.checkCachingResolvableAliases(ctx, u, tokens, opt...); err != nil { + if err == ErrRefreshNotSupported { + // This is expected so no need to propagate the error up + semaphore.(*atomic.Bool).Store(false) + continue + } + retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id)))) + } + semaphore.(*atomic.Bool).Store(false) + } + + cacheKey = fmt.Sprintf("%s-%s", u.Id, Sessions) + semaphore, _ = r.syncSemaphores.LoadOrStore(cacheKey, new(atomic.Bool)) + if semaphore.(*atomic.Bool).CompareAndSwap(false, true) { + if err := r.repo.checkCachingSessions(ctx, u, tokens, opt...); err != nil { + if err == ErrRefreshNotSupported { + semaphore.(*atomic.Bool).Store(false) + // This is expected so no need to propagate the error up + continue + } + retErr = stderrors.Join(retErr, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("for user id %s", u.Id)))) + } + semaphore.(*atomic.Bool).Store(false) + } } return retErr } diff --git a/internal/clientcache/internal/cache/refresh_test.go b/internal/clientcache/internal/cache/refresh_test.go index bd0905de9..0d4518727 100644 --- a/internal/clientcache/internal/cache/refresh_test.go +++ b/internal/clientcache/internal/cache/refresh_test.go @@ -54,6 +54,22 @@ func testStaticResourceRetrievalFunc[T any](t *testing.T, ret [][]T, removed [][ } } +func testTargetStaticResourceRetrievalFunc(inFunc func(ctx context.Context, s1, s2 string, refToken RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error)) TargetRetrievalFunc { + return func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (ret *targets.TargetListResult, refreshToken RefreshTokenValue, err error) { + retTargets, removed, refreshToken, err := inFunc(ctx, addr, authTok, refreshTok) + if err != nil { + return nil, "", err + } + + ret = &targets.TargetListResult{ + Items: retTargets, + RemovedIds: removed, + ResponseType: "complete", + } + return ret, refreshToken, nil + } +} + // testNoRefreshRetrievalFunc simulates a controller that doesn't support refresh // since it does not return any refresh token. func testNoRefreshRetrievalFunc[T any](t *testing.T) func(context.Context, string, string, RefreshTokenValue) ([]T, []string, RefreshTokenValue, error) { @@ -426,7 +442,7 @@ func TestRefreshForSearch(t *testing.T) { opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{ retTargets[:3], retTargets[3:], @@ -435,7 +451,7 @@ func TestRefreshForSearch(t *testing.T) { nil, {retTargets[0].Id, retTargets[1].Id}, }, - )), + ))), } assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...)) @@ -481,7 +497,7 @@ func TestRefreshForSearch(t *testing.T) { opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{ retTargets[:3], retTargets[3:], @@ -490,7 +506,7 @@ func TestRefreshForSearch(t *testing.T) { nil, {retTargets[0].Id, retTargets[1].Id}, }, - )), + ))), } assert.NoError(t, rs.RefreshForSearch(ctx, at.Id, Targets, opts...)) @@ -538,8 +554,8 @@ func TestRefreshForSearch(t *testing.T) { // Get the first set of resources, but no refresh tokens err = rs.Refresh(ctx, WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), - WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), + WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.ErrorContains(t, err, ErrRefreshNotSupported.Error()) got, err := r.ListTargets(ctx, at.Id) @@ -554,14 +570,14 @@ func TestRefreshForSearch(t *testing.T) { // any more. err = rs.Refresh(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), + WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.Nil(t, err) err = rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), + WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.Nil(t, err) got, err = r.ListTargets(ctx, at.Id) @@ -575,8 +591,9 @@ func TestRefreshForSearch(t *testing.T) { // the resources starting to be cached. err = rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{retTargets}, [][]string{{}}))), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{retTargets}, [][]string{{}}))) + ) assert.Nil(t, err, err) got, err = r.ListTargets(ctx, at.Id) @@ -601,7 +618,7 @@ func TestRefreshForSearch(t *testing.T) { } opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, [][]*sessions.Session{ retSess[:3], @@ -653,7 +670,7 @@ func TestRefreshForSearch(t *testing.T) { } opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, [][]*sessions.Session{ retSess[:3], @@ -710,7 +727,7 @@ func TestRefreshForSearch(t *testing.T) { } opts := []Option{ WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, [][]*aliases.Alias{ retAl[:3], @@ -762,7 +779,7 @@ func TestRefreshForSearch(t *testing.T) { } opts := []Option{ WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, [][]*aliases.Alias{ retAls[:3], @@ -839,7 +856,7 @@ func TestRefreshNonBlocking(t *testing.T) { opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{ retTargets[:3], retTargets[3:], @@ -848,7 +865,7 @@ func TestRefreshNonBlocking(t *testing.T) { nil, {retTargets[0].Id, retTargets[1].Id}, }, - )), + ))), } refreshWaitChs := &testRefreshWaitChs{ @@ -899,7 +916,7 @@ func TestRefreshNonBlocking(t *testing.T) { } opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, [][]*sessions.Session{ retSess[:3], @@ -961,7 +978,7 @@ func TestRefreshNonBlocking(t *testing.T) { } opts := []Option{ WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, [][]*aliases.Alias{ retAl[:3], @@ -1041,7 +1058,7 @@ func TestRefresh(t *testing.T) { opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, [][]*targets.Target{ retTargets[:3], retTargets[3:], @@ -1050,7 +1067,7 @@ func TestRefresh(t *testing.T) { nil, {retTargets[0].Id, retTargets[1].Id}, }, - )), + ))), } assert.NoError(t, rs.Refresh(ctx, opts...)) @@ -1082,7 +1099,7 @@ func TestRefresh(t *testing.T) { } opts := []Option{ WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, [][]*sessions.Session{ retSess[:3], @@ -1123,7 +1140,7 @@ func TestRefresh(t *testing.T) { } opts := []Option{ WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, [][]*aliases.Alias{ retAls[:3], @@ -1160,15 +1177,15 @@ func TestRefresh(t *testing.T) { err = rs.Refresh(ctx, WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { require.Equal(t, boundaryAddr, addr) require.Equal(t, at.Token, token) return nil, nil, "", innerErr - })) + }))) assert.ErrorContains(t, err, innerErr.Error()) err = rs.Refresh(ctx, WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))), WithSessionRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*sessions.Session, []string, RefreshTokenValue, error) { require.Equal(t, boundaryAddr, addr) require.Equal(t, at.Token, token) @@ -1202,7 +1219,7 @@ func TestRefresh(t *testing.T) { require.NoError(t, rs.Refresh(ctx, WithAliasRetrievalFunc(testStaticResourceRetrievalFuncForId[*aliases.Alias](t, nil, nil)), WithSessionRetrievalFunc(testStaticResourceRetrievalFunc[*sessions.Session](t, nil, nil)), - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil)))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc[*targets.Target](t, nil, nil))))) ps, err = r.listTokens(ctx, u) require.NoError(t, err) @@ -1246,7 +1263,7 @@ func TestRecheckCachingSupport(t *testing.T) { assert.NoError(t, rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))) got, err := r.ListTargets(ctx, at.Id) require.NoError(t, err) @@ -1258,7 +1275,7 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.Refresh(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))) assert.ErrorIs(t, err, ErrRefreshNotSupported) got, err = r.ListTargets(ctx, at.Id) @@ -1272,7 +1289,7 @@ func TestRecheckCachingSupport(t *testing.T) { assert.NoError(t, rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))))) }) t.Run("sessions", func(t *testing.T) { @@ -1286,7 +1303,7 @@ func TestRecheckCachingSupport(t *testing.T) { assert.NoError(t, rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))) got, err := r.ListSessions(ctx, at.Id) @@ -1298,7 +1315,7 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.Refresh(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.ErrorIs(t, err, ErrRefreshNotSupported) @@ -1311,7 +1328,7 @@ func TestRecheckCachingSupport(t *testing.T) { assert.NoError(t, rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))) got, err = r.ListSessions(ctx, at.Id) require.NoError(t, err) @@ -1332,7 +1349,7 @@ func TestRecheckCachingSupport(t *testing.T) { assert.NoError(t, rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))) got, err := r.ListResolvableAliases(ctx, at.Id) @@ -1344,7 +1361,7 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.Refresh(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.ErrorIs(t, err, ErrRefreshNotSupported) @@ -1357,7 +1374,7 @@ func TestRecheckCachingSupport(t *testing.T) { assert.NoError(t, rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)))) got, err = r.ListResolvableAliases(ctx, at.Id) require.NoError(t, err) @@ -1378,7 +1395,7 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.Refresh(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.ErrorIs(t, err, ErrRefreshNotSupported) @@ -1386,21 +1403,21 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { require.Equal(t, boundaryAddr, addr) require.Equal(t, at.Token, token) return nil, nil, "", innerErr - })) + }))) assert.ErrorContains(t, err, innerErr.Error()) err = rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(func(ctx context.Context, addr, token string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { require.Equal(t, boundaryAddr, addr) require.Equal(t, at.Token, token) return nil, nil, "", innerErr - })) + }))) assert.ErrorContains(t, err, innerErr.Error()) }) @@ -1415,7 +1432,7 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.Refresh(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)), + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t))) assert.ErrorIs(t, err, ErrRefreshNotSupported) @@ -1435,7 +1452,7 @@ func TestRecheckCachingSupport(t *testing.T) { err = rs.RecheckCachingSupport(ctx, WithAliasRetrievalFunc(testNoRefreshRetrievalFuncForId[*aliases.Alias](t)), WithSessionRetrievalFunc(testNoRefreshRetrievalFunc[*sessions.Session](t)), - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))) assert.NoError(t, err) ps, err = r.listTokens(ctx, u) diff --git a/internal/clientcache/internal/cache/repository_implicit_scopes_test.go b/internal/clientcache/internal/cache/repository_implicit_scopes_test.go index f5bc22633..ca5f5c0b6 100644 --- a/internal/clientcache/internal/cache/repository_implicit_scopes_test.go +++ b/internal/clientcache/internal/cache/repository_implicit_scopes_test.go @@ -62,7 +62,7 @@ func TestRepository_ImplicitScopes(t *testing.T) { target("3"), } require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))) for _, t := range ts { expectedScopes = append(expectedScopes, &scopes.Scope{ diff --git a/internal/clientcache/internal/cache/repository_targets.go b/internal/clientcache/internal/cache/repository_targets.go index e5cfa6e1f..56ccc16f2 100644 --- a/internal/clientcache/internal/cache/repository_targets.go +++ b/internal/clientcache/internal/cache/repository_targets.go @@ -21,32 +21,42 @@ import ( // TargetRetrievalFunc is a function that retrieves targets // from the provided boundary addr using the provided token. -type TargetRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) (ret []*targets.Target, removedIds []string, refreshToken RefreshTokenValue, err error) +type TargetRetrievalFunc func(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (ret *targets.TargetListResult, refreshToken RefreshTokenValue, err error) -func defaultTargetFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue) ([]*targets.Target, []string, RefreshTokenValue, error) { +func defaultTargetFunc(ctx context.Context, addr, authTok string, refreshTok RefreshTokenValue, inPage *targets.TargetListResult, opt ...Option) (*targets.TargetListResult, RefreshTokenValue, error) { const op = "cache.defaultTargetFunc" conf, err := api.DefaultConfig() if err != nil { - return nil, nil, "", errors.Wrap(ctx, err, op) + return nil, "", errors.Wrap(ctx, err, op) + } + opts, err := getOpts(opt...) + if err != nil { + return nil, "", errors.Wrap(ctx, err, op) } conf.Addr = addr conf.Token = authTok client, err := api.NewClient(conf) if err != nil { - return nil, nil, "", errors.Wrap(ctx, err, op) + return nil, "", errors.Wrap(ctx, err, op) } tarClient := targets.NewClient(client) - l, err := tarClient.List(ctx, "global", targets.WithRecursive(true), targets.WithListToken(string(refreshTok))) + var l *targets.TargetListResult + switch inPage { + case nil: + l, err = tarClient.List(ctx, "global", targets.WithRecursive(true), targets.WithListToken(string(refreshTok)), targets.WithClientDirectedPagination(!opts.withUseNonPagedListing)) + default: + l, err = tarClient.ListNextPage(ctx, inPage, targets.WithListToken(string(refreshTok))) + } if err != nil { if api.ErrInvalidListToken.Is(err) { - return nil, nil, "", err + return nil, "", err } - return nil, nil, "", errors.Wrap(ctx, err, op) + return nil, "", errors.Wrap(ctx, err, op) } if l.ResponseType == "" { - return nil, nil, "", ErrRefreshNotSupported + return nil, "", ErrRefreshNotSupported } - return l.Items, l.RemovedIds, RefreshTokenValue(l.ListToken), nil + return l, RefreshTokenValue(l.ListToken), nil } // refreshTargets uses attempts to refresh the targets for the provided user @@ -81,13 +91,13 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut // Find and use a token for retrieving targets var gotResponse bool - var resp []*targets.Target - var removedIds []string + var currentPage *targets.TargetListResult var newRefreshToken RefreshTokenValue + var foundAuthToken string var unsupportedCacheRequest bool var retErr error for at, t := range tokens { - resp, removedIds, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal) + currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, oldRefreshTokenVal, currentPage) if api.ErrInvalidListToken.Is(err) { event.WriteSysEvent(ctx, op, "old list token is no longer valid, starting new initial fetch", "user_id", u.Id) if err := r.deleteRefreshToken(ctx, u, resourceType); err != nil { @@ -95,7 +105,7 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut } // try again without the refresh token oldRefreshToken = nil - resp, removedIds, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "") + currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "", currentPage) } if err != nil { if err == ErrRefreshNotSupported { @@ -105,6 +115,7 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut continue } } + foundAuthToken = t gotResponse = true break } @@ -118,44 +129,57 @@ func (r *Repository) refreshTargets(ctx context.Context, u *user, tokens map[Aut } var numDeleted int - _, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error { - var err error - switch { - case oldRefreshToken == nil || unsupportedCacheRequest: - if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id", - []any{sql.Named("fk_user_id", u.Id)}); err != nil { - return err + var numUpserted int + var clearPerformed bool + for { + _, err = r.rw.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(_ db.Reader, w db.Writer) error { + var err error + if (oldRefreshToken == nil || unsupportedCacheRequest) && !clearPerformed { + if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id", + []any{sql.Named("fk_user_id", u.Id)}); err != nil { + return err + } + clearPerformed = true } - case len(removedIds) > 0: - if numDeleted, err = w.Exec(ctx, "delete from target where id in @ids", - []any{sql.Named("ids", removedIds)}); err != nil { - return err + switch { + case unsupportedCacheRequest: + if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil { + return err + } + case newRefreshToken != "": + if err := upsertTargets(ctx, w, u, currentPage.Items); err != nil { + return err + } + numUpserted += len(currentPage.Items) + if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil { + return err + } + default: + // controller supports caching, but doesn't have any resources } + if !unsupportedCacheRequest && len(currentPage.RemovedIds) > 0 { + if numDeleted, err = w.Exec(ctx, "delete from target where id in @ids", + []any{sql.Named("ids", currentPage.RemovedIds)}); err != nil { + return err + } + } + return nil + }) + if unsupportedCacheRequest || currentPage.ResponseType == "" || currentPage.ResponseType == "complete" { + break } - switch { - case unsupportedCacheRequest: - if err := upsertRefreshToken(ctx, w, u, resourceType, sentinelNoRefreshToken); err != nil { - return err - } - case newRefreshToken != "": - if err := upsertTargets(ctx, w, u, resp); err != nil { - return err - } - if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil { - return err - } - default: - // controller supports caching, but doesn't have any resources + currentPage, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, foundAuthToken, newRefreshToken, currentPage) + if err != nil { + break } - return nil - }) + } if err != nil { return errors.Wrap(ctx, err, op) } if unsupportedCacheRequest { return ErrRefreshNotSupported } - event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id) + event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", numUpserted, "user_id", u.Id) return nil } @@ -185,12 +209,12 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma // Find and use a token for retrieving targets var gotResponse bool - var resp []*targets.Target + var resp *targets.TargetListResult var newRefreshToken RefreshTokenValue var unsupportedCacheRequest bool var retErr error for at, t := range tokens { - resp, _, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "") + resp, newRefreshToken, err = opts.withTargetRetrievalFunc(ctx, u.Address, t, "", nil, WithUseNonPagedListing(true)) if err != nil { if err == ErrRefreshNotSupported { unsupportedCacheRequest = true @@ -221,14 +245,15 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma return err } case newRefreshToken != "": - var err error // Now that there is a refresh token, the data can be cached, so - // cache it and store the refresh token for future refreshes. + // cache it and store the refresh token for future refreshes. First + // remove any values, then add the new ones + var err error if numDeleted, err = w.Exec(ctx, "delete from target where fk_user_id = @fk_user_id", []any{sql.Named("fk_user_id", u.Id)}); err != nil { return err } - if err := upsertTargets(ctx, w, u, resp); err != nil { + if err := upsertTargets(ctx, w, u, resp.Items); err != nil { return err } if err := upsertRefreshToken(ctx, w, u, resourceType, newRefreshToken); err != nil { @@ -249,7 +274,7 @@ func (r *Repository) checkCachingTargets(ctx context.Context, u *user, tokens ma if unsupportedCacheRequest { return ErrRefreshNotSupported } - event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp), "user_id", u.Id) + event.WriteSysEvent(ctx, op, "targets updated", "deleted", numDeleted, "upserted", len(resp.Items), "user_id", u.Id) return nil } diff --git a/internal/clientcache/internal/cache/repository_targets_test.go b/internal/clientcache/internal/cache/repository_targets_test.go index a3c705b51..deb87d692 100644 --- a/internal/clientcache/internal/cache/repository_targets_test.go +++ b/internal/clientcache/internal/cache/repository_targets_test.go @@ -141,7 +141,7 @@ func TestRepository_refreshTargets(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { err := r.refreshTargets(ctx, tc.u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{tc.targets}, [][]string{nil}))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{tc.targets}, [][]string{nil})))) if tc.errorContains == "" { assert.NoError(t, err) rw := db.New(s) @@ -213,12 +213,12 @@ func TestRepository_RefreshTargets_InvalidListTokenError(t *testing.T) { } require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(invalidAuthTokenFunc))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(invalidAuthTokenFunc)))) // This time an invalid auth token should be returned, and refreshTargets should fall back // to requesting without one. require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(invalidAuthTokenFunc))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(invalidAuthTokenFunc)))) assert.Equal(t, 1, withRefreshToken) assert.Equal(t, 2, withoutRefreshToken) @@ -261,7 +261,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) { } require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))) got, err := r.ListTargets(ctx, at.Id) require.NoError(t, err) @@ -270,7 +270,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) { // Refreshing again uses the refresh token and get additional sessions, appending // them to the response require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))) assert.NoError(t, err) got, err = r.ListTargets(ctx, at.Id) @@ -280,7 +280,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) { // Refreshing again wont return any more resources, but also none should be // removed require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, ts, [][]string{nil, nil}))))) assert.NoError(t, err) got, err = r.ListTargets(ctx, at.Id) @@ -289,7 +289,7 @@ func TestRepository_RefreshTargets_withRefreshTokens(t *testing.T) { // Refresh again with the refresh token being reported as invalid. require.NoError(t, r.refreshTargets(ctx, &u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ts[0])))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testErroringForRefreshTokenRetrievalFunc(t, ts[0]))))) assert.NoError(t, err) got, err = r.ListTargets(ctx, at.Id) @@ -345,7 +345,7 @@ func TestRepository_ListTargets(t *testing.T) { target("3"), } require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))) t.Run("wrong user gets no targets", func(t *testing.T) { l, err := r.ListTargets(ctx, kt2.AuthTokenId) @@ -389,7 +389,7 @@ func TestRepository_ListTargetsLimiting(t *testing.T) { ts = append(ts, target("t"+strconv.Itoa(i))) } require.NoError(t, r.refreshTargets(ctx, u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))) searchService, err := NewSearchService(ctx, r) require.NoError(t, err) @@ -512,7 +512,7 @@ func TestRepository_QueryTargets(t *testing.T) { }, } require.NoError(t, r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))) t.Run("wrong token gets no targets", func(t *testing.T) { l, err := r.QueryTargets(ctx, kt2.AuthTokenId, query) @@ -556,7 +556,7 @@ func TestRepository_QueryTargetsLimiting(t *testing.T) { ts = append(ts, target("t"+strconv.Itoa(i))) } require.NoError(t, r.refreshTargets(ctx, u, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))))) searchService, err := NewSearchService(ctx, r) require.NoError(t, err) @@ -606,13 +606,13 @@ func TestDefaultTargetRetrievalFunc(t *testing.T) { require.NoError(t, err) require.NotNil(t, tar2) - got, removed, refTok, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "") + got, refTok, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, "", nil) assert.NoError(t, err) assert.NotEmpty(t, refTok) - assert.Empty(t, removed) + assert.Empty(t, got.RemovedIds) found1 := false found2 := false - for _, t := range got { + for _, t := range got.Items { if t.Id == tar1.Item.Id { found1 = true } @@ -623,10 +623,10 @@ func TestDefaultTargetRetrievalFunc(t *testing.T) { assert.True(t, found1, "expected to find target %s in list", tar1.Item.Id) assert.True(t, found2, "expected to find target %s in list", tar2.Item.Id) - got2, removed2, refTok2, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok) + got2, refTok2, err := defaultTargetFunc(tc.Context(), tc.ApiAddrs()[0], tc.Token().Token, refTok, nil) assert.NoError(t, err) assert.NotEmpty(t, refTok2) assert.NotEqual(t, refTok2, refTok) - assert.Empty(t, removed2) - assert.Empty(t, got2) + assert.Empty(t, got.RemovedIds) + assert.Empty(t, got2.Items) } diff --git a/internal/clientcache/internal/cache/status_test.go b/internal/clientcache/internal/cache/status_test.go index 2a8738454..67e34fc2a 100644 --- a/internal/clientcache/internal/cache/status_test.go +++ b/internal/clientcache/internal/cache/status_test.go @@ -190,11 +190,11 @@ func TestStatus(t *testing.T) { target("4"), } err = r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil}))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts}, [][]string{nil})))) require.NoError(t, err) err = r.refreshTargets(ctx, u2, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts[:2]}, [][]string{nil}))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testStaticResourceRetrievalFunc(t, [][]*targets.Target{ts[:2]}, [][]string{nil})))) require.NoError(t, err) sess := []*sessions.Session{ @@ -312,7 +312,7 @@ func TestStatus_unsupported(t *testing.T) { require.ErrorIs(t, err, ErrRefreshNotSupported) err = r.refreshTargets(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"}, - WithTargetRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t))) + WithTargetRetrievalFunc(testTargetStaticResourceRetrievalFunc(testNoRefreshRetrievalFunc[*targets.Target](t)))) require.ErrorIs(t, err, ErrRefreshNotSupported) err = r.refreshSessions(ctx, u1, map[AuthToken]string{{Id: "id"}: "something"}, diff --git a/internal/clientcache/internal/daemon/options.go b/internal/clientcache/internal/daemon/options.go index daed203df..1a4b15ad5 100644 --- a/internal/clientcache/internal/daemon/options.go +++ b/internal/clientcache/internal/daemon/options.go @@ -20,9 +20,10 @@ type options struct { WithReadyToServeNotificationCh chan struct{} withBoundaryTokenReaderFunc cache.BoundaryTokenReaderFn - withUrl string - withLogger hclog.Logger - withHomeDir string + withUrl string + withLogger hclog.Logger + withHomeDir string + withForceResetSchema bool } // Option - how options are passed as args @@ -110,6 +111,15 @@ func WithBoundaryTokenReaderFunc(_ context.Context, fn cache.BoundaryTokenReader } } +// WithForceResetSchema provides an optional way to force resetting the schema, +// e.g. wiping the cache +func WithForceResetSchema(_ context.Context, force bool) Option { + return func(o *options) error { + o.withForceResetSchema = force + return nil + } +} + // WithReadyToServeNotificationCh provides an optional channel to notify when // the server is ready to serve; mainly used for test timing but exported for // availability. The channel will be closed just before the HTTP server is diff --git a/internal/clientcache/internal/daemon/options_test.go b/internal/clientcache/internal/daemon/options_test.go index 0afcbab1b..4fd8bb4d6 100644 --- a/internal/clientcache/internal/daemon/options_test.go +++ b/internal/clientcache/internal/daemon/options_test.go @@ -108,6 +108,14 @@ func Test_GetOpts(t *testing.T) { testOpts.withHomeDir = "/tmp" assert.Equal(t, opts, testOpts) }) + t.Run("WithForceResetSchema", func(t *testing.T) { + opts, err := getOpts(WithForceResetSchema(ctx, true)) + require.NoError(t, err) + testOpts := getDefaultOptions() + assert.False(t, testOpts.withForceResetSchema) + testOpts.withForceResetSchema = true + assert.Equal(t, opts, testOpts) + }) t.Run("WithReadyToServeNotificationCh", func(t *testing.T) { ch := make(chan struct{}) opts, err := getOpts(WithReadyToServeNotificationCh(ctx, ch)) diff --git a/internal/clientcache/internal/daemon/server.go b/internal/clientcache/internal/daemon/server.go index cc642629b..2c0037c34 100644 --- a/internal/clientcache/internal/daemon/server.go +++ b/internal/clientcache/internal/daemon/server.go @@ -90,6 +90,8 @@ type Config struct { // The maximum amount of time a refresh should block a search request from // completing before it times out. MaxSearchRefreshTimeout time.Duration + // Force resetting the schema, that is, drop all existing data + ForceResetSchema bool } func (sc *Config) validate(ctx context.Context) error { @@ -230,6 +232,7 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e store, err = openStore(ctx, WithUrl(ctx, s.conf.DatabaseUrl), WithLogger(ctx, s.logger), + WithForceResetSchema(ctx, s.conf.ForceResetSchema), ) if err != nil { return errors.Wrap(ctx, err, op) @@ -537,6 +540,7 @@ func openStore(ctx context.Context, opt ...Option) (*db.DB, error) { dbOpts = append(dbOpts, cachedb.WithGormFormatter(opts.withLogger)) } } + dbOpts = append(dbOpts, cachedb.WithForceResetSchema(opts.withForceResetSchema)) store, err := cachedb.Open(ctx, dbOpts...) if err != nil { return nil, errors.Wrap(ctx, err, op) diff --git a/internal/clientcache/internal/daemon/testing.go b/internal/clientcache/internal/daemon/testing.go index 7faa91a8c..0689d9cd4 100644 --- a/internal/clientcache/internal/daemon/testing.go +++ b/internal/clientcache/internal/daemon/testing.go @@ -97,11 +97,13 @@ func (s *TestServer) AddResources(t *testing.T, p *authtokens.AuthToken, alts [] } return alts, nil, "addedaliases", nil } - tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) { + tarFn := func(ctx context.Context, _ string, tok string, _ cache.RefreshTokenValue, inPage *targets.TargetListResult, opt ...cache.Option) (*targets.TargetListResult, cache.RefreshTokenValue, error) { if tok != p.Token { - return nil, nil, "", nil + return nil, "", nil } - return tars, nil, "addedtargets", nil + return &targets.TargetListResult{ + Items: tars, + }, "addedtargets", nil } sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) { if tok != p.Token { @@ -124,13 +126,15 @@ func (s *TestServer) AddUnsupportedCachingData(t *testing.T, p *authtokens.AuthT r, err := cache.NewRepository(ctx, s.CacheServer.store.Load(), &sync.Map{}, s.cmd.ReadTokenFromKeyring, atReadFn) require.NoError(t, err) - tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) { + tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue, inPage *targets.TargetListResult, opt ...cache.Option) (*targets.TargetListResult, cache.RefreshTokenValue, error) { if tok != p.Token { - return nil, nil, "", nil + return &targets.TargetListResult{}, "", nil } - return []*targets.Target{ - {Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"}, - }, nil, "", cache.ErrRefreshNotSupported + return &targets.TargetListResult{ + Items: []*targets.Target{ + {Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"}, + }, + }, "", cache.ErrRefreshNotSupported } sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) { if tok != p.Token { diff --git a/internal/clientcache/internal/db/db.go b/internal/clientcache/internal/db/db.go index 4690c6e10..9063cf372 100644 --- a/internal/clientcache/internal/db/db.go +++ b/internal/clientcache/internal/db/db.go @@ -69,7 +69,7 @@ func Open(ctx context.Context, opt ...Option) (*db.DB, error) { if err != nil { return nil, errors.Wrap(ctx, err, op) } - if !ok { + if !ok || opts.withForceResetSchema { if err := resetSchema(ctx, conn); err != nil { return nil, errors.Wrap(ctx, err, op) } diff --git a/internal/clientcache/internal/db/options.go b/internal/clientcache/internal/db/options.go index 30186c90b..6300ba840 100644 --- a/internal/clientcache/internal/db/options.go +++ b/internal/clientcache/internal/db/options.go @@ -9,11 +9,12 @@ import ( ) type options struct { - withSchemaVersion string - withDebug bool - withUrl string - withDbType dbw.DbType - withGormFormatter hclog.Logger + withSchemaVersion string + withDebug bool + withUrl string + withDbType dbw.DbType + withGormFormatter hclog.Logger + withForceResetSchema bool } // Option - how options are passed as args @@ -67,3 +68,11 @@ func WithDebug(debug bool) Option { return nil } } + +// WithForceResetSchema provides an optional way to force resetting the cache +func WithForceResetSchema(debug bool) Option { + return func(o *options) error { + o.withForceResetSchema = debug + return nil + } +} diff --git a/internal/clientcache/internal/db/options_test.go b/internal/clientcache/internal/db/options_test.go index b67096b08..251f20c3f 100644 --- a/internal/clientcache/internal/db/options_test.go +++ b/internal/clientcache/internal/db/options_test.go @@ -45,4 +45,12 @@ func Test_GetOpts(t *testing.T) { testOpts.withSchemaVersion = version assert.Equal(t, opts, testOpts) }) + t.Run("WithForceResetSchema", func(t *testing.T) { + opts, err := getOpts(WithForceResetSchema(true)) + require.NoError(t, err) + testOpts := getDefaultOptions() + assert.False(t, testOpts.withForceResetSchema) + testOpts.withForceResetSchema = true + assert.Equal(t, opts, testOpts) + }) } diff --git a/internal/tests/api/targets/target_test.go b/internal/tests/api/targets/target_test.go index 0a6263abe..72a6bb1ea 100644 --- a/internal/tests/api/targets/target_test.go +++ b/internal/tests/api/targets/target_test.go @@ -336,6 +336,39 @@ func TestListWithListToken(t *testing.T) { require.Empty(res.Items) } +func TestListWithPageSize(t *testing.T) { + // Set database read timeout to avoid duplicates in response + oldReadTimeout := globals.RefreshReadLookbackDuration + globals.RefreshReadLookbackDuration = 0 + t.Cleanup(func() { + globals.RefreshReadLookbackDuration = oldReadTimeout + }) + require := require.New(t) + tc := controller.NewTestController(t, nil) + defer tc.Shutdown() + + client := tc.Client() + token := tc.Token() + client.SetToken(token.Token) + _, proj := iam.TestScopes(t, tc.IamRepo(), iam.WithUserId(token.UserId)) + + tarClient := targets.NewClient(client) + _, err := tarClient.Create(tc.Context(), "tcp", proj.GetPublicId(), targets.WithName("1"), targets.WithTcpTargetDefaultPort(2)) + require.NoError(err) + _, err = tarClient.Create(tc.Context(), "tcp", proj.GetPublicId(), targets.WithName("2"), targets.WithTcpTargetDefaultPort(2)) + require.NoError(err) + + // Refresh tokens recursive listing over global scope + res, err := tarClient.List(tc.Context(), "global", targets.WithRecursive(true), targets.WithPageSize(2)) + require.NoError(err) + require.Len(res.Items, 4, "expected the 2 targets created above and the 2 auto created for the test controller") + refTok := res.ListToken + + res, err = tarClient.List(tc.Context(), "global", targets.WithRecursive(true), targets.WithListToken(refTok)) + require.NoError(err) + require.Empty(res.Items) +} + func TestTarget_AddressMutualExclusiveRelationship(t *testing.T) { tc := controller.NewTestController(t, nil)