CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Paginate targets in cache (#5101)

Browse Source 
* Update Go API to support pagination
* Implement pagination of targets in cache

(cherry picked from commit cd25fad42f325fc050b7ea3bec3336780d2b5e16)
This commit is contained in:
Jeff Mitchell authored and Timothy Messier committed 2024-09-24 14:18:53 +00:00
1 parent 2a87dcad48
commit 3a4ee032d9
62 files changed
+3875 -1533
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

api/managedgroups/managedgroups.gen.go
+133 -54
View File
@@ -11,6 +11,7 @@ import (
"fmt"
"net/url"
"slices"
"strconv"
"time"
"github.com/hashicorp/boundary/api"
@@ -68,6 +69,13 @@ type ManagedGroupListResult struct {
ListToken string `json:"list_token,omitempty"`
ResponseType string `json:"response_type,omitempty"`
Response *api.Response
// The following fields are used for cached information when client-directed
// pagination is used.
recursive bool
pageSize uint32
authMethodId string
allRemovedIds []string
}
func (n ManagedGroupListResult) GetItems() []*ManagedGroup {
@@ -340,96 +348,91 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
return nil, apiErr
}
target.Response = resp
if target.ResponseType == "complete" || target.ResponseType == "" {
return target, nil
}
// In case we shortcut out due to client directed pagination, ensure these
// are set
target.pageSize = opts.withPageSize
target.authMethodId = authMethodId
target.allRemovedIds = target.RemovedIds
if opts.withClientDirectedPagination {
return target, nil
}
allItems := make([]*ManagedGroup, 0, target.EstItemCount)
allItems = append(allItems, target.Items...)
// If there are more results, automatically fetch the rest of the results.
// idToIndex keeps a map from the ID of an item to its index in target.Items.
// This is used to update updated items in-place and remove deleted items
// from the result after pagination is done.
idToIndex := map[string]int{}
for i, item := range target.Items {
for i, item := range allItems {
idToIndex[item.Id] = i
}
// If we're here there are more pages and the client does not want to
// paginate on their own; fetch them as this call returns all values.
currentPage := target
for {
req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
if err != nil {
return nil, fmt.Errorf("error creating List request: %w", err)
return nil, fmt.Errorf("error getting next page in List call: %w", err)
}
opts.queryMap["list_token"] = target.ListToken
if len(opts.queryMap) > 0 {
q := url.Values{}
for k, v := range opts.queryMap {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
}
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("error performing client request during List call: %w", err)
}
page := new(ManagedGroupListResult)
apiErr, err := resp.Decode(page)
if err != nil {
return nil, fmt.Errorf("error decoding List response: %w", err)
}
if apiErr != nil {
return nil, apiErr
}
for _, item := range page.Items {
for _, item := range nextPage.Items {
if i, ok := idToIndex[item.Id]; ok {
// Item has already been seen at index i, update in-place
target.Items[i] = item
allItems[i] = item
} else {
target.Items = append(target.Items, item)
idToIndex[item.Id] = len(target.Items) - 1
allItems = append(allItems, item)
idToIndex[item.Id] = len(allItems) - 1
}
}
// RemovedIds contain any ManagedGroup that were deleted since the last response.
target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
target.EstItemCount = page.EstItemCount
target.ListToken = page.ListToken
target.ResponseType = page.ResponseType
target.Response = resp
if target.ResponseType == "complete" {
currentPage = nextPage
if currentPage.ResponseType == "complete" {
break
}
}
// For now, removedIds will only be populated if this pagination cycle was the result of a
// "refresh" operation (i.e., the caller provided a list token option to this call).
//
// Sort to make response deterministic
slices.Sort(target.RemovedIds)
// Remove any duplicates
target.RemovedIds = slices.Compact(target.RemovedIds)
// The current page here is the final page of the results, that is, the
// response type is "complete"
// Remove items that were deleted since the end of the last iteration.
// If a ManagedGroup has been updated and subsequently removed, we don't want
// it to appear both in the Items and RemovedIds, so we remove it from the Items.
for _, removedId := range target.RemovedIds {
for _, removedId := range currentPage.RemovedIds {
if i, ok := idToIndex[removedId]; ok {
// Remove the item at index i without preserving order
// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
target.Items[i] = target.Items[len(target.Items)-1]
target.Items = target.Items[:len(target.Items)-1]
allItems[i] = allItems[len(allItems)-1]
allItems = allItems[:len(allItems)-1]
// Update the index of the previously last element
idToIndex[target.Items[i].Id] = i
idToIndex[allItems[i].Id] = i
}
}
// Sort the results again since in-place updates and deletes
// may have shuffled items. We sort by created time descending
// (most recently created first), same as the API.
slices.SortFunc(allItems, func(i, j *ManagedGroup) int {
return j.CreatedTime.Compare(i.CreatedTime)
})
// Since we paginated to the end, we can avoid confusion
// for the user by setting the estimated item count to the
// length of the items slice. If we don't set this here, it
// will equal the value returned in the last response, which is
// often much smaller than the total number returned.
target.EstItemCount = uint(len(target.Items))
// Sort the results again since in-place updates and deletes
// may have shuffled items. We sort by created time descending
// (most recently created first), same as the API.
slices.SortFunc(target.Items, func(i, j *ManagedGroup) int {
return j.CreatedTime.Compare(i.CreatedTime)
})
currentPage.EstItemCount = uint(len(allItems))
// Set items to the full list we have collected here
currentPage.Items = allItems
// Set the returned value to the last page with calculated values
target = currentPage
// Finally, since we made at least 2 requests to the server to fulfill this
// function call, resp.Body and resp.Map will only contain the most recent response.
// Overwrite them with the true response.
@@ -443,4 +446,80 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) (
// Note: the HTTP response body is consumed by resp.Decode in the loop,
// so it doesn't need to be updated (it will always be, and has always been, empty).
return target, nil
}
func (c *Client) ListNextPage(ctx context.Context, currentPage *ManagedGroupListResult, opt ...Option) (*ManagedGroupListResult, error) {
if currentPage == nil {
return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
}
if currentPage.authMethodId == "" {
return nil, fmt.Errorf("empty authMethodId value in currentPage passed into ListNextPage request")
}
if c.client == nil {
return nil, fmt.Errorf("nil client")
}
if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
return nil, fmt.Errorf("no more pages available in ListNextPage request")
}
opts, apiOpts := getOpts(opt...)
opts.queryMap["auth_method_id"] = currentPage.authMethodId
if currentPage.pageSize != 0 {
opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
}
req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...)
if err != nil {
return nil, fmt.Errorf("error creating List request: %w", err)
}
opts.queryMap["list_token"] = currentPage.ListToken
if len(opts.queryMap) > 0 {
q := url.Values{}
for k, v := range opts.queryMap {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
}
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
}
nextPage := new(ManagedGroupListResult)
apiErr, err := resp.Decode(nextPage)
if err != nil {
return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
}
if apiErr != nil {
return nil, apiErr
}
// Ensure values are carried forward to the next call
nextPage.authMethodId = currentPage.authMethodId
nextPage.pageSize = currentPage.pageSize
// Cache the removed IDs from this page
nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
// Set the response body to the current response
nextPage.Response = resp
// If we're done iterating, pull the full set of removed IDs into the last
// response
if nextPage.ResponseType == "complete" {
// Collect up the last values
nextPage.RemovedIds = nextPage.allRemovedIds
// For now, removedIds will only be populated if this pagination cycle
// was the result of a "refresh" operation (i.e., the caller provided a
// list token option to this call).
//
// Sort to make response deterministic
slices.Sort(nextPage.RemovedIds)
// Remove any duplicates
nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
}
return nextPage, nil
}