CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Paginate targets in cache (#5101)

Browse Source 
* Update Go API to support pagination
* Implement pagination of targets in cache

(cherry picked from commit cd25fad42f325fc050b7ea3bec3336780d2b5e16)
This commit is contained in:
Jeff Mitchell authored and Timothy Messier committed 2024-09-24 14:18:53 +00:00
1 parent 2a87dcad48
commit 3a4ee032d9
62 files changed
+3875 -1533
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/api/genapi/input.go
+5
View File
@@ -142,6 +142,10 @@ type structInfo struct {
// fields
fieldFilter []string
// nonPaginatedListing indicates a collection that does not support
// pagination
nonPaginatedListing bool
allowEmpty bool
}
@@ -1426,5 +1430,6 @@ var inputStructs = []*structInfo{
createResponseTypes: []string{CreateResponseType, ReadResponseType, UpdateResponseType, DeleteResponseType, ListResponseType},
recursiveListing: true,
versionEnabled: true,
nonPaginatedListing: true,
},
}
internal/api/genapi/templates.go
+169 -54
View File
@@ -49,6 +49,7 @@ type templateInput struct {
SliceSubtypes map[string]sliceSubtypeInfo
ExtraFields []fieldInfo
VersionEnabled bool
NonPaginatedListing bool
CreateResponseTypes []string
SkipListFiltering bool
RecursiveListing bool
@@ -69,6 +70,7 @@ func fillTemplates() {
ParentTypeName: in.parentTypeName,
ExtraFields: in.extraFields,
VersionEnabled: in.versionEnabled,
NonPaginatedListing: in.nonPaginatedListing,
CreateResponseTypes: in.createResponseTypes,
SkipListFiltering: in.skipListFiltering,
RecursiveListing: in.recursiveListing,
@@ -268,96 +270,96 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
return nil, apiErr
}
target.Response = resp
{{ if .NonPaginatedListing }}
return target, nil
{{ end }}
{{ if ( not ( .NonPaginatedListing ) ) }}
if target.ResponseType == "complete" || target.ResponseType == "" {
return target, nil
}
// In case we shortcut out due to client directed pagination, ensure these
// are set
{{ if .RecursiveListing }}
target.recursive = opts.withRecursive
{{ end }}
target.pageSize = opts.withPageSize
target.{{ .CollectionFunctionArg }} = {{ .CollectionFunctionArg }}
target.allRemovedIds = target.RemovedIds
if opts.withClientDirectedPagination {
return target, nil
}
allItems := make([]*{{ .Name }}, 0, target.EstItemCount)
allItems = append(allItems, target.Items...)
// If there are more results, automatically fetch the rest of the results.
// idToIndex keeps a map from the ID of an item to its index in target.Items.
// This is used to update updated items in-place and remove deleted items
// from the result after pagination is done.
idToIndex := map[string]int{}
for i, item := range target.Items {
for i, item := range allItems {
idToIndex[item.Id] = i
}
// If we're here there are more pages and the client does not want to
// paginate on their own; fetch them as this call returns all values.
currentPage := target
for {
req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...)
nextPage, err := c.ListNextPage(ctx, currentPage, opt...)
if err != nil {
return nil, fmt.Errorf("error creating List request: %w", err)
return nil, fmt.Errorf("error getting next page in List call: %w", err)
}
opts.queryMap["list_token"] = target.ListToken
if len(opts.queryMap) > 0 {
q := url.Values{}
for k, v := range opts.queryMap {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
}
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("error performing client request during List call: %w", err)
}
page := new({{ .Name }}ListResult)
apiErr, err := resp.Decode(page)
if err != nil {
return nil, fmt.Errorf("error decoding List response: %w", err)
}
if apiErr != nil {
return nil, apiErr
}
for _, item := range page.Items {
for _, item := range nextPage.Items {
if i, ok := idToIndex[item.Id]; ok {
// Item has already been seen at index i, update in-place
target.Items[i] = item
allItems[i] = item
} else {
target.Items = append(target.Items, item)
idToIndex[item.Id] = len(target.Items) - 1
allItems = append(allItems, item)
idToIndex[item.Id] = len(allItems) - 1
}
}
// RemovedIds contain any {{ .Name }} that were deleted since the last response.
target.RemovedIds = append(target.RemovedIds, page.RemovedIds...)
target.EstItemCount = page.EstItemCount
target.ListToken = page.ListToken
target.ResponseType = page.ResponseType
target.Response = resp
if target.ResponseType == "complete" {
currentPage = nextPage
if currentPage.ResponseType == "complete" {
break
}
}
// For now, removedIds will only be populated if this pagination cycle was the result of a
// "refresh" operation (i.e., the caller provided a list token option to this call).
//
// Sort to make response deterministic
slices.Sort(target.RemovedIds)
// Remove any duplicates
target.RemovedIds = slices.Compact(target.RemovedIds)
// The current page here is the final page of the results, that is, the
// response type is "complete"
// Remove items that were deleted since the end of the last iteration.
// If a {{ .Name }} has been updated and subsequently removed, we don't want
// it to appear both in the Items and RemovedIds, so we remove it from the Items.
for _, removedId := range target.RemovedIds {
for _, removedId := range currentPage.RemovedIds {
if i, ok := idToIndex[removedId]; ok {
// Remove the item at index i without preserving order
// https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order
target.Items[i] = target.Items[len(target.Items)-1]
target.Items = target.Items[:len(target.Items)-1]
allItems[i] = allItems[len(allItems)-1]
allItems = allItems[:len(allItems)-1]
// Update the index of the previously last element
idToIndex[target.Items[i].Id] = i
idToIndex[allItems[i].Id] = i
}
}
// Sort the results again since in-place updates and deletes
// may have shuffled items. We sort by created time descending
// (most recently created first), same as the API.
slices.SortFunc(allItems, func(i, j *{{ .Name }}) int {
return j.CreatedTime.Compare(i.CreatedTime)
})
// Since we paginated to the end, we can avoid confusion
// for the user by setting the estimated item count to the
// length of the items slice. If we don't set this here, it
// will equal the value returned in the last response, which is
// often much smaller than the total number returned.
target.EstItemCount = uint(len(target.Items))
// Sort the results again since in-place updates and deletes
// may have shuffled items. We sort by created time descending
// (most recently created first), same as the API.
slices.SortFunc(target.Items, func(i, j *{{ .Name }}) int {
return j.CreatedTime.Compare(i.CreatedTime)
})
currentPage.EstItemCount = uint(len(allItems))
// Set items to the full list we have collected here
currentPage.Items = allItems
// Set the returned value to the last page with calculated values
target = currentPage
// Finally, since we made at least 2 requests to the server to fulfill this
// function call, resp.Body and resp.Map will only contain the most recent response.
// Overwrite them with the true response.
@@ -371,7 +373,93 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string,
// Note: the HTTP response body is consumed by resp.Decode in the loop,
// so it doesn't need to be updated (it will always be, and has always been, empty).
return target, nil
{{ end }}
}
{{ if ( not ( .NonPaginatedListing ) ) }}
func (c *Client) ListNextPage(ctx context.Context, currentPage *{{ .Name }}ListResult, opt ...Option) (*{{ .Name }}ListResult, error) {
if currentPage == nil {
return nil, fmt.Errorf("empty currentPage value passed into ListNextPage request")
}
if currentPage.{{ .CollectionFunctionArg }} == "" {
return nil, fmt.Errorf("empty {{ .CollectionFunctionArg }} value in currentPage passed into ListNextPage request")
}
if c.client == nil {
return nil, fmt.Errorf("nil client")
}
if currentPage.ResponseType == "complete" || currentPage.ResponseType == "" {
return nil, fmt.Errorf("no more pages available in ListNextPage request")
}
opts, apiOpts := getOpts(opt...)
opts.queryMap["{{ snakeCase .CollectionFunctionArg }}"] = currentPage.{{ .CollectionFunctionArg }}
{{ if .RecursiveListing }}
// Don't require them to re-specify recursive
if currentPage.recursive {
opts.queryMap["recursive"] = "true"
}
{{ end }}
if currentPage.pageSize != 0 {
opts.queryMap["page_size"] = strconv.FormatUint(uint64(currentPage.pageSize), 10)
}
req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...)
if err != nil {
return nil, fmt.Errorf("error creating List request: %w", err)
}
opts.queryMap["list_token"] = currentPage.ListToken
if len(opts.queryMap) > 0 {
q := url.Values{}
for k, v := range opts.queryMap {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
}
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("error performing client request during List call during ListNextPage: %w", err)
}
nextPage := new({{ .Name }}ListResult)
apiErr, err := resp.Decode(nextPage)
if err != nil {
return nil, fmt.Errorf("error decoding List response during ListNextPage: %w", err)
}
if apiErr != nil {
return nil, apiErr
}
// Ensure values are carried forward to the next call
nextPage.{{ .CollectionFunctionArg }} = currentPage.{{ .CollectionFunctionArg }}
{{ if .RecursiveListing }}
nextPage.recursive = currentPage.recursive
{{ end }}
nextPage.pageSize = currentPage.pageSize
// Cache the removed IDs from this page
nextPage.allRemovedIds = append(currentPage.allRemovedIds, nextPage.RemovedIds...)
// Set the response body to the current response
nextPage.Response = resp
// If we're done iterating, pull the full set of removed IDs into the last
// response
if nextPage.ResponseType == "complete" {
// Collect up the last values
nextPage.RemovedIds = nextPage.allRemovedIds
// For now, removedIds will only be populated if this pagination cycle
// was the result of a "refresh" operation (i.e., the caller provided a
// list token option to this call).
//
// Sort to make response deterministic
slices.Sort(nextPage.RemovedIds)
// Remove any duplicates
nextPage.RemovedIds = slices.Compact(nextPage.RemovedIds)
}
return nextPage, nil
}
{{ end }}
`))
var readTemplate = template.Must(template.New("").Parse(`
@@ -764,6 +852,14 @@ type {{ .Name }}ListResult struct {
ListToken string `, "`json:\"list_token,omitempty\"`", `
ResponseType string `, "`json:\"response_type,omitempty\"`", `
Response *api.Response
// The following fields are used for cached information when client-directed
// pagination is used.
recursive bool
pageSize uint32
{{ .CollectionFunctionArg }} string
allRemovedIds []string
}
func (n {{ .Name }}ListResult) GetItems() []*{{ .Name }} {
@@ -845,6 +941,8 @@ type options struct {
withSkipCurlOutput bool
withFilter string
withListToken string
withClientDirectedPagination bool
withPageSize uint32
{{ if .RecursiveListing }} withRecursive bool {{ end }}
}
@@ -875,6 +973,9 @@ func getOpts(opt ...Option) (options, []api.Option) {
if opts.withRecursive {
opts.queryMap["recursive"] = strconv.FormatBool(opts.withRecursive)
} {{ end }}
if opts.withPageSize != 0 {
opts.queryMap["page_size"] = strconv.FormatUint(uint64(opts.withPageSize), 10)
}
return opts, apiOpts
}
@@ -916,6 +1017,20 @@ func WithFilter(filter string) Option {
}
}
{{ end }}
// WithClientDirectedPagination tells the List function to return only the first
// page, if more pages are available
func WithClientDirectedPagination(with bool) Option {
return func(o *options) {
o.withClientDirectedPagination = with
}
}
// WithPageSize controls the size of pages used during List
func WithPageSize(with uint32) Option {
return func(o *options) {
o.withPageSize = with
}
}
{{ if .RecursiveListing }}
// WithRecursive tells the API to use recursion for listing operations on this
// resource