From acc048a1d1742a577dc7745ecc7c7a1fd6ff31dc Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Mon, 6 Nov 2023 15:22:57 -0800 Subject: [PATCH] internal/genapi: correct list response body and map When using the new auto-pagination logic, we would previously return the Body and Map of the last response, instead of one corresponding to the returned result. Correct this by overwriting the Body and Map after pagination has finished. --- api/accounts/account.gen.go | 38 +++++++++++++++-- api/authmethods/authmethods.gen.go | 38 +++++++++++++++-- api/authtokens/authtokens.gen.go | 38 +++++++++++++++-- .../credential_library.gen.go | 38 +++++++++++++++-- api/credentials/credential.gen.go | 38 +++++++++++++++-- api/credentialstores/credential_store.gen.go | 38 +++++++++++++++-- api/groups/group.gen.go | 38 +++++++++++++++-- api/hostcatalogs/host_catalog.gen.go | 38 +++++++++++++++-- api/hosts/host.gen.go | 38 +++++++++++++++-- api/hostsets/host_set.gen.go | 38 +++++++++++++++-- api/managedgroups/managedgroups.gen.go | 38 +++++++++++++++-- api/roles/role.gen.go | 38 +++++++++++++++-- api/scopes/scope.gen.go | 38 +++++++++++++++-- .../session_recording.gen.go | 38 +++++++++++++++-- api/sessions/session.gen.go | 38 +++++++++++++++-- api/storagebuckets/storage_bucket.gen.go | 38 +++++++++++++++-- api/targets/session_secret.gen.go | 4 +- api/targets/target.gen.go | 38 +++++++++++++++-- api/users/user.gen.go | 38 +++++++++++++++-- api/workers/worker.gen.go | 38 +++++++++++++++-- internal/api/genapi/templates.go | 42 +++++++++++++++++-- 21 files changed, 687 insertions(+), 81 deletions(-) diff --git a/api/accounts/account.gen.go b/api/accounts/account.gen.go index f547c3c25..46170e84c 100644 --- a/api/accounts/account.gen.go +++ b/api/accounts/account.gen.go @@ -6,6 +6,7 @@ package accounts import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -352,6 +353,9 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "accounts", nil, apiOpts...) if err != nil { @@ -389,7 +393,9 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -408,10 +414,34 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Account) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/authmethods/authmethods.gen.go b/api/authmethods/authmethods.gen.go index cf2fd2cb9..942480d97 100644 --- a/api/authmethods/authmethods.gen.go +++ b/api/authmethods/authmethods.gen.go @@ -6,6 +6,7 @@ package authmethods import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -358,6 +359,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "auth-methods", nil, apiOpts...) if err != nil { @@ -395,7 +399,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -414,10 +420,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *AuthMethod) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/authtokens/authtokens.gen.go b/api/authtokens/authtokens.gen.go index 1e5b960bc..bd30ffe72 100644 --- a/api/authtokens/authtokens.gen.go +++ b/api/authtokens/authtokens.gen.go @@ -6,6 +6,7 @@ package authtokens import ( "context" + "encoding/json" "fmt" "net/url" "slices" @@ -243,6 +244,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "auth-tokens", nil, apiOpts...) if err != nil { @@ -280,7 +284,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -299,10 +305,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Auth idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *AuthToken) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/credentiallibraries/credential_library.gen.go b/api/credentiallibraries/credential_library.gen.go index 0f2d0d6c5..9f11aada2 100644 --- a/api/credentiallibraries/credential_library.gen.go +++ b/api/credentiallibraries/credential_library.gen.go @@ -6,6 +6,7 @@ package credentiallibraries import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -358,6 +359,9 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "credential-libraries", nil, apiOpts...) if err != nil { @@ -395,7 +399,9 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -414,10 +420,34 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *CredentialLibrary) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/credentials/credential.gen.go b/api/credentials/credential.gen.go index 840d3a31c..4835f3e78 100644 --- a/api/credentials/credential.gen.go +++ b/api/credentials/credential.gen.go @@ -6,6 +6,7 @@ package credentials import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -356,6 +357,9 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "credentials", nil, apiOpts...) if err != nil { @@ -393,7 +397,9 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -412,10 +418,34 @@ func (c *Client) List(ctx context.Context, credentialStoreId string, opt ...Opti idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Credential) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/credentialstores/credential_store.gen.go b/api/credentialstores/credential_store.gen.go index b6eb7c549..88601f5da 100644 --- a/api/credentialstores/credential_store.gen.go +++ b/api/credentialstores/credential_store.gen.go @@ -6,6 +6,7 @@ package credentialstores import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -357,6 +358,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "credential-stores", nil, apiOpts...) if err != nil { @@ -394,7 +398,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -413,10 +419,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Cred idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *CredentialStore) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/groups/group.gen.go b/api/groups/group.gen.go index 5469c7a5f..1372ed40a 100644 --- a/api/groups/group.gen.go +++ b/api/groups/group.gen.go @@ -6,6 +6,7 @@ package groups import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -351,6 +352,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "groups", nil, apiOpts...) if err != nil { @@ -388,7 +392,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -407,11 +413,35 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Grou idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Group) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/hostcatalogs/host_catalog.gen.go b/api/hostcatalogs/host_catalog.gen.go index 4d5184a8e..536268a48 100644 --- a/api/hostcatalogs/host_catalog.gen.go +++ b/api/hostcatalogs/host_catalog.gen.go @@ -6,6 +6,7 @@ package hostcatalogs import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -362,6 +363,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "host-catalogs", nil, apiOpts...) if err != nil { @@ -399,7 +403,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -418,10 +424,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Host idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *HostCatalog) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/hosts/host.gen.go b/api/hosts/host.gen.go index 049c41fdd..0ce434581 100644 --- a/api/hosts/host.gen.go +++ b/api/hosts/host.gen.go @@ -6,6 +6,7 @@ package hosts import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -358,6 +359,9 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "hosts", nil, apiOpts...) if err != nil { @@ -395,7 +399,9 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -414,10 +420,34 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Host) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/hostsets/host_set.gen.go b/api/hostsets/host_set.gen.go index 109acbf56..a30430c72 100644 --- a/api/hostsets/host_set.gen.go +++ b/api/hostsets/host_set.gen.go @@ -6,6 +6,7 @@ package hostsets import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -356,6 +357,9 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "host-sets", nil, apiOpts...) if err != nil { @@ -393,7 +397,9 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -412,11 +418,35 @@ func (c *Client) List(ctx context.Context, hostCatalogId string, opt ...Option) idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *HostSet) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/managedgroups/managedgroups.gen.go b/api/managedgroups/managedgroups.gen.go index d72b54653..f56e99d74 100644 --- a/api/managedgroups/managedgroups.gen.go +++ b/api/managedgroups/managedgroups.gen.go @@ -6,6 +6,7 @@ package managedgroups import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -352,6 +353,9 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "managed-groups", nil, apiOpts...) if err != nil { @@ -389,7 +393,9 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -408,10 +414,34 @@ func (c *Client) List(ctx context.Context, authMethodId string, opt ...Option) ( idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *ManagedGroup) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/roles/role.gen.go b/api/roles/role.gen.go index c5857a342..283ba80d3 100644 --- a/api/roles/role.gen.go +++ b/api/roles/role.gen.go @@ -6,6 +6,7 @@ package roles import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -354,6 +355,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "roles", nil, apiOpts...) if err != nil { @@ -391,7 +395,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -410,11 +416,35 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Role idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Role) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/scopes/scope.gen.go b/api/scopes/scope.gen.go index 47332ac52..f03bc0017 100644 --- a/api/scopes/scope.gen.go +++ b/api/scopes/scope.gen.go @@ -6,6 +6,7 @@ package scopes import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -351,6 +352,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "scopes", nil, apiOpts...) if err != nil { @@ -388,7 +392,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -407,10 +413,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Scop idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Scope) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/sessionrecordings/session_recording.gen.go b/api/sessionrecordings/session_recording.gen.go index 382b576e6..da00e7fd3 100644 --- a/api/sessionrecordings/session_recording.gen.go +++ b/api/sessionrecordings/session_recording.gen.go @@ -6,6 +6,7 @@ package sessionrecordings import ( "context" + "encoding/json" "fmt" "net/url" "slices" @@ -193,6 +194,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "session-recordings", nil, apiOpts...) if err != nil { @@ -230,7 +234,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -249,10 +255,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *SessionRecording) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/sessions/session.gen.go b/api/sessions/session.gen.go index 585b94e80..d243ec4e3 100644 --- a/api/sessions/session.gen.go +++ b/api/sessions/session.gen.go @@ -6,6 +6,7 @@ package sessions import ( "context" + "encoding/json" "fmt" "net/url" "slices" @@ -210,6 +211,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "sessions", nil, apiOpts...) if err != nil { @@ -247,7 +251,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -266,10 +272,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Sess idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Session) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/storagebuckets/storage_bucket.gen.go b/api/storagebuckets/storage_bucket.gen.go index 218b01b1c..df947d59e 100644 --- a/api/storagebuckets/storage_bucket.gen.go +++ b/api/storagebuckets/storage_bucket.gen.go @@ -6,6 +6,7 @@ package storagebuckets import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -359,6 +360,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "storage-buckets", nil, apiOpts...) if err != nil { @@ -396,7 +400,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -415,10 +421,34 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Stor idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *StorageBucket) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/targets/session_secret.gen.go b/api/targets/session_secret.gen.go index 4665b276e..f79d3d7da 100644 --- a/api/targets/session_secret.gen.go +++ b/api/targets/session_secret.gen.go @@ -4,7 +4,9 @@ package targets -import "encoding/json" +import ( + "encoding/json" +) type SessionSecret struct { Raw json.RawMessage `json:"raw,omitempty"` diff --git a/api/targets/target.gen.go b/api/targets/target.gen.go index dd6a9d220..e5b1054bf 100644 --- a/api/targets/target.gen.go +++ b/api/targets/target.gen.go @@ -6,6 +6,7 @@ package targets import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -368,6 +369,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "targets", nil, apiOpts...) if err != nil { @@ -405,7 +409,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -424,11 +430,35 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Targ idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Target) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/users/user.gen.go b/api/users/user.gen.go index 799f8d1da..885457d59 100644 --- a/api/users/user.gen.go +++ b/api/users/user.gen.go @@ -6,6 +6,7 @@ package users import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -355,6 +356,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "users", nil, apiOpts...) if err != nil { @@ -392,7 +396,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -411,11 +417,35 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*User idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *User) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/api/workers/worker.gen.go b/api/workers/worker.gen.go index 50cf95a7b..922af4fa8 100644 --- a/api/workers/worker.gen.go +++ b/api/workers/worker.gen.go @@ -6,6 +6,7 @@ package workers import ( "context" + "encoding/json" "errors" "fmt" "net/url" @@ -409,6 +410,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "workers", nil, apiOpts...) if err != nil { @@ -446,7 +450,9 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work idToIndex[item.Id] = len(target.Items) - 1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -465,11 +471,35 @@ func (c *Client) List(ctx context.Context, scopeId string, opt ...Option) (*Work idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *Worker) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } diff --git a/internal/api/genapi/templates.go b/internal/api/genapi/templates.go index 7391a2c61..35f25acb4 100644 --- a/internal/api/genapi/templates.go +++ b/internal/api/genapi/templates.go @@ -276,6 +276,9 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string, for i, item := range target.Items { idToIndex[item.Id] = i } + // Removed IDs in the response may contain duplicates, + // maintain a set to avoid returning duplicates to the user. + removedIds := map[string]struct{}{} for { req, err := c.client.NewRequest(ctx, "GET", "{{ .CollectionPath }}", nil, apiOpts...) if err != nil { @@ -313,7 +316,9 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string, idToIndex[item.Id] = len(target.Items)-1 } } - target.RemovedIds = append(target.RemovedIds, page.RemovedIds...) + for _, removedId := range page.RemovedIds { + removedIds[removedId] = struct{}{} + } target.EstItemCount = page.EstItemCount target.ListToken = page.ListToken target.ResponseType = page.ResponseType @@ -332,11 +337,35 @@ func (c *Client) List(ctx context.Context, {{ .CollectionFunctionArg }} string, idToIndex[target.Items[i].Id] = i } } - // Finally, sort the results again since in-place updates and deletes - // may have shuffled items. + for deletedId := range removedIds { + target.RemovedIds = append(target.RemovedIds, deletedId) + } + // Sort to make response deterministic + slices.Sort(target.RemovedIds) + // Since we paginated to the end, we can avoid confusion + // for the user by setting the estimated item count to the + // length of the items slice. If we don't set this here, it + // will equal the value returned in the last response, which is + // often much smaller than the total number returned. + target.EstItemCount = uint(len(target.Items)) + // Sort the results again since in-place updates and deletes + // may have shuffled items. We sort by created time descending + // (most recently created first), same as the API. slices.SortFunc(target.Items, func(i, j *{{ .Name }}) int { - return i.UpdatedTime.Compare(j.UpdatedTime) + return j.CreatedTime.Compare(i.CreatedTime) }) + // Finally, since we made at least 2 requests to the server to fulfill this + // function call, resp.Body and resp.Map will only contain the most recent response. + // Overwrite them with the true response. + target.response.Body.Reset() + if err := json.NewEncoder(target.response.Body).Encode(target); err != nil { + return nil, fmt.Errorf("error encoding final JSON list response: %w", err) + } + if err := json.Unmarshal(target.response.Body.Bytes(), &target.response.Map); err != nil { + return nil, fmt.Errorf("error encoding final map list response: %w", err) + } + // Note: the HTTP response body is consumed by resp.Decode in the loop, + // so it doesn't need to be updated (it will always be, and has always been, empty). return target, nil } `)) @@ -671,8 +700,13 @@ var structTemplate = template.Must(template.New("").Funcs( package {{ .Package }} import ( + "bytes" "context" + "encoding/json" + "errors" "fmt" + "io" + "net/url" "slices" "time"