From c6e80dbe3254197940e48b25b538f72753c422ac Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Thu, 30 Nov 2023 22:21:55 -0800 Subject: [PATCH] internal/credential: add library pagination logic --- internal/credential/service_list_libraries.go | 63 ++ .../service_list_libraries_ext_test.go | 575 ++++++++++++++++++ .../credential/service_list_libraries_page.go | 65 ++ .../service_list_libraries_refresh.go | 72 +++ .../service_list_libraries_refresh_page.go | 77 +++ 5 files changed, 852 insertions(+) create mode 100644 internal/credential/service_list_libraries.go create mode 100644 internal/credential/service_list_libraries_ext_test.go create mode 100644 internal/credential/service_list_libraries_page.go create mode 100644 internal/credential/service_list_libraries_refresh.go create mode 100644 internal/credential/service_list_libraries_refresh_page.go diff --git a/internal/credential/service_list_libraries.go b/internal/credential/service_list_libraries.go new file mode 100644 index 000000000..f68c4e532 --- /dev/null +++ b/internal/credential/service_list_libraries.go @@ -0,0 +1,63 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/pagination" +) + +// LibraryService defines the interface expected +// to list, estimate the count of and list deleted items of +// credential libraries. +type LibraryService interface { + EstimatedLibraryCount(context.Context) (int, error) + ListDeletedLibraryIds(context.Context, time.Time) ([]string, time.Time, error) + ListLibraries(context.Context, string, ...Option) ([]Library, time.Time, error) + ListLibrariesRefresh(context.Context, string, time.Time, ...Option) ([]Library, time.Time, error) +} + +// ListLibraries lists up to page size credential libraries, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credential libraries from the database, at page size chunks, to fill the page. +// It returns a new list token used to continue pagination or refresh items. +// Credential libraries are ordered by create time descending (most recently created first). +func ListLibraries( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Library], + service LibraryService, + credentialStoreId string, +) (*pagination.ListResponse[Library], error) { + const op = "credential.ListLibraries" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + case credentialStoreId == "": + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID") + } + + listItemsFn := func(ctx context.Context, lastPageItem Library, limit int) ([]Library, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } + return service.ListLibraries(ctx, credentialStoreId, opts...) + } + + return pagination.List(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedLibraryCount) +} diff --git a/internal/credential/service_list_libraries_ext_test.go b/internal/credential/service_list_libraries_ext_test.go new file mode 100644 index 000000000..69271dc91 --- /dev/null +++ b/internal/credential/service_list_libraries_ext_test.go @@ -0,0 +1,575 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential_test + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hashicorp/boundary/globals" + "github.com/hashicorp/boundary/internal/credential" + "github.com/hashicorp/boundary/internal/credential/vault" + "github.com/hashicorp/boundary/internal/credential/vault/store" + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/timestamp" + "github.com/hashicorp/boundary/internal/iam" + "github.com/hashicorp/boundary/internal/kms" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/scheduler" + "github.com/hashicorp/boundary/internal/types/resource" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestLibraryService_List(t *testing.T) { + // Set database read timeout to avoid duplicates in response + oldReadTimeout := globals.RefreshReadLookbackDuration + globals.RefreshReadLookbackDuration = 0 + t.Cleanup(func() { + globals.RefreshReadLookbackDuration = oldReadTimeout + }) + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + sqlDb, err := conn.SqlDB(ctx) + require.NoError(t, err) + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kms := kms.TestKms(t, conn, wrapper) + sche := scheduler.TestScheduler(t, conn, wrapper) + _, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + fiveDaysAgo := time.Now().AddDate(0, 0, -5) + credStore := vault.TestCredentialStores(t, conn, wrapper, prj.GetPublicId(), 1)[0] + libs := []credential.Library{ + vault.TestCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0], + vault.TestSSHCertificateCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0], + vault.TestCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0], + vault.TestSSHCertificateCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0], + vault.TestSSHCertificateCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0], + } + + // since we sort by create time descending, we need to reverse the slice + slices.Reverse(libs) + + repo, err := vault.NewRepository(ctx, rw, rw, kms, sche) + require.NoError(t, err) + + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported( + vault.CredentialLibrary{}, + vault.SSHCertificateCredentialLibrary{}, + store.SSHCertificateCredentialLibrary{}, + store.CredentialLibrary{}, + timestamp.Timestamp{}, + timestamppb.Timestamp{}, + ), + } + + t.Run("ListLibraries validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err := credential.ListLibraries(ctx, nil, 1, filterFunc, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err := credential.ListLibraries(ctx, []byte("some hash"), 0, filterFunc, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err := credential.ListLibraries(ctx, []byte("some hash"), -1, filterFunc, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + _, err := credential.ListLibraries(ctx, []byte("some hash"), 1, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err := credential.ListLibraries(ctx, []byte("some hash"), 1, filterFunc, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store ID", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err := credential.ListLibraries(ctx, []byte("some hash"), 1, filterFunc, repo, "") + require.ErrorContains(t, err, "missing credential store ID") + }) + }) + t.Run("ListLibrariesPage validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "token did not have a pagination token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store ID", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "") + require.ErrorContains(t, err, "missing store id") + }) + }) + t.Run("ListLibrariesRefresh validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefresh(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefresh(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefresh(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err = credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing token") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store ID", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "") + require.ErrorContains(t, err, "missing credential store ID") + }) + }) + t.Run("ListLibrariesRefreshPage validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefreshPage(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + _, err = credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "token did not have a refresh token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, l credential.Library) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.CredentialLibrary, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + }) + + t.Run("simple pagination", func(t *testing.T) { + filterFunc := func(context.Context, credential.Library) (bool, error) { + return true, nil + } + resp, err := credential.ListLibraries(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 5) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], libs[0], cmpOpts...)) + + resp2, err := credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 5) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 1) + require.Empty(t, cmp.Diff(resp2.Items[0], libs[1], cmpOpts...)) + + resp3, err := credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 5) + require.Empty(t, resp3.DeletedIds) + require.Len(t, resp3.Items, 1) + require.Empty(t, cmp.Diff(resp3.Items[0], libs[2], cmpOpts...)) + + resp4, err := credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp4.CompleteListing) + require.Equal(t, resp4.EstimatedItemCount, 5) + require.Empty(t, resp4.DeletedIds) + require.Len(t, resp4.Items, 1) + require.Empty(t, cmp.Diff(resp4.Items[0], libs[3], cmpOpts...)) + + resp5, err := credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp5.CompleteListing) + require.Equal(t, resp5.EstimatedItemCount, 5) + require.Empty(t, resp5.DeletedIds) + require.Len(t, resp5.Items, 1) + require.Empty(t, cmp.Diff(resp5.Items[0], libs[4], cmpOpts...)) + + // Finished initial pagination phase, request refresh + // Expect no results. + resp6, err := credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp5.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp6.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp6.CompleteListing) + require.Equal(t, resp6.EstimatedItemCount, 5) + require.Empty(t, resp6.DeletedIds) + require.Empty(t, resp6.Items) + + // Create some new libraries + newLib1 := vault.TestCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0] + newLib2 := vault.TestSSHCertificateCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 1)[0] + t.Cleanup(func() { + _, err := repo.DeleteCredentialLibrary(ctx, credStore.ProjectId, newLib1.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteSSHCertificateCredentialLibrary(ctx, credStore.ProjectId, newLib2.GetPublicId()) + require.NoError(t, err) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + }) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + // Refresh again, should get newLib2 + resp7, err := credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp7.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp7.CompleteListing) + require.Equal(t, resp7.EstimatedItemCount, 7) + require.Empty(t, resp7.DeletedIds) + require.Len(t, resp7.Items, 1) + require.Empty(t, cmp.Diff(resp7.Items[0], newLib2, cmpOpts...)) + + // Refresh again, should get newLib1 + resp8, err := credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp8.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp8.CompleteListing) + require.Equal(t, resp8.EstimatedItemCount, 7) + require.Empty(t, resp8.DeletedIds) + require.Len(t, resp8.Items, 1) + require.Empty(t, cmp.Diff(resp8.Items[0], newLib1, cmpOpts...)) + + // Refresh again, should get no results + resp9, err := credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp9.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp9.CompleteListing) + require.Equal(t, resp9.EstimatedItemCount, 7) + require.Empty(t, resp9.DeletedIds) + require.Empty(t, resp9.Items) + }) + + t.Run("simple pagination with aggressive filtering", func(t *testing.T) { + filterFunc := func(ctx context.Context, l credential.Library) (bool, error) { + return l.GetPublicId() == libs[1].GetPublicId() || + l.GetPublicId() == libs[len(libs)-1].GetPublicId(), nil + } + resp, err := credential.ListLibraries(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 5) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], libs[1], cmpOpts...)) + + resp2, err := credential.ListLibrariesPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp2.ListToken) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 5) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 1) + require.Empty(t, cmp.Diff(resp2.Items[0], libs[len(libs)-1], cmpOpts...)) + + // request a refresh, nothing should be returned + resp3, err := credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 5) + require.Empty(t, resp3.DeletedIds) + require.Empty(t, resp3.Items) + + // Create some new libraries + newLibs1 := vault.TestCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 2) + newLibs2 := vault.TestSSHCertificateCredentialLibraries(t, conn, wrapper, credStore.GetPublicId(), 2) + t.Cleanup(func() { + _, err := repo.DeleteCredentialLibrary(ctx, credStore.ProjectId, newLibs1[0].GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteCredentialLibrary(ctx, credStore.ProjectId, newLibs1[1].GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteSSHCertificateCredentialLibrary(ctx, credStore.ProjectId, newLibs2[0].GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteSSHCertificateCredentialLibrary(ctx, credStore.ProjectId, newLibs2[1].GetPublicId()) + require.NoError(t, err) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + }) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + filterFunc = func(_ context.Context, l credential.Library) (bool, error) { + return l.GetPublicId() == newLibs2[0].GetPublicId() || + l.GetPublicId() == newLibs1[0].GetPublicId(), nil + } + // Refresh again, should get newLibs2[0] + resp4, err := credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp4.CompleteListing) + require.Equal(t, resp4.EstimatedItemCount, 9) + require.Empty(t, resp4.DeletedIds) + require.Len(t, resp4.Items, 1) + require.Empty(t, cmp.Diff(resp4.Items[0], newLibs2[0], cmpOpts...)) + + // Refresh again, should get newLibs1[0] + resp5, err := credential.ListLibrariesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp5.CompleteListing) + require.Equal(t, resp5.EstimatedItemCount, 9) + require.Empty(t, resp5.DeletedIds) + require.Len(t, resp5.Items, 1) + require.Empty(t, cmp.Diff(resp5.Items[0], newLibs1[0], cmpOpts...)) + }) + + t.Run("simple pagination with deletion", func(t *testing.T) { + filterFunc := func(context.Context, credential.Library) (bool, error) { + return true, nil + } + deletedCredentialLibraryId := libs[0].GetPublicId() + _, err := repo.DeleteSSHCertificateCredentialLibrary(ctx, prj.GetPublicId(), deletedCredentialLibraryId) + require.NoError(t, err) + libs = libs[1:] + + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + resp, err := credential.ListLibraries(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 4) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], libs[0], cmpOpts...)) + + // request remaining results + resp2, err := credential.ListLibrariesPage(ctx, []byte("some hash"), 3, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 4) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 3) + require.Empty(t, cmp.Diff(resp2.Items, libs[1:], cmpOpts...)) + + deletedCredentialLibraryId = libs[0].GetPublicId() + _, err = repo.DeleteSSHCertificateCredentialLibrary(ctx, prj.GetPublicId(), deletedCredentialLibraryId) + require.NoError(t, err) + libs = libs[1:] + + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + // request a refresh, nothing should be returned except the deleted id + resp3, err := credential.ListLibrariesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 3) + require.Contains(t, resp3.DeletedIds, deletedCredentialLibraryId) + require.Empty(t, resp3.Items) + }) +} diff --git a/internal/credential/service_list_libraries_page.go b/internal/credential/service_list_libraries_page.go new file mode 100644 index 000000000..901fbfe0f --- /dev/null +++ b/internal/credential/service_list_libraries_page.go @@ -0,0 +1,65 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/pagination" +) + +// ListLibrariesPage lists up to page size credential libraries, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credential libraries from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Credential libraries are ordered by create time descending (most recently created first). +func ListLibrariesPage( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Library], + tok *listtoken.Token, + service LibraryService, + credentialStoreId string, +) (*pagination.ListResponse[Library], error) { + const op = "credential.ListLibrariesPage" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + } + if _, ok := tok.Subtype.(*listtoken.PaginationToken); !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a pagination token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem Library, limit int) ([]Library, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } else { + lastItem, err := tok.LastItem(ctx) + if err != nil { + return nil, time.Time{}, err + } + opts = append(opts, WithStartPageAfterItem(lastItem)) + } + return service.ListLibraries(ctx, credentialStoreId, opts...) + } + + return pagination.ListPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedLibraryCount, tok) +} diff --git a/internal/credential/service_list_libraries_refresh.go b/internal/credential/service_list_libraries_refresh.go new file mode 100644 index 000000000..ff29020fd --- /dev/null +++ b/internal/credential/service_list_libraries_refresh.go @@ -0,0 +1,72 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/globals" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/pagination" +) + +// ListRefresh lists up to page size credential libraries, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credential libraries from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Credential libraries are ordered by update time descending (most recently updated first). +// Credential libraries may contain items that were already returned during the initial +// pagination phase. It also returns a list of any credential libraries deleted since the +// start of the initial pagination phase or last response. +func ListLibrariesRefresh( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Library], + tok *listtoken.Token, + service LibraryService, + credentialStoreId string, +) (*pagination.ListResponse[Library], error) { + const op = "credential.ListLibrariesRefresh" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + case credentialStoreId == "": + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID") + } + rt, ok := tok.Subtype.(*listtoken.StartRefreshToken) + if !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a start-refresh token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem Library, limit int) ([]Library, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } + // Add the database read timeout to account for any creations missed due to concurrent + // transactions in the initial pagination phase. + return service.ListLibrariesRefresh(ctx, credentialStoreId, rt.PreviousPhaseUpperBound.Add(-globals.RefreshReadLookbackDuration), opts...) + } + listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) { + // Add the database read timeout to account for any deletions missed due to concurrent + // transactions in previous requests. + return service.ListDeletedLibraryIds(ctx, since.Add(-globals.RefreshReadLookbackDuration)) + } + return pagination.ListRefresh(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedLibraryCount, listDeletedIdsFn, tok) +} diff --git a/internal/credential/service_list_libraries_refresh_page.go b/internal/credential/service_list_libraries_refresh_page.go new file mode 100644 index 000000000..b0fb8a0cf --- /dev/null +++ b/internal/credential/service_list_libraries_refresh_page.go @@ -0,0 +1,77 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/globals" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/pagination" +) + +// ListRefreshPage lists up to page size credential libraries, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credential libraries from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Credential libraries are ordered by update time descending (most recently updated first). +// Credential libraries may contain items that were already returned during the initial +// pagination phase. It also returns a list of any credential libraries deleted since the +// last response. +func ListLibrariesRefreshPage( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Library], + tok *listtoken.Token, + service LibraryService, + credentialStoreId string, +) (*pagination.ListResponse[Library], error) { + const op = "credential.ListLibrariesRefreshPage" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + } + rt, ok := tok.Subtype.(*listtoken.RefreshToken) + if !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a refresh token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem Library, limit int) ([]Library, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } else { + lastItem, err := tok.LastItem(ctx) + if err != nil { + return nil, time.Time{}, err + } + opts = append(opts, WithStartPageAfterItem(lastItem)) + } + // Add the database read timeout to account for any creations missed due to concurrent + // transactions in the original list pagination phase. + return service.ListLibrariesRefresh(ctx, credentialStoreId, rt.PhaseLowerBound.Add(-globals.RefreshReadLookbackDuration), opts...) + } + listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) { + // Add the database read timeout to account for any deletes missed due to concurrent + // transactions in the original list pagination phase. + return service.ListDeletedLibraryIds(ctx, since.Add(-globals.RefreshReadLookbackDuration)) + } + + return pagination.ListRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedLibraryCount, listDeletedIdsFn, tok) +}