From 4716c1a788ee976669734d4bdd9476e8459a3037 Mon Sep 17 00:00:00 2001 From: Todd Date: Thu, 2 Jun 2022 09:28:34 -0700 Subject: [PATCH] Update List Workers Repo function to allow limits and provide scopes (#2121) * Update List Workers Repo function to allow limits and provide scopes Co-authored-by: Irena Rindos Co-authored-by: Michael Gaffney --- .../handlers/targets/target_service.go | 2 +- internal/servers/repository.go | 17 +++- internal/servers/repository_worker.go | 46 ++++++----- internal/servers/repository_worker_test.go | 82 +++++++++++++++++-- 4 files changed, 117 insertions(+), 30 deletions(-) diff --git a/internal/daemon/controller/handlers/targets/target_service.go b/internal/daemon/controller/handlers/targets/target_service.go index 25dfea85a..efa23f89e 100644 --- a/internal/daemon/controller/handlers/targets/target_service.go +++ b/internal/daemon/controller/handlers/targets/target_service.go @@ -895,7 +895,7 @@ func (s Service) AuthorizeSession(ctx context.Context, req *pbs.AuthorizeSession // worker IDs below is used to contain their IDs in the same order. This is // used to fetch tags for filtering. But we avoid allocation unless we // actually need it. - selectedWorkers, err := serversRepo.ListWorkers(ctx) + selectedWorkers, err := serversRepo.ListWorkers(ctx, []string{scope.Global.String()}) if err != nil { return nil, err } diff --git a/internal/servers/repository.go b/internal/servers/repository.go index d482364d4..0fbcf292c 100644 --- a/internal/servers/repository.go +++ b/internal/servers/repository.go @@ -26,11 +26,13 @@ type Repository struct { reader db.Reader writer db.Writer kms *kms.Kms + // defaultLimit provides a default for limiting the number of results returned from the repo + defaultLimit int } // NewRepository creates a new servers Repository. Supports the options: WithLimit // which sets a default limit on results returned by repo operations. -func NewRepository(r db.Reader, w db.Writer, kms *kms.Kms) (*Repository, error) { +func NewRepository(r db.Reader, w db.Writer, kms *kms.Kms, opt ...Option) (*Repository, error) { const op = "servers.NewRepository" if r == nil { return nil, errors.NewDeprecated(errors.InvalidParameter, op, "nil reader") @@ -41,10 +43,17 @@ func NewRepository(r db.Reader, w db.Writer, kms *kms.Kms) (*Repository, error) if kms == nil { return nil, errors.NewDeprecated(errors.InvalidParameter, op, "nil kms") } + + opts := getOpts(opt...) + if opts.withLimit == 0 { + // zero signals the boundary defaults should be used. + opts.withLimit = db.DefaultLimit + } return &Repository{ - reader: r, - writer: w, - kms: kms, + reader: r, + writer: w, + kms: kms, + defaultLimit: opts.withLimit, }, nil } diff --git a/internal/servers/repository_worker.go b/internal/servers/repository_worker.go index 18b70c94a..787521afe 100644 --- a/internal/servers/repository_worker.go +++ b/internal/servers/repository_worker.go @@ -114,39 +114,47 @@ func lookupWorker(ctx context.Context, reader db.Reader, id string) (*Worker, er return w, nil } -// ListWorkers is a passthrough to listWorkersWithReader that uses the repo's normal reader. -func (r *Repository) ListWorkers(ctx context.Context, opt ...Option) ([]*Worker, error) { - return r.listWorkersWithReader(ctx, r.reader, opt...) -} - -// listWorkersWithReader will return a listing of resources and honor the -// WithLimit option. If WithLiveness is zero the default liveness value is used, -// if it is negative then the last status update time is ignored. This method -// accepts a reader, allowing it to be used within a transaction or without. -func (r *Repository) listWorkersWithReader(ctx context.Context, reader db.Reader, opt ...Option) ([]*Worker, error) { - const op = "workers.listWorkersWithReader" +// ListWorkers will return a listing of Workers and honor the WithLimit option. +// If WithLiveness is zero the default liveness value is used, if it is negative +// then the last status update time is ignored. +// If WithLimit < 0, then unlimited results are returned. If WithLimit == 0, then +// default limits are used for results. +func (r *Repository) ListWorkers(ctx context.Context, scopeIds []string, opt ...Option) ([]*Worker, error) { + const op = "servers.(Repository).ListWorkers" switch { - case isNil(reader): - return nil, errors.New(ctx, errors.InvalidParameter, op, "reader is nil") + case len(scopeIds) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "no scope ids set") } + opts := getOpts(opt...) liveness := opts.withLiveness if liveness == 0 { liveness = DefaultLiveness } - var where string + var where []string + var whereArgs []interface{} if liveness > 0 { - where = fmt.Sprintf("last_status_time > now() - interval '%d seconds'", uint32(liveness.Seconds())) + where = append(where, fmt.Sprintf("last_status_time > now() - interval '%d seconds'", uint32(liveness.Seconds()))) + } + if len(scopeIds) > 0 { + where = append(where, "scope_id in (?)") + whereArgs = append(whereArgs, scopeIds) + } + + limit := r.defaultLimit + if opts.withLimit != 0 { + // non-zero signals an override of the default limit for the repo. + limit = opts.withLimit } var wAggs []*workerAggregate - if err := reader.SearchWhere( + if err := r.reader.SearchWhere( ctx, &wAggs, - where, - []interface{}{}, - db.WithLimit(opts.withLimit), + strings.Join(where, " and "), + whereArgs, + db.WithLimit(limit), ); err != nil { return nil, errors.Wrap(ctx, err, op, errors.WithMsg("error searching for workers")) } diff --git a/internal/servers/repository_worker_test.go b/internal/servers/repository_worker_test.go index 351f28b80..5096f786f 100644 --- a/internal/servers/repository_worker_test.go +++ b/internal/servers/repository_worker_test.go @@ -291,7 +291,7 @@ func TestUpsertWorkerStatus(t *testing.T) { tc.errAssert(t, err) // Still only the original worker exists. - workers, err := repo.ListWorkers(ctx) + workers, err := repo.ListWorkers(ctx, []string{scope.Global.String()}) require.NoError(t, err) assert.Len(t, workers, 1) }) @@ -304,7 +304,7 @@ func TestUpsertWorkerStatus(t *testing.T) { _, err = repo.UpsertWorkerStatus(ctx, anotherStatus) require.NoError(t, err) { - workers, err := repo.ListWorkers(ctx) + workers, err := repo.ListWorkers(ctx, []string{scope.Global.String()}) require.NoError(t, err) assert.Len(t, workers, 2) } @@ -366,7 +366,77 @@ func TestTagUpdatingListing(t *testing.T) { assert.ElementsMatch(t, []string{"value21", "value22"}, worker1.CanonicalTags()["tag22"]) } -func TestListWorkersWithLiveness(t *testing.T) { +func TestListWorkers(t *testing.T) { + t.Parallel() + conn, _ := db.TestSetup(t, "postgres") + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kms := kms.TestKms(t, conn, wrapper) + const testLimit = 10 + repo, err := servers.NewRepository(rw, rw, kms, servers.WithLimit(testLimit)) + require.NoError(t, err) + ctx := context.Background() + + tests := []struct { + name string + createCnt int + reqScopes []string + opts []servers.Option + wantCnt int + wantErr bool + }{ + { + name: "no-limit", + createCnt: testLimit + 1, + reqScopes: []string{scope.Global.String()}, + opts: []servers.Option{servers.WithLimit(-1)}, + wantCnt: testLimit + 1, + wantErr: false, + }, + { + name: "default-limit", + createCnt: testLimit + 1, + reqScopes: []string{scope.Global.String()}, + wantCnt: testLimit, + wantErr: false, + }, + { + name: "custom-limit", + createCnt: testLimit + 1, + reqScopes: []string{scope.Global.String()}, + opts: []servers.Option{servers.WithLimit(3)}, + wantCnt: 3, + wantErr: false, + }, + { + name: "no-scope", + createCnt: 1, + reqScopes: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := servers.NewWorker(scope.Global.String()) + db.TestDeleteWhere(t, conn, w, "true") + for i := 0; i < tt.createCnt; i++ { + servers.TestWorker(t, conn, wrapper) + } + // the purpose of these tests isn't to check liveness, so disable + // liveness checking. + opts := append(tt.opts, servers.WithLiveness(-1)) + got, err := repo.ListWorkers(ctx, tt.reqScopes, opts...) + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Len(t, got, tt.wantCnt) + }) + } +} + +func TestListWorkers_WithLiveness(t *testing.T) { t.Parallel() require := require.New(t) conn, _ := db.TestSetup(t, "postgres") @@ -407,7 +477,7 @@ func TestListWorkersWithLiveness(t *testing.T) { } // Default liveness, should only list 1 - result, err := serversRepo.ListWorkers(ctx) + result, err := serversRepo.ListWorkers(ctx, []string{scope.Global.String()}) require.NoError(err) require.Len(result, 1) requireIds([]string{worker1.GetPublicId()}, result) @@ -420,13 +490,13 @@ func TestListWorkersWithLiveness(t *testing.T) { // Static liveness. Should get two, so long as this did not take // more than 5s to execute. - result, err = serversRepo.ListWorkers(ctx, servers.WithLiveness(time.Second*5)) + result, err = serversRepo.ListWorkers(ctx, []string{scope.Global.String()}, servers.WithLiveness(time.Second*5)) require.NoError(err) require.Len(result, 2) requireIds([]string{worker1.GetPublicId(), worker2.GetPublicId()}, result) // Liveness disabled, should get all three workers. - result, err = serversRepo.ListWorkers(ctx, servers.WithLiveness(-1)) + result, err = serversRepo.ListWorkers(ctx, []string{scope.Global.String()}, servers.WithLiveness(-1)) require.NoError(err) require.Len(result, 3) requireIds([]string{worker1.GetPublicId(), worker2.GetPublicId(), worker3.GetPublicId()}, result)