Add create, list, lookup functions for plugin based host catalogs (#1515)

Create the repository for plugin host catalogs.  A large TODO is left in the code to interface with the plugin system once ready.

internal/host/plugin/host_catalog.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/host/plugin/store"
+	"github.com/hashicorp/boundary/internal/oplog"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -17,21 +18,24 @@ import (
 type HostCatalog struct {
 	*store.HostCatalog
 	tableName string `gorm:"-"`
+
+	secrets map[string]interface{} `gorm:"-"`
 }
 
 // NewHostCatalog creates a new in memory HostCatalog assigned to a scopeId
 // and pluginId. Name and description are the only valid options. All other
 // options are ignored.
-func NewHostCatalog(ctx context.Context, pluginId, scopeId string, opt ...Option) (*HostCatalog, error) {
+func NewHostCatalog(ctx context.Context, scopeId, pluginId string, opt ...Option) (*HostCatalog, error) {
 	const op = "plugin.NewHostCatalog"
 	opts := getOpts(opt...)
 	hc := &HostCatalog{
 		HostCatalog: &store.HostCatalog{
-			PluginId:    pluginId,
 			ScopeId:     scopeId,
+			PluginId:    pluginId,
 			Name:        opts.withName,
 			Description: opts.withDescription,
 		},
+		secrets: opts.withSecrets,
 	}
 
 	if opts.withAttributes != nil {
@@ -50,11 +54,22 @@ func allocHostCatalog() *HostCatalog {
 	}
 }
 
+// clone provides a deep copy of the HostCatalog with the exception of the
+// secret.  The secret shallow copied.
 func (c *HostCatalog) clone() *HostCatalog {
 	cp := proto.Clone(c.HostCatalog)
-	return &HostCatalog{
+
+	hc := &HostCatalog{
 		HostCatalog: cp.(*store.HostCatalog),
 	}
+	if c.secrets != nil {
+		newSecret := make(map[string]interface{}, len(c.secrets))
+		for k, v := range c.secrets {
+			newSecret[k] = v
+		}
+		hc.secrets = newSecret
+	}
+	return hc
 }
 
 // TableName returns the table name for the host catalog.
@@ -70,3 +85,15 @@ func (c *HostCatalog) TableName() string {
 func (c *HostCatalog) SetTableName(n string) {
 	c.tableName = n
 }
+
+func (s *HostCatalog) oplog(op oplog.OpType) oplog.Metadata {
+	metadata := oplog.Metadata{
+		"resource-public-id": []string{s.PublicId},
+		"resource-type":      []string{"plugin-host-catalog"},
+		"op-type":            []string{op.String()},
+	}
+	if s.ScopeId != "" {
+		metadata["scope-id"] = []string{s.ScopeId}
+	}
+	return metadata
+}

internal/host/plugin/host_catalog_secret.go

@@ -4,10 +4,13 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/hashicorp/boundary/internal/db"
 	"github.com/hashicorp/boundary/internal/errors"
 	"github.com/hashicorp/boundary/internal/host/plugin/store"
+	"github.com/hashicorp/boundary/internal/oplog"
 	wrapping "github.com/hashicorp/go-kms-wrapping"
 	"github.com/hashicorp/go-kms-wrapping/structwrapping"
+	"google.golang.org/protobuf/proto"
 )
 
 // HostCatalogSecret contains the encrypted secret for a host catalog.
@@ -19,7 +22,7 @@ type HostCatalogSecret struct {
 
 // newHostCatalogSecret creates an in memory host catalog secret.
 // All options are ignored.
-func newHostCatalogSecret(ctx context.Context, catalogId string, attributes map[string]interface{}, _ ...Option) (*HostCatalogSecret, error) {
+func newHostCatalogSecret(ctx context.Context, catalogId string, secret map[string]interface{}, _ ...Option) (*HostCatalogSecret, error) {
 	const op = "plugin.newHostCatlogSecret"
 	hcs := &HostCatalogSecret{
 		HostCatalogSecret: &store.HostCatalogSecret{
@@ -27,8 +30,8 @@ func newHostCatalogSecret(ctx context.Context, catalogId string, attributes map[
 		},
 	}
 
-	if attributes != nil {
-		attrs, err := json.Marshal(attributes)
+	if secret != nil {
+		attrs, err := json.Marshal(secret)
 		if err != nil {
 			return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter))
 		}
@@ -37,6 +40,19 @@ func newHostCatalogSecret(ctx context.Context, catalogId string, attributes map[
 	return hcs, nil
 }
 
+func allocHostCatalogSecret() *HostCatalogSecret {
+	return &HostCatalogSecret{
+		HostCatalogSecret: &store.HostCatalogSecret{},
+	}
+}
+
+func (c *HostCatalogSecret) clone() *HostCatalogSecret {
+	cp := proto.Clone(c.HostCatalogSecret)
+	return &HostCatalogSecret{
+		HostCatalogSecret: cp.(*store.HostCatalogSecret),
+	}
+}
+
 // TableName returns the table name for the host catalog.
 func (c *HostCatalogSecret) TableName() string {
 	if c.tableName != "" {
@@ -90,3 +106,19 @@ func (c *HostCatalogSecret) deleteQuery() (query string, queryValues []interface
 	}
 	return
 }
+
+func (c *HostCatalogSecret) oplogMessage(opType db.OpType) *oplog.Message {
+	msg := oplog.Message{
+		Message:  c.clone(),
+		TypeName: c.TableName(),
+	}
+	switch opType {
+	case db.CreateOp:
+		msg.OpType = oplog.OpType_OP_TYPE_CREATE
+	case db.UpdateOp:
+		msg.OpType = oplog.OpType_OP_TYPE_UPDATE
+	case db.DeleteOp:
+		msg.OpType = oplog.OpType_OP_TYPE_DELETE
+	}
+	return &msg
+}

internal/host/plugin/host_catalog_secret_test.go

@@ -24,7 +24,7 @@ func TestHostCatalogSecret_New(t *testing.T) {
 
 	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 	plg := host.TestPlugin(t, conn, "test", "test")
-	cat := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
+	cat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
 
 	type args struct {
 		catalogId string
@@ -120,7 +120,7 @@ func TestHostCatalogSecret_Custom_Queries(t *testing.T) {
 
 	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 	plg := host.TestPlugin(t, conn, "test", "prefix")
-	cat := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
+	cat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
 	databaseWrapper, err := kkms.GetWrapper(ctx, prj.PublicId, kms.KeyPurposeDatabase)
 
 	hcs, err := newHostCatalogSecret(ctx, cat.GetPublicId(), map[string]interface{}{"foo": "bar"})

internal/host/plugin/host_catalog_test.go

@@ -141,14 +141,14 @@ func TestHostCatalog_Create(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewHostCatalog(ctx, tt.args.pluginId, tt.args.scopeId, tt.args.opts...)
+			got, err := NewHostCatalog(ctx, tt.args.scopeId, tt.args.pluginId, tt.args.opts...)
 			require.NoError(t, err)
 			require.NotNil(t, got)
 
 			assert.Emptyf(t, got.PublicId, "PublicId set")
 			assert.Equal(t, tt.want, got)
 
-			id, err := newHostCatalogId(plg.GetIdPrefix())
+			id, err := newHostCatalogId(ctx, plg.GetIdPrefix())
 			assert.NoError(t, err)
 
 			tt.want.PublicId = id
@@ -182,28 +182,28 @@ func TestHostCatalog_Create_DuplicateNames(t *testing.T) {
 	plg := host.TestPlugin(t, conn, "test1", "prefix1")
 	plg2 := host.TestPlugin(t, conn, "test2", "prefix2")
 
-	got, err := NewHostCatalog(ctx, plg.GetPublicId(), prj.GetPublicId(), WithName("duplicate"))
+	got, err := NewHostCatalog(ctx, prj.GetPublicId(), plg.GetPublicId(), WithName("duplicate"))
 	require.NoError(t, err)
-	got.PublicId, err = newHostCatalogId(plg.GetIdPrefix())
+	got.PublicId, err = newHostCatalogId(ctx, plg.GetIdPrefix())
 	require.NoError(t, err)
 	w.Create(ctx, got)
 
-	// Can't create another resource with the same pluginName in the same scope
-	got.PublicId, err = newHostCatalogId(plg.GetIdPrefix())
+	// Can't create another resource with the same name in the same scope
+	got.PublicId, err = newHostCatalogId(ctx, plg.GetIdPrefix())
 	require.NoError(t, err)
 	assert.Error(t, w.Create(ctx, got))
 
-	// Can't create another resource with same pluginName in same scope even for different plugin
-	got, err = NewHostCatalog(ctx, plg2.GetPublicId(), prj.GetPublicId(), WithName("duplicate"))
+	// Can't create another resource with same name in same scope even for different plugin
+	got, err = NewHostCatalog(ctx, prj.GetPublicId(), plg2.GetPublicId(), WithName("duplicate"))
 	require.NoError(t, err)
-	got.PublicId, err = newHostCatalogId(plg2.GetIdPrefix())
+	got.PublicId, err = newHostCatalogId(ctx, plg2.GetIdPrefix())
 	require.NoError(t, err)
 	assert.Error(t, w.Create(ctx, got))
 
-	// Can create another resource with same pluginName in different scope even for same plugin
-	got, err = NewHostCatalog(ctx, plg.GetPublicId(), prj2.GetPublicId(), WithName("duplicate"))
+	// Can create another resource with same name in different scope even for same plugin
+	got, err = NewHostCatalog(ctx, prj2.GetPublicId(), plg.GetPublicId(), WithName("duplicate"))
 	require.NoError(t, err)
-	got.PublicId, err = newHostCatalogId(plg.GetIdPrefix())
+	got.PublicId, err = newHostCatalogId(ctx, plg.GetIdPrefix())
 	require.NoError(t, err)
 	assert.NoError(t, w.Create(ctx, got))
 }
@@ -213,8 +213,8 @@ func TestHostCatalog_Delete(t *testing.T) {
 	wrapper := db.TestWrapper(t)
 	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 	plg := host.TestPlugin(t, conn, "test", "prefix")
-	cat := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
-	ignoredCat := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
+	cat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
+	ignoredCat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
 	_ = ignoredCat
 	tests := []struct {
 		name        string
@@ -264,7 +264,7 @@ func TestHostCatalog_Delete_Cascading(t *testing.T) {
 	t.Run("delete-scope", func(t *testing.T) {
 		_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 		plg := host.TestPlugin(t, conn, "deletescope", "deletescope")
-		cat := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
+		cat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
 
 		deleted, err := w.Delete(ctx, prj)
 		require.NoError(t, err)
@@ -278,7 +278,7 @@ func TestHostCatalog_Delete_Cascading(t *testing.T) {
 	t.Run("delete-plugin", func(t *testing.T) {
 		_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 		plg := host.TestPlugin(t, conn, "deleteplugin", "deleteplugin")
-		cat := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
+		cat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
 
 		deleted, err := w.Delete(ctx, plg)
 		require.NoError(t, err)

internal/host/plugin/host_set_test.go

@@ -17,8 +17,8 @@ func TestHostSet_Create(t *testing.T) {
 	wrapper := db.TestWrapper(t)
 	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 	plg := host.TestPlugin(t, conn, "test", "prefix")
-	cat := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
-	cat2 := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	cat := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
+	cat2 := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 
 	type args struct {
 		catalogId string
@@ -145,7 +145,7 @@ func TestHostSet_Create(t *testing.T) {
 			assert.Emptyf(t, got.PublicId, "PublicId set")
 			assert.Equal(t, tt.want, got)
 
-			id, err := newHostSetId(plg.GetIdPrefix())
+			id, err := newHostSetId(ctx, plg.GetIdPrefix())
 			assert.NoError(t, err)
 
 			tt.want.PublicId = id

internal/host/plugin/ids.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/hashicorp/boundary/internal/db"
@@ -24,20 +25,20 @@ const (
 	Subtype = subtypes.Subtype("plugin")
 )
 
-func newHostCatalogId(upre string) (string, error) {
+func newHostCatalogId(ctx context.Context, upre string) (string, error) {
 	prefix := fmt.Sprintf("%s_%s", HostCatalogPrefix, upre)
 	id, err := db.NewPublicId(prefix)
 	if err != nil {
-		return "", errors.WrapDeprecated(err, "plugin.newHostCatalogId")
+		return "", errors.Wrap(ctx, err, "plugin.newHostCatalogId")
 	}
 	return id, nil
 }
 
-func newHostSetId(upre string) (string, error) {
+func newHostSetId(ctx context.Context, upre string) (string, error) {
 	prefix := fmt.Sprintf("%s_%s", HostSetPrefix, upre)
 	id, err := db.NewPublicId(prefix)
 	if err != nil {
-		return "", errors.WrapDeprecated(err, "plugin.newHostSetId")
+		return "", errors.Wrap(ctx, err, "plugin.newHostSetId")
 	}
 	return id, nil
 }

internal/host/plugin/immutable_fields_test.go

@@ -24,7 +24,7 @@ func TestPluginCatalog_ImmutableFields(t *testing.T) {
 	o, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 	plg := host.TestPlugin(t, conn, "test", "prefix")
 	plg2 := host.TestPlugin(t, conn, "test2", "prefix2")
-	new := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	new := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 
 	tests := []struct {
 		name      string
@@ -98,7 +98,7 @@ func TestPluginSet_ImmutableFields(t *testing.T) {
 	ts := timestamp.Timestamp{Timestamp: &timestamppb.Timestamp{Seconds: 0, Nanos: 0}}
 	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
 	plg := host.TestPlugin(t, conn, "test", "prefix")
-	cat := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	cat := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 	new := TestSet(t, conn, cat.GetPublicId())
 
 	tests := []struct {

internal/host/plugin/options.go

@@ -18,6 +18,7 @@ type options struct {
 	withDescription string
 	withLimit       int
 	withAttributes  map[string]interface{}
+	withSecrets     map[string]interface{}
 }
 
 func getDefaultOptions() options {
@@ -49,6 +50,13 @@ func WithAttributes(attrs map[string]interface{}) Option {
 	}
 }
 
+// WithSecrets provides an optional secrets field.
+func WithSecrets(secrets map[string]interface{}) Option {
+	return func(o *options) {
+		o.withSecrets = secrets
+	}
+}
+
 // WithLimit provides an option to provide a limit. Intentionally allowing
 // negative integers. If WithLimit < 0, then unlimited results are
 // returned. If WithLimit == 0, then default limits are used for results.

internal/host/plugin/repository.go

@@ -1,3 +1,6 @@
+// Package plugin provides a plugin host catalog, and plugin host set resource
+// which are used to interact with a host plugin as well as a repository to
+// perform CRUDL and custom actions on these resources.
 package plugin
 
 import (

internal/host/plugin/repository_host_catalog.go

@@ -0,0 +1,166 @@
+package plugin
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/oplog"
+	hostplg "github.com/hashicorp/boundary/internal/plugin/host"
+)
+
+// CreateCatalog inserts c into the repository and returns a new
+// HostCatalog containing the catalog's PublicId. c must contain a valid
+// ScopeID and PluginID. c must not contain a PublicId. The PublicId is
+// generated and assigned by this method. opt is ignored.
+//
+// c.Secret, c.Name and c.Description are optional. If c.Name is set, it must be
+// unique within c.ScopeID.  If c.Secret is set, it will be stored encrypted but
+// not included in the returned *HostCatalog.
+//
+// Both c.CreateTime and c.UpdateTime are ignored.
+func (r *Repository) CreateCatalog(ctx context.Context, c *HostCatalog, _ ...Option) (*HostCatalog, error) {
+	const op = "plugin.(Repository).CreateCatalog"
+	if c == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "nil HostCatalog")
+	}
+	if c.HostCatalog == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "nil embedded HostCatalog")
+	}
+	if c.ScopeId == "" {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "no scope id")
+	}
+	if c.PublicId != "" {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "public id not empty")
+	}
+	if c.PluginId == "" {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "no plugin id")
+	}
+	if c.Attributes == nil {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "nil attributes")
+	}
+	c = c.clone()
+
+	// TODO: Capture this in a plugin manager and call the plugin's OnCreateCatalog function
+	plg := hostplg.NewPlugin("", "")
+	plg.PublicId = c.PluginId
+	if err := r.reader.LookupByPublicId(ctx, plg); err != nil {
+		if errors.IsNotFoundError(err) {
+			return nil, errors.Wrap(ctx, err, op, errors.WithCode(errors.InvalidParameter), errors.WithMsg(fmt.Sprintf("can't find plugin with id: %q", c.GetPluginId())))
+		}
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("unable to get host plugin with id %q", c.PluginId)))
+	}
+
+	id, err := newHostCatalogId(ctx, plg.GetIdPrefix())
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+	c.PublicId = id
+
+	oplogWrapper, err := r.kms.GetWrapper(ctx, c.ScopeId, kms.KeyPurposeOplog)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("unable to get oplog wrapper"))
+	}
+	dbWrapper, err := r.kms.GetWrapper(ctx, c.ScopeId, kms.KeyPurposeDatabase)
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("unable to get db wrapper"))
+	}
+
+	var hcSecret *HostCatalogSecret
+	if c.secrets != nil {
+		// TODO: Create the secret using the returned value from the call to the plugin.
+		hcSecret, err = newHostCatalogSecret(ctx, id, c.secrets)
+		if err != nil {
+			return nil, errors.Wrap(ctx, err, op)
+		}
+		hcSecret.encrypt(ctx, dbWrapper)
+	}
+
+	var newHostCatalog *HostCatalog
+	_, err = r.writer.DoTx(
+		ctx,
+		db.StdRetryCnt,
+		db.ExpBackoff{},
+		func(_ db.Reader, w db.Writer) error {
+			msgs := make([]*oplog.Message, 0, 3)
+			ticket, err := w.GetTicket(c)
+			if err != nil {
+				return errors.Wrap(ctx, err, op, errors.WithMsg("unable to get ticket"))
+			}
+
+			newHostCatalog = c.clone()
+			var cOplogMsg oplog.Message
+			if err := w.Create(ctx, newHostCatalog, db.NewOplogMsg(&cOplogMsg)); err != nil {
+				return errors.Wrap(ctx, err, op)
+			}
+			msgs = append(msgs, &cOplogMsg)
+
+			if hcSecret != nil {
+				newSecret := hcSecret.clone()
+				q, v := newSecret.insertQuery()
+				rows, err := w.Exec(ctx, q, v)
+				if err != nil {
+					return errors.Wrap(ctx, err, op)
+				}
+				if rows > 1 {
+					return errors.New(ctx, errors.MultipleRecords, op, "more than 1 catalog secret would have been created")
+				}
+				msgs = append(msgs, newSecret.oplogMessage(db.CreateOp))
+			}
+
+			metadata := c.oplog(oplog.OpType_OP_TYPE_CREATE)
+			if err := w.WriteOplogEntryWith(ctx, oplogWrapper, ticket, metadata, msgs); err != nil {
+				return errors.Wrap(ctx, err, op, errors.WithMsg("unable to write oplog"))
+			}
+			return nil
+		},
+	)
+
+	if err != nil {
+		if errors.IsUniqueError(err) {
+			return nil, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("in scope: %s: name %s already exists", c.ScopeId, c.Name)))
+		}
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("in scope: %s", c.ScopeId)))
+	}
+	return newHostCatalog, nil
+}
+
+// LookupCatalog returns the HostCatalog for id. Returns nil, nil if no
+// HostCatalog is found for id.
+func (r *Repository) LookupCatalog(ctx context.Context, id string, _ ...Option) (*HostCatalog, error) {
+	const op = "plugin.(Repository).LookupCatalog"
+	if id == "" {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "no public id")
+	}
+	c := allocHostCatalog()
+	c.PublicId = id
+	if err := r.reader.LookupByPublicId(ctx, c); err != nil {
+		if errors.IsNotFoundError(err) {
+			return nil, nil
+		}
+		return nil, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("failed for: %s", id)))
+	}
+	return c, nil
+}
+
+// ListCatalogs returns a slice of HostCatalogs for the scope IDs. WithLimit is the only option supported.
+func (r *Repository) ListCatalogs(ctx context.Context, scopeIds []string, opt ...Option) ([]*HostCatalog, error) {
+	const op = "plugin.(Repository).ListCatalogs"
+	if len(scopeIds) == 0 {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "no scope id")
+	}
+	opts := getOpts(opt...)
+	limit := r.defaultLimit
+	if opts.withLimit != 0 {
+		// non-zero signals an override of the default limit for the repo.
+		limit = opts.withLimit
+	}
+	var hostCatalogs []*HostCatalog
+	err := r.reader.SearchWhere(ctx, &hostCatalogs, "scope_id in (?)", []interface{}{scopeIds}, db.WithLimit(limit))
+	if err != nil {
+		return nil, errors.Wrap(ctx, err, op)
+	}
+	return hostCatalogs, nil
+}

internal/host/plugin/repository_host_catalog_test.go

@@ -0,0 +1,377 @@
+package plugin
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/host/plugin/store"
+	"github.com/hashicorp/boundary/internal/iam"
+	"github.com/hashicorp/boundary/internal/kms"
+	"github.com/hashicorp/boundary/internal/oplog"
+	hostplg "github.com/hashicorp/boundary/internal/plugin/host"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRepository_CreateCatalog(t *testing.T) {
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+	plg := hostplg.TestPlugin(t, conn, "test", "test")
+
+	tests := []struct {
+		name       string
+		in         *HostCatalog
+		opts       []Option
+		want       *HostCatalog
+		wantSecret []byte
+		wantIsErr  errors.Code
+	}{
+		{
+			name:      "nil-catalog",
+			wantIsErr: errors.InvalidParameter,
+		},
+		{
+			name:      "nil-embedded-catalog",
+			in:        &HostCatalog{},
+			wantIsErr: errors.InvalidParameter,
+		},
+		{
+			name: "no-scope",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					PluginId:   plg.GetPublicId(),
+					Attributes: []byte("{}"),
+				},
+			},
+			wantIsErr: errors.InvalidParameter,
+		},
+		{
+			name: "no-plugin",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					ScopeId:    prj.GetPublicId(),
+					Attributes: []byte("{}"),
+				},
+			},
+			wantIsErr: errors.InvalidParameter,
+		},
+		{
+			name: "no-attributes",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					ScopeId:  prj.GetPublicId(),
+					PluginId: plg.GetPublicId(),
+				},
+			},
+			wantIsErr: errors.InvalidParameter,
+		},
+		{
+			name: "valid-no-options",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					ScopeId:    prj.GetPublicId(),
+					PluginId:   plg.GetPublicId(),
+					Attributes: []byte("{}"),
+				},
+			},
+			want: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					ScopeId:    prj.GetPublicId(),
+					PluginId:   plg.GetPublicId(),
+					Attributes: []byte("{}"),
+				},
+			},
+		},
+		{
+			name: "not-found-plugin",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					ScopeId:    prj.GetPublicId(),
+					PluginId:   "unknown_plugin",
+					Attributes: []byte("{}"),
+				},
+			},
+			wantIsErr: errors.InvalidParameter,
+		},
+		{
+			name: "valid-with-name",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					Name:       "test-name-repo",
+					ScopeId:    prj.GetPublicId(),
+					PluginId:   plg.GetPublicId(),
+					Attributes: []byte("{}"),
+				},
+			},
+			want: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					Name:       "test-name-repo",
+					ScopeId:    prj.GetPublicId(),
+					PluginId:   plg.GetPublicId(),
+					Attributes: []byte("{}"),
+				},
+			},
+		},
+		{
+			name: "valid-with-description",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					Description: "test-description-repo",
+					ScopeId:     prj.GetPublicId(),
+					PluginId:    plg.GetPublicId(),
+					Attributes:  []byte("{}"),
+				},
+			},
+			want: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					Description: "test-description-repo",
+					ScopeId:     prj.GetPublicId(),
+					PluginId:    plg.GetPublicId(),
+					Attributes:  []byte("{}"),
+				},
+			},
+		},
+		{
+			name: "valid-with-secret",
+			in: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					Description: "test-description-repo",
+					ScopeId:     prj.GetPublicId(),
+					PluginId:    plg.GetPublicId(),
+					Attributes:  []byte("{}"),
+				},
+				secrets: map[string]interface{}{
+					"k1": "v1",
+					"k2": 2,
+					"k3": nil,
+				},
+			},
+			want: &HostCatalog{
+				HostCatalog: &store.HostCatalog{
+					Description: "test-description-repo",
+					ScopeId:     prj.GetPublicId(),
+					PluginId:    plg.GetPublicId(),
+					Attributes:  []byte("{}"),
+				},
+			},
+			wantSecret: []byte(`{"k1":"v1","k2":2,"k3":null}`),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			assert := assert.New(t)
+			kmsCache := kms.TestKms(t, conn, wrapper)
+			repo, err := NewRepository(rw, rw, kmsCache)
+			assert.NoError(err)
+			assert.NotNil(repo)
+			got, err := repo.CreateCatalog(ctx, tt.in, tt.opts...)
+			if tt.wantIsErr != 0 {
+				assert.Truef(errors.Match(errors.T(tt.wantIsErr), err), "want err: %q got: %q", tt.wantIsErr, err)
+				assert.Nil(got)
+				return
+			}
+			require.NoError(t, err)
+			assert.Empty(tt.in.PublicId)
+			assert.NotNil(got)
+			assertPluginBasedPublicId(t, HostCatalogPrefix, got.PublicId)
+			assert.NotSame(tt.in, got)
+			assert.Equal(tt.want.Name, got.Name)
+			assert.Equal(tt.want.Description, got.Description)
+			assert.Equal(got.CreateTime, got.UpdateTime)
+
+			assert.NoError(db.TestVerifyOplog(t, rw, got.PublicId, db.WithOperation(oplog.OpType_OP_TYPE_CREATE), db.WithCreateNotBefore(10*time.Second)))
+
+			cSecret := allocHostCatalogSecret()
+			err = rw.LookupWhere(ctx, &cSecret, "catalog_id=?", got.GetPublicId())
+			if tt.wantSecret == nil {
+				assert.Nil(got.secrets)
+				require.Error(t, err)
+				require.True(t, errors.IsNotFoundError(err))
+				return
+			}
+			require.NoError(t, err)
+			require.Empty(t, cSecret.Secret)
+			require.NotEmpty(t, cSecret.CtSecret)
+
+			dbWrapper, err := kmsCache.GetWrapper(ctx, got.GetScopeId(), kms.KeyPurposeDatabase)
+			require.NoError(t, err)
+			cSecret.decrypt(ctx, dbWrapper)
+			assert.Equal(string(tt.wantSecret), string(cSecret.Secret))
+		})
+	}
+
+	t.Run("invalid-duplicate-names", func(t *testing.T) {
+		assert := assert.New(t)
+		kms := kms.TestKms(t, conn, wrapper)
+		repo, err := NewRepository(rw, rw, kms)
+		assert.NoError(err)
+		assert.NotNil(repo)
+		_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+		in := &HostCatalog{
+			HostCatalog: &store.HostCatalog{
+				ScopeId:    prj.GetPublicId(),
+				Name:       "test-name-repo",
+				PluginId:   plg.GetPublicId(),
+				Attributes: []byte("{}"),
+			},
+		}
+
+		got, err := repo.CreateCatalog(context.Background(), in)
+		assert.NoError(err)
+		assert.NotNil(got)
+		assertPluginBasedPublicId(t, HostCatalogPrefix, got.PublicId)
+		assert.NotSame(in, got)
+		assert.Equal(in.Name, got.Name)
+		assert.Equal(in.Description, got.Description)
+		assert.Equal(got.CreateTime, got.UpdateTime)
+
+		got2, err := repo.CreateCatalog(context.Background(), in)
+		assert.Truef(errors.Match(errors.T(errors.NotUnique), err), "want err code: %v got err: %v", errors.NotUnique, err)
+		assert.Nil(got2)
+	})
+
+	t.Run("valid-duplicate-names-diff-scopes", func(t *testing.T) {
+		assert := assert.New(t)
+		kms := kms.TestKms(t, conn, wrapper)
+		repo, err := NewRepository(rw, rw, kms)
+		assert.NoError(err)
+		assert.NotNil(repo)
+		org, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+		in := &HostCatalog{
+			HostCatalog: &store.HostCatalog{
+				Name:       "test-name-repo",
+				PluginId:   plg.GetPublicId(),
+				Attributes: []byte("{}"),
+			},
+		}
+		in2 := in.clone()
+
+		in.ScopeId = prj.GetPublicId()
+		got, err := repo.CreateCatalog(context.Background(), in)
+		assert.NoError(err)
+		assert.NotNil(got)
+		assertPluginBasedPublicId(t, HostCatalogPrefix, got.PublicId)
+		assert.NotSame(in, got)
+		assert.Equal(in.Name, got.Name)
+		assert.Equal(in.Description, got.Description)
+		assert.Equal(got.CreateTime, got.UpdateTime)
+
+		in2.ScopeId = org.GetPublicId()
+		got2, err := repo.CreateCatalog(context.Background(), in2)
+		assert.NoError(err)
+		assert.NotNil(got2)
+		assertPluginBasedPublicId(t, HostCatalogPrefix, got2.PublicId)
+		assert.NotSame(in2, got2)
+		assert.Equal(in2.Name, got2.Name)
+		assert.Equal(in2.Description, got2.Description)
+		assert.Equal(got2.CreateTime, got2.UpdateTime)
+	})
+}
+
+func assertPluginBasedPublicId(t *testing.T, prefix, actual string) {
+	t.Helper()
+	assert.NotEmpty(t, actual)
+	parts := strings.Split(actual, "_")
+	assert.Equalf(t, 3, len(parts), "want two '_' in PublicId, got %d in %q", len(parts), actual)
+	assert.Equalf(t, prefix, parts[0], "PublicId want prefix: %q, got: %q in %q", prefix, parts[0], actual)
+}
+
+func TestRepository_LookupCatalog(t *testing.T) {
+	ctx := context.Background()
+	conn, _ := db.TestSetup(t, "postgres")
+	rw := db.New(conn)
+	wrapper := db.TestWrapper(t)
+	_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+	plg := hostplg.TestPlugin(t, conn, "test", "test")
+	cat := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
+	badId, err := newHostCatalogId(ctx, plg.GetIdPrefix())
+	assert.NoError(t, err)
+	assert.NotNil(t, badId)
+
+	tests := []struct {
+		name    string
+		id      string
+		want    *HostCatalog
+		wantErr errors.Code
+	}{
+		{
+			name: "found",
+			id:   cat.GetPublicId(),
+			want: cat,
+		},
+		{
+			name: "not-found",
+			id:   badId,
+			want: nil,
+		},
+		{
+			name:    "bad-public-id",
+			id:      "",
+			want:    nil,
+			wantErr: errors.InvalidParameter,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			assert := assert.New(t)
+			kms := kms.TestKms(t, conn, wrapper)
+			repo, err := NewRepository(rw, rw, kms)
+			assert.NoError(err)
+			assert.NotNil(repo)
+
+			got, err := repo.LookupCatalog(ctx, tt.id)
+			if tt.wantErr != 0 {
+				assert.Truef(errors.Match(errors.T(tt.wantErr), err), "want err: %q got: %q", tt.wantErr, err)
+				return
+			}
+			assert.NoError(err)
+
+			switch {
+			case tt.want == nil:
+				assert.Nil(got)
+			case tt.want != nil:
+				assert.NotNil(got)
+				assert.Equal(got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRepository_ListCatalogs_Multiple_Scopes(t *testing.T) {
+	t.Parallel()
+	conn, _ := db.TestSetup(t, "postgres")
+	wrapper := db.TestWrapper(t)
+	rw := db.New(conn)
+	kms := kms.TestKms(t, conn, wrapper)
+	repo, err := NewRepository(rw, rw, kms)
+	assert.NoError(t, err)
+	assert.NotNil(t, repo)
+	plg := hostplg.TestPlugin(t, conn, "test", "test")
+
+	const numPerScope = 10
+	var projs []string
+	var total int
+	for i := 0; i < numPerScope; i++ {
+		_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
+		projs = append(projs, prj.GetPublicId())
+		for j := 0; j < numPerScope; j++ {
+			TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
+			total++
+		}
+	}
+
+	got, err := repo.ListCatalogs(context.Background(), projs)
+	require.NoError(t, err)
+	assert.Equal(t, total, len(got))
+}

internal/host/plugin/repository_host_set.go

@@ -54,7 +54,7 @@ func (r *Repository) CreateSet(ctx context.Context, scopeId string, s *HostSet,
 		return nil, errors.Wrap(ctx, err, op, errors.WithMsg("unable to get host plugin"))
 	}
 
-	id, err := newHostSetId(plg.GetIdPrefix())
+	id, err := newHostSetId(ctx, plg.GetIdPrefix())
 	if err != nil {
 		return nil, errors.Wrap(ctx, err, op)
 	}

internal/host/plugin/repository_host_set_test.go

@@ -28,7 +28,7 @@ func TestRepository_CreateSet(t *testing.T) {
 	iamRepo := iam.TestRepo(t, conn, wrapper)
 	_, prj := iam.TestScopes(t, iamRepo)
 	plg := hostplg.TestPlugin(t, conn, "create", "create")
-	catalog := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	catalog := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 	attrs := []byte("{}")
 
 	tests := []struct {
@@ -159,7 +159,7 @@ func TestRepository_CreateSet(t *testing.T) {
 		require.NotNil(repo)
 
 		_, prj := iam.TestScopes(t, iamRepo)
-		catalog := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+		catalog := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 
 		in := &HostSet{
 			HostSet: &store.HostSet{
@@ -190,8 +190,8 @@ func TestRepository_CreateSet(t *testing.T) {
 		require.NotNil(repo)
 
 		_, prj := iam.TestScopes(t, iamRepo)
-		catalogA := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
-		catalogB := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+		catalogA := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
+		catalogB := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 
 		in := &HostSet{
 			HostSet: &store.HostSet{
@@ -224,6 +224,7 @@ func TestRepository_CreateSet(t *testing.T) {
 }
 
 func TestRepository_LookupSet(t *testing.T) {
+	ctx := context.Background()
 	conn, _ := db.TestSetup(t, "postgres")
 	rw := db.New(conn)
 	wrapper := db.TestWrapper(t)
@@ -232,9 +233,9 @@ func TestRepository_LookupSet(t *testing.T) {
 	_, prj := iam.TestScopes(t, iamRepo)
 	plg := hostplg.TestPlugin(t, conn, "lookup", "lookup")
 
-	catalog := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	catalog := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 	hostSet := TestSet(t, conn, catalog.PublicId)
-	hostSetId, err := newHostSetId(plg.GetIdPrefix())
+	hostSetId, err := newHostSetId(ctx, plg.GetIdPrefix())
 	require.NoError(t, err)
 
 	tests := []struct {
@@ -265,7 +266,7 @@ func TestRepository_LookupSet(t *testing.T) {
 			repo, err := NewRepository(rw, rw, kms)
 			assert.NoError(err)
 			require.NotNil(repo)
-			got, err := repo.LookupSet(context.Background(), tt.in)
+			got, err := repo.LookupSet(ctx, tt.in)
 			if tt.wantIsErr != 0 {
 				assert.Truef(errors.Match(errors.T(tt.wantIsErr), err), "want err: %q got: %q", tt.wantIsErr, err)
 				assert.Nil(got)
@@ -286,8 +287,8 @@ func TestRepository_ListSets(t *testing.T) {
 
 	_, prj := iam.TestScopes(t, iamRepo)
 	plg := hostplg.TestPlugin(t, conn, "list", "list")
-	catalogA := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
-	catalogB := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	catalogA := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
+	catalogB := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 
 	hostSets := []*HostSet{
 		TestSet(t, conn, catalogA.PublicId),
@@ -350,7 +351,7 @@ func TestRepository_ListSets_Limits(t *testing.T) {
 
 	_, prj := iam.TestScopes(t, iamRepo)
 	plg := hostplg.TestPlugin(t, conn, "listlimit", "listlimit")
-	catalog := TestCatalog(t, conn, plg.GetPublicId(), prj.PublicId)
+	catalog := TestCatalog(t, conn, prj.PublicId, plg.GetPublicId())
 	count := 10
 	var hostSets []*HostSet
 	for i := 0; i < count; i++ {

internal/host/plugin/testing.go

@@ -14,12 +14,12 @@ import (
 // TestCatalog creates count number of plugin host catalogs to the provided DB
 // with the provided scope id.  If any errors are encountered during the creation of
 // the host catalog, the test will fail.
-func TestCatalog(t *testing.T, conn *gorm.DB, pluginId, scopeId string, opt ...Option) *HostCatalog {
+func TestCatalog(t *testing.T, conn *gorm.DB, scopeId, pluginId string, opt ...Option) *HostCatalog {
 	t.Helper()
 	ctx := context.Background()
 	w := db.New(conn)
 
-	cat, err := NewHostCatalog(ctx, pluginId, scopeId, opt...)
+	cat, err := NewHostCatalog(ctx, scopeId, pluginId, opt...)
 	require.NoError(t, err)
 	assert.NotNil(t, cat)
 
@@ -27,7 +27,7 @@ func TestCatalog(t *testing.T, conn *gorm.DB, pluginId, scopeId string, opt ...O
 	plg.PublicId = pluginId
 	require.NoError(t, w.LookupByPublicId(ctx, plg))
 
-	id, err := newHostCatalogId(plg.GetIdPrefix())
+	id, err := newHostCatalogId(ctx, plg.GetIdPrefix())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id)
 	cat.PublicId = id
@@ -57,7 +57,7 @@ func TestSet(t *testing.T, conn *gorm.DB, catalogId string, opt ...Option) *Host
 	plg.PublicId = cg.GetPluginId()
 	require.NoError(t, w.LookupByPublicId(ctx, plg))
 
-	id, err := newHostSetId(plg.GetIdPrefix())
+	id, err := newHostSetId(ctx, plg.GetIdPrefix())
 	assert.NoError(err)
 	assert.NotEmpty(id)
 	set.PublicId = id

internal/host/plugin/testing_test.go

@@ -24,7 +24,7 @@ func Test_TestCatalogs(t *testing.T) {
 	require.NotNil(plg)
 	assert.NotEmpty(plg.GetPublicId())
 
-	cs := TestCatalog(t, conn, plg.GetPublicId(), proj.GetPublicId(), WithName("foo"), WithDescription("bar"))
+	cs := TestCatalog(t, conn, proj.GetPublicId(), plg.GetPublicId(), WithName("foo"), WithDescription("bar"))
 	assert.NotEmpty(cs.GetPublicId())
 	assert.True(strings.HasPrefix(cs.GetPublicId(), fmt.Sprintf("%s_%s", HostCatalogPrefix, "prefix")))
 	assert.Equal("foo", cs.GetName())
@@ -43,7 +43,7 @@ func Test_TestSet(t *testing.T) {
 	require.NotNil(plg)
 	assert.NotEmpty(plg.GetPublicId())
 
-	c := TestCatalog(t, conn, plg.GetPublicId(), prj.GetPublicId())
+	c := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
 	set := TestSet(t, conn, c.GetPublicId(), WithName("foo"), WithDescription("bar"))
 	assert.NotEmpty(set.GetPublicId())
 	assert.True(strings.HasPrefix(set.GetPublicId(), fmt.Sprintf("%s_%s", HostSetPrefix, "prefix")))