CyberCoveLLC/threshold
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(host/plugin): Always send plugin information on DHC requests

Browse Source
This commit is contained in:
Hugo committed 2024-09-30 21:24:18 +01:00
1 parent c3d03f69d8
commit 60dc0ec3ff
11 files changed
+440 -289
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/host/plugin/host_catalog.go
+24
View File
@@ -15,6 +15,8 @@ import (
"github.com/hashicorp/boundary/internal/host/plugin/store"
"github.com/hashicorp/boundary/internal/libs/crypto"
"github.com/hashicorp/boundary/internal/oplog"
"github.com/hashicorp/boundary/internal/plugin"
plgstore "github.com/hashicorp/boundary/internal/plugin/store"
"github.com/hashicorp/boundary/internal/types/resource"
wrapping "github.com/hashicorp/go-kms-wrapping/v2"
"google.golang.org/protobuf/proto"
@@ -142,6 +144,8 @@ func (c *HostCatalog) oplog(op oplog.OpType) oplog.Metadata {
return metadata
}
// TODO: Refactor repository (catalogAgg + getPlugin) uses to just use
// catalogAgg + catalogAgg.plugin().
type catalogAgg struct {
PublicId string `gorm:"primary_key"`
ProjectId string
@@ -158,6 +162,12 @@ type catalogAgg struct {
KeyId string
PersistedCreateTime *timestamp.Timestamp
PersistedUpdateTime *timestamp.Timestamp
PluginScopeId string
PluginName string
PluginDescription string
PluginCreateTime *timestamp.Timestamp
PluginUpdateTime *timestamp.Timestamp
PluginVersion uint32
}
func (agg *catalogAgg) toCatalogAndPersisted() (*HostCatalog, *HostCatalogSecret) {
@@ -189,6 +199,20 @@ func (agg *catalogAgg) toCatalogAndPersisted() (*HostCatalog, *HostCatalogSecret
return c, s
}
func (agg *catalogAgg) plugin() *plugin.Plugin {
return &plugin.Plugin{
Plugin: &plgstore.Plugin{
PublicId: agg.PluginId,
ScopeId: agg.PluginScopeId,
Name: agg.PluginName,
Description: agg.PluginDescription,
CreateTime: agg.PluginCreateTime,
UpdateTime: agg.PluginUpdateTime,
Version: agg.PluginVersion,
},
}
}
// TableName returns the table name for gorm
func (agg *catalogAgg) TableName() string {
return "host_plugin_catalog_with_secret"
internal/host/plugin/host_catalog_test.go
+20
View File
@@ -462,3 +462,23 @@ func TestHostCatalog_SecretsHmac(t *testing.T) {
})
}
}
func TestCatalogAgg(t *testing.T) {
ctx := context.Background()
conn, _ := db.TestSetup(t, "postgres")
rw := db.New(conn)
_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, db.TestWrapper(t)))
plg := plugin.TestPlugin(t, conn, "test")
hc := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
ca := catalogAgg{}
ca.PublicId = hc.GetPublicId()
require.NoError(t, rw.LookupByPublicId(ctx, &ca))
outHc, _ := ca.toCatalogAndPersisted()
require.NotNil(t, outHc)
require.Empty(t, cmp.Diff(hc, outHc, protocmp.Transform()))
require.NotNil(t, ca.plugin())
require.Empty(t, cmp.Diff(plg, ca.plugin(), protocmp.Transform()))
}
internal/host/plugin/job_set_sync.go
+1 -1
View File
@@ -262,7 +262,7 @@ func (r *SetSyncJob) syncSets(ctx context.Context, setAggs []*hostSetAgg) error
if !ok {
return errors.New(ctx, errors.NotSpecificIntegrity, op, "catalog returned when no set requested it")
}
plgCat, err := toPluginCatalog(ctx, c)
plgCat, err := toPluginCatalog(ctx, c, ca.plugin())
if err != nil {
return errors.Wrap(ctx, err, op, errors.WithMsg("storage to plugin catalog conversion"))
}
internal/host/plugin/job_set_sync_test.go
+3 -1
View File
@@ -165,7 +165,8 @@ func TestSetSyncJob_Run(t *testing.T) {
cat := TestCatalog(t, conn, prj.GetPublicId(), plg.GetPublicId())
plgServer.ListHostsFn = func(_ context.Context, _ *plgpb.ListHostsRequest) (*plgpb.ListHostsResponse, error) {
plgServer.ListHostsFn = func(_ context.Context, req *plgpb.ListHostsRequest) (*plgpb.ListHostsResponse, error) {
require.NotNil(req.GetCatalog().GetPlugin())
return &plgpb.ListHostsResponse{}, nil
}
// Start with a set with a member that should be removed
@@ -189,6 +190,7 @@ func TestSetSyncJob_Run(t *testing.T) {
set1 := TestSet(t, conn, kmsCache, sched, cat, plgm)
counter := new(uint32)
plgServer.ListHostsFn = func(ctx context.Context, req *plgpb.ListHostsRequest) (*plgpb.ListHostsResponse, error) {
require.NotNil(req.GetCatalog().GetPlugin())
assert.GreaterOrEqual(1, len(req.GetSets()))
var setIds []string
for _, s := range req.GetSets() {
internal/host/plugin/repository_host_catalog.go
+35 -13
View File
@@ -19,6 +19,7 @@ import (
"github.com/hashicorp/boundary/internal/util"
pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
pbset "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/plugins"
plgpb "github.com/hashicorp/boundary/sdk/pbs/plugin"
"github.com/mr-tron/base58"
"google.golang.org/grpc/codes"
@@ -37,12 +38,15 @@ func normalizeCatalogAttributes(ctx context.Context, plgClient plgpb.HostPluginS
return errors.New(ctx, errors.InvalidParameter, op, "plugin client is nil")
case plgHc == nil:
return errors.New(ctx, errors.InvalidParameter, op, "host catalog is nil")
case plgHc.GetWorkerFilter().GetValue() != "" && plgHc.GetPlugin() == nil:
return errors.New(ctx, errors.InvalidParameter, op, "plugin data is not available on host catalog with worker filter")
case plgHc.GetAttributes() == nil:
return nil
}
ret, err := plgClient.NormalizeCatalogData(ctx, &plgpb.NormalizeCatalogDataRequest{
Attributes: plgHc.GetAttributes(),
Plugin: plgHc.GetPlugin(),
})
switch {
case err == nil:
@@ -116,7 +120,11 @@ func (r *Repository) CreateCatalog(ctx context.Context, c *HostCatalog, _ ...Opt
}
}
plgHc, err := toPluginCatalog(ctx, c)
plg, err := r.getPlugin(ctx, c.GetPluginId())
if err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
plgHc, err := toPluginCatalog(ctx, c, plg)
if err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
@@ -209,10 +217,6 @@ func (r *Repository) CreateCatalog(ctx context.Context, c *HostCatalog, _ ...Opt
}
return nil, nil, errors.Wrap(ctx, err, op, errors.WithMsg(fmt.Sprintf("in project: %s", c.ProjectId)))
}
plg, err := r.getPlugin(ctx, newHostCatalog.GetPluginId())
if err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
return newHostCatalog, plg, nil
}
@@ -345,10 +349,10 @@ func (r *Repository) UpdateCatalog(ctx context.Context, c *HostCatalog, version
needSetSync = true
}
// Get the plugin for the host catalog - this is to return it back
// after the update is complete. We don't actually do anything with
// the record otherwise. Fetch it here so that if there's an
// integrity error, we don't call the plugin.
// Get the plugin for the host catalog - this is to return it back after the
// update is complete, as well as forwarding it to the actual plugin given
// that it is necessary if we're using a worker filter. Fetch it here so
// that if there's an integrity error, we don't call the plugin.
plg, err := r.getPlugin(ctx, currentCatalog.GetPluginId())
if err != nil {
return nil, nil, db.NoRowsAffected, errors.Wrap(ctx, err, op)
@@ -362,11 +366,11 @@ func (r *Repository) UpdateCatalog(ctx context.Context, c *HostCatalog, version
// Convert the catalog values to API protobuf values, which is what
// we use for the plugin hook calls.
currPlgHc, err := toPluginCatalog(ctx, currentCatalog)
currPlgHc, err := toPluginCatalog(ctx, currentCatalog, plg)
if err != nil {
return nil, nil, db.NoRowsAffected, errors.Wrap(ctx, err, op)
}
newPlgHc, err := toPluginCatalog(ctx, newCatalog)
newPlgHc, err := toPluginCatalog(ctx, newCatalog, plg)
if err != nil {
return nil, nil, db.NoRowsAffected, errors.Wrap(ctx, err, op)
}
@@ -612,7 +616,12 @@ func (r *Repository) DeleteCatalog(ctx context.Context, id string, _ ...Option)
if c == nil {
return db.NoRowsAffected, nil
}
plgHc, err := toPluginCatalog(ctx, c)
plg, err := r.getPlugin(ctx, c.GetPluginId())
if err != nil {
return db.NoRowsAffected, errors.Wrap(ctx, err, op)
}
plgHc, err := toPluginCatalog(ctx, c, plg)
if err != nil {
return db.NoRowsAffected, errors.Wrap(ctx, err, op)
}
@@ -718,7 +727,7 @@ func (r *Repository) getPlugin(ctx context.Context, plgId string) (*plg.Plugin,
// toPluginCatalog returns a host catalog, with it's secret if available, in the format expected
// by the host plugin system.
func toPluginCatalog(ctx context.Context, in *HostCatalog) (*pb.HostCatalog, error) {
func toPluginCatalog(ctx context.Context, in *HostCatalog, plg *plg.Plugin) (*pb.HostCatalog, error) {
const op = "plugin.toPluginCatalog"
if in == nil {
return nil, errors.New(ctx, errors.InvalidParameter, op, "nil storage plugin")
@@ -740,6 +749,7 @@ func toPluginCatalog(ctx context.Context, in *HostCatalog) (*pb.HostCatalog, err
Name: name,
Description: description,
WorkerFilter: workerFilter,
Plugin: toPluginInfo(plg),
}
if len(in.GetSecretsHmac()) > 0 {
hc.SecretsHmac = base58.Encode(in.GetSecretsHmac())
@@ -759,6 +769,18 @@ func toPluginCatalog(ctx context.Context, in *HostCatalog) (*pb.HostCatalog, err
return hc, nil
}
// toPluginInfo converts a Plugin object into PluginInfo.
func toPluginInfo(plg *plg.Plugin) *plugins.PluginInfo {
if plg == nil {
return nil
}
return &plugins.PluginInfo{
Id: plg.GetPublicId(),
Name: plg.GetName(),
Description: plg.GetDescription(),
}
}
// toPluginPersistedData converts a *HostCatalogSecret from storage to a
// *plgpb.HostCatalogPersisted expected by a plugin. Project Id must be set.
func toPluginPersistedData(ctx context.Context, kmsCache *kms.Kms, projectId string, cSecret *HostCatalogSecret) (*plgpb.HostCatalogPersisted, error) {
internal/host/plugin/repository_host_catalog_test.go
+28 -2
View File
@@ -22,8 +22,10 @@ import (
"github.com/hashicorp/boundary/internal/oplog"
"github.com/hashicorp/boundary/internal/plugin"
"github.com/hashicorp/boundary/internal/plugin/loopback"
plgstore "github.com/hashicorp/boundary/internal/plugin/store"
"github.com/hashicorp/boundary/internal/scheduler"
"github.com/hashicorp/boundary/internal/scheduler/job"
"github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/plugins"
plgpb "github.com/hashicorp/boundary/sdk/pbs/plugin"
"github.com/mitchellh/mapstructure"
"github.com/stretchr/testify/assert"
@@ -139,7 +141,7 @@ func TestRepository_CreateCatalog(t *testing.T) {
Attributes: []byte{},
},
},
wantIsErr: errors.InvalidParameter,
wantIsErr: errors.RecordNotFound,
},
{
name: "valid-with-name",
@@ -313,11 +315,13 @@ func TestRepository_CreateCatalog(t *testing.T) {
retAttrs.Fields[normalizeToSliceKey] = structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{structpb.NewStringValue(attrs.NormalizeToSlice)},
})
require.NotNil(req.GetPlugin())
return &plgpb.NormalizeCatalogDataResponse{Attributes: retAttrs}, nil
},
OnCreateCatalogFn: func(_ context.Context, req *plgpb.OnCreateCatalogRequest) (*plgpb.OnCreateCatalogResponse, error) {
pluginCalled = true
gotPluginAttrs = req.GetCatalog().GetAttributes()
require.NotNil(req.GetCatalog().GetPlugin())
return &plgpb.OnCreateCatalogResponse{Persisted: &plgpb.HostCatalogPersisted{Secrets: req.GetCatalog().GetSecrets()}}, nil
},
},
@@ -540,6 +544,7 @@ func TestRepository_UpdateCatalog(t *testing.T) {
retAttrs.Fields[normalizeToSliceKey] = structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{structpb.NewStringValue(attrs.NormalizeToSlice)},
})
require.NotNil(t, req.GetPlugin())
return &plgpb.NormalizeCatalogDataResponse{Attributes: retAttrs}, nil
},
OnUpdateCatalogFn: func(_ context.Context, req *plgpb.OnUpdateCatalogRequest) (*plgpb.OnUpdateCatalogResponse, error) {
@@ -549,7 +554,8 @@ func TestRepository_UpdateCatalog(t *testing.T) {
respSecrets = nil
setRespSecretsNil = false
}
require.NotNil(t, req.GetCurrentCatalog().GetPlugin())
require.NotNil(t, req.GetNewCatalog().GetPlugin())
return &plgpb.OnUpdateCatalogResponse{Persisted: &plgpb.HostCatalogPersisted{Secrets: respSecrets}}, pluginError
},
},
@@ -1453,6 +1459,7 @@ func TestRepository_DeleteCatalog(t *testing.T) {
tt := tt
t.Run(tt.name, func(t *testing.T) {
pluginInstance.OnDeleteCatalogFn = func(_ context.Context, request *plgpb.OnDeleteCatalogRequest) (*plgpb.OnDeleteCatalogResponse, error) {
require.NotNil(t, request.GetCatalog().GetPlugin())
return nil, tt.pluginChecker(t, request)
}
got, err := repo.DeleteCatalog(context.Background(), tt.id)
@@ -1662,6 +1669,25 @@ func TestRepository_UpdateCatalog_SyncSets(t *testing.T) {
require.Less(t, time.Until(jj.GetNextScheduledRun().GetTimestamp().AsTime()), time.Second)
}
func TestToPluginInfo(t *testing.T) {
require.Nil(t, toPluginInfo(nil))
plg := &plugin.Plugin{
Plugin: &plgstore.Plugin{
PublicId: "plg_1234567890",
Name: "A Plugin Name",
Description: "A Plugin Description",
},
}
pi := toPluginInfo(plg)
require.NotNil(t, pi)
require.Equal(t, &plugins.PluginInfo{
Id: "plg_1234567890",
Name: "A Plugin Name",
Description: "A Plugin Description",
}, pi)
}
func assertPluginBasedPublicId(t *testing.T, prefix, actual string) {
t.Helper()
assert.NotEmpty(t, actual)
internal/host/plugin/repository_host_set.go
+14 -10
View File
@@ -21,6 +21,7 @@ import (
"github.com/hashicorp/boundary/internal/plugin"
"github.com/hashicorp/boundary/internal/scheduler"
"github.com/hashicorp/boundary/internal/util"
hcpb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostcatalogs"
pb "github.com/hashicorp/boundary/sdk/pbs/controller/api/resources/hostsets"
plgpb "github.com/hashicorp/boundary/sdk/pbs/plugin"
"google.golang.org/grpc/codes"
@@ -32,19 +33,22 @@ import (
// normalizeCatalogAttributes allows a plugin to normalize set attributes before
// they are saved
func normalizeSetAttributes(ctx context.Context, plgClient plgpb.HostPluginServiceClient, plgHs *pb.HostSet) error {
func normalizeSetAttributes(ctx context.Context, plgClient plgpb.HostPluginServiceClient, plgHc *hcpb.HostCatalog, plgHs *pb.HostSet) error {
const op = "plugin.(Repository).normalizeSetAttributes"
switch {
case util.IsNil(plgClient):
return errors.New(ctx, errors.InvalidParameter, op, "plugin client is nil")
case plgHs == nil:
return errors.New(ctx, errors.InvalidParameter, op, "host set is nil")
case plgHc.GetWorkerFilter().GetValue() != "" && plgHc.GetPlugin() == nil:
return errors.New(ctx, errors.InvalidParameter, op, "plugin data is not available on host catalog with worker filter")
case plgHs.GetAttributes() == nil:
return nil
}
ret, err := plgClient.NormalizeSetData(ctx, &plgpb.NormalizeSetDataRequest{
Attributes: plgHs.GetAttributes(),
Plugin: plgHc.GetPlugin(),
})
switch {
case err == nil:
@@ -120,7 +124,11 @@ func (r *Repository) CreateSet(ctx context.Context, projectId string, s *HostSet
return nil, nil, errors.New(ctx, errors.Internal, op, fmt.Sprintf("plugin %q not available", c.GetPluginId()))
}
plgHc, err := toPluginCatalog(ctx, c)
plg, err := r.getPlugin(ctx, c.GetPluginId())
if err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
plgHc, err := toPluginCatalog(ctx, c, plg)
if err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
@@ -130,7 +138,7 @@ func (r *Repository) CreateSet(ctx context.Context, projectId string, s *HostSet
}
if plgHs.GetAttributes() != nil {
if err := normalizeSetAttributes(ctx, plgClient, plgHs); err != nil {
if err := normalizeSetAttributes(ctx, plgClient, plgHc, plgHs); err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
if s.Attributes, err = proto.Marshal(plgHs.GetAttributes()); err != nil {
@@ -213,10 +221,6 @@ func (r *Repository) CreateSet(ctx context.Context, projectId string, s *HostSet
// The set now exists in the plugin, sync it immediately.
_ = r.scheduler.UpdateJobNextRunInAtLeast(ctx, setSyncJobName, 0, scheduler.WithRunNow(true))
plg, err := r.getPlugin(ctx, c.GetPluginId())
if err != nil {
return nil, nil, errors.Wrap(ctx, err, op)
}
return returnedHostSet, plg, nil
}
@@ -367,7 +371,7 @@ func (r *Repository) UpdateSet(ctx context.Context, projectId string, s *HostSet
// Convert the catalog values to API protobuf values, which is what
// we use for the plugin hook calls.
plgHc, err := toPluginCatalog(ctx, catalog)
plgHc, err := toPluginCatalog(ctx, catalog, plg)
if err != nil {
return nil, nil, nil, db.NoRowsAffected, errors.Wrap(ctx, err, op)
}
@@ -382,7 +386,7 @@ func (r *Repository) UpdateSet(ctx context.Context, projectId string, s *HostSet
if updateAttributes {
if newPlgSet.GetAttributes() != nil {
if err := normalizeSetAttributes(ctx, plgClient, newPlgSet); err != nil {
if err := normalizeSetAttributes(ctx, plgClient, plgHc, newPlgSet); err != nil {
return nil, nil, nil, db.NoRowsAffected, errors.Wrap(ctx, err, op)
}
if newSet.Attributes, err = proto.Marshal(newPlgSet.GetAttributes()); err != nil {
@@ -737,7 +741,7 @@ func (r *Repository) DeleteSet(ctx context.Context, projectId string, publicId s
if !ok || plgClient == nil {
return 0, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("plugin %q not available", c.GetPluginId()))
}
plgHc, err := toPluginCatalog(ctx, c)
plgHc, err := toPluginCatalog(ctx, c, plg)
if err != nil {
return 0, errors.Wrap(ctx, err, op)
}
internal/host/plugin/repository_host_set_test.go
+4
View File
@@ -303,11 +303,13 @@ func TestRepository_CreateSet(t *testing.T) {
retAttrs.Fields[normalizeToSliceKey] = structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{structpb.NewStringValue(attrs.NormalizeToSlice)},
})
require.NotNil(req.GetPlugin())
return &plgpb.NormalizeSetDataResponse{Attributes: retAttrs}, nil
},
OnCreateSetFn: func(ctx context.Context, req *plgpb.OnCreateSetRequest) (*plgpb.OnCreateSetResponse, error) {
pluginCalled = true
pluginReceivedAttrs = req.GetSet().GetAttributes()
require.NotNil(req.GetCatalog().GetPlugin())
return &plgpb.OnCreateSetResponse{}, nil
},
}),
@@ -1226,6 +1228,7 @@ func TestRepository_UpdateSet(t *testing.T) {
retAttrs.Fields[normalizeToSliceKey] = structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{structpb.NewStringValue(attrs.NormalizeToSlice)},
})
require.NotNil(req.GetPlugin())
return &plgpb.NormalizeSetDataResponse{Attributes: retAttrs}, nil
},
OnUpdateSetFn: func(_ context.Context, req *plgpb.OnUpdateSetRequest) (*plgpb.OnUpdateSetResponse, error) {
@@ -1233,6 +1236,7 @@ func TestRepository_UpdateSet(t *testing.T) {
for _, check := range tt.wantCheckPluginReqFuncs {
check(t, req)
}
require.NotNil(req.GetCatalog().GetPlugin())
return &plgpb.OnUpdateSetResponse{}, tt.withPluginError
},
},