Add -filter to CLI listing and website docs (#967)

* Add -filter to CLI listing.

This doesn't include docs/Changelog yet, that will come shortly.

* Add changelog

* Add website docs and fix a curl output bug

CHANGELOG.md

@@ -24,6 +24,13 @@ All of these changes are from [PR
 
 ### New and Improved
 
+* list filtering: Listing now supports filtering results before being returned
+  to the user. The filtering takes place server side and uses boolean
+  expressions against the JSON representation of returned items. See [the
+  documentation](https://www.boundaryproject.io/docs/concepts/filtering/resource-listing)
+  for more details. ([PR](https://github.com/hashicorp/boundary/pull/952))
+  ([PR](https://github.com/hashicorp/boundary/pull/957))
+  ([PR](https://github.com/hashicorp/boundary/pull/967))
 * server: Officially support reloading TLS parameters on `SIGHUP`. (This likely
   worked before but wasn't fully tested.)
   ([PR](https://github.com/hashicorp/boundary/pull/959))

api/output_string.go

@@ -57,7 +57,7 @@ func (d *OutputStringError) parseRequest() {
 					h = fmt.Sprintf("Bearer $(boundary config get-token -keyring-type %s -token-name %s)", keyringType, tokenName)
 				}
 			}
-			d.parsedCurlString = fmt.Sprintf("%s-H \"%s: %s\" ", d.parsedCurlString, k, h)
+			d.parsedCurlString = fmt.Sprintf("%s-H \"%s: %s\"", d.parsedCurlString, k, h)
 		}
 	}
 
@@ -65,10 +65,11 @@ func (d *OutputStringError) parseRequest() {
 		// We need to escape single quotes since that's what we're using to
 		// quote the body
 		escapedBody := strings.Replace(string(body), "'", "'\"'\"'", -1)
-		d.parsedCurlString = fmt.Sprintf("%s-d '%s' ", d.parsedCurlString, escapedBody)
+		d.parsedCurlString = fmt.Sprintf(" %s-d '%s'", d.parsedCurlString, escapedBody)
 	}
 
-	d.parsedCurlString = fmt.Sprintf("%s%s", d.parsedCurlString, d.Request.URL.String())
+	// Filters can have shell characters so we use single quotes to surround the URL
+	d.parsedCurlString = fmt.Sprintf("%s '%s'", d.parsedCurlString, d.Request.URL.String())
 }
 
 func (d *OutputStringError) CurlString() string {

internal/cmd/base/base.go

@@ -77,6 +77,7 @@ type Command struct {
 	FlagHostCatalogId string
 	FlagVersion       int
 	FlagRecursive     bool
+	FlagFilter        string
 
 	client *api.Client
 }

internal/cmd/commands/accountscmd/accounts.gen.go

@@ -94,7 +94,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"auth-method-id"},
+	"list": {"auth-method-id", "filter"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -176,6 +176,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, accounts.WithDescription(c.FlagDescription))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, accounts.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/accountscmd/password_accounts.gen.go

@@ -159,6 +159,10 @@ func (c *PasswordCommand) Run(args []string) int {
 		opts = append(opts, accounts.WithDescription(c.FlagDescription))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, accounts.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/authmethodscmd/authmethods.gen.go

@@ -92,7 +92,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -163,6 +163,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, authmethods.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, authmethods.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 {

internal/cmd/commands/authmethodscmd/password_authmethods.gen.go

@@ -164,6 +164,10 @@ func (c *PasswordCommand) Run(args []string) int {
 		opts = append(opts, authmethods.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, authmethods.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/authtokenscmd/authtokens.gen.go

@@ -92,7 +92,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -159,6 +159,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, authtokens.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, authtokens.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 {

internal/cmd/commands/groupscmd/groups.gen.go

@@ -104,7 +104,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, groups.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, groups.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/hostcatalogscmd/hostcatalogs.gen.go

@@ -92,7 +92,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -163,6 +163,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, hostcatalogs.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, hostcatalogs.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 {

internal/cmd/commands/hostcatalogscmd/static_hostcatalogs.gen.go

@@ -162,6 +162,10 @@ func (c *StaticCommand) Run(args []string) int {
 		opts = append(opts, hostcatalogs.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, hostcatalogs.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/hostscmd/hosts.gen.go

@@ -92,7 +92,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"host-catalog-id"},
+	"list": {"host-catalog-id", "filter"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -174,6 +174,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, hosts.WithDescription(c.FlagDescription))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, hosts.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 {

internal/cmd/commands/hostscmd/static_hosts.gen.go

@@ -159,6 +159,10 @@ func (c *StaticCommand) Run(args []string) int {
 		opts = append(opts, hosts.WithDescription(c.FlagDescription))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, hosts.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/hostsetscmd/hostsets.gen.go

@@ -94,7 +94,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"host-catalog-id"},
+	"list": {"host-catalog-id", "filter"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -176,6 +176,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, hostsets.WithDescription(c.FlagDescription))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, hostsets.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/hostsetscmd/static_hostsets.gen.go

@@ -157,6 +157,10 @@ func (c *StaticCommand) Run(args []string) int {
 		opts = append(opts, hostsets.WithDescription(c.FlagDescription))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, hostsets.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/rolescmd/roles.gen.go

@@ -104,7 +104,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, roles.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, roles.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/scopescmd/scopes.gen.go

@@ -104,7 +104,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, scopes.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, scopes.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/server/listener_reload_test.go

@@ -104,10 +104,10 @@ func TestServer_ReloadListener(t *testing.T) {
 	// Setup initial certs
 	inBytes, err := ioutil.ReadFile(wd + "bundle1.pem")
 	require.NoError(err)
-	require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0777))
+	require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0o777))
 
 	relHcl := fmt.Sprintf(reloadConfig, cmd.DatabaseUrl, controllerKey, workerAuthKey, recoveryKey, td, td)
-	require.NoError(ioutil.WriteFile(td+"/reload.hcl", []byte(relHcl), 0777))
+	require.NoError(ioutil.WriteFile(td+"/reload.hcl", []byte(relHcl), 0o777))
 
 	// Populate CA pool
 	inBytes, _ = ioutil.ReadFile(td + "/bundle.pem")
@@ -146,7 +146,7 @@ func TestServer_ReloadListener(t *testing.T) {
 
 	inBytes, err = ioutil.ReadFile(wd + "bundle2.pem")
 	require.NoError(err)
-	require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0777))
+	require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0o777))
 
 	cmd.SighupCh <- struct{}{}
 	select {

internal/cmd/commands/sessionscmd/sessions.gen.go

@@ -86,7 +86,7 @@ var flagsMap = map[string][]string{
 
 	"read": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -153,6 +153,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, sessions.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, sessions.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/targetscmd/targets.gen.go

@@ -95,7 +95,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -182,6 +182,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, targets.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, targets.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/targetscmd/tcp_targets.gen.go

@@ -164,6 +164,10 @@ func (c *TcpCommand) Run(args []string) int {
 		opts = append(opts, targets.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, targets.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/commands/userscmd/users.gen.go

@@ -104,7 +104,7 @@ var flagsMap = map[string][]string{
 
 	"delete": {"id"},
 
-	"list": {"scope-id", "recursive"},
+	"list": {"scope-id", "filter", "recursive"},
 }
 
 func (c *Command) Flags() *base.FlagSets {
@@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int {
 		opts = append(opts, users.WithRecursive(true))
 	}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, users.WithFilter(c.FlagFilter))
+	}
+
 	var version uint32
 
 	switch c.Func {

internal/cmd/common/flags.go

@@ -71,6 +71,12 @@ func PopulateCommonFlags(c *base.Command, f *base.FlagSet, resourceType string,
 				Target: &c.FlagRecursive,
 				Usage:  "If set, the list operation will be applied recursively into child scopes, if supported by the type.",
 			})
+		case "filter":
+			f.StringVar(&base.StringVar{
+				Name:   "filter",
+				Target: &c.FlagFilter,
+				Usage:  "If set, the list operation will be filtered before being returned. The filter operates against each item in the list. Using single quotes is recommended as filters contain double quotes. See https://www.boundaryproject.io/docs/concepts/filtering/resource-listing for details.",
+			})
 		}
 	}
 }

internal/cmd/gencli/templates.go

@@ -183,7 +183,7 @@ var flags{{ camelCase .SubActionPrefix }}Map = map[string][]string{
 	"delete": {"id"},
 	{{ end }}
 	{{ if eq $action "list" }}
-	"list": { "{{ kebabCase $input.Container }}-id" {{ if (eq $input.Container "Scope") }}, "recursive"{{ end }} },
+	"list": { "{{ kebabCase $input.Container }}-id", "filter" {{ if (eq $input.Container "Scope") }}, "recursive"{{ end }} },
 	{{ end }}
 	{{ end }}
 }
@@ -296,6 +296,10 @@ func (c *{{ camelCase .SubActionPrefix }}Command) Run(args []string) int {
 	}
 	{{ end }}
 
+	if c.FlagFilter != "" {
+		opts = append(opts, {{ .Pkg }}.WithFilter(c.FlagFilter))
+	}
+
 	{{ if .HasScopeName }}
 	switch c.FlagScopeName {
 	case "":

website/content/docs/concepts/filtering/resource-listing.mdx

@@ -0,0 +1,88 @@
+---
+layout: docs
+page_title: Filtering - Resource Listing
+sidebar_title: Resource Listing
+description: |-
+  How to use filter list responses coming back from Boundary.
+---
+
+This page describes how to use filters when listing resources. This can be used
+to reduce the returned set of resources when performing a list operation.
+
+~> This feature is intended to provide a userful service to clients; it does not
+affect the database queries generated for the operation and as such is not
+designed to provide greater efficiency.
+
+# List Filtering
+
+Starting in Boundary 0.1.8, when running a list action, a filter can be
+specified. It uses the standard [filter syntax](/docs/concepts/filtering) used
+elsewhere in Boundary. Unless otherwise specified for a given list endpoint, the
+list of items being returned is looped through and the filter is run on the JSON
+representation of that item. A good way to see what that data looks like is by
+looking at representative JSON output on the command line; for example, the
+following is the output of `boundary targets list -scope-id p_1234567890 -format json` on a dev instance (piped through `jq` for readability):
+
+```json
+[
+  {
+    "id": "ttcp_1234567890",
+    "scope_id": "p_1234567890",
+    "scope": {
+      "id": "p_1234567890",
+      "type": "project",
+      "name": "Generated project scope",
+      "description": "Provides an initial project scope in Boundary",
+      "parent_scope_id": "o_1234567890"
+    },
+    "name": "Generated target",
+    "description": "Provides an initial target in Boundary",
+    "created_time": "2021-02-24T22:19:50.640476Z",
+    "updated_time": "2021-02-24T22:19:50.640476Z",
+    "version": 1,
+    "type": "tcp",
+    "session_max_seconds": 28800,
+    "session_connection_limit": -1,
+    "attributes": {
+      "default_port": 22
+    },
+    "authorized_actions": [
+      "read",
+      "update",
+      "delete",
+      "add-host-sets",
+      "set-host-sets",
+      "remove-host-sets",
+      "authorize-session"
+    ]
+  }
+]
+```
+
+As the filter tests each entry being returned, it places the data under test
+within the filter at `/item`.
+
+On the CLI a filter can be given via `-filter`.
+
+~> Double quotes are part of the filter syntax; when using the CLI, it is likely
+easier to surround the filter with single quotes than to deal with escaping
+double quotes.
+
+When using the HTTP API, it is a `filter` query parameter.
+
+~> Ensure that the query parameter is properly escaped! Most HTTP libraries will
+do this for you. If you're having trouble, try using the `-output-curl-string`
+flag with the Boundary CLI:
+
+```
+$ boundary targets list -scope-id p_1234567890 -format json -filter '"authorize-session" in "/item/authorized_actions"' -output-curl-string
+curl -H "Authorization: Bearer $(boundary config get-token -keyring-type pass -token-name default)"-H "Content-Type: application/json" 'http://127.0.0.1:9200/v1/targets?filter=%22authorize-session%22+in+%22%2Fitem%2Fauthorized_actions%22&scope_id=p_1234567890'
+```
+
+Following are some examples.
+
+- Resources in which the user is allowed to run an "update" action:
+  `"update" in "/item/authorized_actions"`
+
+- Resources matching a name pattern, but only those within an organization
+  scope: `"/item/name" matches "groupa-*" and "/item/scope/type" == "org"`

website/data/docs-navigation.js

@@ -44,7 +44,7 @@ export default [
       },
       {
         category: 'filtering',
-        content: ['worker-tags'],
+        content: ['resource-listing', 'worker-tags'],
       },
     ],
   },