Restore soft deleted users on authentication (#5549)

* internal/clientcache: restore user on login

A user that previously had its last auth token deleted
(for example, if they logged out), should be restored
again when they successfully authenticate again.

* internal/clientcache: delete keyringless expired auth tokens

Previously, we would only correctly delete auth tokens
that were expired (or deleted) if they were also in the
keyring.

* internal/clientcache: remove tautological condition

At this stage we already know that at is non-nil, because
the case above would have been true if it was nil.

* internal/clientcache: improve error handling

The previous error handling would have ignored API errors that were not ErrUnauthorized or ErrNotFound.
Handle all API errors the same as other unexpected errors.

* internal/clientcache: make errors distinct

These two error conditions are otherwise impossible
to tell apart.

* test(e2e): Update search test to catch bug (#5551)

---------

Co-authored-by: Michael Li <michael.li@hashicorp.com>

testing/internal/e2e/tests/base/search_test.go

@@ -285,4 +285,131 @@ func TestCliSearch(t *testing.T) {
 	err = json.Unmarshal(output.Stdout, &searchResult)
 	require.NoError(t, err)
 	require.Len(t, searchResult.Item.Sessions, 0)
+
+	// Log out and confirm search does not work
+	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("logout"))
+	require.NoError(t, output.Err, string(output.Stderr))
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"search",
+			"-resource", "targets",
+			"-format", "json",
+			"-query", fmt.Sprintf(`name %% "%s" and scope_id = "%s"`, targetPrefix, projectId),
+		),
+	)
+	require.Error(t, output.Err)
+
+	// Log back in and confirm search works
+	boundary.AuthenticateAdminCli(t, ctx)
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"search",
+			"-resource", "targets",
+			"-format", "json",
+			"-query", fmt.Sprintf(`name %% "%s" and scope_id = "%s"`, targetPrefix, projectId),
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	searchResult = clientcache.SearchResult{}
+	err = json.Unmarshal(output.Stdout, &searchResult)
+	require.NoError(t, err)
+	require.Len(t, searchResult.Item.Targets, len(targetIds))
+
+	// Restart cache and confirm search works
+	t.Log("Restarting cache...")
+	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop"))
+	require.NoError(t, output.Err, string(output.Stderr))
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"cache", "start",
+			"-refresh-interval", "5s",
+			"-background",
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	err = backoff.RetryNotify(
+		func() error {
+			output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json"))
+			if output.Err != nil {
+				return errors.New(strings.TrimSpace(string(output.Stderr)))
+			}
+
+			err = json.Unmarshal(output.Stdout, &statusResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	require.Equal(t, statusResult.StatusCode, 200)
+	require.GreaterOrEqual(t, statusResult.Item.Uptime, 0*time.Second)
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"search",
+			"-resource", "targets",
+			"-format", "json",
+			"-query", fmt.Sprintf(`name %% "%s" and scope_id = "%s"`, targetPrefix, projectId),
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	searchResult = clientcache.SearchResult{}
+	err = json.Unmarshal(output.Stdout, &searchResult)
+	require.NoError(t, err)
+	require.Len(t, searchResult.Item.Targets, len(targetIds))
+
+	// Log out and restart cache. Log in and confirm search works
+	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("logout"))
+	t.Log("Restarting cache...")
+	output = e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "stop"))
+	require.NoError(t, output.Err, string(output.Stderr))
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"cache", "start",
+			"-refresh-interval", "5s",
+			"-background",
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	err = backoff.RetryNotify(
+		func() error {
+			output := e2e.RunCommand(ctx, "boundary", e2e.WithArgs("cache", "status", "-format", "json"))
+			if output.Err != nil {
+				return errors.New(strings.TrimSpace(string(output.Stderr)))
+			}
+
+			err = json.Unmarshal(output.Stdout, &statusResult)
+			if err != nil {
+				return backoff.Permanent(err)
+			}
+
+			return nil
+		},
+		backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 5),
+		func(err error, td time.Duration) {
+			t.Logf("%s. Retrying...", err.Error())
+		},
+	)
+	require.NoError(t, err)
+	require.Equal(t, statusResult.StatusCode, 200)
+	require.GreaterOrEqual(t, statusResult.Item.Uptime, 0*time.Second)
+	boundary.AuthenticateAdminCli(t, ctx)
+	output = e2e.RunCommand(ctx, "boundary",
+		e2e.WithArgs(
+			"search",
+			"-resource", "targets",
+			"-format", "json",
+			"-query", fmt.Sprintf(`name %% "%s" and scope_id = "%s"`, targetPrefix, projectId),
+		),
+	)
+	require.NoError(t, output.Err, string(output.Stderr))
+	searchResult = clientcache.SearchResult{}
+	err = json.Unmarshal(output.Stdout, &searchResult)
+	require.NoError(t, err)
+	require.Len(t, searchResult.Item.Targets, len(targetIds))
 }