Ensure only one refresh is running for a user/resource at a time (#5054)

This ensures we don't have competing update processes running. When doing the
inline RefreshForSearch, a specific error will be returned so we can hint in
the API that we're in the middle of a refresh.

(cherry picked from commit 54776091e32af2f2f110f2bf10d0617bd5677aba)

internal/clientcache/internal/daemon/search_handler.go

@@ -6,6 +6,7 @@ package daemon
 import (
 	"context"
 	"encoding/json"
+	stderrors "errors"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -22,6 +23,22 @@ import (
 	"github.com/hashicorp/go-hclog"
 )
 
+type RefreshStatus string
+
+const (
+	// Not refreshing means the result is complete, that is, the cache is not in
+	// the process of being built or refreshed
+	NotRefreshing RefreshStatus = "not-refreshing"
+	// Refreshing means that the cache is in the process of being refreshed, so
+	// the result may not be complete and the caller should try again later for
+	// more complete results
+	Refreshing RefreshStatus = "refreshing"
+	// RefreshError means that there was an error refreshing the cache. It says
+	// nothing about the completeness of the result, only that when attempting
+	// to refresh the cache in-line with the search an error was encountered.
+	RefreshError RefreshStatus = "refresh-error"
+)
+
 // SearchResult is the struct returned to search requests.
 type SearchResult struct {
 	ResolvableAliases []*aliases.Alias    `json:"resolvable_aliases,omitempty"`
@@ -29,6 +46,8 @@ type SearchResult struct {
 	Sessions          []*sessions.Session `json:"sessions,omitempty"`
 	ImplicitScopes    []*scopes.Scope     `json:"implicit_scopes,omitempty"`
 	Incomplete        bool                `json:"incomplete,omitempty"`
+	RefreshStatus     RefreshStatus       `json:"refresh_status,omitempty"`
+	RefreshError      string              `json:"refresh_error,omitempty"`
 }
 
 const (
@@ -131,13 +150,20 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 			opts = append(opts, cache.WithIgnoreSearchStaleness(true))
 		}
 
+		var refreshError error
 		// Refresh the resources for the provided user, if possible. This is best
 		// effort, so if there is any problem refreshing, we just log the error
 		// and move on to handling the search request.
 		switch searchableResource {
 		case cache.ImplicitScopes:
+			// This is not able to be refreshed, so continue on
 		default:
-			if err := refreshService.RefreshForSearch(reqCtx, authTokenId, searchableResource, opts...); err != nil {
+			refreshError = refreshService.RefreshForSearch(reqCtx, authTokenId, searchableResource, opts...)
+			switch {
+			case refreshError == nil,
+				stderrors.Is(refreshError, cache.ErrRefreshInProgress):
+				// Don't event in these cases
+			default:
 				// we don't stop the search, we just log that the inline refresh failed
 				event.WriteError(ctx, op, err, event.WithInfoMsg("when refreshing the resources inline for search", "auth_token_id", authTokenId, "resource", searchableResource))
 			}
@@ -166,6 +192,16 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 		}
 
 		apiRes := toApiResult(res)
+		switch {
+		case refreshError == nil:
+			apiRes.RefreshStatus = NotRefreshing
+		case stderrors.Is(refreshError, cache.ErrRefreshInProgress):
+			apiRes.RefreshStatus = Refreshing
+		default:
+			apiRes.RefreshStatus = RefreshError
+			apiRes.RefreshError = refreshError.Error()
+		}
+
 		j, err := json.Marshal(apiRes)
 		if err != nil {
 			event.WriteError(ctx, op, err, event.WithInfoMsg("when marshaling search result to JSON", "auth_token_id", authTokenId, "resource", searchableResource, "query", query, "filter", filter))