Refresh specific resources before searching when they are stale (#4046)

validate versions for all handlers. Refresh specific user before searching

Pass flags on to the background process

Allow the search refresh timeout to be configured

internal/clientcache/internal/daemon/search_handler.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/internal/clientcache/internal/cache"
 	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/event"
 	"github.com/hashicorp/boundary/internal/util"
 )
 
@@ -31,11 +32,13 @@ const (
 	authTokenIdKey  = "auth_token_id"
 )
 
-func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository) (http.HandlerFunc, error) {
+func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshService *cache.RefreshService) (http.HandlerFunc, error) {
 	const op = "daemon.newSearchHandlerFunc"
 	switch {
 	case util.IsNil(repo):
 		return nil, errors.New(ctx, errors.InvalidParameter, op, "repository is missing")
+	case util.IsNil(refreshService):
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "refresh service is missing")
 	}
 
 	s, err := cache.NewSearchService(ctx, repo)
@@ -67,6 +70,14 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository) (http.Han
 			return
 		}
 
+		// Refresh the resources for the provided user, if possible. This is best
+		// effort, so if there is any problem refreshing, we just log the error
+		// and move on to handling the search request.
+		if err := refreshService.RefreshForSearch(ctx, authTokenId, searchableResource); err != nil {
+			// we don't stop the search, we just log that the inline refresh failed
+			event.WriteError(ctx, op, err, event.WithInfoMsg("when refreshing the resources inline for search", "auth_token_id", authTokenId, "resource", searchableResource))
+		}
+
 		query := r.URL.Query().Get(queryKey)
 		filter := r.URL.Query().Get(filterKey)
 

internal/clientcache/internal/daemon/server.go

@@ -31,6 +31,16 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 )
 
+const (
+	// DefaultSearchStaleness is the default amount of time that can pass before
+	// a search request causes a blocking refresh to happen for the resources
+	// being searched for.
+	DefaultSearchStaleness = 60 * time.Second
+	// DefaultSearchRefreshTimeout is the default amount of time that a refresh
+	// that is blocking a search request can run before timing out.
+	DefaultSearchRefreshTimeout = 1 * time.Second
+)
+
 // Commander is an interface that provides a way to get an apiClient
 // and retrieve the keyring and token information used by a command.
 type Commander interface {
@@ -68,6 +78,12 @@ type Config struct {
 	LogFormat         string
 	LogWriter         io.Writer
 	DotDirectory      string
+	// The amount of time since the last refresh that must have passed for a
+	// search query to trigger an inline refresh.
+	MaxSearchStaleness time.Duration
+	// The maximum amount of time a refresh should block a search request from
+	// completeing before it times out.
+	MaxSearchRefreshTimeout time.Duration
 }
 
 func (sc *Config) validate(ctx context.Context) error {
@@ -79,6 +95,12 @@ func (sc *Config) validate(ctx context.Context) error {
 		return errors.New(ctx, errors.InvalidParameter, op, "missing contextCancel")
 	case sc.DotDirectory == "":
 		return errors.New(ctx, errors.InvalidParameter, op, "missing dot directory")
+	case sc.RefreshInterval < 0:
+		return errors.New(ctx, errors.InvalidParameter, op, "negative refresh interval")
+	case sc.FullFetchInterval < 0:
+		return errors.New(ctx, errors.InvalidParameter, op, "negative full fetch interval")
+	case sc.MaxSearchStaleness < 0:
+		return errors.New(ctx, errors.InvalidParameter, op, "negative max search staleness")
 	}
 	return nil
 }
@@ -196,8 +218,36 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 	if s.store, s.storeUrl, err = openStore(ctx, s.conf.DatabaseUrl, s.conf.StoreDebug); err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
-	s.info["Database URL"] = s.storeUrl
-	s.infoKeys = append(s.infoKeys, "Database URL")
+	if s.storeUrl != "" {
+		s.info["Database URL"] = s.storeUrl
+		s.infoKeys = append(s.infoKeys, "Database URL")
+	}
+	maxSearchRefreshTimeout := DefaultSearchRefreshTimeout
+	if s.conf.MaxSearchRefreshTimeout > 0 {
+		maxSearchRefreshTimeout = s.conf.MaxSearchRefreshTimeout
+	}
+	maxSearchStaleness := DefaultSearchStaleness
+	if s.conf.MaxSearchStaleness > 0 {
+		maxSearchStaleness = s.conf.MaxSearchStaleness
+	}
+	s.info["Max Search Staleness"] = maxSearchStaleness.String()
+	s.infoKeys = append(s.infoKeys, "Max Search Staleness")
+	s.info["Max Search Refresh Timeout"] = maxSearchRefreshTimeout.String()
+	s.infoKeys = append(s.infoKeys, "Max Search Refresh Timeout")
+	s.info["Refresh Interval"] = DefaultRefreshInterval.String()
+	s.infoKeys = append(s.infoKeys, "Refresh Interval")
+	s.info["Full Fetch Interval"] = DefaultFullFetchInterval.String()
+	s.infoKeys = append(s.infoKeys, "Full Fetch Interval")
+
+	ticOptions := []Option{}
+	if s.conf.RefreshInterval > 0 {
+		s.info["Refresh Interval"] = s.conf.RefreshInterval.String()
+		ticOptions = append(ticOptions, withRefreshInterval(ctx, s.conf.RefreshInterval))
+	}
+	if s.conf.FullFetchInterval > 0 {
+		s.info["Full Fetch Interval"] = s.conf.FullFetchInterval.String()
+		ticOptions = append(ticOptions, withFullFetchInterval(ctx, s.conf.FullFetchInterval))
+	}
 
 	s.printInfo(ctx)
 
@@ -206,17 +256,10 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 		return errors.Wrap(ctx, err, op)
 	}
 
-	refreshService, err := cache.NewRefreshService(ctx, repo)
+	refreshService, err := cache.NewRefreshService(ctx, repo, maxSearchStaleness, maxSearchRefreshTimeout)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
-	ticOptions := []Option{}
-	if s.conf.RefreshInterval > 0 {
-		ticOptions = append(ticOptions, withRefreshInterval(ctx, s.conf.RefreshInterval))
-	}
-	if s.conf.FullFetchInterval > 0 {
-		ticOptions = append(ticOptions, withFullFetchInterval(ctx, s.conf.FullFetchInterval))
-	}
 	tic, err := newRefreshTicker(ctx, refreshService, ticOptions...)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
@@ -236,23 +279,23 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 	}()
 
 	mux := http.NewServeMux()
-	searchFn, err := newSearchHandlerFunc(ctx, repo)
+	searchFn, err := newSearchHandlerFunc(ctx, repo, refreshService)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
-	mux.HandleFunc("/v1/search", searchFn)
+	mux.Handle("/v1/search", versionEnforcement(searchFn))
 
 	statusFn, err := newStatusHandlerFunc(ctx, repo, l.Addr().String())
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
-	mux.HandleFunc("/v1/status", statusFn)
+	mux.Handle("/v1/status", versionEnforcement(statusFn))
 
 	tokenFn, err := newTokenHandlerFunc(ctx, repo, tic)
 	if err != nil {
 		return errors.Wrap(ctx, err, op)
 	}
-	mux.HandleFunc("/v1/tokens", tokenFn)
+	mux.Handle("/v1/tokens", versionEnforcement(tokenFn))
 
 	stopFn, err := newStopHandlerFunc(ctx, s.conf.ContextCancel)
 	if err != nil {

internal/clientcache/internal/daemon/testing.go

@@ -34,8 +34,8 @@ func NewTestServer(t *testing.T, cmd Commander, opt ...Option) *TestServer {
 
 	cfg := &Config{
 		ContextCancel:     cancel,
-		RefreshInterval:   defaultRefreshInterval,
-		FullFetchInterval: defaultFullFetchInterval,
+		RefreshInterval:   DefaultRefreshInterval,
+		FullFetchInterval: DefaultFullFetchInterval,
 		StoreDebug:        opts.withDebug,
 		LogWriter:         io.Discard,
 		DotDirectory:      dotDir,
@@ -101,7 +101,7 @@ func (s *TestServer) AddResources(t *testing.T, p *authtokens.AuthToken, tars []
 		}
 		return sess, nil, "", nil
 	}
-	rs, err := cache.NewRefreshService(ctx, r)
+	rs, err := cache.NewRefreshService(ctx, r, 0, 0)
 	require.NoError(t, err)
 	require.NoError(t, rs.Refresh(ctx, cache.WithTargetRetrievalFunc(tarFn), cache.WithSessionRetrievalFunc(sessFn)))
 }

internal/clientcache/internal/daemon/ticker.go

@@ -15,11 +15,13 @@ import (
 )
 
 const (
-	DefaultRefreshIntervalSeconds = 60
-	defaultRefreshInterval        = DefaultRefreshIntervalSeconds * time.Second
-
-	DefaultFullFetchIntervalSeconds = 60 * 5
-	defaultFullFetchInterval        = DefaultFullFetchIntervalSeconds * time.Second
+	// DefaultRefreshInterval is the default interval that resources that come
+	// from controllers that do support refresh tokens are retrieved and
+	// refreshed.
+	DefaultRefreshInterval = 60 * time.Second
+	// DefaultFullFetchInterval is the default interval that resources that come
+	// from controllers that do not support refresh tokens are fetched.
+	DefaultFullFetchInterval = 60 * 5 * time.Second
 
 	defaultRandomizationFactor float64 = 0.2
 )
@@ -58,12 +60,12 @@ func newRefreshTicker(ctx context.Context, refresh refreshService, opt ...Option
 		return nil, errors.Wrap(ctx, err, op)
 	}
 
-	refreshInterval := defaultRefreshInterval
+	refreshInterval := DefaultRefreshInterval
 	if opts.withRefreshInterval != 0 {
 		refreshInterval = opts.withRefreshInterval
 	}
 
-	fullFetchInterval := defaultFullFetchInterval
+	fullFetchInterval := DefaultFullFetchInterval
 	if opts.withFullFetchInterval != 0 {
 		fullFetchInterval = opts.withFullFetchInterval
 	}

internal/clientcache/internal/daemon/token_handler.go

@@ -100,7 +100,7 @@ func newTokenHandlerFunc(ctx context.Context, repo *cache.Repository, refresher
 			}
 		}
 
-		tok, err := repo.LookupToken(ctx, perReq.AuthTokenId)
+		oldTok, err := repo.LookupToken(ctx, perReq.AuthTokenId)
 		if err != nil {
 			writeError(w, "error performing auth token lookup", http.StatusInternalServerError)
 			return
@@ -124,9 +124,15 @@ func newTokenHandlerFunc(ctx context.Context, repo *cache.Repository, refresher
 			}
 		}
 
+		newTok, err := repo.LookupToken(ctx, perReq.AuthTokenId)
+		if err != nil {
+			writeError(w, "error performing follow up auth token lookup", http.StatusInternalServerError)
+			return
+		}
+
 		w.WriteHeader(http.StatusNoContent)
 
-		if tok == nil {
+		if oldTok == nil && newTok != nil {
 			refresher.refresh()
 		}
 	}, nil