Do not cache or allow searching for old controllers (#4136)

* Do not cache or allow searching for old controllers

* Identify resourceless responses as coming from supported or unsupported controllers

* Do not report refresh token status info for unsupported controllers

internal/clientcache/internal/daemon/options.go

@@ -14,7 +14,7 @@ import (
 type options struct {
 	withDebug                              bool
 	withRefreshInterval                    time.Duration
-	withFullFetchInterval                  time.Duration
+	withRecheckSupportInterval             time.Duration
 	testWithIntervalRandomizationFactor    float64
 	testWithIntervalRandomizationFactorSet bool
 	withBoundaryTokenReaderFunc            cache.BoundaryTokenReaderFn
@@ -57,13 +57,13 @@ func withRefreshInterval(_ context.Context, d time.Duration) Option {
 	}
 }
 
-// withFullFetchInterval provides an optional full fetch interval.
-func withFullFetchInterval(_ context.Context, d time.Duration) Option {
+// withRecheckSupportInterval provides an optional full fetch interval.
+func withRecheckSupportInterval(_ context.Context, d time.Duration) Option {
 	return func(o *options) error {
 		if d <= 0 {
-			return fmt.Errorf("provided full fetch interval %q must be positive", d)
+			return fmt.Errorf("provided recheck support interval %q must be positive", d)
 		}
-		o.withFullFetchInterval = d
+		o.withRecheckSupportInterval = d
 		return nil
 	}
 }

internal/clientcache/internal/daemon/options_test.go

@@ -44,15 +44,15 @@ func Test_GetOpts(t *testing.T) {
 		testOpts := getDefaultOptions()
 		assert.Equal(t, opts, testOpts)
 	})
-	t.Run("withFullFetchInterval", func(t *testing.T) {
-		opts, err := getOpts(withFullFetchInterval(ctx, time.Second))
+	t.Run("withRecheckSupportInterval", func(t *testing.T) {
+		opts, err := getOpts(withRecheckSupportInterval(ctx, time.Second))
 		require.NoError(t, err)
 		testOpts := getDefaultOptions()
-		testOpts.withFullFetchInterval = time.Second
+		testOpts.withRecheckSupportInterval = time.Second
 		assert.Equal(t, opts, testOpts)
 	})
-	t.Run("withFullFetchInterval_negative", func(t *testing.T) {
-		opts, err := getOpts(withFullFetchInterval(ctx, -time.Second))
+	t.Run("withRecheckSupportInterval_negative", func(t *testing.T) {
+		opts, err := getOpts(withRecheckSupportInterval(ctx, -time.Second))
 		require.ErrorContains(t, err, "must be positive")
 		testOpts := getDefaultOptions()
 		assert.Equal(t, opts, testOpts)

internal/clientcache/internal/daemon/search_handler.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/hashicorp/boundary/api"
 	"github.com/hashicorp/boundary/api/sessions"
 	"github.com/hashicorp/boundary/api/targets"
 	"github.com/hashicorp/boundary/internal/clientcache/internal/cache"
@@ -70,10 +71,22 @@ func newSearchHandlerFunc(ctx context.Context, repo *cache.Repository, refreshSe
 			return
 		}
 
+		supported, err := s.Supported(ctx, t)
+		if err != nil {
+			event.WriteError(ctx, op, err, event.WithInfoMsg("when checking if search is supported for the provided auth token", "auth_token_id", authTokenId))
+			writeError(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if !supported {
+			writeUnsupportedError(w)
+			return
+		}
+
 		var opts []cache.Option
 		if b, err := strconv.ParseBool(r.URL.Query().Get(forceRefreshKey)); err == nil && b {
 			opts = append(opts, cache.WithIgnoreSearchStaleness(true))
 		}
+
 		// Refresh the resources for the provided user, if possible. This is best
 		// effort, so if there is any problem refreshing, we just log the error
 		// and move on to handling the search request.
@@ -123,3 +136,17 @@ func toApiResult(sr *cache.SearchResult) *SearchResult {
 		Sessions: sr.Sessions,
 	}
 }
+
+var errSearchNotSupported = &api.Error{
+	Kind:    "Unsupported Search Request",
+	Message: "The request is for a Boundary instance that doesn't support search. The Boundary instance must support refresh tokens for search to be supported.",
+}
+
+func writeUnsupportedError(w http.ResponseWriter) {
+	b, err := json.Marshal(errSearchNotSupported)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Unable to marshal error %v into api error format: %s", errSearchNotSupported, err.Error()), http.StatusInternalServerError)
+		return
+	}
+	http.Error(w, string(b), http.StatusBadRequest)
+}

internal/clientcache/internal/daemon/server.go

@@ -69,15 +69,15 @@ type CacheServer struct {
 }
 
 type Config struct {
-	ContextCancel     context.CancelFunc
-	RefreshInterval   time.Duration
-	FullFetchInterval time.Duration
-	DatabaseUrl       string
-	StoreDebug        bool
-	LogLevel          string
-	LogFormat         string
-	LogWriter         io.Writer
-	DotDirectory      string
+	ContextCancel          context.CancelFunc
+	RefreshInterval        time.Duration
+	RecheckSupportInterval time.Duration
+	DatabaseUrl            string
+	StoreDebug             bool
+	LogLevel               string
+	LogFormat              string
+	LogWriter              io.Writer
+	DotDirectory           string
 	// The amount of time since the last refresh that must have passed for a
 	// search query to trigger an inline refresh.
 	MaxSearchStaleness time.Duration
@@ -97,8 +97,8 @@ func (sc *Config) validate(ctx context.Context) error {
 		return errors.New(ctx, errors.InvalidParameter, op, "missing dot directory")
 	case sc.RefreshInterval < 0:
 		return errors.New(ctx, errors.InvalidParameter, op, "negative refresh interval")
-	case sc.FullFetchInterval < 0:
-		return errors.New(ctx, errors.InvalidParameter, op, "negative full fetch interval")
+	case sc.RecheckSupportInterval < 0:
+		return errors.New(ctx, errors.InvalidParameter, op, "negative recheck support interval")
 	case sc.MaxSearchStaleness < 0:
 		return errors.New(ctx, errors.InvalidParameter, op, "negative max search staleness")
 	}
@@ -236,7 +236,7 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 	s.infoKeys = append(s.infoKeys, "Max Search Refresh Timeout")
 	s.info["Refresh Interval"] = DefaultRefreshInterval.String()
 	s.infoKeys = append(s.infoKeys, "Refresh Interval")
-	s.info["Full Fetch Interval"] = DefaultFullFetchInterval.String()
+	s.info["Full Fetch Interval"] = DefaultRecheckSupportInterval.String()
 	s.infoKeys = append(s.infoKeys, "Full Fetch Interval")
 
 	ticOptions := []Option{}
@@ -244,9 +244,9 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 		s.info["Refresh Interval"] = s.conf.RefreshInterval.String()
 		ticOptions = append(ticOptions, withRefreshInterval(ctx, s.conf.RefreshInterval))
 	}
-	if s.conf.FullFetchInterval > 0 {
-		s.info["Full Fetch Interval"] = s.conf.FullFetchInterval.String()
-		ticOptions = append(ticOptions, withFullFetchInterval(ctx, s.conf.FullFetchInterval))
+	if s.conf.RecheckSupportInterval > 0 {
+		s.info["Recheck Support Interval"] = s.conf.RecheckSupportInterval.String()
+		ticOptions = append(ticOptions, withRecheckSupportInterval(ctx, s.conf.RecheckSupportInterval))
 	}
 
 	s.printInfo(ctx)
@@ -275,7 +275,7 @@ func (s *CacheServer) Serve(ctx context.Context, cmd Commander, opt ...Option) e
 	}()
 	go func() {
 		defer tickerWg.Done()
-		tic.startFullFetch(tickingCtx)
+		tic.startRecheckCachingSupport(tickingCtx)
 	}()
 
 	mux := http.NewServeMux()

internal/clientcache/internal/daemon/status_handler.go

@@ -42,12 +42,22 @@ type AuthTokenStatus struct {
 	KeyringlessReferences int    `json:"keyringless_references,omitempty"`
 }
 
+// Information about the boundary controller the data is being cached from.
+type BoundaryStatus struct {
+	// The boundary address for this user
+	Address string `json:"address,omitempty"`
+	// Whether the controller responses are supported by the cache
+	CacheSupport string `json:"cache_support,omitempty"`
+	// How long ago the Boundary instance was checked for cache compatability
+	LastSupportCheck time.Duration `json:"last_support_check,omitempty"`
+}
+
 // UserStatus contains the status of a specific user tracked by the cache
 type UserStatus struct {
 	// The Id of the user this status is for
 	Id string `json:"id,omitempty"`
-	// The boundary address for this user
-	Address string `json:"address,omitempty"`
+	// The boundary instance this user connects to
+	BoundaryInstance BoundaryStatus `json:"boundary_instance,omitempty"`
 	// The auth tokens used by this user to authenticate with the boundary instance
 	AuthTokens []AuthTokenStatus `json:"auth_tokens,omitempty"`
 	// The resources tracked by the cache for this user
@@ -116,8 +126,12 @@ func toApiStatus(in *cache.Status, started time.Time, socketAddr string) *Status
 
 	for _, inU := range in.Users {
 		outU := UserStatus{
-			Id:      inU.Id,
-			Address: inU.Address,
+			Id: inU.Id,
+			BoundaryInstance: BoundaryStatus{
+				Address:          inU.BoundaryStatus.Address,
+				CacheSupport:     string(inU.BoundaryStatus.CachingSupported),
+				LastSupportCheck: inU.BoundaryStatus.LastSupportCheck,
+			},
 		}
 		for _, inAt := range inU.AuthTokens {
 			outU.AuthTokens = append(outU.AuthTokens, AuthTokenStatus{

internal/clientcache/internal/daemon/testing.go

@@ -33,12 +33,12 @@ func NewTestServer(t *testing.T, cmd Commander, opt ...Option) *TestServer {
 	dotDir := t.TempDir()
 
 	cfg := &Config{
-		ContextCancel:     cancel,
-		RefreshInterval:   DefaultRefreshInterval,
-		FullFetchInterval: DefaultFullFetchInterval,
-		StoreDebug:        opts.withDebug,
-		LogWriter:         io.Discard,
-		DotDirectory:      dotDir,
+		ContextCancel:          cancel,
+		RefreshInterval:        DefaultRefreshInterval,
+		RecheckSupportInterval: DefaultRecheckSupportInterval,
+		StoreDebug:             opts.withDebug,
+		LogWriter:              io.Discard,
+		DotDirectory:           dotDir,
 	}
 
 	s, err := New(ctx, cfg)
@@ -93,15 +93,45 @@ func (s *TestServer) AddResources(t *testing.T, p *authtokens.AuthToken, tars []
 		if tok != p.Token {
 			return nil, nil, "", nil
 		}
-		return tars, nil, "", nil
+		return tars, nil, "addedtargets", nil
 	}
 	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) {
 		if tok != p.Token {
 			return nil, nil, "", nil
 		}
-		return sess, nil, "", nil
+		return sess, nil, "addedsessions", nil
 	}
 	rs, err := cache.NewRefreshService(ctx, r, 0, 0)
 	require.NoError(t, err)
 	require.NoError(t, rs.Refresh(ctx, cache.WithTargetRetrievalFunc(tarFn), cache.WithSessionRetrievalFunc(sessFn)))
 }
+
+// AddUnsupportedCachingData provides data in a way that simulates it coming from
+// a boundary instance that does not support refresh tokens. Since refresh tokens
+// are required for caching, this has the effect of the data not being cached and
+// the user being identified as not supported in the cache.
+func (s *TestServer) AddUnsupportedCachingData(t *testing.T, p *authtokens.AuthToken, atReadFn cache.BoundaryTokenReaderFn) {
+	t.Helper()
+	ctx := context.Background()
+	r, err := cache.NewRepository(ctx, s.CacheServer.store, &sync.Map{}, s.cmd.ReadTokenFromKeyring, atReadFn)
+	require.NoError(t, err)
+
+	tarFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*targets.Target, []string, cache.RefreshTokenValue, error) {
+		if tok != p.Token {
+			return nil, nil, "", nil
+		}
+		return []*targets.Target{
+			{Id: "ttcp_unsupported", Name: "unsupported", Description: "not supported"},
+		}, nil, "", cache.ErrRefreshNotSupported
+	}
+	sessFn := func(ctx context.Context, _, tok string, _ cache.RefreshTokenValue) ([]*sessions.Session, []string, cache.RefreshTokenValue, error) {
+		if tok != p.Token {
+			return nil, nil, "", nil
+		}
+		return []*sessions.Session{}, nil, "", cache.ErrRefreshNotSupported
+	}
+	rs, err := cache.NewRefreshService(ctx, r, 0, 0)
+	require.NoError(t, err)
+	err = rs.Refresh(ctx, cache.WithTargetRetrievalFunc(tarFn), cache.WithSessionRetrievalFunc(sessFn))
+	require.ErrorContains(t, err, "not supported for this controller")
+}

internal/clientcache/internal/daemon/ticker.go

@@ -18,31 +18,32 @@ const (
 	// DefaultRefreshInterval is the default interval that resources that come
 	// from controllers that do support refresh tokens are retrieved and
 	// refreshed.
-	DefaultRefreshInterval = 60 * time.Second
-	// DefaultFullFetchInterval is the default interval that resources that come
-	// from controllers that do not support refresh tokens are fetched.
-	DefaultFullFetchInterval = 60 * 5 * time.Second
+	DefaultRefreshInterval = 5 * time.Minute
+	// DefaultRecheckSupportInterval is the default interval that a boundary
+	// instance which is not supported in the cache is rechecked to see if it
+	// has become supported. Support is based on if it supports refresh tokens.
+	DefaultRecheckSupportInterval = 1 * time.Hour
 
 	defaultRandomizationFactor float64 = 0.2
 )
 
 type refreshService interface {
 	Refresh(context.Context, ...cache.Option) error
-	FullFetch(context.Context, ...cache.Option) error
+	RecheckCachingSupport(context.Context, ...cache.Option) error
 }
 
 type refreshTicker struct {
-	tickerCtx           context.Context
-	refreshInterval     time.Duration
-	fullFetchInterval   time.Duration
-	randomizationFactor float64
-	rand                *rand.Rand
-	refreshChan         chan struct{}
-	refresher           refreshService
+	tickerCtx              context.Context
+	refreshInterval        time.Duration
+	recheckSupportInterval time.Duration
+	randomizationFactor    float64
+	rand                   *rand.Rand
+	refreshChan            chan struct{}
+	refresher              refreshService
 }
 
 // newRefreshTicker creates a new refresh ticker. Accepted options are
-// withRefreshInterval, and withFullFetchInterval.
+// withRefreshInterval, and withRecheckSupportInterval.
 // testWithIntervalRandomizationFactor is provided for tests.
 // The intervals can deviate +/- randomizationFactor * configured interval.
 // For example, an interval of 10 seconds and a randomization factor of 0.2 will
@@ -65,9 +66,9 @@ func newRefreshTicker(ctx context.Context, refresh refreshService, opt ...Option
 		refreshInterval = opts.withRefreshInterval
 	}
 
-	fullFetchInterval := DefaultFullFetchInterval
-	if opts.withFullFetchInterval != 0 {
-		fullFetchInterval = opts.withFullFetchInterval
+	recheckSupportInterval := DefaultRecheckSupportInterval
+	if opts.withRecheckSupportInterval != 0 {
+		recheckSupportInterval = opts.withRecheckSupportInterval
 	}
 
 	randomizationFactor := defaultRandomizationFactor
@@ -76,11 +77,11 @@ func newRefreshTicker(ctx context.Context, refresh refreshService, opt ...Option
 	}
 
 	return &refreshTicker{
-		refreshInterval:     refreshInterval,
-		fullFetchInterval:   fullFetchInterval,
-		randomizationFactor: randomizationFactor,
-		rand:                rand.New(rand.NewSource(time.Now().UnixNano())),
-		refresher:           refresh,
+		refreshInterval:        refreshInterval,
+		recheckSupportInterval: recheckSupportInterval,
+		randomizationFactor:    randomizationFactor,
+		rand:                   rand.New(rand.NewSource(time.Now().UnixNano())),
+		refresher:              refresh,
 
 		// We make this channel size 1 so if something happens midway through the refresh
 		// we can immediately refresh again immediately to pick up something that might have been
@@ -100,8 +101,11 @@ func (rt *refreshTicker) refresh() {
 	}
 }
 
-func (rt *refreshTicker) startFullFetch(ctx context.Context) {
-	const op = "daemon.(refreshTicker).startFullFetch"
+// startRecheckCachingSupport repeatedly calls RecheckCachingSupport which
+// checks all users that are marked as not supported by the cache to see if
+// the controller version has been updated to support refresh tokens/caching.
+func (rt *refreshTicker) startRecheckCachingSupport(ctx context.Context) {
+	const op = "daemon.(refreshTicker).startRecheckCachingSupport"
 	timer := time.NewTimer(0)
 	for {
 		// full fetch doesn't listen to the refreshChan since completely new
@@ -112,10 +116,10 @@ func (rt *refreshTicker) startFullFetch(ctx context.Context) {
 			return
 		case <-timer.C:
 		}
-		if err := rt.refresher.FullFetch(ctx); err != nil {
+		if err := rt.refresher.RecheckCachingSupport(ctx); err != nil {
 			event.WriteError(rt.tickerCtx, op, err)
 		}
-		timer.Reset(rt.nextIntervalWithRandomness(rt.fullFetchInterval))
+		timer.Reset(rt.nextIntervalWithRandomness(rt.recheckSupportInterval))
 	}
 }
 

internal/clientcache/internal/daemon/ticker_test.go

@@ -47,7 +47,7 @@ func TestTickerRefresh(t *testing.T) {
 	}
 }
 
-func TestTickerFullFetch(t *testing.T) {
+func TestTickerRecheckCachingSupport(t *testing.T) {
 	ctx := context.Background()
 
 	refresher := &fakeRefresher{make(chan struct{})}
@@ -56,7 +56,7 @@ func TestTickerFullFetch(t *testing.T) {
 	require.NoError(t, err)
 
 	tickerCtx, tickerCancel := context.WithCancel(ctx)
-	go rt.startFullFetch(tickerCtx)
+	go rt.startRecheckCachingSupport(tickerCtx)
 	t.Cleanup(func() {
 		tickerCancel()
 	})
@@ -106,7 +106,7 @@ func (r *fakeRefresher) Refresh(context.Context, ...cache.Option) error {
 	return nil
 }
 
-func (r *fakeRefresher) FullFetch(context.Context, ...cache.Option) error {
+func (r *fakeRefresher) RecheckCachingSupport(context.Context, ...cache.Option) error {
 	r.called <- struct{}{}
 	return nil
 }