Restore soft deleted users on authentication (#5549)

* internal/clientcache: restore user on login

A user that previously had its last auth token deleted
(for example, if they logged out), should be restored
again when they successfully authenticate again.

* internal/clientcache: delete keyringless expired auth tokens

Previously, we would only correctly delete auth tokens
that were expired (or deleted) if they were also in the
keyring.

* internal/clientcache: remove tautological condition

At this stage we already know that at is non-nil, because
the case above would have been true if it was nil.

* internal/clientcache: improve error handling

The previous error handling would have ignored API errors that were not ErrUnauthorized or ErrNotFound.
Handle all API errors the same as other unexpected errors.

* internal/clientcache: make errors distinct

These two error conditions are otherwise impossible
to tell apart.

* test(e2e): Update search test to catch bug (#5551)

---------

Co-authored-by: Michael Li <michael.li@hashicorp.com>

internal/clientcache/internal/cache/refresh.go

@@ -94,17 +94,16 @@ func (r *RefreshService) cleanAndPickAuthTokens(ctx context.Context, u *user) (m
 				if err := r.repo.deleteKeyringToken(ctx, *kt); err != nil {
 					return nil, errors.Wrap(ctx, err, op, errors.WithMsg("for user %q, auth token %q", u.Id, t.Id))
 				}
-			case at != nil:
+			default:
 				_, err := r.repo.tokenReadFromBoundaryFn(ctx, u.Address, at.Token)
-				var apiErr *api.Error
 				switch {
 				case err != nil && (api.ErrUnauthorized.Is(err) || api.ErrNotFound.Is(err)):
 					if err := r.repo.deleteKeyringToken(ctx, *kt); err != nil {
 						return nil, errors.Wrap(ctx, err, op, errors.WithMsg("for user %q, auth token %q", u.Id, t.Id))
 					}
-					event.WriteSysEvent(ctx, op, "Removed auth token from cache because it was not found to be valid in boundary", "auth token id", at.Id)
+					event.WriteSysEvent(ctx, op, "Removed auth token from db because it was not found to be valid in boundary", "auth token id", at.Id)
 					continue
-				case err != nil && !errors.Is(err, apiErr):
+				case err != nil:
 					event.WriteError(ctx, op, err, event.WithInfoMsg("validating keyring stored token against boundary", "auth token id", at.Id))
 					continue
 				}
@@ -114,17 +113,18 @@ func (r *RefreshService) cleanAndPickAuthTokens(ctx context.Context, u *user) (m
 		if atv, ok := r.repo.idToKeyringlessAuthToken.Load(t.Id); ok {
 			if at, ok := atv.(*authtokens.AuthToken); ok {
 				_, err := r.repo.tokenReadFromBoundaryFn(ctx, u.Address, at.Token)
-				var apiErr *api.Error
 				switch {
 				case err != nil && (api.ErrUnauthorized.Is(err) || api.ErrNotFound.Is(err)):
 					r.repo.idToKeyringlessAuthToken.Delete(t.Id)
-					event.WriteSysEvent(ctx, op, "Removed auth token from cache because it was not found to be valid in boundary", "auth token id", at.Id)
+					event.WriteSysEvent(ctx, op, "Removed auth token from in memory cache because it was not found to be valid in boundary", "auth token id", at.Id)
+					if err := r.repo.cleanExpiredOrOrphanedAuthTokens(ctx); err != nil {
+						return nil, errors.Wrap(ctx, err, op, errors.WithMsg("for user %q, auth token %q", u.Id, t.Id))
+					}
 					continue
-				case err != nil && !errors.Is(err, apiErr):
+				case err != nil:
 					event.WriteError(ctx, op, err, event.WithInfoMsg("validating in memory stored token against boundary", "auth token id", at.Id))
 					continue
 				}
-
 				ret[*t] = at.Token
 			}
 		}

internal/clientcache/internal/cache/repository_token.go

@@ -51,7 +51,8 @@ func upsertUserAndAuthToken(ctx context.Context, reader db.Reader, writer db.Wri
 		}
 		onConflict := &db.OnConflict{
 			Target: db.Columns{"id"},
-			Action: db.DoNothing(true),
+			// Unset the deleted_at column if it was set to un-delete the user
+			Action: db.SetColumnValues(map[string]any{"deleted_at": "infinity"}),
 		}
 		if err := writer.Create(ctx, u, db.WithOnConflict(onConflict)); err != nil {
 			return errors.Wrap(ctx, err, op)