feat: 🎸 Initial setup of IndexedDB (#2064)

* feat: 🎸 Initial setup of IndexedDB

* fix: 🐛 Rethrow errors up from indexed db handler

addons/api/addon/adapters/application.js

@@ -115,6 +115,7 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
    * @return {Promise} promise
    */
   async query(store, schema, query) {
+    const queryObject = structuredClone(query);
     let result;
     let data = [];
 
@@ -128,12 +129,12 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
     // Run this loop as long as the response_type is delta,
     // which indicates that there are more items in the list
     do {
-      result = await super.query(store, schema, query);
+      result = await super.query(store, schema, queryObject);
       //add the result items to a data array
       if (result && result.items) {
         data.push(...result.items);
         //pass in the list token for subsequent calls to fetch the remaining list items
-        query.list_token = result.list_token;
+        queryObject.list_token = result.list_token;
       }
     } while (result && result.response_type === 'delta');
 
@@ -211,7 +212,10 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
   handleResponse(status, headers, payload, requestData) {
     if (this.isInvalid(status, headers, payload)) {
       const detailedMessage = this.generatedDetailedMessage(...arguments);
-      const transformedPayload = this.transformValidationErrors(payload);
+      const transformedPayload = this.transformValidationErrors(
+        payload,
+        status,
+      );
       return new InvalidError(transformedPayload.errors, detailedMessage);
     }
     return super.handleResponse(status, headers, payload, requestData);
@@ -269,15 +273,16 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
    * error format.
    *
    * @param {object} payload
+   * @param status
    * @return {object}
    */
-  transformValidationErrors(payload) {
+  transformValidationErrors(payload, status) {
     const fieldErrors = get(payload, 'details.request_fields') || [];
     // Normalize the primary error message into "base", which is an error
     // that applies to the whole model instance
     const baseError = {
       isInvalid: true,
-      status: payload.source,
+      status,
       code: payload.code,
       detail: payload.message,
       source: { pointer: '/data' },

addons/api/addon/handlers/indexed-db-handler.js

@@ -0,0 +1,121 @@
+import { modelIndexes } from 'api/services/indexed-db';
+import { inject as service } from '@ember/service';
+import { getOwner, setOwner } from '@ember/application';
+import { pluralize } from 'ember-inflector';
+
+/**
+ * Handler to sit in front of the API request layer
+ * so we can handle any caching with indexedDb first
+ */
+export default class IndexedDbHandler {
+  @service indexedDb;
+
+  constructor(context) {
+    setOwner(this, getOwner(context));
+  }
+
+  async request(context, next) {
+    switch (context.request.op) {
+      case 'query': {
+        const { store, data } = context.request;
+        const { type, query, options: { pushToStore = true } = {} } = data;
+        const supportedModels = Object.keys(modelIndexes);
+        const { db: indexedDb } = this.indexedDb ?? {};
+        const pluralizedType = pluralize(type);
+
+        // Go through normal flow if we don't yet support the model
+        // or if we don't have an indexedDb instance
+        if (!supportedModels.includes(pluralizedType) || !indexedDb) {
+          return next(context.request);
+        }
+
+        const listToken = await indexedDb.tokens.get(pluralizedType);
+        const adapter = store.adapterFor(type);
+        const schema = store.modelFor(type);
+
+        let payload;
+        try {
+          payload = await adapter.query(store, schema, {
+            ...query,
+            scope_id: 'global',
+            recursive: true,
+            list_token: listToken?.token,
+          });
+        } catch (e) {
+          // If we get any error that's not a rate limiting error,
+          // assume the token is not good anymore. We'll delete the token and
+          // clear the DB and try again without a token.
+          // TODO: Probably should check for 400 and bad token error specifically instead?
+          if (e?.errors[0].status !== 429) {
+            await indexedDb[pluralizedType].clear();
+            await indexedDb.tokens.delete(pluralizedType);
+
+            payload = await adapter.query(store, schema, {
+              ...query,
+              scope_id: 'global',
+              recursive: true,
+              list_token: null,
+            });
+          } else {
+            throw e;
+          }
+        }
+
+        // Store the token we just got back from the payload
+        await indexedDb.tokens.put({
+          id: pluralizedType,
+          token: payload.list_token,
+        });
+
+        // Remove any records from the DB if the API indicates they've been deleted
+        if (payload.removed_ids?.length > 0) {
+          await indexedDb[pluralizedType].bulkDelete(payload.removed_ids);
+        }
+
+        const serializer = store.serializerFor(type);
+        const normalizedPayload = serializer.normalizeResponse(
+          store,
+          schema,
+          payload,
+          null,
+          'query',
+        );
+
+        const { data: payloadData } = normalizedPayload;
+
+        // Store the new data we just got back from the API refresh
+        await indexedDb[pluralizedType].bulkPut(
+          payloadData.map((datum) => this.indexedDb.normalizeData(datum, true)),
+        );
+
+        // TODO: Filter here, this is just a placeholder for now
+        const dbRecords = (
+          await indexedDb[pluralizedType]
+            .where('attributes.scope.scope_id')
+            .equals(query.scope_id)
+            .toArray()
+        ).map(this.indexedDb.normalizeData);
+        const recordCount = await indexedDb[pluralizedType].count();
+
+        // Return the raw data if we don't push to the store.
+        let records = dbRecords.map((record) => ({
+          ...record.attributes,
+          id: record.id,
+        }));
+
+        if (pushToStore) {
+          records = store.push({ data: dbRecords });
+        }
+
+        // Set a meta property on the array to store the total items of the results.
+        // This isn't conventional but is better than returning an ArrayProxy
+        // or EmberArray since the ember store query method asserts it has to be an array
+        // so we can't just return an object.
+        records.meta = { totalItems: recordCount };
+        return records;
+      }
+      default:
+        return next(context.request);
+    }
+  }
+}

addons/api/addon/services/indexed-db.js

@@ -0,0 +1,114 @@
+import Service, { inject as service } from '@ember/service';
+import Dexie from 'dexie';
+import { typeOf } from '@ember/utils';
+
+// List of resources we support to be cached and searched in
+// indexedDb. Any field that can be searched upon should be added to
+// this index. Increment the version number of the database whenever
+// indexes change.
+export const modelIndexes = {
+  tokens: '&id, token',
+  targets: '&id, attributes.type, attributes.name, attributes.scope.scope_id',
+  sessions: '&id, attributes.type, attributes.scope.scope_id',
+};
+
+export const formatDbName = (userId, clusterUrl) =>
+  `boundary-${userId}-${clusterUrl}`;
+
+/**
+ * Recursively copy over the object and convert any booleans to "true" or "false" so they can be indexed
+ * @param result
+ * @param object
+ * @returns {*}
+ */
+const convertBooleansInObject = (result, object) => {
+  Object.entries(object).forEach(([key, value]) => {
+    if (typeOf(value) === 'object') {
+      result[key] = convertBooleansInObject({}, value);
+    } else if (typeOf(value) === 'boolean') {
+      result[key] = value ? 'true' : 'false';
+    } else {
+      result[key] = value;
+    }
+  });
+
+  return result;
+};
+
+/**
+ * Recursively copy over the object and convert any string "true" or "false" back to booleans
+ * @param result
+ * @param object
+ * @returns {*}
+ */
+const unconvertBooleansInObject = (result, object) => {
+  Object.entries(object).forEach(([key, value]) => {
+    if (typeOf(value) === 'object') {
+      result[key] = unconvertBooleansInObject({}, value);
+    } else if (value === 'true' || value === 'false') {
+      result[key] = value === 'true' ? true : false;
+    } else {
+      result[key] = value;
+    }
+  });
+
+  return result;
+};
+
+/**
+ * Service to encapsulate the IndexedDB implementation. To use this service, call
+ * `setup` from the application root and
+ */
+export default class IndexedDbService extends Service {
+  // =services
+  @service session;
+
+  // =attributes
+  #db;
+  get db() {
+    return this.#db;
+  }
+
+  setup(dbName) {
+    // Don't run setup again if we already have one or if we didn't get a name
+    if (this.#db || !dbName) {
+      return;
+    }
+
+    this.#db = new Dexie(dbName);
+    this.#db.version(1).stores(modelIndexes);
+  }
+
+  /**
+   * Cleanup the data by converting all booleans to "false" or "true".
+   * IndexedDB doesn't support indexes on boolean values so we have to
+   * convert them to be able to support using boolean values.
+   *
+   * Data is assumed to be in POJOs and that our normalized data are not
+   * ember objects, otherwise they should also be converted.
+   *
+   * Undo the operation by setting `convertBooleans` to false.
+   *
+   * @param {Object} data
+   * @param {Boolean} convertBooleans
+   * @return {{id, type, attributes: {}, relationships: {}}}
+   */
+  normalizeData(data, convertBooleans) {
+    if (!data) {
+      return data;
+    }
+
+    let { id, type, attributes = {}, relationships = {} } = data;
+
+    return {
+      id,
+      type,
+      attributes: convertBooleans
+        ? convertBooleansInObject({}, attributes)
+        : unconvertBooleansInObject({}, attributes),
+      relationships: convertBooleans
+        ? convertBooleansInObject({}, relationships)
+        : unconvertBooleansInObject({}, relationships),
+    };
+  }
+}

addons/api/addon/services/store.js

@@ -6,6 +6,7 @@
 import Store, { CacheHandler } from '@ember-data/store';
 import RequestManager from '@ember-data/request';
 import { LegacyNetworkHandler } from '@ember-data/legacy-compat';
+import IndexedDbHandler from 'api/handlers/indexed-db-handler';
 
 export default class extends Store {
   requestManager = new RequestManager();
@@ -13,7 +14,9 @@ export default class extends Store {
   constructor(args) {
     super(args);
 
-    this.requestManager.use([LegacyNetworkHandler]);
+    const indexedDbHandler = new IndexedDbHandler(this);
+
+    this.requestManager.use([indexedDbHandler, LegacyNetworkHandler]);
     this.requestManager.useCache(CacheHandler);
   }
 }