feat: 🎸 Add indexedDb parser (#2068)

* feat: 🎸 Add indexedDb parser

✅ Closes: https://hashicorp.atlassian.net/browse/ICU-12164

---------

Co-authored-by: Lisbet Alvarez <lisbet.alvarez@hashicorp.com>

addons/api/addon/handlers/client-daemon-handler.js

@@ -7,6 +7,7 @@ import { inject as service } from '@ember/service';
 import { getOwner, setOwner } from '@ember/application';
 import { pluralize } from 'ember-inflector';
 import { generateMQLExpression } from '../utils/mql-query';
+import { paginateResults } from '../utils/paginate-results';
 
 /**
  * Not all types are yet supported by the client daemon so we'll
@@ -19,31 +20,6 @@ const supportedTypes = {
   session: ['id', 'type', 'status', 'endpoint', 'scope_id', 'target_id'],
 };
 
-/**
- * Takes an array and the current page and pagesize to calculate the correct
- * number of results to return to the caller.
- *
- * @param array
- * @param page
- * @param pageSize
- * @returns {[*]}
- */
-const paginateResults = (array, page, pageSize) => {
-  const length = array?.length;
-  if (!array || length === 0) {
-    return [];
-  }
-  if (!page || !pageSize) {
-    return array;
-  }
-
-  const offset = (page - 1) * pageSize;
-  const start = Math.min(length - 1, offset);
-  const end = Math.min(length, offset + pageSize);
-
-  return array.slice(start, end);
-};
-
 const fetchControllerData = async (context, next) => {
   const { data } = context.request;
   const { query: originalQuery } = data;

addons/api/addon/handlers/indexed-db-handler.js

@@ -1,7 +1,8 @@
 import { modelIndexes } from 'api/services/indexed-db';
 import { inject as service } from '@ember/service';
 import { getOwner, setOwner } from '@ember/application';
-import { pluralize } from 'ember-inflector';
+import { queryIndexedDb } from '../utils/indexed-db-query';
+import { paginateResults } from '../utils/paginate-results';
 
 /**
  * Handler to sit in front of the API request layer
@@ -21,22 +22,23 @@ export default class IndexedDbHandler {
         const { type, query, options: { pushToStore = true } = {} } = data;
         const supportedModels = Object.keys(modelIndexes);
         const { db: indexedDb } = this.indexedDb ?? {};
-        const pluralizedType = pluralize(type);
 
         // Go through normal flow if we don't yet support the model
         // or if we don't have an indexedDb instance
-        if (!supportedModels.includes(pluralizedType) || !indexedDb) {
+        if (!supportedModels.includes(type) || !indexedDb) {
           return next(context.request);
         }
 
-        const listToken = await indexedDb.tokens.get(pluralizedType);
+        let { page, pageSize, query: queryObj, ...remainingQuery } = query;
+
+        const listToken = await indexedDb.token.get(type);
         const adapter = store.adapterFor(type);
         const schema = store.modelFor(type);
 
         let payload;
         try {
           payload = await adapter.query(store, schema, {
-            ...query,
+            ...remainingQuery,
             scope_id: 'global',
             recursive: true,
             list_token: listToken?.token,
@@ -47,11 +49,11 @@ export default class IndexedDbHandler {
           // clear the DB and try again without a token.
           // TODO: Probably should check for 400 and bad token error specifically instead?
           if (e?.errors[0].status !== 429) {
-            await indexedDb[pluralizedType].clear();
-            await indexedDb.tokens.delete(pluralizedType);
+            await indexedDb[type].clear();
+            await indexedDb.token.delete(type);
 
             payload = await adapter.query(store, schema, {
-              ...query,
+              ...remainingQuery,
               scope_id: 'global',
               recursive: true,
               list_token: null,
@@ -62,14 +64,14 @@ export default class IndexedDbHandler {
         }
 
         // Store the token we just got back from the payload
-        await indexedDb.tokens.put({
-          id: pluralizedType,
+        await indexedDb.token.put({
+          id: type,
           token: payload.list_token,
         });
 
         // Remove any records from the DB if the API indicates they've been deleted
         if (payload.removed_ids?.length > 0) {
-          await indexedDb[pluralizedType].bulkDelete(payload.removed_ids);
+          await indexedDb[type].bulkDelete(payload.removed_ids);
         }
 
         const serializer = store.serializerFor(type);
@@ -84,18 +86,20 @@ export default class IndexedDbHandler {
         const { data: payloadData } = normalizedPayload;
 
         // Store the new data we just got back from the API refresh
-        await indexedDb[pluralizedType].bulkPut(
-          payloadData.map((datum) => this.indexedDb.normalizeData(datum, true)),
+        const items = payloadData.map((datum) =>
+          this.indexedDb.normalizeData(datum, true),
+        );
+        await indexedDb[type].bulkPut(items);
+
+        const indexedDbResults = await queryIndexedDb(
+          indexedDb,
+          type,
+          queryObj,
         );
 
-        // TODO: Filter here, this is just a placeholder for now
-        const dbRecords = (
-          await indexedDb[pluralizedType]
-            .where('attributes.scope.scope_id')
-            .equals(query.scope_id)
-            .toArray()
-        ).map(this.indexedDb.normalizeData);
-        const recordCount = await indexedDb[pluralizedType].count();
+        const dbRecords = paginateResults(indexedDbResults, page, pageSize).map(
+          this.indexedDb.normalizeData,
+        );
 
         // Return the raw data if we don't push to the store.
         let records = dbRecords.map((record) => ({
@@ -111,7 +115,7 @@ export default class IndexedDbHandler {
         // This isn't conventional but is better than returning an ArrayProxy
         // or EmberArray since the ember store query method asserts it has to be an array
         // so we can't just return an object.
-        records.meta = { totalItems: recordCount };
+        records.meta = { totalItems: indexedDbResults.length };
         return records;
       }
       default:

addons/api/addon/services/indexed-db.js

@@ -1,4 +1,4 @@
-import Service, { inject as service } from '@ember/service';
+import Service from '@ember/service';
 import Dexie from 'dexie';
 import { typeOf } from '@ember/utils';
 
@@ -7,9 +7,11 @@ import { typeOf } from '@ember/utils';
 // this index. Increment the version number of the database whenever
 // indexes change.
 export const modelIndexes = {
-  tokens: '&id, token',
-  targets: '&id, attributes.type, attributes.name, attributes.scope.scope_id',
-  sessions: '&id, attributes.type, attributes.scope.scope_id',
+  token: '&id, token',
+  target:
+    '&id, attributes.created_time, attributes.type, attributes.name, attributes.description, attributes.address, attributes.scope.scope_id',
+  session:
+    '&id, attributes.created_time, attributes.type, attributes.status, attributes.endpoint, attributes.target_id, attributes.user_id, attributes.scope.scope_id',
 };
 
 export const formatDbName = (userId, clusterUrl) =>
@@ -46,7 +48,7 @@ const unconvertBooleansInObject = (result, object) => {
     if (typeOf(value) === 'object') {
       result[key] = unconvertBooleansInObject({}, value);
     } else if (value === 'true' || value === 'false') {
-      result[key] = value === 'true' ? true : false;
+      result[key] = value === 'true';
     } else {
       result[key] = value;
     }
@@ -60,9 +62,6 @@ const unconvertBooleansInObject = (result, object) => {
  * `setup` from the application root and
  */
 export default class IndexedDbService extends Service {
-  // =services
-  @service session;
-
   // =attributes
   #db;
   get db() {

addons/api/addon/utils/indexed-db-query.js

@@ -0,0 +1,263 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+import { get } from '@ember/object';
+import { modelIndexes } from '../services/indexed-db';
+
+/**
+ * Takes a POJO representing a filter query and builds a query to
+ * query indexedDb with. Note that the first filter is important as that
+ * will use the actual index in indexedDb while any subsequent ones will be less efficient.
+ *
+ * @example
+ *   const filter = queryIndexedDb(indexedDb, type, {
+ *     search: generated',
+ *     filters: {
+ *       scope_id: {
+ *         logicalOperator: 'and',
+ *         values: [{ notEquals: 'scope1' }, { notEquals: 'scope2' }],
+ *       },
+ *     },
+ *   });
+ */
+export async function queryIndexedDb(indexedDb, resource, query) {
+  let { search, filters = {} } = query ?? {};
+  let filterCollection = indexedDb[resource];
+
+  search = getSearchObject(search, resource);
+  filters = cleanObject(filters);
+
+  if (Object.keys(filters).length) {
+    // Remove the first item to use as the initial collection and then reduce the remaining ones
+    const entries = Object.entries(filters);
+    const [firstKey, firstFilterArrayOrObject] = entries.shift();
+
+    filterCollection = await entries.reduce(
+      async (currCollection, [key, filterArrayOrObject]) => {
+        // Grab the results from the set of one filters to be `AND` together with the rest of the filters
+        const filterResults = await buildInitialWhereClause({
+          table: indexedDb[resource],
+          key,
+          filterArrayOrObject,
+        }).primaryKeys();
+        const resultIdsSet = new Set(filterResults);
+
+        return (await currCollection).and((record) => {
+          return resultIdsSet.has(record.id);
+        });
+      },
+      buildInitialWhereClause({
+        key: firstKey,
+        table: indexedDb[resource],
+        filterArrayOrObject: firstFilterArrayOrObject,
+      }),
+    );
+  }
+
+  if (search) {
+    const { text, fields } = search;
+
+    filterCollection = filterCollection.filter((record) =>
+      fields
+        .map((key) => get(record, key))
+        .some((element) => {
+          if (typeof element === 'string') {
+            return element?.toLowerCase().includes(text?.toLowerCase());
+          }
+        }),
+    );
+  }
+
+  // OrderBy has to be used on where clauses while if it's a
+  // collection already we have to use sortBy. We'll also sort by
+  // descending order based on created time to match the API as any
+  // "or" clause can change the order
+  if (filterCollection.orderBy) {
+    return filterCollection.orderBy(getKey('created_time')).reverse().toArray();
+  }
+  return filterCollection.reverse().sortBy(getKey('created_time'));
+}
+
+const buildInitialWhereClause = ({ filterArrayOrObject, table, key }) => {
+  const filterValueArray = Array.isArray(filterArrayOrObject)
+    ? filterArrayOrObject
+    : filterArrayOrObject.values;
+
+  const { logicalOperator } = filterArrayOrObject;
+
+  // Short circuit if all the filter types were equals or notEquals since indexedDb
+  // can more efficiently query if we pass all values at once
+  const checkAllFilterType = (type) =>
+    filterValueArray.every((obj) => Object.keys(obj)[0] === type);
+  if (checkAllFilterType('equals')) {
+    return table
+      .where(getKey(key))
+      .anyOf(filterValueArray.map((obj) => obj['equals']));
+  }
+  if (checkAllFilterType('notEquals')) {
+    return table
+      .where(getKey(key))
+      .noneOf(filterValueArray.map((obj) => obj['notEquals']));
+  }
+
+  // Remove the first item to use as the initial where clause and then reduce the remaining ones
+  const [firstOperation, firstFilterValue] = Object.entries(
+    filterValueArray[0],
+  )[0];
+
+  // Not all are filters are equals or notEquals,
+  // manually build the initial where clause and resulting collections
+  return filterValueArray.slice(1).reduce(
+    (result, currObj) => {
+      const [operation, value] = Object.entries(currObj)[0];
+
+      return buildIndexedDbCollection({
+        key: getKey(key),
+        value,
+        collection: result,
+        operation,
+        logicalOperator,
+      });
+    },
+    buildIndexedDbWhere({
+      table,
+      key: getKey(key),
+      operation: firstOperation,
+      value: firstFilterValue,
+    }),
+  );
+};
+
+/**
+ * Builds the initial where clause for indexed DB to filter on. Note that this makes
+ * the first filter important as that will do the real filtering from indexed DB while
+ * any subsequent filter has to be done in memory.
+ */
+const buildIndexedDbWhere = ({ table, key, value, operation }) => {
+  const baseWhereClause = table.where(key);
+  switch (operation) {
+    case 'contains':
+      // We return an error due to not being able to support contains on first filter due to performance.
+      throw new Error('Contains is not supported as the first filter.');
+    case 'gt':
+      return baseWhereClause.above(value);
+    case 'gte':
+      return baseWhereClause.aboveOrEqual(value);
+    case 'lt':
+      return baseWhereClause.below(value);
+    case 'lte':
+      return baseWhereClause.belowOrEqual(value);
+    case 'notEquals':
+      return baseWhereClause.notEqual(value);
+    default:
+      return baseWhereClause.equals(value);
+  }
+};
+
+/**
+ * Continues building the queries for indexed DB to filter on. Note that these are not true
+ * indexed DB queries and in fact just standard JS filters for logical "and" queries and parallel queries
+ * to indexed DB for logical "or" queries that get deduped.
+ */
+const buildIndexedDbCollection = ({
+  key,
+  value,
+  collection,
+  operation,
+  logicalOperator,
+}) => {
+  if (logicalOperator === 'and') {
+    switch (operation) {
+      case 'contains':
+        return collection.and((record) => get(record, key)?.includes(value));
+      case 'gt':
+        return collection.and((record) => get(record, key) > value);
+      case 'gte':
+        return collection.and((record) => get(record, key) >= value);
+      case 'lt':
+        return collection.and((record) => get(record, key) < value);
+      case 'lte':
+        return collection.and((record) => get(record, key) <= value);
+      case 'notEquals':
+        return collection.and((record) => get(record, key) !== value);
+      default:
+        return collection.and((record) => get(record, key) === value);
+    }
+  }
+
+  switch (operation) {
+    case 'contains':
+      // We return an error due to not being able to support contains on this filter due to performance.
+      throw new Error(
+        'Contains is not supported as a filter option with "or" operator.',
+      );
+    case 'gt':
+      return collection.or(key).above(value);
+    case 'gte':
+      return collection.or(key).aboveOrEqual(value);
+    case 'lt':
+      return collection.or(key).below(value);
+    case 'lte':
+      return collection.or(key).belowOrEqual(value);
+    case 'notEquals':
+      return collection.or(key).notEqual(value);
+    default:
+      return collection.or(key).equals(value);
+  }
+};
+
+/**
+ * Translates the fields passed in from the UI for resources to the keys indexed DB knows them as.
+ * Usually it's the path of the actual normalized model.
+ */
+const getKey = (key) => {
+  switch (key) {
+    case 'scope_id':
+      return `attributes.scope.${key}`;
+    case 'id':
+      return key;
+    default:
+      return `attributes.${key}`;
+  }
+};
+
+/**
+ * Cleans the object to remove any keys that have a value of an empty array. This makes
+ * it easier so we don't have to worry about empty arrays getting queried in indexed db.
+ */
+const cleanObject = (obj) => {
+  if (!obj) {
+    return {};
+  }
+
+  return Object.entries(obj).reduce((result, [key, value]) => {
+    if (Array.isArray(value)) {
+      if (value.length > 0) {
+        result[key] = value;
+      }
+    } else if (value.values.length > 0) {
+      result[key] = value;
+    }
+
+    return result;
+  }, {});
+};
+
+/**
+ * Makes the search object that is expected when a user searches. Uses
+ * the indexes we setup in indexed db.
+ */
+const getSearchObject = (text, resource) => {
+  if (!text) {
+    return null;
+  }
+
+  return {
+    text,
+    fields: modelIndexes[resource]
+      .split(', ')
+      .map((index) => index.replace('&', '')),
+  };
+};

addons/api/addon/utils/mql-query.js

@@ -9,8 +9,8 @@
  *
  * @example
  *   const filter = generateMQLFilterExpression({
- *     name: [{ contains: 'search text' }],
- *     scope: [{ equals: 'scope1'}, { equals: 'scope2' }],
+ *     name: { logicalOperator: 'and', values: [{ notEquals: 'search text' }, { notEquals: 'test' }] },
+ *     scope_id: [{ equals: 'scope1'}, { equals: 'scope2' }],
  *   });
  *
  *   // (name % "search text") and (scope_id = "scope1" or scope_id = "scope2")
@@ -21,10 +21,10 @@ export function generateMQLFilterExpression(filterObj) {
   }
 
   const expressions = Object.entries(filterObj)
-    .map(([key, filterObjValue]) => {
-      const filterValueArray = Array.isArray(filterObjValue)
-        ? filterObjValue
-        : filterObjValue.values;
+    .map(([key, filterArrayOrObject]) => {
+      const filterValueArray = Array.isArray(filterArrayOrObject)
+        ? filterArrayOrObject
+        : filterArrayOrObject.values;
 
       let clauses;
       const filterValues = filterValueArray.map((filterObjValue) => {
@@ -53,7 +53,7 @@ export function generateMQLFilterExpression(filterObj) {
       });
 
       // Default to `or` if we don't have a logical operator
-      const { logicalOperator } = filterObjValue;
+      const { logicalOperator } = filterArrayOrObject;
       if (logicalOperator === 'and') {
         clauses = and(filterValues);
       } else {

addons/api/addon/utils/paginate-results.js

@@ -0,0 +1,24 @@
+/**
+ * Takes an array and the current page and pagesize to calculate the correct
+ * number of results to return to the caller.
+ *
+ * @param array
+ * @param page
+ * @param pageSize
+ * @returns {[*]}
+ */
+export const paginateResults = (array, page, pageSize) => {
+  const length = array?.length;
+  if (!array || length === 0) {
+    return [];
+  }
+  if (!page || !pageSize) {
+    return array;
+  }
+
+  const offset = (page - 1) * pageSize;
+  const start = Math.min(length - 1, offset);
+  const end = Math.min(length, offset + pageSize);
+
+  return array.slice(start, end);
+};