From bd5e3da1f41f178ff70c50c3a139a599515f18b1 Mon Sep 17 00:00:00 2001 From: Zhihe Li Date: Mon, 5 Feb 2024 15:59:58 -0500 Subject: [PATCH] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20Initial=20setup=20of=20I?= =?UTF-8?q?ndexedDB=20(#2064)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 🎸 Initial setup of IndexedDB * fix: 🐛 Rethrow errors up from indexed db handler --- .../addon-test-support/helpers/indexed-db.js | 30 +++++ addons/api/addon/adapters/application.js | 15 ++- .../api/addon/handlers/indexed-db-handler.js | 121 ++++++++++++++++++ addons/api/addon/services/indexed-db.js | 114 +++++++++++++++++ addons/api/addon/services/store.js | 5 +- addons/api/app/services/indexed-db.js | 1 + addons/api/package.json | 1 + .../tests/unit/services/indexed-db-test.js | 17 +++ ui/admin/app/routes/application.js | 14 ++ ui/admin/app/services/session.js | 22 ++++ ui/admin/ember-cli-build.js | 3 +- ui/admin/tests/unit/services/session-test.js | 12 ++ yarn.lock | 5 + 13 files changed, 353 insertions(+), 7 deletions(-) create mode 100644 addons/api/addon-test-support/helpers/indexed-db.js create mode 100644 addons/api/addon/handlers/indexed-db-handler.js create mode 100644 addons/api/addon/services/indexed-db.js create mode 100644 addons/api/app/services/indexed-db.js create mode 100644 addons/api/tests/unit/services/indexed-db-test.js create mode 100644 ui/admin/app/services/session.js create mode 100644 ui/admin/tests/unit/services/session-test.js diff --git a/addons/api/addon-test-support/helpers/indexed-db.js b/addons/api/addon-test-support/helpers/indexed-db.js new file mode 100644 index 000000000..5efe92343 --- /dev/null +++ b/addons/api/addon-test-support/helpers/indexed-db.js @@ -0,0 +1,30 @@ +/** + * This test helper can help set up and cleanup the database in your tests. + * + * ```js + * import { setupIndexedDb } from 'api/test-support/helpers/indexed-db'; + * + * module('Acceptance | my test', function(hooks) { + * setupApplicationTest(hooks); + * setupIndexedDb(hooks); + * + * // add your actual tests here + * }); + * ``` + * + * @param hooks + */ +export function setupIndexedDb(hooks) { + hooks.beforeEach(function () { + const indexedDb = this.owner.lookup('service:indexed-db'); + indexedDb.setup(`test-indexed-db-${Date.now()}`); + }); + + hooks.afterEach(async function () { + const indexedDb = this.owner.lookup('service:indexed-db'); + + await indexedDb.db.delete(); + }); +} + +export default setupIndexedDb; diff --git a/addons/api/addon/adapters/application.js b/addons/api/addon/adapters/application.js index 30561606c..65c8c2c6d 100644 --- a/addons/api/addon/adapters/application.js +++ b/addons/api/addon/adapters/application.js @@ -115,6 +115,7 @@ export default class ApplicationAdapter extends RESTAdapter.extend( * @return {Promise} promise */ async query(store, schema, query) { + const queryObject = structuredClone(query); let result; let data = []; @@ -128,12 +129,12 @@ export default class ApplicationAdapter extends RESTAdapter.extend( // Run this loop as long as the response_type is delta, // which indicates that there are more items in the list do { - result = await super.query(store, schema, query); + result = await super.query(store, schema, queryObject); //add the result items to a data array if (result && result.items) { data.push(...result.items); //pass in the list token for subsequent calls to fetch the remaining list items - query.list_token = result.list_token; + queryObject.list_token = result.list_token; } } while (result && result.response_type === 'delta'); @@ -211,7 +212,10 @@ export default class ApplicationAdapter extends RESTAdapter.extend( handleResponse(status, headers, payload, requestData) { if (this.isInvalid(status, headers, payload)) { const detailedMessage = this.generatedDetailedMessage(...arguments); - const transformedPayload = this.transformValidationErrors(payload); + const transformedPayload = this.transformValidationErrors( + payload, + status, + ); return new InvalidError(transformedPayload.errors, detailedMessage); } return super.handleResponse(status, headers, payload, requestData); @@ -269,15 +273,16 @@ export default class ApplicationAdapter extends RESTAdapter.extend( * error format. * * @param {object} payload + * @param status * @return {object} */ - transformValidationErrors(payload) { + transformValidationErrors(payload, status) { const fieldErrors = get(payload, 'details.request_fields') || []; // Normalize the primary error message into "base", which is an error // that applies to the whole model instance const baseError = { isInvalid: true, - status: payload.source, + status, code: payload.code, detail: payload.message, source: { pointer: '/data' }, diff --git a/addons/api/addon/handlers/indexed-db-handler.js b/addons/api/addon/handlers/indexed-db-handler.js new file mode 100644 index 000000000..a10802c11 --- /dev/null +++ b/addons/api/addon/handlers/indexed-db-handler.js @@ -0,0 +1,121 @@ +import { modelIndexes } from 'api/services/indexed-db'; +import { inject as service } from '@ember/service'; +import { getOwner, setOwner } from '@ember/application'; +import { pluralize } from 'ember-inflector'; + +/** + * Handler to sit in front of the API request layer + * so we can handle any caching with indexedDb first + */ +export default class IndexedDbHandler { + @service indexedDb; + + constructor(context) { + setOwner(this, getOwner(context)); + } + + async request(context, next) { + switch (context.request.op) { + case 'query': { + const { store, data } = context.request; + const { type, query, options: { pushToStore = true } = {} } = data; + const supportedModels = Object.keys(modelIndexes); + const { db: indexedDb } = this.indexedDb ?? {}; + const pluralizedType = pluralize(type); + + // Go through normal flow if we don't yet support the model + // or if we don't have an indexedDb instance + if (!supportedModels.includes(pluralizedType) || !indexedDb) { + return next(context.request); + } + + const listToken = await indexedDb.tokens.get(pluralizedType); + const adapter = store.adapterFor(type); + const schema = store.modelFor(type); + + let payload; + try { + payload = await adapter.query(store, schema, { + ...query, + scope_id: 'global', + recursive: true, + list_token: listToken?.token, + }); + } catch (e) { + // If we get any error that's not a rate limiting error, + // assume the token is not good anymore. We'll delete the token and + // clear the DB and try again without a token. + // TODO: Probably should check for 400 and bad token error specifically instead? + if (e?.errors[0].status !== 429) { + await indexedDb[pluralizedType].clear(); + await indexedDb.tokens.delete(pluralizedType); + + payload = await adapter.query(store, schema, { + ...query, + scope_id: 'global', + recursive: true, + list_token: null, + }); + } else { + throw e; + } + } + + // Store the token we just got back from the payload + await indexedDb.tokens.put({ + id: pluralizedType, + token: payload.list_token, + }); + + // Remove any records from the DB if the API indicates they've been deleted + if (payload.removed_ids?.length > 0) { + await indexedDb[pluralizedType].bulkDelete(payload.removed_ids); + } + + const serializer = store.serializerFor(type); + const normalizedPayload = serializer.normalizeResponse( + store, + schema, + payload, + null, + 'query', + ); + + const { data: payloadData } = normalizedPayload; + + // Store the new data we just got back from the API refresh + await indexedDb[pluralizedType].bulkPut( + payloadData.map((datum) => this.indexedDb.normalizeData(datum, true)), + ); + + // TODO: Filter here, this is just a placeholder for now + const dbRecords = ( + await indexedDb[pluralizedType] + .where('attributes.scope.scope_id') + .equals(query.scope_id) + .toArray() + ).map(this.indexedDb.normalizeData); + const recordCount = await indexedDb[pluralizedType].count(); + + // Return the raw data if we don't push to the store. + let records = dbRecords.map((record) => ({ + ...record.attributes, + id: record.id, + })); + + if (pushToStore) { + records = store.push({ data: dbRecords }); + } + + // Set a meta property on the array to store the total items of the results. + // This isn't conventional but is better than returning an ArrayProxy + // or EmberArray since the ember store query method asserts it has to be an array + // so we can't just return an object. + records.meta = { totalItems: recordCount }; + return records; + } + default: + return next(context.request); + } + } +} diff --git a/addons/api/addon/services/indexed-db.js b/addons/api/addon/services/indexed-db.js new file mode 100644 index 000000000..30e9aadd1 --- /dev/null +++ b/addons/api/addon/services/indexed-db.js @@ -0,0 +1,114 @@ +import Service, { inject as service } from '@ember/service'; +import Dexie from 'dexie'; +import { typeOf } from '@ember/utils'; + +// List of resources we support to be cached and searched in +// indexedDb. Any field that can be searched upon should be added to +// this index. Increment the version number of the database whenever +// indexes change. +export const modelIndexes = { + tokens: '&id, token', + targets: '&id, attributes.type, attributes.name, attributes.scope.scope_id', + sessions: '&id, attributes.type, attributes.scope.scope_id', +}; + +export const formatDbName = (userId, clusterUrl) => + `boundary-${userId}-${clusterUrl}`; + +/** + * Recursively copy over the object and convert any booleans to "true" or "false" so they can be indexed + * @param result + * @param object + * @returns {*} + */ +const convertBooleansInObject = (result, object) => { + Object.entries(object).forEach(([key, value]) => { + if (typeOf(value) === 'object') { + result[key] = convertBooleansInObject({}, value); + } else if (typeOf(value) === 'boolean') { + result[key] = value ? 'true' : 'false'; + } else { + result[key] = value; + } + }); + + return result; +}; + +/** + * Recursively copy over the object and convert any string "true" or "false" back to booleans + * @param result + * @param object + * @returns {*} + */ +const unconvertBooleansInObject = (result, object) => { + Object.entries(object).forEach(([key, value]) => { + if (typeOf(value) === 'object') { + result[key] = unconvertBooleansInObject({}, value); + } else if (value === 'true' || value === 'false') { + result[key] = value === 'true' ? true : false; + } else { + result[key] = value; + } + }); + + return result; +}; + +/** + * Service to encapsulate the IndexedDB implementation. To use this service, call + * `setup` from the application root and + */ +export default class IndexedDbService extends Service { + // =services + @service session; + + // =attributes + #db; + get db() { + return this.#db; + } + + setup(dbName) { + // Don't run setup again if we already have one or if we didn't get a name + if (this.#db || !dbName) { + return; + } + + this.#db = new Dexie(dbName); + this.#db.version(1).stores(modelIndexes); + } + + /** + * Cleanup the data by converting all booleans to "false" or "true". + * IndexedDB doesn't support indexes on boolean values so we have to + * convert them to be able to support using boolean values. + * + * Data is assumed to be in POJOs and that our normalized data are not + * ember objects, otherwise they should also be converted. + * + * Undo the operation by setting `convertBooleans` to false. + * + * @param {Object} data + * @param {Boolean} convertBooleans + * @return {{id, type, attributes: {}, relationships: {}}} + */ + normalizeData(data, convertBooleans) { + if (!data) { + return data; + } + + let { id, type, attributes = {}, relationships = {} } = data; + + return { + id, + type, + attributes: convertBooleans + ? convertBooleansInObject({}, attributes) + : unconvertBooleansInObject({}, attributes), + relationships: convertBooleans + ? convertBooleansInObject({}, relationships) + : unconvertBooleansInObject({}, relationships), + }; + } +} diff --git a/addons/api/addon/services/store.js b/addons/api/addon/services/store.js index a416162c4..89ed36e49 100644 --- a/addons/api/addon/services/store.js +++ b/addons/api/addon/services/store.js @@ -6,6 +6,7 @@ import Store, { CacheHandler } from '@ember-data/store'; import RequestManager from '@ember-data/request'; import { LegacyNetworkHandler } from '@ember-data/legacy-compat'; +import IndexedDbHandler from 'api/handlers/indexed-db-handler'; export default class extends Store { requestManager = new RequestManager(); @@ -13,7 +14,9 @@ export default class extends Store { constructor(args) { super(args); - this.requestManager.use([LegacyNetworkHandler]); + const indexedDbHandler = new IndexedDbHandler(this); + + this.requestManager.use([indexedDbHandler, LegacyNetworkHandler]); this.requestManager.useCache(CacheHandler); } } diff --git a/addons/api/app/services/indexed-db.js b/addons/api/app/services/indexed-db.js new file mode 100644 index 000000000..19338bae0 --- /dev/null +++ b/addons/api/app/services/indexed-db.js @@ -0,0 +1 @@ +export { default } from 'api/services/indexed-db'; diff --git a/addons/api/package.json b/addons/api/package.json index f839ab4cc..08fadd5f6 100644 --- a/addons/api/package.json +++ b/addons/api/package.json @@ -23,6 +23,7 @@ "doc:toc": "doctoc README.md" }, "dependencies": { + "dexie": "^3.2.4", "ember-auto-import": "^2.6.3", "ember-can": "^4.2.0", "ember-cli-babel": "^7.26.10", diff --git a/addons/api/tests/unit/services/indexed-db-test.js b/addons/api/tests/unit/services/indexed-db-test.js new file mode 100644 index 000000000..3002231e0 --- /dev/null +++ b/addons/api/tests/unit/services/indexed-db-test.js @@ -0,0 +1,17 @@ +import { module, test } from 'qunit'; +import { setupTest } from 'dummy/tests/helpers'; +import Service from '@ember/service'; + +class SessionService extends Service {} + +module('Unit | Service | indexed-db', function (hooks) { + setupTest(hooks); + + // TODO: Replace this with your real tests. + test('it exists', function (assert) { + this.owner.register('service:session', SessionService); + + let service = this.owner.lookup('service:indexed-db'); + assert.ok(service); + }); +}); diff --git a/ui/admin/app/routes/application.js b/ui/admin/app/routes/application.js index 7177b17f9..acac88d62 100644 --- a/ui/admin/app/routes/application.js +++ b/ui/admin/app/routes/application.js @@ -8,6 +8,7 @@ import { getOwner } from '@ember/application'; import { inject as service } from '@ember/service'; import { action } from '@ember/object'; import { A } from '@ember/array'; +import { formatDbName } from 'api/services/indexed-db'; /** * Entry route for the application. @@ -19,6 +20,10 @@ export default class ApplicationRoute extends Route { @service confirm; @service router; @service intl; + @service features; + @service featureEdition; + @service indexedDb; + @service('browser/window') window; // =attributes @@ -60,6 +65,15 @@ export default class ApplicationRoute extends Route { await this.session.setup(); const theme = this.session.get('data.theme'); this.toggleTheme(theme); + + // Setup the DB from a successful authentication restoration + if (this.session.isAuthenticated) { + const userId = this.session.data?.authenticated?.user_id; + const hostUrl = this.window.location.host; + if (userId && hostUrl) { + this.indexedDb.setup(formatDbName(userId, hostUrl)); + } + } } /** diff --git a/ui/admin/app/services/session.js b/ui/admin/app/services/session.js new file mode 100644 index 000000000..5761b6019 --- /dev/null +++ b/ui/admin/app/services/session.js @@ -0,0 +1,22 @@ +import BaseSessionService from 'ember-simple-auth/services/session'; +import { inject as service } from '@ember/service'; +import { formatDbName } from 'api/services/indexed-db'; + +export default class SessionService extends BaseSessionService { + @service indexedDb; + @service('browser/window') window; + + /** + * Extend ember simple auth's handleAuthentication method + * so we can hook in and setup the DB after a successful authentication + */ + handleAuthentication() { + super.handleAuthentication(...arguments); + + const userId = this.data?.authenticated?.user_id; + const hostUrl = this.window.host; + if (userId && hostUrl) { + this.indexedDb.setup(formatDbName(userId, hostUrl)); + } + } +} diff --git a/ui/admin/ember-cli-build.js b/ui/admin/ember-cli-build.js index 9b3d0244f..ed72c5efa 100644 --- a/ui/admin/ember-cli-build.js +++ b/ui/admin/ember-cli-build.js @@ -34,8 +34,9 @@ module.exports = function (defaults) { }, }, // TODO: Update to 4.12 when deprecations are resolved + // as multiple things break when forcing compatability on > 4.6 emberData: { - compatWith: '4.4', + compatWith: '4.6', }, }); diff --git a/ui/admin/tests/unit/services/session-test.js b/ui/admin/tests/unit/services/session-test.js new file mode 100644 index 000000000..c542abb82 --- /dev/null +++ b/ui/admin/tests/unit/services/session-test.js @@ -0,0 +1,12 @@ +import { module, test } from 'qunit'; +import { setupTest } from 'admin/tests/helpers'; + +module('Unit | Service | session', function (hooks) { + setupTest(hooks); + + // TODO: Replace this with your real tests. + test('it exists', function (assert) { + let service = this.owner.lookup('service:session'); + assert.ok(service); + }); +}); diff --git a/yarn.lock b/yarn.lock index cac9184bc..f3ad11c70 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9635,6 +9635,11 @@ detect-node@^2.0.4: resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1" integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== +dexie@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/dexie/-/dexie-3.2.4.tgz#b22a9729be1102acb2eee16102ea6e2bc76454cf" + integrity sha512-VKoTQRSv7+RnffpOJ3Dh6ozknBqzWw/F3iqMdsZg958R0AS8AnY9x9d1lbwENr0gzeGJHXKcGhAMRaqys6SxqA== + dezalgo@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456"