From 834058ff45dc35bce28939ed906480fac4f81813 Mon Sep 17 00:00:00 2001 From: Zhihe Li Date: Tue, 1 Apr 2025 14:04:31 -0400 Subject: [PATCH] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20Batch=20writes=20to?= =?UTF-8?q?=20indexedDB=20(#2752)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✅ Closes: https://hashicorp.atlassian.net/browse/ICU-16795 --- addons/api/addon/adapters/application.js | 9 +- .../api/addon/handlers/indexed-db-handler.js | 194 ++++++++++++------ 2 files changed, 134 insertions(+), 69 deletions(-) diff --git a/addons/api/addon/adapters/application.js b/addons/api/addon/adapters/application.js index aec4f99fb..6dd7daae7 100644 --- a/addons/api/addon/adapters/application.js +++ b/addons/api/addon/adapters/application.js @@ -115,7 +115,7 @@ export default class ApplicationAdapter extends RESTAdapter.extend( * @return {Promise} promise */ async query(store, schema, query) { - const queryObject = structuredClone(query); + const { batchLimit, ...queryObject } = query; let result; let data = []; @@ -126,8 +126,10 @@ export default class ApplicationAdapter extends RESTAdapter.extend( result = await super.query(store, schema, query); return prenormalizeArrayResponse(result); } + // Run this loop as long as the response_type is delta, // which indicates that there are more items in the list + // or the batch size has not been reached do { result = await super.query(store, schema, queryObject); //add the result items to a data array @@ -136,7 +138,10 @@ export default class ApplicationAdapter extends RESTAdapter.extend( //pass in the list token for subsequent calls to fetch the remaining list items queryObject.list_token = result.list_token; } - } while (result && result.response_type === 'delta'); + } while ( + result?.response_type === 'delta' && + (!batchLimit || data.length < batchLimit) + ); result.items = data; return prenormalizeArrayResponse(result); diff --git a/addons/api/addon/handlers/indexed-db-handler.js b/addons/api/addon/handlers/indexed-db-handler.js index fe29810a1..35f124e21 100644 --- a/addons/api/addon/handlers/indexed-db-handler.js +++ b/addons/api/addon/handlers/indexed-db-handler.js @@ -10,6 +10,8 @@ import { queryIndexedDb } from '../utils/indexed-db-query'; import { paginateResults } from '../utils/paginate-results'; import { hashCode } from '../utils/hash-code'; +const BATCH_LIMIT = 10000; + /** * Handler to sit in front of the API request layer * so we can handle any caching with indexedDb first @@ -52,84 +54,54 @@ export default class IndexedDbHandler { if (!peekIndexedDB) { const tokenKey = `${type}-${hashCode(remainingQuery)}`; - + let payload; let listToken; + let writeToIndexedDbPromise; + if (storeToken) { - listToken = await indexedDb.token.get(tokenKey); + const tokenObj = await indexedDb.token.get(tokenKey); + listToken = tokenObj?.token; } - let payload; - try { - payload = await adapter.query(store, schema, { - ...remainingQuery, - list_token: listToken?.token, - }); - } catch (e) { - // If we get an invalid list token, we'll delete the token and - // clear the DB and try again without a token. - if ( - e?.errors[0].status === 400 && - e?.errors[0].code === 'invalid list token' - ) { - await indexedDb[type].clear(); - // Delete all tokens of the same resource type so we don't keep clearing the DB. - // Even if other tokens may still be valid, we might have cleared data that they - // depended on so we should clear all tokens of the same type when clearing the DB. - const tokenKeys = await indexedDb.token - .where(':id') - .startsWith(type) - .primaryKeys(); - - await indexedDb.token.bulkDelete(tokenKeys); - + do { + try { payload = await adapter.query(store, schema, { ...remainingQuery, + list_token: listToken, + batchLimit: BATCH_LIMIT, + }); + + // await the previous writeToIndexedDbPromise before writing to indexedDb again + if (writeToIndexedDbPromise) { + await writeToIndexedDbPromise; + } + } catch (err) { + payload = await this.retryQueryFailure({ + err, + indexedDb, + type, + adapter, + store, + schema, + remainingQuery, }); - } else { - throw e; } - } + listToken = payload.list_token; - // Store the token we just got back from the payload if it exists - if (payload.list_token && storeToken) { - await indexedDb.token.put({ - id: tokenKey, - token: payload.list_token, - }); - } - - // Remove any records from the DB if the API indicates they've been deleted - if (payload.removed_ids?.length > 0) { - await indexedDb[type].bulkDelete(payload.removed_ids); - } - // This is a temporary fix of clearing the DB (specifically for auth-methods) - // since we are not storing the token we do not get back a list of removed_ids - // from the API call and we do not want deleted items from showing in list view. - if (!storeToken) { - await indexedDb[type].clear(); - } - - const normalizedPayload = serializer.normalizeResponse( - store, - schema, - payload, - null, - 'query', - ); - - const { data: payloadData } = normalizedPayload; - - // Store the new data we just got back from the API refresh - const items = payloadData.map((datum) => - this.indexedDb.normalizeData({ - data: datum, - cleanData: true, - schema, + // Don't await immediately and let the next query run while we write to indexedDb + writeToIndexedDbPromise = this.writeToIndexedDb({ + payload, + storeToken, + indexedDb, + tokenKey, + type, serializer, - }), - ); + store, + schema, + }); + } while (payload.response_type === 'delta'); - await indexedDb[type].bulkPut(items); + await writeToIndexedDbPromise; } const indexedDbResults = await queryIndexedDb( @@ -163,10 +135,98 @@ export default class IndexedDbHandler { // or EmberArray since the ember store query method asserts it has to be an array // so we can't just return an object. records.meta = { totalItems: indexedDbResults.length }; + return records; } default: return next(context.request); } } + + async retryQueryFailure({ + err, + indexedDb, + type, + adapter, + store, + schema, + remainingQuery, + }) { + // If we get an invalid list token, we'll delete the token and + // clear the DB and try again without a token. + if ( + err?.errors[0].status === 400 && + err?.errors[0].code === 'invalid list token' + ) { + await indexedDb[type].clear(); + // Delete all tokens of the same resource type so we don't keep clearing the DB. + // Even if other tokens may still be valid, we might have cleared data that they + // depended on so we should clear all tokens of the same type when clearing the DB. + const tokenKeys = await indexedDb.token + .where(':id') + .startsWith(type) + .primaryKeys(); + + await indexedDb.token.bulkDelete(tokenKeys); + + return await adapter.query(store, schema, { + ...remainingQuery, + }); + } else { + throw err; + } + } + + async writeToIndexedDb({ + payload, + storeToken, + indexedDb, + tokenKey, + type, + serializer, + store, + schema, + }) { + // Store the token we just got back from the payload if it exists + if (payload.list_token && storeToken) { + await indexedDb.token.put({ + id: tokenKey, + token: payload.list_token, + }); + } + + // Remove any records from the DB if the API indicates they've been deleted + if (payload.removed_ids?.length > 0) { + await indexedDb[type].bulkDelete(payload.removed_ids); + } + + // This is a temporary fix of clearing the DB (specifically for auth-methods) + // since we are not storing the token we do not get back a list of removed_ids + // from the API call and we do not want deleted items from showing in list view. + if (!storeToken) { + await indexedDb[type].clear(); + } + + const normalizedPayload = serializer.normalizeResponse( + store, + schema, + payload, + null, + 'query', + ); + + const { data: payloadData } = normalizedPayload; + + // Store the new data we just got back from the API refresh + const items = payloadData.map((datum) => + this.indexedDb.normalizeData({ + data: datum, + cleanData: true, + schema, + serializer, + }), + ); + + await indexedDb[type].bulkPut(items); + } }