CyberCoveLLC/threshold-ui
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 💡 Batch writes to indexedDB (#2752)

Browse Source 
✅ Closes: https://hashicorp.atlassian.net/browse/ICU-16795
This commit is contained in:
Zhihe Li authored and GitHub committed 2025-04-01 14:04:31 -04:00
1 parent 0f41a0a576
commit 834058ff45
2 files changed
+134 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

addons/api/addon/adapters/application.js
+7 -2
View File
@@ -115,7 +115,7 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
* @return {Promise} promise
*/
async query(store, schema, query) {
const queryObject = structuredClone(query);
const { batchLimit, ...queryObject } = query;
let result;
let data = [];
@@ -126,8 +126,10 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
result = await super.query(store, schema, query);
return prenormalizeArrayResponse(result);
}
// Run this loop as long as the response_type is delta,
// which indicates that there are more items in the list
// or the batch size has not been reached
do {
result = await super.query(store, schema, queryObject);
//add the result items to a data array
@@ -136,7 +138,10 @@ export default class ApplicationAdapter extends RESTAdapter.extend(
//pass in the list token for subsequent calls to fetch the remaining list items
queryObject.list_token = result.list_token;
}
} while (result && result.response_type === 'delta');
} while (
result?.response_type === 'delta' &&
(!batchLimit || data.length < batchLimit)
);
result.items = data;
return prenormalizeArrayResponse(result);
addons/api/addon/handlers/indexed-db-handler.js
+127 -67
View File
@@ -10,6 +10,8 @@ import { queryIndexedDb } from '../utils/indexed-db-query';
import { paginateResults } from '../utils/paginate-results';
import { hashCode } from '../utils/hash-code';
const BATCH_LIMIT = 10000;
/**
* Handler to sit in front of the API request layer
* so we can handle any caching with indexedDb first
@@ -52,84 +54,54 @@ export default class IndexedDbHandler {
if (!peekIndexedDB) {
const tokenKey = `${type}-${hashCode(remainingQuery)}`;
let payload;
let listToken;
let writeToIndexedDbPromise;
if (storeToken) {
listToken = await indexedDb.token.get(tokenKey);
const tokenObj = await indexedDb.token.get(tokenKey);
listToken = tokenObj?.token;
}
let payload;
try {
payload = await adapter.query(store, schema, {
...remainingQuery,
list_token: listToken?.token,
});
} catch (e) {
// If we get an invalid list token, we'll delete the token and
// clear the DB and try again without a token.
if (
e?.errors[0].status === 400 &&
e?.errors[0].code === 'invalid list token'
) {
await indexedDb[type].clear();
// Delete all tokens of the same resource type so we don't keep clearing the DB.
// Even if other tokens may still be valid, we might have cleared data that they
// depended on so we should clear all tokens of the same type when clearing the DB.
const tokenKeys = await indexedDb.token
.where(':id')
.startsWith(type)
.primaryKeys();
await indexedDb.token.bulkDelete(tokenKeys);
do {
try {
payload = await adapter.query(store, schema, {
...remainingQuery,
list_token: listToken,
batchLimit: BATCH_LIMIT,
});
// await the previous writeToIndexedDbPromise before writing to indexedDb again
if (writeToIndexedDbPromise) {
await writeToIndexedDbPromise;
}
} catch (err) {
payload = await this.retryQueryFailure({
err,
indexedDb,
type,
adapter,
store,
schema,
remainingQuery,
});
} else {
throw e;
}
}
listToken = payload.list_token;
// Store the token we just got back from the payload if it exists
if (payload.list_token && storeToken) {
await indexedDb.token.put({
id: tokenKey,
token: payload.list_token,
});
}
// Remove any records from the DB if the API indicates they've been deleted
if (payload.removed_ids?.length > 0) {
await indexedDb[type].bulkDelete(payload.removed_ids);
}
// This is a temporary fix of clearing the DB (specifically for auth-methods)
// since we are not storing the token we do not get back a list of removed_ids
// from the API call and we do not want deleted items from showing in list view.
if (!storeToken) {
await indexedDb[type].clear();
}
const normalizedPayload = serializer.normalizeResponse(
store,
schema,
payload,
null,
'query',
);
const { data: payloadData } = normalizedPayload;
// Store the new data we just got back from the API refresh
const items = payloadData.map((datum) =>
this.indexedDb.normalizeData({
data: datum,
cleanData: true,
schema,
// Don't await immediately and let the next query run while we write to indexedDb
writeToIndexedDbPromise = this.writeToIndexedDb({
payload,
storeToken,
indexedDb,
tokenKey,
type,
serializer,
}),
);
store,
schema,
});
} while (payload.response_type === 'delta');
await indexedDb[type].bulkPut(items);
await writeToIndexedDbPromise;
}
const indexedDbResults = await queryIndexedDb(
@@ -163,10 +135,98 @@ export default class IndexedDbHandler {
// or EmberArray since the ember store query method asserts it has to be an array
// so we can't just return an object.
records.meta = { totalItems: indexedDbResults.length };
return records;
}
default:
return next(context.request);
}
}
async retryQueryFailure({
err,
indexedDb,
type,
adapter,
store,
schema,
remainingQuery,
}) {
// If we get an invalid list token, we'll delete the token and
// clear the DB and try again without a token.
if (
err?.errors[0].status === 400 &&
err?.errors[0].code === 'invalid list token'
) {
await indexedDb[type].clear();
// Delete all tokens of the same resource type so we don't keep clearing the DB.
// Even if other tokens may still be valid, we might have cleared data that they
// depended on so we should clear all tokens of the same type when clearing the DB.
const tokenKeys = await indexedDb.token
.where(':id')
.startsWith(type)
.primaryKeys();
await indexedDb.token.bulkDelete(tokenKeys);
return await adapter.query(store, schema, {
...remainingQuery,
});
} else {
throw err;
}
}
async writeToIndexedDb({
payload,
storeToken,
indexedDb,
tokenKey,
type,
serializer,
store,
schema,
}) {
// Store the token we just got back from the payload if it exists
if (payload.list_token && storeToken) {
await indexedDb.token.put({
id: tokenKey,
token: payload.list_token,
});
}
// Remove any records from the DB if the API indicates they've been deleted
if (payload.removed_ids?.length > 0) {
await indexedDb[type].bulkDelete(payload.removed_ids);
}
// This is a temporary fix of clearing the DB (specifically for auth-methods)
// since we are not storing the token we do not get back a list of removed_ids
// from the API call and we do not want deleted items from showing in list view.
if (!storeToken) {
await indexedDb[type].clear();
}
const normalizedPayload = serializer.normalizeResponse(
store,
schema,
payload,
null,
'query',
);
const { data: payloadData } = normalizedPayload;
// Store the new data we just got back from the API refresh
const items = payloadData.map((datum) =>
this.indexedDb.normalizeData({
data: datum,
cleanData: true,
schema,
serializer,
}),
);
await indexedDb[type].bulkPut(items);
}
}