feat: 🎸 Add sqlite handler and parser (#2933)
* feat: 🎸 Add sqlite handler and parser ✅ Closes: https://hashicorp.atlassian.net/browse/ICU-17410
This commit is contained in:
Zhihe Li committed
1 parent
8abed1a809
commit
e0982da744
33 files changed
+1692
-111
No files matched your search
@@ -4,16 +4,203 @@
|
||||
*/
|
||||
|
||||
import { getOwner, setOwner } from '@ember/owner';
|
||||
import { service } from '@ember/service';
|
||||
import { modelMapping } from 'api/services/sqlite';
|
||||
import { get } from '@ember/object';
|
||||
import { typeOf } from '@ember/utils';
|
||||
import { hashCode } from '../utils/hash-code';
|
||||
import { generateSQLExpressions } from '../utils/sqlite-query';
|
||||
|
||||
export default class SqliteHandler {
|
||||
@service sqlite;
|
||||
|
||||
batchLimit = 5_000;
|
||||
|
||||
constructor(context) {
|
||||
setOwner(this, getOwner(context));
|
||||
}
|
||||
|
||||
async request(context, next) {
|
||||
switch (context.request.op) {
|
||||
case 'query': {
|
||||
const { store, data } = context.request;
|
||||
const {
|
||||
type,
|
||||
query,
|
||||
options: {
|
||||
pushToStore = true,
|
||||
peekDb = false,
|
||||
storeToken = true,
|
||||
} = {},
|
||||
} = data;
|
||||
const supportedModels = Object.keys(modelMapping);
|
||||
|
||||
// Go through normal flow if we don't yet support the model
|
||||
// or if we don't have a sqlite db instance
|
||||
if (!supportedModels.includes(type) || !this.sqlite) {
|
||||
return next(context.request);
|
||||
}
|
||||
|
||||
const adapter = store.adapterFor(type);
|
||||
const schema = store.modelFor(type);
|
||||
const serializer = store.serializerFor(type);
|
||||
|
||||
let { page, pageSize, query: queryObj, ...remainingQuery } = query;
|
||||
let payload,
|
||||
listToken,
|
||||
writeToDbPromise,
|
||||
totalInsert = 0;
|
||||
|
||||
if (!peekDb) {
|
||||
const tokenKey = `${type}-${hashCode(remainingQuery)}`;
|
||||
if (storeToken) {
|
||||
const [tokenObj] = await this.sqlite.fetchResource({
|
||||
...generateSQLExpressions('token', {
|
||||
filters: { id: [{ equals: tokenKey }] },
|
||||
}),
|
||||
});
|
||||
listToken = tokenObj?.token;
|
||||
}
|
||||
|
||||
do {
|
||||
try {
|
||||
payload = await adapter.query(store, schema, {
|
||||
...remainingQuery,
|
||||
list_token: listToken,
|
||||
batchLimit: this.batchLimit,
|
||||
});
|
||||
totalInsert += payload.items?.length ?? 0;
|
||||
|
||||
// await the previous writeToDbPromise before writing to db again
|
||||
if (writeToDbPromise) {
|
||||
await writeToDbPromise;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error retrieving data:', err);
|
||||
throw err;
|
||||
}
|
||||
listToken = payload.list_token;
|
||||
|
||||
writeToDbPromise = this.writeToDb(
|
||||
payload,
|
||||
storeToken,
|
||||
tokenKey,
|
||||
serializer,
|
||||
store,
|
||||
schema,
|
||||
type,
|
||||
);
|
||||
} while (payload.response_type === 'delta');
|
||||
|
||||
await writeToDbPromise;
|
||||
}
|
||||
|
||||
// TODO: Remove all timers once we're ready to merge back to main
|
||||
console.time(`SQLite fetch ${type}`);
|
||||
|
||||
if (totalInsert > 0) {
|
||||
// If there were any inserts, let sqlite handle running analyze on the DB
|
||||
await this.sqlite.analyzeDatabase();
|
||||
console.timeLog(`SQLite fetch ${type}`, 'analyze');
|
||||
}
|
||||
|
||||
const { sql, parameters } = generateSQLExpressions(type, queryObj, {
|
||||
page,
|
||||
pageSize,
|
||||
select: ['data'],
|
||||
});
|
||||
|
||||
const rows = await this.sqlite.fetchResource({
|
||||
sql,
|
||||
parameters,
|
||||
});
|
||||
console.timeLog(`SQLite fetch ${type}`, 'rows');
|
||||
|
||||
const { sql: countSql, parameters: countParams } =
|
||||
generateSQLExpressions(type, queryObj, {
|
||||
select: ['count(*) as total'],
|
||||
});
|
||||
const count = await this.sqlite.fetchResource({
|
||||
sql: countSql,
|
||||
parameters: countParams,
|
||||
});
|
||||
console.timeLog(`SQLite fetch ${type}`, 'count');
|
||||
|
||||
const results = rows.map((item) => JSON.parse(item.data));
|
||||
|
||||
// If we are not pushing to the store, use the raw data with id property
|
||||
const records = pushToStore
|
||||
? store.push({ data: results })
|
||||
: results.map((record) => ({
|
||||
...record.attributes,
|
||||
id: record.id,
|
||||
}));
|
||||
|
||||
// Set a meta property on the array to store the total items of the results.
|
||||
// This isn't conventional but is better than returning an ArrayProxy
|
||||
// or EmberArray since the ember store query method asserts it has to be an array
|
||||
// so we can't just return an object.
|
||||
console.timeEnd(`SQLite fetch ${type}`);
|
||||
records.meta = { totalItems: count[0].total };
|
||||
return records;
|
||||
}
|
||||
default:
|
||||
return next(context.request);
|
||||
}
|
||||
}
|
||||
|
||||
async writeToDb(
|
||||
payload,
|
||||
storeToken,
|
||||
tokenKey,
|
||||
serializer,
|
||||
store,
|
||||
schema,
|
||||
type,
|
||||
) {
|
||||
if (payload.list_token && storeToken) {
|
||||
await this.sqlite.insertResource('token', [
|
||||
[tokenKey, payload.list_token],
|
||||
]);
|
||||
}
|
||||
|
||||
// Remove any records from the DB if the API indicates they've been deleted
|
||||
if (payload.removed_ids?.length > 0) {
|
||||
await this.sqlite.deleteResource(type, payload.removed_ids);
|
||||
}
|
||||
|
||||
const normalizedPayload = serializer.normalizeResponse(
|
||||
store,
|
||||
schema,
|
||||
payload,
|
||||
null,
|
||||
'query',
|
||||
);
|
||||
|
||||
const { data: payloadData } = normalizedPayload;
|
||||
|
||||
// Store the new data we just got back from the API refresh
|
||||
const items = payloadData.map((datum) => {
|
||||
const params = Object.values(modelMapping[type]).map((key) => {
|
||||
let value;
|
||||
|
||||
if (key === 'id') {
|
||||
value = datum.id;
|
||||
} else {
|
||||
value = get(datum, `attributes.${key}`);
|
||||
}
|
||||
|
||||
if (typeOf(value) === 'date') {
|
||||
return value.toISOString();
|
||||
}
|
||||
return value;
|
||||
});
|
||||
|
||||
return [...params, JSON.stringify(datum)];
|
||||
});
|
||||
|
||||
if (items.length > 0) console.time(`SQLite insert`);
|
||||
await this.sqlite.insertResource(type, items);
|
||||
if (items.length > 0) console.timeEnd(`SQLite insert`);
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
/**
|
||||
* Copyright (c) HashiCorp, Inc.
|
||||
* SPDX-License-Identifier: BUSL-1.1
|
||||
*/
|
||||
|
||||
import Service from '@ember/service';
|
||||
import { PWBHost } from 'promise-worker-bi';
|
||||
|
||||
export default class SqliteDbService extends Service {
|
||||
// =attributes
|
||||
|
||||
// We need a handle on the original web worker to properly clean it up.
|
||||
webWorker;
|
||||
// This will be the worker we interact with that is wrapped by PWBHost.
|
||||
worker;
|
||||
|
||||
// TODO: Actually use the name that gets passed into setup later
|
||||
setup() {
|
||||
if (this.worker) {
|
||||
// If the worker already exists, we don't need to do anything.
|
||||
return;
|
||||
}
|
||||
|
||||
// In production, our JS files will be fingerprinted with an MD5 hash
|
||||
// so we need a way to map the worker file name to be able to find it.
|
||||
//
|
||||
// Ember is automagically replacing any URL path with the fingerprinted
|
||||
// asset, and this is done by broccoli-asset-rev.
|
||||
//
|
||||
// The problem is it's a bit finicky as seen in this issue
|
||||
// https://github.com/ember-cli/broccoli-asset-rewrite/issues/66
|
||||
// and will not work properly if it's part of a template string or interpolated.
|
||||
// Therefore, we need the URL path in a string somewhere to be replaced properly.
|
||||
this.webWorker = new Worker('/workers/sqlite-worker.js', {
|
||||
type: 'module',
|
||||
});
|
||||
|
||||
// We use PWBHost to simplify communication so we can just send and receive
|
||||
// messages with promises rather than having to manually set up event listeners
|
||||
this.worker = new PWBHost(this.webWorker);
|
||||
}
|
||||
|
||||
willDestroy() {
|
||||
this.webWorker?.terminate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Copyright (c) HashiCorp, Inc.
|
||||
* SPDX-License-Identifier: BUSL-1.1
|
||||
*/
|
||||
|
||||
import Service from '@ember/service';
|
||||
import { PWBHost } from 'promise-worker-bi';
|
||||
|
||||
// A mapping of columns to their expected model attributes for supported models
|
||||
// The JSON data column is not included here but is assumed to always be last.
|
||||
export const modelMapping = {
|
||||
target: {
|
||||
id: 'id',
|
||||
type: 'type',
|
||||
name: 'name',
|
||||
description: 'description',
|
||||
address: 'address',
|
||||
scope_id: 'scope.scope_id',
|
||||
created_time: 'created_time',
|
||||
},
|
||||
};
|
||||
|
||||
// A list of tables that we support searching using FTS5 in SQLite.
|
||||
export const searchTables = new Set(['target']);
|
||||
|
||||
export default class SqliteDbService extends Service {
|
||||
// =attributes
|
||||
|
||||
// We need a handle on the original web worker to properly clean it up.
|
||||
webWorker;
|
||||
// This will be the worker we interact with that is wrapped by PWBHost.
|
||||
worker;
|
||||
|
||||
setup(dbName) {
|
||||
if (this.worker) {
|
||||
// If the worker already exists, we don't need to do anything.
|
||||
return;
|
||||
}
|
||||
|
||||
// In production, our JS files will be fingerprinted with an MD5 hash
|
||||
// so we need a way to map the worker file name to be able to find it.
|
||||
//
|
||||
// Ember is automagically replacing any URL path with the fingerprinted
|
||||
// asset, and this is done by broccoli-asset-rev.
|
||||
//
|
||||
// The problem is it's a bit finicky as seen in this issue
|
||||
// https://github.com/ember-cli/broccoli-asset-rewrite/issues/66
|
||||
// and will not work properly if it's part of a template string or interpolated.
|
||||
// Therefore, we need the URL path in a string somewhere to be replaced properly.
|
||||
this.webWorker = new Worker('/workers/sqlite-worker.js', {
|
||||
type: 'module',
|
||||
});
|
||||
|
||||
// We use PWBHost to simplify communication so we can just send and receive
|
||||
// messages with promises rather than having to manually set up event listeners
|
||||
this.worker = new PWBHost(this.webWorker);
|
||||
this.worker.register(async ({ method }) => {
|
||||
if (method === 'getDatabaseName') {
|
||||
return dbName;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fetchResource({ sql, parameters }) {
|
||||
return this.worker.postMessage({
|
||||
method: 'fetchResource',
|
||||
payload: { sql, parameters },
|
||||
});
|
||||
}
|
||||
|
||||
insertResource(resource, items) {
|
||||
return this.worker.postMessage({
|
||||
method: 'insertResource',
|
||||
payload: { resource, items, modelMapping },
|
||||
});
|
||||
}
|
||||
|
||||
deleteResource(resource, ids) {
|
||||
return this.worker.postMessage({
|
||||
method: 'deleteResource',
|
||||
payload: { resource, ids },
|
||||
});
|
||||
}
|
||||
|
||||
analyzeDatabase() {
|
||||
return this.worker.postMessage({ method: 'analyzeDatabase' });
|
||||
}
|
||||
|
||||
async downloadDatabase() {
|
||||
const byteArray = await this.worker.postMessage({
|
||||
method: 'downloadDatabase',
|
||||
});
|
||||
|
||||
const blob = new Blob([byteArray.buffer], {
|
||||
type: 'application/x-sqlite3',
|
||||
});
|
||||
|
||||
// Create temporary elements to initialize the download
|
||||
const a = window.document.createElement('a');
|
||||
window.document.body.appendChild(a);
|
||||
a.href = window.URL.createObjectURL(blob);
|
||||
a.download = 'boundary.db';
|
||||
a.addEventListener('click', function () {
|
||||
setTimeout(function () {
|
||||
window.URL.revokeObjectURL(a.href);
|
||||
a.remove();
|
||||
}, 500);
|
||||
});
|
||||
a.click();
|
||||
}
|
||||
|
||||
async clearDatabase() {
|
||||
await this.worker.postMessage({ method: 'clearDatabase' });
|
||||
}
|
||||
|
||||
async deleteDatabase() {
|
||||
await this.worker.postMessage({ method: 'deleteDatabase' });
|
||||
}
|
||||
|
||||
willDestroy() {
|
||||
this.webWorker?.terminate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* Copyright (c) HashiCorp, Inc.
|
||||
* SPDX-License-Identifier: BUSL-1.1
|
||||
*/
|
||||
|
||||
import { modelMapping } from 'api/services/sqlite';
|
||||
import { searchTables } from 'api/services/sqlite';
|
||||
import { typeOf } from '@ember/utils';
|
||||
|
||||
/**
|
||||
* Takes a POJO representing a filter query and builds a SQL query.
|
||||
*
|
||||
* @example
|
||||
* const filter = generateSQLExpressions(type, {
|
||||
* search: generated',
|
||||
* filters: {
|
||||
* scope_id: {
|
||||
* logicalOperator: 'and',
|
||||
* values: [{ notEquals: 'scope1' }, { notEquals: 'scope2' }],
|
||||
* },
|
||||
* },
|
||||
* });
|
||||
*/
|
||||
export function generateSQLExpressions(
|
||||
resource,
|
||||
query = {},
|
||||
{ page, pageSize, select } = {},
|
||||
) {
|
||||
let { search, filters, sort = {} } = query;
|
||||
const conditions = [];
|
||||
const parameters = [];
|
||||
|
||||
addFilterConditions(filters, parameters, conditions);
|
||||
addSearchConditions(search, resource, parameters, conditions);
|
||||
|
||||
const { attribute, direction } = sort;
|
||||
let orderByClause = '';
|
||||
if (attribute) {
|
||||
// We have to check if the attribute is valid for the resource
|
||||
// as we can't use parameterized queries for ORDER BY
|
||||
const validAttributes = Object.keys(modelMapping[resource]);
|
||||
if (validAttributes.includes(attribute)) {
|
||||
orderByClause = `ORDER BY ${attribute} ${direction === 'desc' ? 'DESC' : 'ASC'}`;
|
||||
}
|
||||
} else if (modelMapping[resource]?.created_time) {
|
||||
orderByClause = `ORDER BY created_time DESC`;
|
||||
}
|
||||
|
||||
const whereClause = conditions.length ? `WHERE ${and(conditions)}` : '';
|
||||
|
||||
const paginationClause = page && pageSize ? `LIMIT ? OFFSET ?` : '';
|
||||
if (paginationClause) {
|
||||
parameters.push(pageSize, (page - 1) * pageSize);
|
||||
}
|
||||
|
||||
const selectClause = `SELECT ${select ? select.join(', ') : '*'} FROM ${resource}`;
|
||||
|
||||
return {
|
||||
// Replace any empty newlines or leading whitespace on each line to be consistent with formatting
|
||||
// This is mainly to help us read and test the generated SQL as it has no effect on the actual SQL execution.
|
||||
sql: `
|
||||
${selectClause}
|
||||
${whereClause}
|
||||
${orderByClause}
|
||||
${paginationClause}`
|
||||
.replace(/^\s+/gm, '')
|
||||
.trim(),
|
||||
parameters,
|
||||
};
|
||||
}
|
||||
|
||||
function addFilterConditions(filters, parameters, conditions) {
|
||||
if (!filters) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, filterArrayOrObject] of Object.entries(filters)) {
|
||||
// Filter can be an array or an object with a `values` property
|
||||
const filterValueArray = Array.isArray(filterArrayOrObject)
|
||||
? filterArrayOrObject
|
||||
: filterArrayOrObject.values;
|
||||
|
||||
if (!filterValueArray || !filterValueArray.length) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const filterConditions = filterValueArray
|
||||
.filter((f) => f)
|
||||
.map((filterObjValue) => {
|
||||
let [operation, value] = Object.entries(filterObjValue)[0];
|
||||
|
||||
// SQLite needs to be working with ISO strings
|
||||
if (typeOf(value) === 'date') {
|
||||
value = value.toISOString();
|
||||
}
|
||||
|
||||
// Handle LIKE operator separately
|
||||
if (operation === 'contains') {
|
||||
parameters.push(`%${value}%`);
|
||||
} else {
|
||||
parameters.push(value);
|
||||
}
|
||||
|
||||
return `${key}${OPERATORS[operation]}`;
|
||||
});
|
||||
|
||||
const { logicalOperator } = filterArrayOrObject;
|
||||
// Default to `or` if we don't have a logical operator
|
||||
conditions.push(
|
||||
parenthetical(
|
||||
logicalOperator === 'and'
|
||||
? and(filterConditions)
|
||||
: or(filterConditions),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function addSearchConditions(search, resource, parameters, conditions) {
|
||||
if (!search) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (searchTables.has(resource)) {
|
||||
// Use the special prefix indicator "*" for full-text search
|
||||
parameters.push(`"${search}"*`);
|
||||
// Use a subquery to match against the FTS table with rowids as SQLite is
|
||||
// much more efficient with FTS queries when using rowids or MATCH (or both).
|
||||
// We could have also used a join here but a subquery is simpler.
|
||||
conditions.push(
|
||||
`rowid IN (SELECT rowid FROM ${resource}_fts WHERE ${resource}_fts MATCH ?)`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const fields = Object.keys(modelMapping[resource]);
|
||||
const searchConditions = parenthetical(
|
||||
or(
|
||||
fields.map((field) => {
|
||||
parameters.push(`%${search}%`);
|
||||
return `${field}${OPERATORS['contains']}`;
|
||||
}),
|
||||
),
|
||||
);
|
||||
conditions.push(searchConditions);
|
||||
}
|
||||
|
||||
// Comparison Operators
|
||||
const OPERATORS = {
|
||||
equals: ' = ?',
|
||||
notEquals: ' != ?',
|
||||
gt: ' > ?',
|
||||
gte: ' >= ?',
|
||||
lt: ' < ?',
|
||||
lte: ' <= ?',
|
||||
contains: ' LIKE ?',
|
||||
};
|
||||
|
||||
// Logical Operators
|
||||
const and = (clauses) => clauses.join(' AND ');
|
||||
const or = (clauses) => clauses.join(' OR ');
|
||||
const parenthetical = (expression) => (expression ? `(${expression})` : '');
|
||||
@@ -6,21 +6,171 @@
|
||||
import { PWBWorker } from 'promise-worker-bi';
|
||||
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
||||
import sharedServiceInit from './utils/shared-service';
|
||||
import {
|
||||
CLEAR_DB,
|
||||
CREATE_TABLES,
|
||||
DELETE_STATEMENT,
|
||||
INSERT_STATEMENTS,
|
||||
} from './utils/schema';
|
||||
|
||||
/** @type {import('@sqlite.org/sqlite-wasm').OpfsSAHPoolDatabase} */
|
||||
let db;
|
||||
/** @type {import('@sqlite.org/sqlite-wasm').SAHPoolUtil} */
|
||||
let poolUtil;
|
||||
/** @type {import('@sqlite.org/sqlite-wasm').Sqlite3Static} */
|
||||
let sqlite3;
|
||||
|
||||
// Maximum number of host parameters for SQLite prepared statements.
|
||||
// See "Maximum Number Of Host Parameters In A Single SQL Statement" in
|
||||
// https://www.sqlite.org/limits.html
|
||||
const MAX_HOST_PARAMETERS = 32766;
|
||||
const SCHEMA_VERSION = 1;
|
||||
|
||||
const methods = {
|
||||
initializeSQLite: async () => {
|
||||
try {
|
||||
const sqlite3 = await sqlite3InitModule({
|
||||
print: console.log,
|
||||
printErr: console.error,
|
||||
});
|
||||
console.log(`SQLite Version: ${sqlite3?.version.libVersion}`);
|
||||
const poolUtil = await sqlite3.installOpfsSAHPoolVfs();
|
||||
new poolUtil.OpfsSAHPoolDb('/boundary');
|
||||
} catch (err) {
|
||||
console.error('Initialization error:', err.name, err.message);
|
||||
sqlite3 = await sqlite3InitModule({
|
||||
print: console.log,
|
||||
printErr: console.error,
|
||||
});
|
||||
console.log(`SQLite Version: ${sqlite3?.version.libVersion}`);
|
||||
const dbName = await promiseWorker.postMessage({
|
||||
method: 'getDatabaseName',
|
||||
});
|
||||
|
||||
let done = false;
|
||||
while (!done) {
|
||||
try {
|
||||
poolUtil = await sqlite3.installOpfsSAHPoolVfs();
|
||||
db = new poolUtil.OpfsSAHPoolDb(dbName);
|
||||
|
||||
const [row] = db.exec('PRAGMA user_version;', { rowMode: 'object' });
|
||||
// Check if we're on a newer schema version and clear the database if so
|
||||
if (SCHEMA_VERSION > row.user_version) {
|
||||
db.exec(CLEAR_DB);
|
||||
}
|
||||
|
||||
db.exec(CREATE_TABLES(SCHEMA_VERSION));
|
||||
// Force an analyze on initial connection
|
||||
db.exec('PRAGMA optimize=0x10002;');
|
||||
done = true;
|
||||
} catch (err) {
|
||||
// If we get a SQLite3Error and the pool is at capacity, we'll assume the error is
|
||||
// because we hit the cap so we add more capacity and see if that solves it.
|
||||
if (
|
||||
err.name === 'SQLite3Error' &&
|
||||
poolUtil.getCapacity() === poolUtil.getFileCount()
|
||||
) {
|
||||
await poolUtil.addCapacity(1);
|
||||
|
||||
// If we have more than 20 files, that might indicate a problem so let's just
|
||||
// remove the VFS and wipe all the client data across users and restart.
|
||||
if (poolUtil.getCapacity() > 20) {
|
||||
await poolUtil.removeVfs();
|
||||
}
|
||||
} else {
|
||||
console.error('SQLite initialization error:', err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
analyzeDatabase: () => {
|
||||
db.exec('PRAGMA optimize');
|
||||
},
|
||||
clearDatabase: () => {
|
||||
db.exec(CLEAR_DB);
|
||||
db.exec(CREATE_TABLES(SCHEMA_VERSION));
|
||||
},
|
||||
deleteDatabase: () => {
|
||||
const name = db.dbFilename();
|
||||
if (db) {
|
||||
db.close();
|
||||
db = null;
|
||||
}
|
||||
|
||||
poolUtil.unlink(`/${name}`);
|
||||
},
|
||||
downloadDatabase: () => {
|
||||
return sqlite3.capi.sqlite3_js_db_export(db);
|
||||
},
|
||||
fetchResource: ({ sql, parameters = [] }) => {
|
||||
try {
|
||||
return db.exec({
|
||||
sql,
|
||||
bind: parameters,
|
||||
rowMode: 'object',
|
||||
});
|
||||
} catch (err) {
|
||||
// TODO: Add error handling to rest of methods and properly return the error back to caller
|
||||
console.error('Fetch resource error:', err, sql, parameters);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
insertResource: ({ resource, items, modelMapping }) => {
|
||||
if (items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const numberOfParameters = items.reduce((acc, item) => {
|
||||
return acc + item.length;
|
||||
}, 0);
|
||||
|
||||
// We want to use as many VALUES as part of the insert statement as it's quicker
|
||||
// than doing multiple inserts. If we have too many rows to insert, we'll manually
|
||||
// chunk them up to avoid hitting the maximum number of host parameters
|
||||
// and execute these separate insert statements in a transaction
|
||||
if (numberOfParameters > MAX_HOST_PARAMETERS) {
|
||||
const chunkSize = Math.floor(MAX_HOST_PARAMETERS / items[0].length);
|
||||
|
||||
return db.transaction(() => {
|
||||
const results = [];
|
||||
for (let i = 0; i < items.length; i += chunkSize) {
|
||||
const chunk = items.slice(i, i + chunkSize);
|
||||
const result = db.exec({
|
||||
sql: INSERT_STATEMENTS(resource, chunk, modelMapping),
|
||||
bind: chunk.flat(),
|
||||
rowMode: 'object',
|
||||
});
|
||||
results.push(result);
|
||||
}
|
||||
return results;
|
||||
});
|
||||
}
|
||||
|
||||
db.exec({
|
||||
sql: INSERT_STATEMENTS(resource, items, modelMapping),
|
||||
bind: items.flat(),
|
||||
rowMode: 'object',
|
||||
});
|
||||
},
|
||||
deleteResource: ({ resource, ids }) => {
|
||||
if (ids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Check if we have too many parameters for the deletion and chunk if so
|
||||
if (ids.length > MAX_HOST_PARAMETERS) {
|
||||
return db.transaction(() => {
|
||||
const results = [];
|
||||
for (let i = 0; i < ids.length; i += MAX_HOST_PARAMETERS) {
|
||||
const chunk = ids.slice(i, i + MAX_HOST_PARAMETERS);
|
||||
const result = db.exec({
|
||||
sql: DELETE_STATEMENT(resource, chunk),
|
||||
bind: chunk,
|
||||
rowMode: 'object',
|
||||
});
|
||||
results.push(result);
|
||||
}
|
||||
return results;
|
||||
});
|
||||
}
|
||||
|
||||
return db.exec({
|
||||
sql: DELETE_STATEMENT(resource, ids),
|
||||
bind: ids,
|
||||
rowMode: 'object',
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const promiseWorker = new PWBWorker();
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Copyright (c) HashiCorp, Inc.
|
||||
* SPDX-License-Identifier: BUSL-1.1
|
||||
*/
|
||||
|
||||
export const CLEAR_DB = `
|
||||
PRAGMA writable_schema = 1;
|
||||
DELETE FROM sqlite_master;
|
||||
PRAGMA writable_schema = 0;
|
||||
VACUUM;
|
||||
PRAGMA integrity_check;`;
|
||||
|
||||
export const CREATE_TABLES = (version) => `
|
||||
BEGIN;
|
||||
|
||||
PRAGMA user_version = ${version};
|
||||
|
||||
CREATE TABLE IF NOT EXISTS token (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
token TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS target (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
address TEXT,
|
||||
scope_id TEXT NOT NULL,
|
||||
created_time TEXT NOT NULL,
|
||||
data TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_target_scope_id_created_time ON target(scope_id, created_time DESC);
|
||||
|
||||
-- Create a contentless FTS table as we will only use the rowids.
|
||||
-- Note that this only creates the FTS index and cannot reference the target content
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS target_fts USING fts5(
|
||||
id,
|
||||
type,
|
||||
name,
|
||||
description,
|
||||
address,
|
||||
scope_id,
|
||||
created_time,
|
||||
content='',
|
||||
);
|
||||
|
||||
-- Create triggers to keep the FTS table in sync with the target table
|
||||
-- Note: The FTS table does not have an update trigger, as it's updated only on
|
||||
-- INSERT and DELETE as we do an INSERT OR REPLACE INTO on the table
|
||||
-- which replaces the row and doesn't execute an UPDATE trigger.
|
||||
CREATE TRIGGER IF NOT EXISTS target_ai AFTER INSERT ON target BEGIN
|
||||
INSERT INTO target_fts(
|
||||
id, type, name, description, address, scope_id, created_time
|
||||
) VALUES (
|
||||
new.id, new.type, new.name, new.description, new.address, new.scope_id, new.created_time
|
||||
);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS target_ad AFTER DELETE ON target BEGIN
|
||||
INSERT INTO target_fts(target_fts, rowid, id, type, name, description, address, scope_id, created_time)
|
||||
VALUES('delete', old.rowid, old.id, old.type, old.name, old.description, old.address, old.scope_id, old.created_time);
|
||||
END;
|
||||
|
||||
COMMIT;`;
|
||||
|
||||
export const INSERT_STATEMENTS = (resource, items, modelMapping) => {
|
||||
if (resource === 'token') {
|
||||
return `REPLACE INTO token (id, token) VALUES ${items.map(() => '(?, ?)').join(', ')};`;
|
||||
}
|
||||
|
||||
if (!modelMapping?.[resource]) {
|
||||
throw new Error(`modelMapping is required for ${resource} insertions.`);
|
||||
}
|
||||
|
||||
const columns = Object.keys(modelMapping[resource]);
|
||||
// Extra column for the data column
|
||||
const numColumns = Array(columns.length + 1);
|
||||
const placeholders = `(${numColumns.fill('?').join(', ')})`;
|
||||
|
||||
return `REPLACE INTO ${resource} (${columns.join(', ')}, data) VALUES ${items.map(() => placeholders).join(', ')};`;
|
||||
};
|
||||
|
||||
export const DELETE_STATEMENT = (resource, ids) =>
|
||||
`DELETE FROM ${resource} WHERE id IN (${ids.map(() => '?').join(',')})`;
|
||||
Reference in new issue
Block a user