feat: 🎸 Allow building and bundling of web workers in ember (#2923)

* feat: 🎸 Allow building and bundling of web workers in ember

* refactor: 💡 Cleanup code and add comments

addons/api/.eslintrc.js

@@ -45,6 +45,7 @@ module.exports = {
         './blueprints/*/index.js',
         './config/**/*.js',
         './tests/dummy/config/**/*.js',
+        './broccoli-plugins/**/*.js',
       ],
       parserOptions: {
         sourceType: 'script',

addons/api/addon/handlers/sqlite-handler.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { getOwner, setOwner } from '@ember/owner';
+
+export default class SqliteHandler {
+  constructor(context) {
+    setOwner(this, getOwner(context));
+  }
+
+  async request(context, next) {
+    switch (context.request.op) {
+      default:
+        return next(context.request);
+    }
+  }
+}

addons/api/addon/services/sqlite-db.js

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Service from '@ember/service';
+import { PWBHost } from 'promise-worker-bi';
+
+export default class SqliteDbService extends Service {
+  // =attributes
+
+  // We need a handle on the original web worker to properly clean it up.
+  webWorker;
+  // This will be the worker we interact with that is wrapped by PWBHost.
+  worker;
+
+  // TODO: Actually use the name that gets passed into setup later
+  setup() {
+    if (this.worker) {
+      // If the worker already exists, we don't need to do anything.
+      return;
+    }
+
+    // In production, our JS files will be fingerprinted with am MD5 hash
+    // so we need a way to map the worker file name to be able to find it.
+    //
+    // Ember is automagically replacing any URL path with the fingerprinted
+    // asset, and this is done by broccoli-asset-rev.
+    //
+    // The problem is it's a bit finicky as seen in this issue
+    // https://github.com/ember-cli/broccoli-asset-rewrite/issues/66
+    // and will not work properly if it's part of a template string or interpolated.
+    // Therefore, we need the URL path in a string somewhere to be replaced properly.
+    this.webWorker = new Worker('/workers/web-worker.js', {
+      type: 'module',
+    });
+
+    // We use PWBHost to simplify communication so we can just send and receive
+    // messages with promises rather than having to manually set up event listeners
+    this.worker = new PWBHost(this.webWorker);
+  }
+
+  willDestroy() {
+    this.webWorker?.terminate();
+  }
+}

addons/api/addon/services/store.js

@@ -11,6 +11,7 @@ import { CacheHandler } from '@ember-data/store';
 import RequestManager from '@ember-data/request';
 import { LegacyNetworkHandler } from '@ember-data/legacy-compat';
 import IndexedDbHandler from 'api/handlers/indexed-db-handler';
+import SqliteHandler from 'api/handlers/sqlite-handler';
 
 export default class extends Store {
   requestManager = new RequestManager();
@@ -19,8 +20,13 @@ export default class extends Store {
     super(args);
 
     const indexedDbHandler = new IndexedDbHandler(this);
+    const sqliteHandler = new SqliteHandler(this);
 
-    this.requestManager.use([indexedDbHandler, LegacyNetworkHandler]);
+    this.requestManager.use([
+      sqliteHandler,
+      indexedDbHandler,
+      LegacyNetworkHandler,
+    ]);
     this.requestManager.useCache(CacheHandler);
   }
 }

addons/api/addon/workers/web-worker.js

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { PWBWorker } from 'promise-worker-bi';
+import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
+
+const promiseWorker = new PWBWorker();
+promiseWorker.register((message) => {
+  console.log(`Received from main: ${message}`);
+});
+
+const initializeSQLite = async () => {
+  try {
+    const sqlite3 = await sqlite3InitModule({
+      print: console.log,
+      printErr: console.error,
+    });
+    console.log(`SQLite Version: ${sqlite3?.version.libVersion}`);
+    const poolUtil = await sqlite3.installOpfsSAHPoolVfs();
+    const db = new poolUtil.OpfsSAHPoolDb('/boundary');
+    console.log(db);
+  } catch (err) {
+    console.error('Initialization error:', err.name, err.message);
+  }
+};
+
+(async () => {
+  await initializeSQLite();
+})();

addons/api/app/services/sqlite-db.js

@@ -0,0 +1,6 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+export { default } from 'api/services/sqlite-db';

addons/api/broccoli-plugins/build-workers.js

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+const Plugin = require('broccoli-plugin');
+
+const path = require('path');
+const fs = require('fs');
+const esbuild = require('esbuild');
+
+module.exports = class BuildWorkers extends Plugin {
+  build() {
+    this.inputPaths.forEach((inputPath) => {
+      this.#buildWorkers(inputPath);
+    });
+  }
+
+  #buildWorkers(inputPath) {
+    const workers = this.#getWorkers(inputPath);
+
+    const workerBuilder = this.#configureWorkerBuilder({
+      isProduction: this.isProduction,
+      buildDir: this.outputPath,
+    });
+
+    Object.entries(workers).map(workerBuilder);
+  }
+
+  #getWorkers(inputPath) {
+    let workers = {};
+    let dir = fs.readdirSync(inputPath);
+
+    dir.forEach((name) => {
+      workers[name] = path.join(inputPath, name);
+    });
+
+    return workers;
+  }
+
+  #configureWorkerBuilder({ isProduction, buildDir }) {
+    return ([name, entryPath]) => {
+      esbuild.buildSync({
+        loader: { '.js': 'js' },
+        entryPoints: [entryPath],
+        bundle: true,
+        outfile: path.join(buildDir, name),
+        format: 'esm',
+        minify: isProduction,
+        sourcemap: !isProduction,
+      });
+    };
+  }
+};

addons/api/index.js

@@ -5,6 +5,68 @@
 
 'use strict';
 
+const { WatchedDir } = require('broccoli-source');
+const funnel = require('broccoli-funnel');
+const merge = require('broccoli-merge-trees');
+const path = require('path');
+
+const buildWorkers = require('./broccoli-plugins/build-workers');
+
+const workerRoot = path.join(__dirname, 'addon', 'workers');
+
 module.exports = {
   name: require('./package').name,
+
+  options: {
+    // Don't automatically import sqlite, we need to manually control how it gets used
+    autoImport: {
+      exclude: ['@sqlite.org/sqlite-wasm'],
+    },
+  },
+
+  treeForAddon() {
+    // Exclude anything in the workers folder from being bundled in the final
+    // build as we're manually bundling the files ourselves below.
+    const tree = this._super.treeForAddon.apply(this, arguments);
+    return funnel(tree, {
+      exclude: ['api/workers/**/*'],
+    });
+  },
+
+  treeForPublic(tree) {
+    const env = this.parent?.app?.env ?? 'production';
+    const { enableSqlite } = this.parent?.app?.options[this.name] ?? {};
+
+    // If sqlite is not enabled, we don't need to build the workers
+    if (!enableSqlite) {
+      return tree;
+    }
+
+    // We need to process and build our workers separately from ember as these
+    // workers are not part of the app and are not getting properly bundled otherwise
+    let workerTree = new WatchedDir(workerRoot);
+    workerTree = new buildWorkers([workerTree]);
+    workerTree.isProduction = env === 'production';
+
+    const workers = funnel(workerTree, {
+      destDir: 'workers',
+    });
+
+    // Also include the sqlite wasm file directly to be placed next to the workers
+    const sqliteWasmDirectoryPath = path.resolve(
+      require.resolve('@sqlite.org/sqlite-wasm/sqlite3.wasm'),
+      '..',
+    );
+    const wasmFile = funnel(sqliteWasmDirectoryPath, {
+      files: ['sqlite3.wasm'],
+      destDir: 'workers',
+    });
+
+    const trees = [workers, wasmFile];
+    if (tree) {
+      trees.push(tree);
+    }
+
+    return merge(trees);
+  },
 };

addons/api/package.json

@@ -27,6 +27,11 @@
   "dependencies": {
     "@babel/core": "^7.26.10",
     "@ember/string": "^4.0.0",
+    "@sqlite.org/sqlite-wasm": "^3.49.1-build4",
+    "broccoli-funnel": "^3.0.8",
+    "broccoli-merge-trees": "^4.2.0",
+    "broccoli-plugin": "^4.0.7",
+    "broccoli-source": "^3.0.1",
     "dexie": "^3.2.4",
     "ember-auto-import": "^2.10.0",
     "ember-can": "^4.2.0",
@@ -34,7 +39,9 @@
     "ember-cli-htmlbars": "^6.3.0",
     "ember-cli-mirage": "^3.0.3",
     "ember-data": "~5.3.12",
+    "esbuild": "^0.25.4",
     "miragejs": "^0.1.48",
+    "promise-worker-bi": "^5.0.1",
     "tracked-built-ins": "^4.0.0"
   },
   "devDependencies": {

addons/api/tests/unit/services/sqlite-db-test.js

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { module, test } from 'qunit';
+import { setupTest } from 'dummy/tests/helpers';
+
+module('Unit | Service | sqlite-db', function (hooks) {
+  setupTest(hooks);
+
+  // TODO: Replace this with your real tests.
+  test('it exists', function (assert) {
+    let service = this.owner.lookup('service:sqlite-db');
+    assert.ok(service);
+  });
+});

package.json

@@ -58,6 +58,7 @@
     "ignoredBuiltDependencies": [
       "@parcel/watcher",
       "core-js",
+      "esbuild",
       "tesseract.js",
       "@nestjs/core",
       "@openapitools/openapi-generator-cli"

ui/admin/app/routes/application.js

@@ -22,6 +22,7 @@ export default class ApplicationRoute extends Route {
   @service features;
   @service featureEdition;
   @service indexedDb;
+  @service sqliteDb;
   @service('browser/window') window;
 
   // =attributes
@@ -73,6 +74,7 @@ export default class ApplicationRoute extends Route {
       const userId = this.session.data?.authenticated?.user_id;
       const hostUrl = this.window.location.host;
       if (userId && hostUrl) {
+        await this.sqliteDb.setup(formatDbName(userId, hostUrl));
         await this.indexedDb.setup(formatDbName(userId, hostUrl));
       }
     }

ui/admin/app/services/session.js

@@ -9,6 +9,7 @@ import { formatDbName } from 'api/services/indexed-db';
 
 export default class SessionService extends BaseSessionService {
   @service indexedDb;
+  @service sqliteDb;
   @service('browser/window') window;
 
   /**
@@ -19,6 +20,7 @@ export default class SessionService extends BaseSessionService {
     const userId = this.data?.authenticated?.user_id;
     const hostUrl = this.window.location?.host;
     if (userId && hostUrl) {
+      await this.sqliteDb.setup(formatDbName(userId, hostUrl));
       await this.indexedDb.setup(formatDbName(userId, hostUrl));
     }
 

ui/admin/ember-cli-build.js

@@ -28,6 +28,9 @@ module.exports = async function (defaults) {
     svg: {
       paths: ['../../addons/core/public'],
     },
+    api: {
+      enableSqlite: true,
+    },
   });
 
   // TODO: The deprecations object can be removed in ember-data 6.0.