feat: 🎸 Add ability to search other resources as well (#3021)

This commit adds the ability to search across multiple tables with joins which unlocks a better search experience in the case of tables that source their data from multiple tables.

addons/api/addon/utils/sqlite-query.js

@@ -211,26 +211,68 @@ function addSearchConditions({
   if (!search || !modelMapping[resource]) {
     return;
   }
+  const searchSelect = search?.select ?? 'rowid';
+  const searchSql = `SELECT ${searchSelect} FROM ${tableName}_fts WHERE ${tableName}_fts MATCH ?`;
+  const getParameter = (fields, text) =>
+    fields?.length > 0
+      ? or(fields.map((field) => `${field}:"${text}"*`))
+      : `"${text}"*`;
 
   // Use the special prefix indicator "*" for full-text search
   if (typeOf(search) === 'object') {
-    if (!search?.text) {
+    if (!search.text) {
       return;
     }
 
-    parameters.push(
-      or(search.fields.map((field) => `${field}:"${search.text}"*`)),
-    );
+    const parameter = getParameter(search.fields, search.text);
+    parameters.push(parameter);
   } else {
     parameters.push(`"${search}"*`);
   }
 
+  // If there are extra related searches on other tables, add them too
+  if (search?.relatedSearches?.length > 0) {
+    const relatedSearchQueries = [];
+
+    search.relatedSearches.forEach((relatedSearch) => {
+      const { resource: relatedResource, fields, join } = relatedSearch;
+      const relatedTableName = underscore(relatedResource);
+
+      const parameter = getParameter(fields, search.text);
+      parameters.push(parameter);
+
+      // Build the related FTS query with optional join
+      let relatedQuery = [];
+      relatedQuery.push(
+        `SELECT "${tableName}".${searchSelect} FROM ${relatedTableName}_fts`,
+      );
+
+      if (join) {
+        const { joinFrom = 'id', joinOn } = join;
+        relatedQuery.push(
+          `JOIN "${tableName}" ON "${tableName}".${joinFrom} = ${relatedResource}_fts.${joinOn}`,
+        );
+      }
+      relatedQuery.push(`WHERE ${relatedTableName}_fts MATCH ?`);
+
+      relatedSearchQueries.push(relatedQuery.join(' '));
+    });
+
+    // Add the original search first since we've already added the original parameter first
+    const conditionStatement = [searchSql, ...relatedSearchQueries].join(
+      '\nUNION ',
+    );
+    conditions.push(
+      `"${tableName}".${searchSelect} IN (${conditionStatement})`,
+    );
+
+    return;
+  }
+
   // Use a subquery to match against the FTS table with rowids as SQLite is
   // much more efficient with FTS queries when using rowids or MATCH (or both).
   // We could have also used a join here but a subquery is simpler.
-  conditions.push(
-    `"${tableName}".rowid IN (SELECT rowid FROM ${tableName}_fts WHERE ${tableName}_fts MATCH ?)`,
-  );
+  conditions.push(`"${tableName}".${searchSelect} IN (${searchSql})`);
 }
 
 function constructSelectClause(select = [{ field: '*' }], tableName) {

addons/api/addon/workers/sqlite-worker.js

@@ -28,7 +28,7 @@ import {
   // See "Maximum Number Of Host Parameters In A Single SQL Statement" in
   // https://www.sqlite.org/limits.html
   const MAX_HOST_PARAMETERS = 32766;
-  const SCHEMA_VERSION = 1;
+  const SCHEMA_VERSION = 2;
 
   // Some browsers do not allow calling getDirectory in private browsing modes even
   // if we're in a secure context. This will cause the SQLite setup to fail so we should

addons/api/addon/workers/utils/schema.js

@@ -25,8 +25,7 @@ const createTargetTables = `
 );
 CREATE INDEX IF NOT EXISTS idx_target_scope_id_created_time ON target(scope_id, created_time DESC);
 
--- Create a contentless FTS table as we will only use the rowids.
--- Note that this only creates the FTS index and cannot reference the target content
+-- Create a content FTS table that references the original table
 CREATE VIRTUAL TABLE IF NOT EXISTS target_fts USING fts5(
     id,
     type,
@@ -35,7 +34,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS target_fts USING fts5(
     address,
     scope_id,
     created_time,
-    content='',
+    content='target',
 );
 
 -- Create triggers to keep the FTS table in sync with the target table
@@ -68,6 +67,7 @@ CREATE TABLE IF NOT EXISTS alias (
     data TEXT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_alias_created_time ON alias(created_time DESC);
+CREATE INDEX IF NOT EXISTS idx_alias_destination_id ON alias(destination_id);
 
 CREATE VIRTUAL TABLE IF NOT EXISTS alias_fts USING fts5(
     id,
@@ -78,7 +78,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS alias_fts USING fts5(
     value,
     scope_id,
     created_time,
-    content='',
+    content='alias',
 );
 
 CREATE TRIGGER IF NOT EXISTS alias_ai AFTER INSERT ON alias BEGIN
@@ -111,7 +111,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS group_fts USING fts5(
     description,
     scope_id,
     created_time,
-    content='',
+    content='group',
 );
 
 CREATE TRIGGER IF NOT EXISTS group_ai AFTER INSERT ON "group" BEGIN
@@ -144,7 +144,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS role_fts USING fts5(
     description,
     scope_id,
     created_time,
-    content='',
+    content='role',
 );
 
 CREATE TRIGGER IF NOT EXISTS role_ai AFTER INSERT ON role BEGIN
@@ -177,7 +177,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS user_fts USING fts5(
     description,
     scope_id,
     created_time,
-    content='',
+    content='user',
 );
 
 CREATE TRIGGER IF NOT EXISTS user_ai AFTER INSERT ON user BEGIN
@@ -212,7 +212,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS credential_store_fts USING fts5(
     description,
     scope_id,
     created_time,
-    content='',
+    content='credential_store',
 );
 
 CREATE TRIGGER IF NOT EXISTS credential_store_ai AFTER INSERT ON credential_store BEGIN
@@ -247,7 +247,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS scope_fts USING fts5(
     description,
     scope_id,
     created_time,
-    content='',
+    content='scope',
 );
 
 CREATE TRIGGER IF NOT EXISTS scope_ai AFTER INSERT ON scope BEGIN
@@ -284,7 +284,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS auth_method_fts USING fts5(
     is_primary,
     scope_id,
     created_time,
-    content='',
+    content='auth_method',
 );
 
 CREATE TRIGGER IF NOT EXISTS auth_method_ai AFTER INSERT ON auth_method BEGIN
@@ -321,7 +321,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS host_catalog_fts USING fts5(
     plugin_name,
     scope_id,
     created_time,
-    content='',
+    content='host_catalog',
 );
 
 CREATE TRIGGER IF NOT EXISTS host_catalog_ai AFTER INSERT ON host_catalog BEGIN
@@ -374,7 +374,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS session_recording_fts USING fts5(
     target_scope_name,
     target_scope_parent_scope_id,
     created_time,
-    content='',
+    content='session_recording',
 );
 
 CREATE TRIGGER IF NOT EXISTS session_recording_ai AFTER INSERT ON session_recording BEGIN
@@ -413,7 +413,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS session_fts USING fts5(
     user_id,
     scope_id,
     created_time,
-    content='',
+    content='session',
 );
 
 CREATE TRIGGER IF NOT EXISTS session_ai AFTER INSERT ON session BEGIN

addons/api/tests/unit/utils/sqlite-query-test.js

@@ -419,6 +419,64 @@ module('Unit | Utility | sqlite-query', function (hooks) {
           ORDER BY "target".created_time DESC`,
         expectedParams: ['name:"favorite"* OR description:"favorite"*'],
       },
+      'search with related searches': {
+        query: {
+          search: {
+            text: 'dev',
+            select: 'id',
+            relatedSearches: [
+              {
+                resource: 'alias',
+                fields: ['name', 'description'],
+                join: {
+                  joinOn: 'destination_id',
+                  joinFrom: 'id',
+                },
+              },
+            ],
+          },
+        },
+        expectedSql: `
+          SELECT * FROM "target"
+          WHERE "target".id IN (SELECT id FROM target_fts WHERE target_fts MATCH ?
+                                UNION SELECT "target".id FROM alias_fts JOIN "target" ON "target".id = alias_fts.destination_id WHERE alias_fts MATCH ?)
+          ORDER BY "target".created_time DESC`,
+        expectedParams: ['"dev"*', 'name:"dev"* OR description:"dev"*'],
+      },
+      'search with multiple related searches': {
+        query: {
+          search: {
+            text: 'dev',
+            relatedSearches: [
+              {
+                resource: 'alias',
+                fields: ['name', 'description'],
+                join: {
+                  joinOn: 'destination_id',
+                },
+              },
+              {
+                resource: 'session',
+                fields: ['name'],
+                join: {
+                  joinOn: 'target_id',
+                },
+              },
+            ],
+          },
+        },
+        expectedSql: `
+          SELECT * FROM "target"
+          WHERE "target".rowid IN (SELECT rowid FROM target_fts WHERE target_fts MATCH ?
+                                   UNION SELECT "target".rowid FROM alias_fts JOIN "target" ON "target".id = alias_fts.destination_id WHERE alias_fts MATCH ?
+                                   UNION SELECT "target".rowid FROM session_fts JOIN "target" ON "target".id = session_fts.target_id WHERE session_fts MATCH ?)
+          ORDER BY "target".created_time DESC`,
+        expectedParams: [
+          '"dev"*',
+          'name:"dev"* OR description:"dev"*',
+          'name:"dev"*',
+        ],
+      },
     },
     function (assert, { query, expectedSql, expectedParams }) {
       const { sql, parameters } = generateSQLExpressions('target', query);