Zhihe Li
92f5a65c00
This commit adds the ability to search across multiple tables with joins which unlocks a better search experience in the case of tables that source their data from multiple tables.
409 lines
13 KiB
JavaScript
409 lines
13 KiB
JavaScript
/**
|
|
* Copyright (c) HashiCorp, Inc.
|
|
* SPDX-License-Identifier: BUSL-1.1
|
|
*/
|
|
|
|
import { modelMapping } from 'api/services/sqlite';
|
|
import { typeOf } from '@ember/utils';
|
|
import { underscore } from '@ember/string';
|
|
import { assert } from '@ember/debug';
|
|
|
|
/**
|
|
* Takes a POJO representing a filter query and builds a SQL query.
|
|
*
|
|
* @example
|
|
* const filter = generateSQLExpressions(type, {
|
|
* search: generated',
|
|
* filters: {
|
|
* scope_id: {
|
|
* logicalOperator: 'and',
|
|
* values: [{ notEquals: 'scope1' }, { notEquals: 'scope2' }],
|
|
* },
|
|
* },
|
|
* });
|
|
*/
|
|
export function generateSQLExpressions(
|
|
resource,
|
|
query = {},
|
|
{ page, pageSize, select } = {},
|
|
) {
|
|
let { search, filters, sort = {} } = query;
|
|
const conditions = [];
|
|
const parameters = [];
|
|
const tableName = underscore(resource);
|
|
const joins = [];
|
|
|
|
addFilterConditions({ filters, parameters, conditions, joins, tableName });
|
|
addSearchConditions({ search, resource, tableName, parameters, conditions });
|
|
|
|
const selectClause = constructSelectClause(select, tableName);
|
|
const joinClause = constructJoinClause(joins);
|
|
const orderByClause = constructOrderByClause(resource, tableName, sort);
|
|
const whereClause = conditions.length ? `WHERE ${and(conditions)}` : '';
|
|
|
|
const paginationClause = page && pageSize ? `LIMIT ? OFFSET ?` : '';
|
|
if (paginationClause) {
|
|
parameters.push(pageSize, (page - 1) * pageSize);
|
|
}
|
|
|
|
return {
|
|
// Replace any empty newlines or leading whitespace on each line to be consistent with formatting
|
|
// This is mainly to help us read and test the generated SQL as it has no effect on the actual SQL execution.
|
|
sql: `
|
|
${selectClause}
|
|
${joinClause}
|
|
${whereClause}
|
|
${orderByClause}
|
|
${paginationClause}`
|
|
.replace(/^\s+/gm, '')
|
|
.trim(),
|
|
parameters,
|
|
};
|
|
}
|
|
|
|
function addFilterConditions({
|
|
filters,
|
|
parameters,
|
|
conditions,
|
|
joins,
|
|
tableName,
|
|
}) {
|
|
if (!filters) {
|
|
return;
|
|
}
|
|
|
|
for (const [key, filterArrayOrObject] of Object.entries(filters)) {
|
|
// Filter can be an array or an object with a `values` property
|
|
const filterValueArray = Array.isArray(filterArrayOrObject)
|
|
? filterArrayOrObject
|
|
: filterArrayOrObject.values;
|
|
|
|
if (!filterValueArray || !filterValueArray.length || key === 'joins') {
|
|
continue;
|
|
}
|
|
|
|
// When attribute conditions are a series of equals/notEquals
|
|
// replace with in/notIn to avoid hitting the sqlite maximum
|
|
// expression tree depth.
|
|
const firstOperator = Object.keys(filterValueArray[0])[0];
|
|
const allOperatorsEqual = filterValueArray
|
|
.flatMap((item) => Object.keys(item))
|
|
.every((op) => op === firstOperator);
|
|
if (
|
|
filterValueArray.length > 1 &&
|
|
(firstOperator === 'equals' || firstOperator === 'notEquals') &&
|
|
allOperatorsEqual
|
|
) {
|
|
const operation = firstOperator === 'equals' ? 'in' : 'notIn';
|
|
const nullOperator =
|
|
firstOperator === 'equals' ? 'IS NULL' : 'IS NOT NULL';
|
|
const values = filterValueArray
|
|
.filter(
|
|
(filterObjValue) =>
|
|
filterObjValue && Object.values(filterObjValue)[0] !== null,
|
|
)
|
|
.map((filterObjValue) => {
|
|
let value = Object.values(filterObjValue)[0];
|
|
if (typeOf(value) === 'date') {
|
|
value = value.toISOString();
|
|
}
|
|
parameters.push(value);
|
|
return value;
|
|
});
|
|
|
|
// Add a check for null values as an IN clause will mistakenly use `=` for null values.
|
|
// We'll add an extra OR clause to handle nulls separately.
|
|
const isNullValue = filterValueArray.some(
|
|
(filterObjValue) => Object.values(filterObjValue)[0] === null,
|
|
);
|
|
|
|
const filterCondition = `"${tableName}".${key}${OPERATORS[operation](values)}${isNullValue ? ` OR "${tableName}".${key} ${nullOperator}` : ''}`;
|
|
conditions.push(parenthetical(filterCondition));
|
|
continue;
|
|
}
|
|
|
|
const filterConditions = filterValueArray
|
|
.filter((f) => f)
|
|
.map((filterObjValue) => {
|
|
let [operation, value] = Object.entries(filterObjValue)[0];
|
|
|
|
// Handle null values: convert equals/notEquals to IS NULL or IS NOT NULL
|
|
if (value === null) {
|
|
if (operation === 'equals') {
|
|
return `"${tableName}".${key} IS NULL`;
|
|
}
|
|
if (operation === 'notEquals') {
|
|
return `"${tableName}".${key} IS NOT NULL`;
|
|
}
|
|
}
|
|
|
|
// SQLite needs to be working with ISO strings
|
|
if (typeOf(value) === 'date') {
|
|
value = value.toISOString();
|
|
}
|
|
|
|
// Handle LIKE operator separately
|
|
if (operation === 'contains') {
|
|
parameters.push(`%${value}%`);
|
|
} else {
|
|
parameters.push(value);
|
|
}
|
|
|
|
return `"${tableName}".${key}${OPERATORS[operation]}`;
|
|
});
|
|
|
|
const { logicalOperator } = filterArrayOrObject;
|
|
// Default to `or` if we don't have a logical operator
|
|
conditions.push(
|
|
parenthetical(
|
|
logicalOperator === 'and'
|
|
? and(filterConditions)
|
|
: or(filterConditions),
|
|
),
|
|
);
|
|
}
|
|
|
|
if (filters.joins?.length > 0) {
|
|
const tableNameIndex = {};
|
|
|
|
filters.joins.forEach((join) => {
|
|
const {
|
|
resource,
|
|
query,
|
|
joinFrom = 'id',
|
|
joinOn,
|
|
joinType = 'INNER',
|
|
} = join;
|
|
const joinTableName = underscore(resource);
|
|
tableNameIndex[resource] ??= 1;
|
|
// Alias in the possible scenario we join the same table more than once
|
|
const alias = `${joinTableName}${tableNameIndex[resource]}`;
|
|
|
|
joins.push({
|
|
type: joinType,
|
|
table: joinTableName,
|
|
alias,
|
|
condition: `"${tableName}".${joinFrom} = ${alias}.${joinOn}`,
|
|
});
|
|
|
|
// Add the conditions from the join query
|
|
if (query?.filters) {
|
|
addFilterConditions({
|
|
filters: query.filters,
|
|
parameters,
|
|
conditions,
|
|
tableName: alias,
|
|
});
|
|
}
|
|
|
|
tableNameIndex[resource]++;
|
|
});
|
|
}
|
|
}
|
|
|
|
function addSearchConditions({
|
|
search,
|
|
resource,
|
|
tableName,
|
|
parameters,
|
|
conditions,
|
|
}) {
|
|
if (!search || !modelMapping[resource]) {
|
|
return;
|
|
}
|
|
const searchSelect = search?.select ?? 'rowid';
|
|
const searchSql = `SELECT ${searchSelect} FROM ${tableName}_fts WHERE ${tableName}_fts MATCH ?`;
|
|
const getParameter = (fields, text) =>
|
|
fields?.length > 0
|
|
? or(fields.map((field) => `${field}:"${text}"*`))
|
|
: `"${text}"*`;
|
|
|
|
// Use the special prefix indicator "*" for full-text search
|
|
if (typeOf(search) === 'object') {
|
|
if (!search.text) {
|
|
return;
|
|
}
|
|
|
|
const parameter = getParameter(search.fields, search.text);
|
|
parameters.push(parameter);
|
|
} else {
|
|
parameters.push(`"${search}"*`);
|
|
}
|
|
|
|
// If there are extra related searches on other tables, add them too
|
|
if (search?.relatedSearches?.length > 0) {
|
|
const relatedSearchQueries = [];
|
|
|
|
search.relatedSearches.forEach((relatedSearch) => {
|
|
const { resource: relatedResource, fields, join } = relatedSearch;
|
|
const relatedTableName = underscore(relatedResource);
|
|
|
|
const parameter = getParameter(fields, search.text);
|
|
parameters.push(parameter);
|
|
|
|
// Build the related FTS query with optional join
|
|
let relatedQuery = [];
|
|
relatedQuery.push(
|
|
`SELECT "${tableName}".${searchSelect} FROM ${relatedTableName}_fts`,
|
|
);
|
|
|
|
if (join) {
|
|
const { joinFrom = 'id', joinOn } = join;
|
|
relatedQuery.push(
|
|
`JOIN "${tableName}" ON "${tableName}".${joinFrom} = ${relatedResource}_fts.${joinOn}`,
|
|
);
|
|
}
|
|
relatedQuery.push(`WHERE ${relatedTableName}_fts MATCH ?`);
|
|
|
|
relatedSearchQueries.push(relatedQuery.join(' '));
|
|
});
|
|
|
|
// Add the original search first since we've already added the original parameter first
|
|
const conditionStatement = [searchSql, ...relatedSearchQueries].join(
|
|
'\nUNION ',
|
|
);
|
|
conditions.push(
|
|
`"${tableName}".${searchSelect} IN (${conditionStatement})`,
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
// Use a subquery to match against the FTS table with rowids as SQLite is
|
|
// much more efficient with FTS queries when using rowids or MATCH (or both).
|
|
// We could have also used a join here but a subquery is simpler.
|
|
conditions.push(`"${tableName}".${searchSelect} IN (${searchSql})`);
|
|
}
|
|
|
|
function constructSelectClause(select = [{ field: '*' }], tableName) {
|
|
const distinctColumns = select.filter(({ isDistinct }) => isDistinct);
|
|
const nonDistinctColumns = select.filter(({ isDistinct }) => !isDistinct);
|
|
|
|
const buildColumnExpression = ({ field, isCount, alias, isDistinct }) => {
|
|
let column = field === '*' ? field : `"${tableName}".${field}`;
|
|
|
|
// If there's just distinct with nothing else, this will be handled separately
|
|
// and we will just return the column back as is
|
|
if (isCount && isDistinct) {
|
|
column = `count(DISTINCT ${column})`;
|
|
} else if (isCount) {
|
|
column = `count(${column})`;
|
|
}
|
|
|
|
if (alias) {
|
|
column = `${column} as ${alias}`;
|
|
}
|
|
|
|
return column;
|
|
};
|
|
|
|
let selectColumns;
|
|
|
|
if (distinctColumns.length > 0) {
|
|
// Check if any columns also have COUNT (or other aggregate function in the future)
|
|
const hasAggregateDistinct = distinctColumns.some((col) => col.isCount);
|
|
|
|
if (hasAggregateDistinct) {
|
|
selectColumns = distinctColumns.map(buildColumnExpression).join(', ');
|
|
|
|
// Add back in the non distinct columns
|
|
if (nonDistinctColumns.length > 0) {
|
|
const regularPart = nonDistinctColumns
|
|
.map(buildColumnExpression)
|
|
.join(', ');
|
|
selectColumns = `${selectColumns}, ${regularPart}`;
|
|
}
|
|
} else {
|
|
assert(
|
|
'Can not combine non-distincts with multi column distincts',
|
|
nonDistinctColumns.length === 0,
|
|
);
|
|
|
|
// Only do multi column DISTINCT with no aggregate functions
|
|
selectColumns = `DISTINCT ${distinctColumns.map(buildColumnExpression).join(', ')}`;
|
|
}
|
|
} else {
|
|
selectColumns = select.map(buildColumnExpression).join(', ');
|
|
}
|
|
|
|
return `SELECT ${selectColumns} FROM "${tableName}"`;
|
|
}
|
|
|
|
function constructJoinClause(joins) {
|
|
if (!joins || joins.length === 0) {
|
|
return '';
|
|
}
|
|
|
|
return joins
|
|
.map(
|
|
({ type, table, alias, condition }) =>
|
|
`${type} JOIN "${table}" ${alias} ON ${condition}`,
|
|
)
|
|
.join(' ');
|
|
}
|
|
|
|
function constructOrderByClause(resource, tableName, sort) {
|
|
const defaultOrderByClause = `ORDER BY "${tableName}".created_time DESC`;
|
|
|
|
const { attributes, customSort, direction, isCoalesced } = sort;
|
|
const sortDirection = direction === 'desc' ? 'DESC' : 'ASC';
|
|
const attributesWithTableName = attributes?.map(
|
|
(attribute) => `"${tableName}".${attribute}`,
|
|
);
|
|
|
|
// We have to check if the attributes are valid for the resource
|
|
// as we can't use parameterized queries for ORDER BY
|
|
const validAttributes = Object.keys(modelMapping[resource] ?? {});
|
|
if (attributes?.some((attr) => !validAttributes.includes(attr))) {
|
|
return modelMapping[resource]?.created_time ? defaultOrderByClause : '';
|
|
}
|
|
|
|
if (customSort?.attributeMap) {
|
|
const whenClauses = Object.keys(customSort.attributeMap).reduce(
|
|
(acc, key) => {
|
|
return acc + `WHEN '${key}' THEN '${customSort.attributeMap[key]}' `;
|
|
},
|
|
'',
|
|
);
|
|
return `ORDER BY CASE ${attributesWithTableName.join(', ')} ${whenClauses}END ${sortDirection}`;
|
|
} else if (attributes?.length > 0) {
|
|
const commaSeparatedVals = attributesWithTableName.join(', ');
|
|
|
|
// In places where `collate nocase` is used, it is to ensure case is ignored on the initial sort.
|
|
// Then, a sort on the same condition is performed to ensure upper-case strings are given preference in a tie.
|
|
if (isCoalesced) {
|
|
return `ORDER BY COALESCE(${commaSeparatedVals}) COLLATE NOCASE ${sortDirection}, COALESCE(${commaSeparatedVals}) ${sortDirection}`;
|
|
}
|
|
|
|
const attributesWithNoCollate = attributes
|
|
.map((attr) => `"${tableName}".${attr} COLLATE NOCASE ${sortDirection}`)
|
|
.join(', ');
|
|
const attributesWithDirection = attributes
|
|
.map((attr) => `"${tableName}".${attr} ${sortDirection}`)
|
|
.join(', ');
|
|
return `ORDER BY ${attributesWithNoCollate}, ${attributesWithDirection}`;
|
|
} else if (modelMapping[resource]?.created_time) {
|
|
return defaultOrderByClause;
|
|
} else {
|
|
return '';
|
|
}
|
|
}
|
|
|
|
// Comparison Operators
|
|
const OPERATORS = {
|
|
equals: ' = ?',
|
|
notEquals: ' != ?',
|
|
gt: ' > ?',
|
|
gte: ' >= ?',
|
|
lt: ' < ?',
|
|
lte: ' <= ?',
|
|
contains: ' LIKE ?',
|
|
in: (values) => ` IN (${values.map(() => '?').join(', ')})`,
|
|
notIn: (values) => ` NOT IN (${values.map(() => '?').join(', ')})`,
|
|
};
|
|
|
|
// Logical Operators
|
|
const and = (clauses) => clauses.join(' AND ');
|
|
const or = (clauses) => clauses.join(' OR ');
|
|
const parenthetical = (expression) => (expression ? `(${expression})` : '');
|