/** * Copyright IBM Corp. 2021, 2025 * SPDX-License-Identifier: BUSL-1.1 */ import { modelMapping } from 'api/services/sqlite'; import { typeOf } from '@ember/utils'; import { underscore } from '@ember/string'; import { assert } from '@ember/debug'; /** * Takes a POJO representing a filter query and builds a SQL query. * * @example * const filter = generateSQLExpressions(type, { * search: generated', * filters: { * scope_id: { * logicalOperator: 'and', * values: [{ notEquals: 'scope1' }, { notEquals: 'scope2' }], * }, * }, * }); */ export function generateSQLExpressions( resource, query = {}, { page, pageSize, select } = {}, ) { let { search, filters, sort = {} } = query; const conditions = []; const parameters = []; const tableName = underscore(resource); const joins = []; addFilterConditions({ filters, parameters, conditions, joins, tableName }); addSearchConditions({ search, resource, tableName, parameters, conditions }); const selectClause = constructSelectClause(select, tableName); const joinClause = constructJoinClause(joins); const orderByClause = constructOrderByClause(resource, tableName, sort); const whereClause = conditions.length ? `WHERE ${and(conditions)}` : ''; const paginationClause = page && pageSize ? `LIMIT ? OFFSET ?` : ''; if (paginationClause) { parameters.push(pageSize, (page - 1) * pageSize); } return { // Replace any empty newlines or leading whitespace on each line to be consistent with formatting // This is mainly to help us read and test the generated SQL as it has no effect on the actual SQL execution. sql: ` ${selectClause} ${joinClause} ${whereClause} ${orderByClause} ${paginationClause}` .replace(/^\s+/gm, '') .trim(), parameters, }; } function addFilterConditions({ filters, parameters, conditions, joins, tableName, }) { if (!filters) { return; } for (const [key, filterArrayOrObject] of Object.entries(filters)) { // Filter can be an array or an object with a `values` property const filterValueArray = Array.isArray(filterArrayOrObject) ? filterArrayOrObject : filterArrayOrObject.values; if (!filterValueArray || !filterValueArray.length || key === 'joins') { continue; } // When attribute conditions are a series of equals/notEquals // replace with in/notIn to avoid hitting the sqlite maximum // expression tree depth. const firstOperator = Object.keys(filterValueArray[0])[0]; const allOperatorsEqual = filterValueArray .flatMap((item) => Object.keys(item)) .every((op) => op === firstOperator); if ( filterValueArray.length > 1 && (firstOperator === 'equals' || firstOperator === 'notEquals') && allOperatorsEqual ) { const operation = firstOperator === 'equals' ? 'in' : 'notIn'; const nullOperator = firstOperator === 'equals' ? 'IS NULL' : 'IS NOT NULL'; const values = filterValueArray .filter( (filterObjValue) => filterObjValue && Object.values(filterObjValue)[0] !== null, ) .map((filterObjValue) => { let value = Object.values(filterObjValue)[0]; if (typeOf(value) === 'date') { value = value.toISOString(); } parameters.push(value); return value; }); // Add a check for null values as an IN clause will mistakenly use `=` for null values. // We'll add an extra OR clause to handle nulls separately. const isNullValue = filterValueArray.some( (filterObjValue) => Object.values(filterObjValue)[0] === null, ); const filterCondition = `"${tableName}".${key}${OPERATORS[operation](values)}${isNullValue ? ` OR "${tableName}".${key} ${nullOperator}` : ''}`; conditions.push(parenthetical(filterCondition)); continue; } const filterConditions = filterValueArray .filter((f) => f) .map((filterObjValue) => { let [operation, value] = Object.entries(filterObjValue)[0]; // Handle null values: convert equals/notEquals to IS NULL or IS NOT NULL if (value === null) { if (operation === 'equals') { return `"${tableName}".${key} IS NULL`; } if (operation === 'notEquals') { return `"${tableName}".${key} IS NOT NULL`; } } // SQLite needs to be working with ISO strings if (typeOf(value) === 'date') { value = value.toISOString(); } // Handle LIKE operator separately if (operation === 'contains') { parameters.push(`%${value}%`); } else { parameters.push(value); } return `"${tableName}".${key}${OPERATORS[operation]}`; }); const { logicalOperator } = filterArrayOrObject; // Default to `or` if we don't have a logical operator conditions.push( parenthetical( logicalOperator === 'and' ? and(filterConditions) : or(filterConditions), ), ); } if (filters.joins?.length > 0) { const tableNameIndex = {}; filters.joins.forEach((join) => { const { resource, query, joinFrom = 'id', joinOn, joinType = 'INNER', } = join; const joinTableName = underscore(resource); tableNameIndex[resource] ??= 1; // Alias in the possible scenario we join the same table more than once const alias = `${joinTableName}${tableNameIndex[resource]}`; joins.push({ type: joinType, table: joinTableName, alias, condition: `"${tableName}".${joinFrom} = ${alias}.${joinOn}`, }); // Add the conditions from the join query if (query?.filters) { addFilterConditions({ filters: query.filters, parameters, conditions, tableName: alias, }); } tableNameIndex[resource]++; }); } } function addSearchConditions({ search, resource, tableName, parameters, conditions, }) { if (!search || !modelMapping[resource]) { return; } const searchSelect = search?.select ?? 'rowid'; const searchSql = `SELECT ${searchSelect} FROM ${tableName}_fts WHERE ${tableName}_fts MATCH ?`; const getParameter = (fields, text) => fields?.length > 0 ? or(fields.map((field) => `${field}:"${text}"*`)) : `"${text}"*`; // Use the special prefix indicator "*" for full-text search if (typeOf(search) === 'object') { if (!search.text) { return; } const parameter = getParameter(search.fields, search.text); parameters.push(parameter); } else { parameters.push(`"${search}"*`); } // If there are extra related searches on other tables, add them too if (search?.relatedSearches?.length > 0) { const relatedSearchQueries = []; search.relatedSearches.forEach((relatedSearch) => { const { resource: relatedResource, fields, join } = relatedSearch; const relatedTableName = underscore(relatedResource); const parameter = getParameter(fields, search.text); parameters.push(parameter); // Build the related FTS query with optional join let relatedQuery = []; relatedQuery.push( `SELECT "${tableName}".${searchSelect} FROM ${relatedTableName}_fts`, ); if (join) { const { joinFrom = 'id', joinOn } = join; relatedQuery.push( `JOIN "${tableName}" ON "${tableName}".${joinFrom} = ${relatedResource}_fts.${joinOn}`, ); } relatedQuery.push(`WHERE ${relatedTableName}_fts MATCH ?`); relatedSearchQueries.push(relatedQuery.join(' ')); }); // Add the original search first since we've already added the original parameter first const conditionStatement = [searchSql, ...relatedSearchQueries].join( '\nUNION ', ); conditions.push( `"${tableName}".${searchSelect} IN (${conditionStatement})`, ); return; } // Use a subquery to match against the FTS table with rowids as SQLite is // much more efficient with FTS queries when using rowids or MATCH (or both). // We could have also used a join here but a subquery is simpler. conditions.push(`"${tableName}".${searchSelect} IN (${searchSql})`); } function constructSelectClause(select = [{ field: '*' }], tableName) { const distinctColumns = select.filter(({ isDistinct }) => isDistinct); const nonDistinctColumns = select.filter(({ isDistinct }) => !isDistinct); const buildColumnExpression = ({ field, isCount, alias, isDistinct }) => { let column = field === '*' ? field : `"${tableName}".${field}`; // If there's just distinct with nothing else, this will be handled separately // and we will just return the column back as is if (isCount && isDistinct) { column = `count(DISTINCT ${column})`; } else if (isCount) { column = `count(${column})`; } if (alias) { column = `${column} as ${alias}`; } return column; }; let selectColumns; if (distinctColumns.length > 0) { // Check if any columns also have COUNT (or other aggregate function in the future) const hasAggregateDistinct = distinctColumns.some((col) => col.isCount); if (hasAggregateDistinct) { selectColumns = distinctColumns.map(buildColumnExpression).join(', '); // Add back in the non distinct columns if (nonDistinctColumns.length > 0) { const regularPart = nonDistinctColumns .map(buildColumnExpression) .join(', '); selectColumns = `${selectColumns}, ${regularPart}`; } } else { assert( 'Can not combine non-distincts with multi column distincts', nonDistinctColumns.length === 0, ); // Only do multi column DISTINCT with no aggregate functions selectColumns = `DISTINCT ${distinctColumns.map(buildColumnExpression).join(', ')}`; } } else { selectColumns = select.map(buildColumnExpression).join(', '); } return `SELECT ${selectColumns} FROM "${tableName}"`; } function constructJoinClause(joins) { if (!joins || joins.length === 0) { return ''; } return joins .map( ({ type, table, alias, condition }) => `${type} JOIN "${table}" ${alias} ON ${condition}`, ) .join(' '); } function constructOrderByClause(resource, tableName, sort) { const defaultOrderByClause = `ORDER BY "${tableName}".created_time DESC`; const { attributes, customSort, direction, isCoalesced } = sort; const sortDirection = direction === 'desc' ? 'DESC' : 'ASC'; const attributesWithTableName = attributes?.map( (attribute) => `"${tableName}".${attribute}`, ); // We have to check if the attributes are valid for the resource // as we can't use parameterized queries for ORDER BY const validAttributes = Object.keys(modelMapping[resource] ?? {}); if (attributes?.some((attr) => !validAttributes.includes(attr))) { return modelMapping[resource]?.created_time ? defaultOrderByClause : ''; } if (customSort?.attributeMap) { const whenClauses = Object.keys(customSort.attributeMap).reduce( (acc, key) => { return acc + `WHEN '${key}' THEN '${customSort.attributeMap[key]}' `; }, '', ); return `ORDER BY CASE ${attributesWithTableName.join(', ')} ${whenClauses}END ${sortDirection}`; } else if (attributes?.length > 0) { const commaSeparatedVals = attributesWithTableName.join(', '); // In places where `collate nocase` is used, it is to ensure case is ignored on the initial sort. // Then, a sort on the same condition is performed to ensure upper-case strings are given preference in a tie. if (isCoalesced) { return `ORDER BY COALESCE(${commaSeparatedVals}) COLLATE NOCASE ${sortDirection}, COALESCE(${commaSeparatedVals}) ${sortDirection}`; } const attributesWithNoCollate = attributes .map((attr) => `"${tableName}".${attr} COLLATE NOCASE ${sortDirection}`) .join(', '); const attributesWithDirection = attributes .map((attr) => `"${tableName}".${attr} ${sortDirection}`) .join(', '); return `ORDER BY ${attributesWithNoCollate}, ${attributesWithDirection}`; } else if (modelMapping[resource]?.created_time) { return defaultOrderByClause; } else { return ''; } } // Comparison Operators const OPERATORS = { equals: ' = ?', notEquals: ' != ?', gt: ' > ?', gte: ' >= ?', lt: ' < ?', lte: ' <= ?', contains: ' LIKE ?', in: (values) => ` IN (${values.map(() => '?').join(', ')})`, notIn: (values) => ` NOT IN (${values.map(() => '?').join(', ')})`, }; // Logical Operators const and = (clauses) => clauses.join(' AND '); const or = (clauses) => clauses.join(' OR '); const parenthetical = (expression) => (expression ? `(${expression})` : '');