diff --git a/CHANGELOG-Unreleased.md b/CHANGELOG-Unreleased.md index 45c5f10d0..34abab6ee 100644 --- a/CHANGELOG-Unreleased.md +++ b/CHANGELOG-Unreleased.md @@ -36,6 +36,9 @@ - You can now also run `.fetchCount()`, `.fetchIds()` on SQL queries - You can now safely pass values for SQL placeholders by passing an array - You can also observe an unsafe raw SQL query -- with some caveats! refer to documentation for more details +- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter: + Add `isFTS` boolean flag to schema column descriptor for creating Full Text Search-able columns + Add `Q.ftsMatch(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater ### Performance diff --git a/docs-master/Query.md b/docs-master/Query.md index 020271f4c..75723c05d 100644 --- a/docs-master/Query.md +++ b/docs-master/Query.md @@ -256,6 +256,11 @@ tasksCollection.query( ) ``` + +#### Full Text Search with `Q.ftsMatch` + +If you are using `SQLite` and used `isFTS` in one or more of your text columns, you can use `Q.where(fieldName, Q.ftsMatch(searchText))`. If you have more than one column with `isFTS`, you can either use `tableName` instead of `fieldName` to search in all fields, or specify `fieldName` to focus the search in only one field. + ## Advanced Queries ### Advanced observing diff --git a/src/QueryDescription/index.d.ts b/src/QueryDescription/index.d.ts index 9fcfea62a..d1c4fee8a 100644 --- a/src/QueryDescription/index.d.ts +++ b/src/QueryDescription/index.d.ts @@ -16,6 +16,7 @@ declare module '@nozbe/watermelondb/QueryDescription' { | 'oneOf' | 'notIn' | 'between' + | 'match' export interface ColumnDescription { column: ColumnName @@ -105,6 +106,7 @@ declare module '@nozbe/watermelondb/QueryDescription' { export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription export function and(...conditions: Condition[]): And export function or(...conditions: Condition[]): Or + export function ftsMatch(value: string): Comparison export function like(value: string): Comparison export function notLike(value: string): Comparison export function experimentalSortBy(sortColumn: ColumnName, sortOrder?: SortOrder): SortBy diff --git a/src/QueryDescription/index.js b/src/QueryDescription/index.js index b8468bf67..44bc41a78 100644 --- a/src/QueryDescription/index.js +++ b/src/QueryDescription/index.js @@ -30,6 +30,7 @@ export type Operator = | 'between' | 'like' | 'notLike' + | 'ftsMatch' export type ColumnDescription = $RE<{ column: ColumnName, type?: symbol }> export type ComparisonRight = @@ -248,6 +249,11 @@ export function sanitizeLikeString(value: string): string { return value.replace(nonLikeSafeRegexp, '_') } +export function ftsMatch(value: string): Comparison { + invariant(typeof value === 'string', 'Value passed to Q.ftsMatch() is not a string') + return { operator: 'ftsMatch', right: { value }, type: comparisonSymbol } +} + export function column(name: ColumnName): ColumnDescription { invariant(typeof name === 'string', 'Name passed to Q.column() is not a string') return { column: checkName(name), type: columnSymbol } diff --git a/src/QueryDescription/test.js b/src/QueryDescription/test.js index 08978a38e..742752fc9 100644 --- a/src/QueryDescription/test.js +++ b/src/QueryDescription/test.js @@ -521,6 +521,26 @@ describe('buildQueryDescription', () => { process.env.NODE_ENV = env } }) + it('supports ftsMatch as fts join', () => { + const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))]) + expect(query).toEqual({ + where: [ + { + type: 'where', + left: 'searchable', + comparison: { + operator: 'ftsMatch', + right: { + value: 'hello world', + }, + }, + }, + ], + joinTables: [], + nestedJoinTables: [], + sortBy: [], + }) + }) it('catches bad types', () => { expect(() => Q.eq({})).toThrow('Invalid value passed to query') // TODO: oneOf/notIn values? @@ -531,6 +551,7 @@ describe('buildQueryDescription', () => { expect(() => Q.notLike(null)).toThrow('not a string') expect(() => Q.notLike({})).toThrow('not a string') expect(() => Q.sanitizeLikeString(null)).toThrow('not a string') + expect(() => Q.ftsMatch(null)).toThrow('not a string') expect(() => Q.column({})).toThrow('not a string') expect(() => Q.experimentalTake('0')).toThrow('not a number') expect(() => Q.experimentalSkip('0')).toThrow('not a number') diff --git a/src/Schema/index.d.ts b/src/Schema/index.d.ts index 296bc68db..5b546d5fb 100644 --- a/src/Schema/index.d.ts +++ b/src/Schema/index.d.ts @@ -17,6 +17,7 @@ declare module '@nozbe/watermelondb/Schema' { type: ColumnType isOptional?: boolean isIndexed?: boolean + isFTS?: boolean } interface ColumnMap { diff --git a/src/Schema/index.js b/src/Schema/index.js index 018d4e4a4..975a055d0 100644 --- a/src/Schema/index.js +++ b/src/Schema/index.js @@ -15,6 +15,7 @@ export type ColumnSchema = $RE<{ type: ColumnType, isOptional?: boolean, isIndexed?: boolean, + isFTS?: boolean, }> export type ColumnMap = { [name: ColumnName]: ColumnSchema } diff --git a/src/__tests__/databaseTests.js b/src/__tests__/databaseTests.js index df53a1f69..c275ecbd0 100644 --- a/src/__tests__/databaseTests.js +++ b/src/__tests__/databaseTests.js @@ -1365,3 +1365,52 @@ export const joinTests = [ skipSqlite: true, }, ] + +export const ftsMatchTests = [ + { + name: 'Can ftsMatch - text1', + query: [Q.where('text1', Q.ftsMatch('bar'))], + matching: [ + { id: 'fts_foo_bar', text1: 'foo bar' }, + { id: 'fts_bar', text1: 'bar' }, + { id: 'fts_bar_baz', text1: 'bar baz' }, + ], + nonMatching: [ + { id: 'fts_foo', text1: 'foo', text2: 'bar baz' }, + { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' }, + { id: 'fts_baz', text1: 'baz', text2: 'foo bar' }, + { id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }, + ], + skipLoki: true, + }, + { + name: 'Can ftsMatch - text2', + query: [Q.where('text2', Q.ftsMatch('bar'))], + matching: [ + { id: 'fts_foo', text1: 'foo', text2: 'bar baz' }, + { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' }, + { id: 'fts_baz', text1: 'baz', text2: 'foo bar' }, + ], + nonMatching: [ + { id: 'fts_foo_bar', text1: 'foo bar' }, + { id: 'fts_bar', text1: 'bar' }, + { id: 'fts_bar_baz', text1: 'bar baz' }, + { id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }, + ], + skipLoki: true, + }, + { + name: 'Can ftsMatch - text1 and text2', + query: [Q.where('tasks', Q.ftsMatch('bar'))], + matching: [ + { id: 'fts_foo_bar', text1: 'foo bar' }, + { id: 'fts_bar', text1: 'bar' }, + { id: 'fts_bar_baz', text1: 'bar baz' }, + { id: 'fts_foo', text1: 'foo', text2: 'bar baz' }, + { id: 'fts_foo_baz', text1: 'foo baz', text2: 'bar' }, + { id: 'fts_baz', text1: 'baz', text2: 'foo bar' }, + ], + nonMatching: [{ id: 'fts_foo_bar_baz', text1: 'foo bar baz', _status: 'deleted' }], + skipLoki: true, + }, +] diff --git a/src/adapters/__tests__/commonTests.js b/src/adapters/__tests__/commonTests.js index 2ab6617b9..d7a9358aa 100644 --- a/src/adapters/__tests__/commonTests.js +++ b/src/adapters/__tests__/commonTests.js @@ -9,7 +9,12 @@ import * as Q from '../../QueryDescription' import { appSchema, tableSchema } from '../../Schema' import { schemaMigrations, createTable, addColumns } from '../../Schema/migrations' -import { matchTests, naughtyMatchTests, joinTests } from '../../__tests__/databaseTests' +import { + matchTests, + naughtyMatchTests, + joinTests, + ftsMatchTests, +} from '../../__tests__/databaseTests' import DatabaseAdapterCompat from '../compat' import { testSchema, @@ -17,6 +22,7 @@ import { mockTaskRaw, performMatchTest, performJoinTest, + performFtsMatchTest, expectSortedEqual, MockTask, mockProjectRaw, @@ -1192,6 +1198,20 @@ export default () => [ } }, ], + ...ftsMatchTests.map((testCase) => [ + `[shared ftsMatch test] ${testCase.name}`, + async (adapter, AdapterClass) => { + const perform = () => performFtsMatchTest(adapter, testCase) + const shouldSkip = + (AdapterClass.name === 'LokiJSAdapter' && testCase.skipLoki) || + (AdapterClass.name === 'SQLiteAdapter' && testCase.skipSqlite) + if (shouldSkip) { + await expect(perform()).rejects.toBeInstanceOf(Error) + } else { + await perform() + } + }, + ]), [ 'can store and retrieve large numbers (regression test)', async (_adapter) => { diff --git a/src/adapters/__tests__/helpers.js b/src/adapters/__tests__/helpers.js index 3c6db77c3..0c4273092 100644 --- a/src/adapters/__tests__/helpers.js +++ b/src/adapters/__tests__/helpers.js @@ -56,8 +56,8 @@ export const testSchema = appSchema({ { name: 'num3', type: 'number' }, { name: 'float1', type: 'number' }, // TODO: Remove me? { name: 'float2', type: 'number' }, - { name: 'text1', type: 'string' }, - { name: 'text2', type: 'string' }, + { name: 'text1', type: 'string', isFTS: true }, + { name: 'text2', type: 'string', isFTS: true }, { name: 'bool1', type: 'boolean' }, { name: 'bool2', type: 'boolean' }, { name: 'order', type: 'number' }, @@ -187,3 +187,9 @@ export const performJoinTest = async (adapter, testCase) => { await allPromises(([table, records]) => insertAll(adapter, table, records), pairs) await performMatchTest(adapter, testCase) } + +export const performFtsMatchTest = async (adapter, testCase) => { + const pairs = toPairs(testCase.extraRecords) + await allPromises(([table, records]) => insertAll(adapter, table, records), pairs) + await performMatchTest(adapter, testCase) +} diff --git a/src/adapters/lokijs/worker/executor.js b/src/adapters/lokijs/worker/executor.js index 4403b877b..5eb90f097 100644 --- a/src/adapters/lokijs/worker/executor.js +++ b/src/adapters/lokijs/worker/executor.js @@ -4,7 +4,13 @@ import logger from '../../../utils/common/logger' import type { CachedQueryResult, CachedFindResult, BatchOperation } from '../../type' -import type { TableName, AppSchema, SchemaVersion, TableSchema } from '../../../Schema' +import type { + TableName, + AppSchema, + SchemaVersion, + TableSchema, + ColumnSchema, +} from '../../../Schema' import type { SchemaMigrations, CreateTableMigrationStep, @@ -298,6 +304,8 @@ export default class LokiExecutor { [], ) + this._warnAboutLackingFTSSupport(columnArray) + this.loki.addCollection(name, { unique: ['id'], indices: ['_status', ...indexedColumns], @@ -401,6 +409,8 @@ export default class LokiExecutor { collection.ensureIndex(column.name) } }) + + this._warnAboutLackingFTSSupport(columns) } // Maps records to their IDs if the record is already cached on JS side @@ -456,4 +466,13 @@ export default class LokiExecutor { // Rethrow error throw error } + + _warnAboutLackingFTSSupport(columns: Array): void { + if (columns.some((column) => column.isFTS)) { + // Warn the user about missing FTS support for the LokiJS adapter + // Please contribute! Here are some pointers: + // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts + logger.warn('[DB][Worker] LokiJS support for FTS is still to be implemented') + } + } } diff --git a/src/adapters/sqlite/encodeQuery/index.js b/src/adapters/sqlite/encodeQuery/index.js index 9ee0cd887..1c050bc20 100644 --- a/src/adapters/sqlite/encodeQuery/index.js +++ b/src/adapters/sqlite/encodeQuery/index.js @@ -53,6 +53,7 @@ const operators: { [Operator]: string } = { between: 'between', like: 'like', notLike: 'not like', + ftsMatch: 'match', } const encodeComparison = (table: TableName, comparison: Comparison) => { @@ -110,6 +111,21 @@ const encodeWhereCondition = ( ) } + if (comparison.operator === 'ftsMatch') { + const srcTable = encodeName(table) + const ftsTable = encodeName(`_fts_${table}`) + const rowid = encodeName('rowid') + const ftsColumn = encodeName(left) + const matchValue = getComparisonRight(table, comparison.right) + const ftsTableColumn = table === left ? `${ftsTable}` : `${ftsTable}.${ftsColumn}` + return ( + `${srcTable}.${rowid} in (` + + `select ${ftsTable}.${rowid} from ${ftsTable} ` + + `where ${ftsTableColumn} match ${matchValue}` + + `)` + ) + } + return `${encodeName(table)}.${encodeName(left)} ${encodeComparison(table, comparison)}` } diff --git a/src/adapters/sqlite/encodeQuery/test.js b/src/adapters/sqlite/encodeQuery/test.js index ace65f158..107002bb2 100644 --- a/src/adapters/sqlite/encodeQuery/test.js +++ b/src/adapters/sqlite/encodeQuery/test.js @@ -71,7 +71,7 @@ describe('SQLite encodeQuery', () => { Q.where('col5', Q.lte(5)), Q.where('col6', Q.notEq(null)), Q.where('col7', Q.oneOf([1, 2, 3])), - Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), + Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), // eslint-disable-line quotes Q.where('col9', Q.between(10, 11)), Q.where('col10', Q.like('%abc')), Q.where('col11', Q.notLike('def%')), @@ -171,6 +171,57 @@ describe('SQLite encodeQuery', () => { ` and "tasks"."_status" is not 'deleted'`, ) }) + it('encodes ftsMatch', () => { + expect(encoded([Q.where('searchable', Q.ftsMatch('hello world'))])).toBe( + `select "tasks".* from "tasks" ` + + `where "tasks"."rowid" in (` + + `select "_fts_tasks"."rowid" from "_fts_tasks" ` + + `where "_fts_tasks"."searchable" match 'hello world'` + + `) and "tasks"."_status" is not 'deleted'`, + ) + expect(encoded([Q.where('tasks', Q.ftsMatch('hello world'))])).toBe( + `select "tasks".* from "tasks" ` + + `where "tasks"."rowid" in (` + + `select "_fts_tasks"."rowid" from "_fts_tasks" ` + + `where "_fts_tasks" match 'hello world'` + + `) and "tasks"."_status" is not 'deleted'`, + ) + }) + it('encodes ftsMatch with other joins', () => { + const query = [ + Q.on('projects', 'team_id', 'abcdef'), + Q.on('projects', 'is_active', true), + Q.on('projects', 'left_column', Q.lte(Q.column('right_column'))), + Q.on('projects', 'left2', Q.weakGt(Q.column('right2'))), + Q.where('left_column', 'right_value'), + Q.on('tag_assignments', 'tag_id', Q.oneOf(['a', 'b', 'c'])), + Q.where('searchable', Q.ftsMatch('hello world')), + ] + const expectedQuery = + `join "projects" on "projects"."id" = "tasks"."project_id"` + + ` join "tag_assignments" on "tag_assignments"."task_id" = "tasks"."id"` + + ` where ("projects"."team_id" is 'abcdef'` + + ` and "projects"."_status" is not 'deleted')` + + ` and ("projects"."is_active" is 1` + + ` and "projects"."_status" is not 'deleted')` + + ` and ("projects"."left_column" <= "projects"."right_column"` + + ` and "projects"."_status" is not 'deleted')` + + ` and (("projects"."left2" > "projects"."right2"` + + ` or ("projects"."left2" is not null` + + ` and "projects"."right2" is null))` + + ` and "projects"."_status" is not 'deleted')` + + ` and "tasks"."left_column" is 'right_value'` + + ` and ("tag_assignments"."tag_id" in ('a', 'b', 'c')` + + ` and "tag_assignments"."_status" is not 'deleted')` + + ` and "tasks"."rowid" in` + + ` (select "_fts_tasks"."rowid" from "_fts_tasks" where` + + ` "_fts_tasks"."searchable" match 'hello world')` + + ` and "tasks"."_status" is not 'deleted'` + expect(encoded(query)).toBe(`select distinct "tasks".* from "tasks" ${expectedQuery}`) + expect(encoded(query, true)).toBe( + `select count(distinct "tasks"."id") as "count" from "tasks" ${expectedQuery}`, + ) + }) it(`encodes on nested in and/or`, () => { expect( encoded([ diff --git a/src/adapters/sqlite/encodeSchema/index.js b/src/adapters/sqlite/encodeSchema/index.js index 0510f061c..dd734b5f4 100644 --- a/src/adapters/sqlite/encodeSchema/index.js +++ b/src/adapters/sqlite/encodeSchema/index.js @@ -8,6 +8,7 @@ import type { AddColumnsMigrationStep, } from '../../../Schema/migrations' import type { SQL } from '../index' +import { invariant } from '../../../utils/common' import encodeName from '../encodeName' import encodeValue from '../encodeValue' @@ -42,7 +43,111 @@ const transform = (sql: string, transformer: ?(string) => string) => transformer ? transformer(sql) : sql const encodeTable: (TableSchema) => SQL = (table) => - transform(encodeCreateTable(table) + encodeTableIndicies(table), table.unsafeSql) + transform( + // eslint-disable-next-line no-use-before-define + encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table), + table.unsafeSql, + ) + +/** FTS Full Text Search */ + +const encodeFTSTrigger: ({ + tableName: string, + ftsTableName: string, + event: 'delete' | 'insert' | 'update', + action: SQL, +}) => SQL = ({ tableName, ftsTableName, event, action }) => { + const triggerName = `${ftsTableName}_${event}` + return `create trigger ${encodeName(triggerName)} after ${event} on ${encodeName( + tableName, + )} begin ${action} end;` +} + +const encodeFTSDeleteTrigger: ({ + tableName: string, + ftsTableName: string, +}) => SQL = ({ tableName, ftsTableName }) => + encodeFTSTrigger({ + tableName, + ftsTableName, + event: 'delete', + action: `delete from ${encodeName(ftsTableName)} where "rowid" = OLD.rowid;`, + }) + +const encodeFTSInsertTrigger: ({ + tableName: string, + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { + const rawColumnNames = ['rowid', ...ftsColumns.map((column) => column.name)] + const columns = rawColumnNames.map(encodeName) + const valueColumns = rawColumnNames.map((column) => `NEW.${encodeName(column)}`) + + const columnsSQL = columns.join(', ') + const valueColumnsSQL = valueColumns.join(', ') + + return encodeFTSTrigger({ + tableName, + ftsTableName, + event: 'insert', + action: `insert into ${encodeName(ftsTableName)} (${columnsSQL}) values (${valueColumnsSQL});`, + }) +} + +const encodeFTSUpdateTrigger: ({ + tableName: string, + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { + const rawColumnNames = ftsColumns.map((column) => column.name) + const assignments = rawColumnNames.map( + (column) => `${encodeName(column)} = NEW.${encodeName(column)}`, + ) + + const assignmentsSQL = assignments.join(', ') + + return encodeFTSTrigger({ + tableName, + ftsTableName, + event: 'update', + action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = NEW."rowid";`, + }) +} + +const encodeFTSTriggers: ({ + tableName: string, + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => { + return ( + encodeFTSDeleteTrigger({ tableName, ftsTableName }) + + encodeFTSInsertTrigger({ tableName, ftsTableName, ftsColumns }) + + encodeFTSUpdateTrigger({ tableName, ftsTableName, ftsColumns }) + ) +} + +const encodeFTSTable: ({ + ftsTableName: string, + ftsColumns: ColumnSchema[], +}) => SQL = ({ ftsTableName, ftsColumns }) => { + const columnsSQL = ftsColumns.map((column) => encodeName(column.name)).join(', ') + return `create virtual table ${encodeName(ftsTableName)} using fts4(${columnsSQL});` +} + +const encodeFTSSearch: (TableSchema) => SQL = (tableSchema) => { + const { name: tableName, columnArray } = tableSchema + const ftsColumns = columnArray.filter((column) => column.isFTS) + if (ftsColumns.length === 0) { + return '' + } + const ftsTableName = `_fts_${tableName}` + return ( + encodeFTSTable({ ftsTableName, ftsColumns }) + + encodeFTSTriggers({ tableName, ftsTableName, ftsColumns }) + ) +} + +/** FTS END */ export const encodeSchema: (AppSchema) => SQL = ({ tables, unsafeSql }) => { const sql = Object.values(tables) @@ -68,6 +173,11 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({ )} = ${encodeValue(nullValue(column))};` const addIndex = encodeIndex(column, table) + invariant( + !column.isFTS, + '[DB][Worker] Support for migrations with isFTS is still to be implemented', + ) + return transform(addColumn + setDefaultValue + addIndex, unsafeSql) }) .join('') diff --git a/src/adapters/sqlite/encodeSchema/test.js b/src/adapters/sqlite/encodeSchema/test.js index 9807ca367..7060cdaed 100644 --- a/src/adapters/sqlite/encodeSchema/test.js +++ b/src/adapters/sqlite/encodeSchema/test.js @@ -64,6 +64,34 @@ describe('encodeSchema', () => { expect(encodeSchema(testSchema)).toBe(expectedSchema) }) + it('encodes schema with FTS', () => { + const testSchema = appSchema({ + version: 1, + tables: [ + tableSchema({ + name: 'tasks', + columns: [ + { name: 'author_id', type: 'string', isIndexed: true }, + { name: 'author_name', type: 'string', isFTS: true }, + { name: 'author_title', type: 'string', isFTS: true }, + { name: 'created_at', type: 'number' }, + ], + }), + ], + }) + + const expectedSchema = + 'create table "tasks" ("id" primary key, "_changed", "_status", "author_id", "author_name", "author_title", "created_at");' + + 'create index "tasks_author_id" on "tasks" ("author_id");' + + 'create index "tasks__status" on "tasks" ("_status");' + + 'create virtual table "_fts_tasks" using fts4("author_name", "author_title");' + + 'create trigger "_fts_tasks_delete" after delete on "tasks" begin delete from "_fts_tasks" where "rowid" = OLD.rowid; end;' + + 'create trigger "_fts_tasks_insert" after insert on "tasks" begin insert into "_fts_tasks" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' + + 'create trigger "_fts_tasks_update" after update on "tasks" begin update "_fts_tasks" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' + + 'create table "local_storage" ("key" varchar(16) primary key not null, "value" text not null);create index "local_storage_key_index" on "local_storage" ("key");' + + expect(encodeSchema(testSchema)).toBe(expectedSchema) + }) it('encodes migrations', () => { const migrationSteps = [ addColumns({