add support for SQLite Full Text Search

CHANGELOG-Unreleased.md

@@ -14,6 +14,11 @@
 
 ### New features
 
+
+- [SQLiteAdapter] Added support for Full Text Search for SQLite adapter:
+  Add `isFTS` boolean flag to schema column descriptor for creating Full Text Search-able columns
+  Add `Q.ftsMatch(value)` that compiles to `match 'value'` SQL for performing Full Text Search using SQLite adpater
+
 ### Performance
 
 ### Changes

src/QueryDescription/index.d.ts

@@ -16,6 +16,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
     | 'oneOf'
     | 'notIn'
     | 'between'
+    | 'match'
 
   export interface ColumnDescription {
     column: ColumnName
@@ -105,6 +106,7 @@ declare module '@nozbe/watermelondb/QueryDescription' {
   export function where(left: ColumnName, valueOrComparison: Value | Comparison): WhereDescription
   export function and(...conditions: Condition[]): And
   export function or(...conditions: Condition[]): Or
+  export function ftsMatch(value: string): Comparison
   export function like(value: string): Comparison
   export function notLike(value: string): Comparison
   export function experimentalSortBy(sortColumn: ColumnName, sortOrder?: SortOrder): SortBy

src/QueryDescription/index.js

@@ -29,6 +29,7 @@ export type Operator =
   | 'between'
   | 'like'
   | 'notLike'
+  | 'match'
 
 export type ColumnDescription = $RE<{ column: ColumnName, type?: symbol }>
 export type ComparisonRight =
@@ -233,6 +234,11 @@ export function sanitizeLikeString(value: string): string {
   return value.replace(nonLikeSafeRegexp, '_')
 }
 
+export function ftsMatch(value: string): Comparison {
+  invariant(typeof value === 'string', 'Value passed to Q.ftsMatch() is not a string')
+  return { operator: 'match', right: { value }, type: comparisonSymbol }
+}
+
 export function column(name: ColumnName): ColumnDescription {
   invariant(typeof name === 'string', 'Name passed to Q.column() is not a string')
   return { column: checkName(name), type: columnSymbol }

src/QueryDescription/test.js

@@ -635,4 +635,25 @@ describe('queryWithoutDeleted', () => {
       ]),
     )
   })
+
+  it('supports ftsMatch as fts join', () => {
+    const query = Q.buildQueryDescription([Q.where('searchable', Q.ftsMatch('hello world'))])
+    expect(query).toEqual({
+      where: [
+        {
+          type: 'where',
+          left: 'searchable',
+          comparison: {
+            operator: 'match',
+            right: {
+              value: 'hello world',
+            },
+          },
+        },
+      ],
+      joinTables: [],
+      nestedJoinTables: [],
+      sortBy: [],
+    })
+  })
 })

src/Schema/index.d.ts

@@ -17,6 +17,7 @@ declare module '@nozbe/watermelondb/Schema' {
     type: ColumnType
     isOptional?: boolean
     isIndexed?: boolean
+    isFTS?: boolean
   }
 
   interface ColumnMap {

src/Schema/index.js

@@ -15,6 +15,7 @@ export type ColumnSchema = $RE<{
   type: ColumnType,
   isOptional?: boolean,
   isIndexed?: boolean,
+  isFTS?: boolean,
 }>
 
 export type ColumnMap = { [name: ColumnName]: ColumnSchema }

src/adapters/lokijs/worker/executor.js

@@ -4,7 +4,13 @@
 import logger from '../../../utils/common/logger'
 
 import type { CachedQueryResult, CachedFindResult, BatchOperation } from '../../type'
-import type { TableName, AppSchema, SchemaVersion, TableSchema } from '../../../Schema'
+import type {
+  TableName,
+  AppSchema,
+  SchemaVersion,
+  TableSchema,
+  ColumnSchema,
+} from '../../../Schema'
 import type {
   SchemaMigrations,
   CreateTableMigrationStep,
@@ -312,6 +318,8 @@ export default class LokiExecutor {
       [],
     )
 
+    this._warnAboutLackingFTSSupport(columnArray)
+
     this.loki.addCollection(name, {
       unique: ['id'],
       indices: ['_status', ...indexedColumns],
@@ -415,6 +423,8 @@ export default class LokiExecutor {
         collection.ensureIndex(column.name)
       }
     })
+
+    this._warnAboutLackingFTSSupport(columns)
   }
 
   // Maps records to their IDs if the record is already cached on JS side
@@ -470,4 +480,14 @@ export default class LokiExecutor {
     // Rethrow error
     throw error
   }
+
+  _warnAboutLackingFTSSupport(columns: Array<ColumnSchema>): void {
+    const searchableColumns = columns.filter(column => column.isFTS)
+    if (searchableColumns.length > 0) {
+      // Warn the user about missing FTS support for the LokiJS adapter
+      // Please contribute! Here are some pointers:
+      // https://github.com/LokiJS-Forge/LokiDB/blob/master/packages/full-text-search/spec/generic/full_text_search.spec.ts
+      logger.warn('[DB][Worker] LokiJS support for FTS is still to be implemented')
+    }
+  }
 }

src/adapters/sqlite/encodeQuery/index.js

@@ -52,6 +52,7 @@ const operators: { [Operator]: string } = {
   between: 'between',
   like: 'like',
   notLike: 'not like',
+  match: 'match',
 }
 
 const encodeComparison = (table: TableName<any>, comparison: Comparison) => {
@@ -109,6 +110,20 @@ const encodeWhereCondition = (
     )
   }
 
+  if (comparison.operator === 'match') {
+    const srcTable = encodeName(table)
+    const ftsTable = encodeName(`_fts_${table}`)
+    const rowid = encodeName('rowid')
+    const ftsColumn = encodeName(left)
+    const matchValue = getComparisonRight(table, comparison.right)
+    return (
+      `${srcTable}.${rowid} in (` +
+      `select ${ftsTable}.${rowid} from ${ftsTable} ` +
+      `where ${ftsTable}.${ftsColumn} match ${matchValue}` +
+      `)`
+    )
+  }
+
   return `${encodeName(table)}.${encodeName(left)} ${encodeComparison(table, comparison)}`
 }
 

src/adapters/sqlite/encodeQuery/test.js

@@ -65,7 +65,7 @@ describe('SQLite encodeQuery', () => {
         Q.where('col5', Q.lte(5)),
         Q.where('col6', Q.notEq(null)),
         Q.where('col7', Q.oneOf([1, 2, 3])),
-        Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])),
+        Q.where('col8', Q.notIn(['"a"', "'b'", 'c'])), // eslint-disable-line quotes
         Q.where('col9', Q.between(10, 11)),
         Q.where('col10', Q.like('%abc')),
         Q.where('col11', Q.notLike('def%')),
@@ -270,4 +270,14 @@ describe('SQLite encodeQuery', () => {
     expect(() => encoded([Q.unsafeLokiExpr({ hi: true })])).toThrow('Unknown clause')
     expect(() => encoded([Q.unsafeLokiTransform(() => {})])).toThrow('not supported')
   })
+  it('encodes JOIN over FTS table', () => {
+    const query = new Query(mockCollection, [Q.where('searchable', Q.ftsMatch('hello world'))])
+    expect(encodeQuery(query)).toBe(
+      `select "tasks".* from "tasks" ` +
+        `where "tasks"."rowid" in (` +
+        `select "_fts_tasks"."rowid" from "_fts_tasks" ` +
+        `where "_fts_tasks"."searchable" match 'hello world'` +
+        `) and "tasks"."_status" is not 'deleted'`,
+    )
+  })
 })

src/adapters/sqlite/encodeSchema/index.js

@@ -8,6 +8,7 @@ import type {
   AddColumnsMigrationStep,
 } from '../../../Schema/migrations'
 import type { SQL } from '../index'
+import { logger } from '../../../utils/common'
 
 import encodeName from '../encodeName'
 import encodeValue from '../encodeValue'
@@ -38,8 +39,112 @@ const encodeTableIndicies: (TableSchema) => SQL = ({ name: tableName, columns })
 const transform = (sql: string, transformer: ?(string) => string) =>
   transformer ? transformer(sql) : sql
 
-const encodeTable: (TableSchema) => SQL = (table) =>
-  transform(encodeCreateTable(table) + encodeTableIndicies(table), table.unsafeSql)
+const encodeTable: TableSchema => SQL = table =>
+  transform(
+    // eslint-disable-next-line no-use-before-define
+    encodeCreateTable(table) + encodeTableIndicies(table) + encodeFTSSearch(table),
+    table.unsafeSql,
+  )
+
+/** FTS Full Text Search */
+
+const encodeFTSTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  event: 'delete' | 'insert' | 'update',
+  action: SQL,
+}) => SQL = ({ tableName, ftsTableName, event, action }) => {
+  const triggerName = `${ftsTableName}_${event}`
+  return `create trigger ${encodeName(triggerName)} after ${event} on ${encodeName(
+    tableName,
+  )} begin ${action} end;`
+}
+
+const encodeFTSDeleteTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+}) => SQL = ({ tableName, ftsTableName }) =>
+  encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'delete',
+    action: `delete from ${encodeName(ftsTableName)} where "rowid" = OLD.rowid;`,
+  })
+
+const encodeFTSInsertTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  const rawColumnNames = ['rowid', ...ftsColumns.map(column => column.name)]
+  const columns = rawColumnNames.map(encodeName)
+  const valueColumns = rawColumnNames.map(column => `NEW.${encodeName(column)}`)
+
+  const columnsSQL = columns.join(', ')
+  const valueColumnsSQL = valueColumns.join(', ')
+
+  return encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'insert',
+    action: `insert into ${encodeName(ftsTableName)} (${columnsSQL}) values (${valueColumnsSQL});`,
+  })
+}
+
+const encodeFTSUpdateTrigger: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  const rawColumnNames = ftsColumns.map(column => column.name)
+  const assignments = rawColumnNames.map(
+    column => `${encodeName(column)} = NEW.${encodeName(column)}`,
+  )
+
+  const assignmentsSQL = assignments.join(', ')
+
+  return encodeFTSTrigger({
+    tableName,
+    ftsTableName,
+    event: 'update',
+    action: `update ${encodeName(ftsTableName)} set ${assignmentsSQL} where "rowid" = NEW."rowid";`,
+  })
+}
+
+const encodeFTSTriggers: ({
+  tableName: string,
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ tableName, ftsTableName, ftsColumns }) => {
+  return (
+    encodeFTSDeleteTrigger({ tableName, ftsTableName }) +
+    encodeFTSInsertTrigger({ tableName, ftsTableName, ftsColumns }) +
+    encodeFTSUpdateTrigger({ tableName, ftsTableName, ftsColumns })
+  )
+}
+
+const encodeFTSTable: ({
+  ftsTableName: string,
+  ftsColumns: ColumnSchema[],
+}) => SQL = ({ ftsTableName, ftsColumns }) => {
+  const columnsSQL = ftsColumns.map(column => encodeName(column.name)).join(', ')
+  return `create virtual table ${encodeName(ftsTableName)} using fts4(${columnsSQL});`
+}
+
+const encodeFTSSearch: TableSchema => SQL = tableSchema => {
+  const { name: tableName, columnArray } = tableSchema
+  const ftsColumns = columnArray.filter(column => column.isFTS)
+  if (ftsColumns.length === 0) {
+    return ''
+  }
+  const ftsTableName = `_fts_${tableName}`
+  return (
+    encodeFTSTable({ ftsTableName, ftsColumns }) +
+    encodeFTSTriggers({ tableName, ftsTableName, ftsColumns })
+  )
+}
+
+/** FTS END */
 
 export const encodeSchema: (AppSchema) => SQL = ({ tables, unsafeSql }) => {
   const sql = Object.values(tables)
@@ -65,6 +170,10 @@ const encodeAddColumnsMigrationStep: (AddColumnsMigrationStep) => SQL = ({
       )} = ${encodeValue(nullValue(column))};`
       const addIndex = encodeIndex(column, table)
 
+      if (column.isFTS) {
+        logger.warn('[DB][Worker] Support for migrations and isFTS is still to be implemented')
+      }
+
       return transform(addColumn + setDefaultValue + addIndex, unsafeSql)
     })
     .join('')

src/adapters/sqlite/encodeSchema/test.js

@@ -57,6 +57,34 @@ describe('encodeSchema', () => {
 
     expect(encodeSchema(testSchema)).toBe(expectedSchema)
   })
+  it('encodes schema with FTS', () => {
+    const testSchema = appSchema({
+      version: 1,
+      tables: [
+        tableSchema({
+          name: 'tasks',
+          columns: [
+            { name: 'author_id', type: 'string', isIndexed: true },
+            { name: 'author_name', type: 'string', isFTS: true },
+            { name: 'author_title', type: 'string', isFTS: true },
+            { name: 'created_at', type: 'number' },
+          ],
+        }),
+      ],
+    })
+
+    const expectedSchema =
+      'create table "tasks" ("id" primary key, "_changed", "_status", "author_id", "author_name", "author_title", "created_at");' +
+      'create index "tasks_author_id" on "tasks" ("author_id");' +
+      'create index "tasks__status" on "tasks" ("_status");' +
+      'create virtual table "_fts_tasks" using fts4("author_name", "author_title");' +
+      'create trigger "_fts_tasks_delete" after delete on "tasks" begin delete from "_fts_tasks" where "rowid" = OLD.rowid; end;' +
+      'create trigger "_fts_tasks_insert" after insert on "tasks" begin insert into "_fts_tasks" ("rowid", "author_name", "author_title") values (NEW."rowid", NEW."author_name", NEW."author_title"); end;' +
+      'create trigger "_fts_tasks_update" after update on "tasks" begin update "_fts_tasks" set "author_name" = NEW."author_name", "author_title" = NEW."author_title" where "rowid" = NEW."rowid"; end;' +
+      ''
+
+    expect(encodeSchema(testSchema)).toBe(expectedSchema)
+  })
   it('encodes migrations', () => {
     const migrationSteps = [
       addColumns({