diff --git a/src/QueryDescription/experimentalOptimize/index.js b/src/QueryDescription/experimentalOptimize/index.js new file mode 100644 index 000000000..6d59ac349 --- /dev/null +++ b/src/QueryDescription/experimentalOptimize/index.js @@ -0,0 +1,210 @@ +// @flow +/* eslint-disable no-use-before-define */ + +import deepFreeze from '../../utils/common/deepFreeze' +import * as Q from '../index' +import type { AppSchema, TableName } from '../../Schema' +import type { QueryDescription, Where, On, And, Or } from '../type' + +// where()s first +// … but where()s with oneOf/notIn last (because it could be a long array) +// then on()s +// … but on()s querying has_manys before on()s querying belongs_tos +// … merge on()s querying the same table + +// Goal: +// - order clauses such that heaviest to execute clauses are last +// - order clauses such that we filter out as many records as possible early +// +// It's hard to do this well without knowing the histogram of different values. SQLite can do this +// out of the box, so we probably don't have to worry about it at all. But for Loki, something +// might still be better than nothing... +// +// However we might be able to use information about schema to guess which fields are cheap to query (indexed) +// +// One simple way to aid in reordering is to allow users to pass Q.likely(), Q.unlikely(), but +// if we want users to add such information, they might as well manually tune the query order +// themselves, no? + +type OptimizeQueryDescriptionOptions = $Exact<{ + query: QueryDescription, + table: TableName, + schema: AppSchema, +}> +export default function optimizeQueryDescription( + options: OptimizeQueryDescriptionOptions, +): QueryDescription { + const { query, table, schema } = options + const optimizedQuery = { ...query } + const optimized = optimizeWhere(query.where, table, schema, 'and') + optimizedQuery.where = getWheres(optimized) + + if (process.env.NODE_ENV !== 'production') { + deepFreeze(optimizedQuery) + } + return optimizedQuery +} + +type score = number // lower number = higher priority +type CondEntry = [Where, score] +type OnEntry = Where[][] +type ListContext = 'and' | 'or' + +const getWheres = (entries: CondEntry[]): Where[] => entries.map(([condition]) => condition) + +const DEFAULT_SCORE = 1 +const INDEXED_MULTIPLIER = 0.1 // rationale: indexed fields are faster to query +const EQ_MULTIPLIER = 0.5 // rationale: equality yields fewer results than lt/gt +const oneOfMultiplier = (length: number) => Math.log2(length) / 2 + 1 +const ON_MULTPLIER = 10 + +function optimizeWhere( + conditions: Where[], + table: TableName, + schema: AppSchema, + listContext: ListContext, +): CondEntry[] { + let optimized: CondEntry[] = [] + const tableSchema = schema.tables[table] + + conditions.forEach((condition) => { + switch (condition.type) { + case 'where': { + const isIndexed = tableSchema.columns[condition.left]?.isIndexed + const isEq = condition.comparison.operator === 'eq' + const isOneOf = condition.comparison.operator === 'oneOf' + optimized.push([ + condition, + DEFAULT_SCORE * + (isIndexed ? INDEXED_MULTIPLIER : 1) * + (isEq ? EQ_MULTIPLIER : 1) * + (isOneOf ? oneOfMultiplier((condition.comparison.right.values: any).length) : 1), + ]) + break + } + case 'and': { + const optimizedInner = optimizeWhere(condition.conditions, table, schema, 'and') + + // a && (b && c) == a && b && c + // a || (b) == a || b + if (listContext === 'and' || optimizedInner.length === 1) { + optimized.push(...optimizedInner) + } else { + optimized.push([ + { ...condition, conditions: getWheres(optimizedInner) }, + // NOTE: we should have a score estimate for this + DEFAULT_SCORE, + ]) + } + break + } + case 'or': { + const optimizedInner = optimizeWhere(condition.conditions, table, schema, 'or') + + // a || (b || c) == a || b || c + // a && (b) == a && b + if (listContext === 'or' || optimizedInner.length === 1) { + optimized.push(...optimizedInner) + } else { + optimized.push([ + { ...condition, conditions: getWheres(optimizedInner) }, + // NOTE: we should have a score estimate for this + DEFAULT_SCORE, + ]) + } + break + } + case 'on': { + // push as is, will be merged later + optimized.push([condition, ON_MULTPLIER]) + break + } + default: { + // SqlExpr, LokiExpr - we don't know how to score these + optimized.push([condition, DEFAULT_SCORE]) + break + } + } + }) + + // merge & optimize ons + // merging needs to be a second pass to be able to merge ons + // originating from flattened and/or expressions + // e.g. (on(a) || on(b)) && on(c) -> on((a || b) && c) + optimized = mergeOns(optimized, table, schema, listContext) + + // sort by score + optimized.sort(([, a], [, b]) => a - b) + + return optimized +} + +function mergeOns( + initial: CondEntry[], + table: TableName, + schema: AppSchema, + listContext: ListContext, +): CondEntry[] { + const optimized: CondEntry[] = [] + const ons: { [table: TableName]: OnEntry } = {} + + // extract all on's, grouped by table + initial.forEach((condEntry) => { + if (condEntry[0].type === 'on') { + const on: On = condEntry[0] + const { table: onTable, conditions: onConditions } = on + const onEntry = ons[onTable] + if (onEntry) { + onEntry.push(onConditions) + } else { + ons[onTable] = [onConditions] + } + } else { + optimized.push(condEntry) + } + }) + + // merge&optimize entries + Object.entries(ons).forEach(([onTable, manyOnConditions]) => { + // merge + // const mergedOnConditions = [] + // manyOnConditions.forEach((conds) => { + // // on(a) && on(b && b) == on(a && b && c) + // // on(a) || on(b && c) == on(a || (b && c)) + // // on(a) || on(b) == on(a || b) + // if (listContext === 'and' || conds.length === 1) { + // mergedOnConditions.push(...conds) + // } else { + // mergedOnConditions.push({ type: 'and', conditions: conds }) + // } + // }) + const mergedOnConditions: Where[] = [ + { + type: (listContext: any), + conditions: manyOnConditions.map((onConds) => ({ type: 'and', conditions: onConds })), + }, + ] + + const optimizedMerged = getWheres( + optimizeWhere(mergedOnConditions, (onTable: any), schema, 'and'), + ) + console.log({ onTable, manyOnConditions, mergedOnConditions, optimizedMerged }) + console.log(mergedOnConditions) + console.log(mergedOnConditions[0].conditions) + console.log(mergedOnConditions[0].conditions[0].conditions) + // wrap in or() if we're transforming on(a) || on(b) to on(a || b) + const wrappedConditions = optimizedMerged + // listContext === 'or' && optimizedMerged.length > 1 + // ? [({ type: 'or', conditions: optimizedMerged }: Or)] + // : optimizedMerged + + const optimizedOn: On = { + type: 'on', + table: (onTable: any), + conditions: wrappedConditions, + } + optimized.push([optimizedOn, ON_MULTPLIER]) + }) + + return optimized +} diff --git a/src/QueryDescription/experimentalOptimize/test.js b/src/QueryDescription/experimentalOptimize/test.js new file mode 100644 index 000000000..839e3a888 --- /dev/null +++ b/src/QueryDescription/experimentalOptimize/test.js @@ -0,0 +1,403 @@ +import { appSchema, tableSchema } from '../../Schema' +import optimizeQueryDescription from './index' +import * as Q from '../index' +import { buildQueryDescription } from '../helpers' + +const standardColumns = [ + { name: 'str', type: 'string' }, + { name: 'num', type: 'number' }, + { name: 'bool', type: 'boolean' }, + { name: 'str_i', type: 'string', isIndexed: true }, + { name: 'num_i', type: 'number', isIndexed: true }, + { name: 'bool_i', type: 'boolean', isIndexed: true }, +] +const schema = appSchema({ + version: 1, + tables: [ + tableSchema({ + name: 'projects', + columns: [...standardColumns], + }), + tableSchema({ + name: 'tasks', + columns: [...standardColumns.map((c) => ({ ...c, name: `t_${c.name}` }))], + }), + tableSchema({ + name: 'comments', + columns: [...standardColumns], + }), + ], +}) + +describe('optimizeQueryDescription', () => { + const optimize = (clauses) => { + const query = buildQueryDescription(clauses) + const optimized = optimizeQueryDescription({ query, table: 'projects', schema }) + expect({ ...optimized, where: [] }).toEqual({ ...query, where: [] }) + return optimized.where + } + it(`empty query`, () => { + expect(optimize([])).toEqual([]) + }) + describe('reorders conditions', () => { + it(`does not reorder conditions if profitability is unknown`, () => { + const orig = [Q.where('foo', 'bar'), Q.unsafeSqlExpr(''), Q.unsafeLokiExpr({})] + expect(optimize(orig)).toEqual(orig) + }) + it(`reorders indexed columns before unindexed`, () => { + expect( + optimize([ + // + Q.where('str', 'bar'), + Q.where('bool_i', 'bar'), + Q.where('str_i', 'bar'), + ]), + ).toEqual([ + // + Q.where('bool_i', 'bar'), + Q.where('str_i', 'bar'), + Q.where('str', 'bar'), + ]) + }) + it(`reorders Q.eq before other comparisons`, () => { + expect( + optimize([ + // + Q.where('str', Q.gt('bar')), + Q.where('str', Q.notEq('bar')), + Q.where('str', 'bar'), + ]), + ).toEqual([ + // + Q.where('str', 'bar'), + Q.where('str', Q.gt('bar')), + Q.where('str', Q.notEq('bar')), + ]) + }) + it(`reorders Q.oneOf depending on number of args`, () => { + expect( + optimize([ + // + Q.where('str', Q.oneOf(Array(10).fill('bar'))), + Q.where('str', Q.oneOf(Array(2).fill('bar'))), + Q.where('str', Q.oneOf(Array(5).fill('bar'))), + Q.where('str', 'bar'), + ]), + ).toEqual([ + // + Q.where('str', 'bar'), + Q.where('str', Q.oneOf(Array(2).fill('bar'))), + Q.where('str', Q.oneOf(Array(5).fill('bar'))), + Q.where('str', Q.oneOf(Array(10).fill('bar'))), + ]) + }) + it(`reorders Q.ons last`, () => { + expect( + optimize([ + // + Q.on('tasks', 'foo', 'bar'), + Q.where('bar', 'baz'), + ]), + ).toEqual([ + // + Q.where('bar', 'baz'), + Q.on('tasks', 'foo', 'bar'), + ]) + }) + }) + describe('flattens inner lists', () => { + it(`flattens Q.and`, () => { + expect( + optimize([ + // + Q.where('str', 'bar'), + Q.and([ + // + Q.where('str', 'bar2'), + Q.and(Q.where('str', 'bar3')), + ]), + ]), + ).toEqual([ + // + Q.where('str', 'bar'), + Q.where('str', 'bar2'), + Q.where('str', 'bar3'), + ]) + }) + it(`flattens Q.or`, () => { + expect( + optimize([ + // + Q.where('str', 'bar'), + Q.or([ + // + Q.where('str', 'bar2'), + Q.or(Q.where('str', 'bar3'), Q.where('str', 'bar4')), + ]), + ]), + ).toEqual([ + // + Q.where('str', 'bar'), + Q.or([Q.where('str', 'bar2'), Q.where('str', 'bar3'), Q.where('str', 'bar4')]), + ]) + }) + it(`does not flatten Q.and(Q.or), Q.or(Q.and)`, () => { + const orig = [ + // + Q.or(Q.where('str', 'bar'), Q.where('str', 'bar2')), + Q.or( + // + Q.where('str', 'bar3'), + Q.and(Q.where('str', 'bar'), Q.where('str', 'bar2')), + ), + ] + expect(optimize(orig)).toEqual(orig) + }) + it(`flattens 1-element Q.and(Q.or), Q.or(Q.and)`, () => { + expect( + optimize([ + // + Q.or(Q.where('str', 'bar')), + Q.or(Q.and(Q.where('str', 'bar')), Q.where('str', 'bar2')), + ]), + ).toEqual([ + // + Q.where('str', 'bar'), + Q.or(Q.where('str', 'bar'), Q.where('str', 'bar2')), + ]) + }) + it.skip(`flattens complex nested conditions`, () => { + expect( + optimize([ + Q.and( + Q.and( + Q.and( + // + Q.on('tasks', Q.on('comments', 'foo', 'bar')), + Q.on('tasks', 'foo', 'bar'), + ), + ), + Q.or( + Q.or( + Q.or( + // + Q.on('tasks', Q.on('comments', 'foo', 'bar')), + Q.on('tasks', 'foo', 'bar'), + ), + ), + ), + ), + ]), + ).toEqual([ + Q.on('tasks', [ + // + Q.where('foo', 'bar'), + Q.on('comments', 'foo', 'bar'), + ]), + // Q.or( + // // + // Q.on('tasks', 'foo', 'bar'), + // Q.on('tasks', Q.on('comments', 'foo', 'bar')), + // ), + ]) + }) + }) + describe('merges Q.on', () => { + it(`merges Q.ons`, () => { + expect( + optimize([ + Q.on('tasks', 'foo', 'bar'), + Q.on('tasks', [ + // + Q.where('bar', 'baz'), + Q.where('baz', 'blah'), + ]), + ]), + ).toEqual([ + Q.on('tasks', [ + // + Q.where('foo', 'bar'), + Q.where('bar', 'baz'), + Q.where('baz', 'blah'), + ]), + ]) + }) + it(`merges inner Q.ons`, () => { + expect( + optimize([ + Q.on('tasks', Q.on('comments', 'foo', 'bar')), + Q.on( + 'tasks', + Q.on('comments', [ + // + Q.where('bar', 'baz'), + Q.where('baz', 'blah'), + ]), + ), + ]), + ).toEqual([ + Q.on( + 'tasks', + Q.on('comments', [ + // + Q.where('foo', 'bar'), + Q.where('bar', 'baz'), + Q.where('baz', 'blah'), + ]), + ), + ]) + }) + it(`merges Q.or(Q.on) into Q.on(Q.or())`, () => { + expect( + optimize([ + Q.or( + Q.where('foo', 'bar'), + Q.on('tasks', 'foo', 'bar'), + Q.on('tasks', [ + // + Q.where('bar', 'baz'), + Q.where('baz', 'blah'), + ]), + ), + ]), + ).toEqual([ + Q.or( + Q.where('foo', 'bar'), + Q.on( + 'tasks', + Q.or( + Q.where('foo', 'bar'), + Q.and([ + // + Q.where('bar', 'baz'), + Q.where('baz', 'blah'), + ]), + ), + ), + ), + ]) + }) + it(`merges Q.on from flattened lists`, () => { + expect( + optimize([ + Q.on('tasks', [ + // + Q.where('baz', 'blah'), + Q.where('fiz', 'buzz'), + ]), + Q.or( + // + Q.on('tasks', 'foo', 'bar'), + Q.on('tasks', 'bar', 'baz'), + ), + ]), + ).toEqual([ + Q.on('tasks', [ + Q.where('baz', 'blah'), + Q.where('fiz', 'buzz'), + Q.or([ + // + Q.where('foo', 'bar'), + Q.where('bar', 'baz'), + ]), + ]), + ]) + }) + }) + describe('optimizes inner lists', () => { + it(`optimizes Q.and`, () => { + expect( + optimize([ + Q.or( + Q.where('str', 'baz'), + Q.and( + // + Q.where('str', 'bar'), + Q.where('bool_i', 'bar'), + Q.where('str_i', 'bar'), + ), + ), + ]), + ).toEqual([ + Q.or( + Q.where('str', 'baz'), + Q.and( + // + Q.where('bool_i', 'bar'), + Q.where('str_i', 'bar'), + Q.where('str', 'bar'), + ), + ), + ]) + }) + it(`optimizes Q.or`, () => { + expect( + optimize([ + Q.or( + // + Q.where('str', 'bar'), + Q.where('bool_i', 'bar'), + Q.where('str_i', 'bar'), + ), + ]), + ).toEqual([ + Q.or( + // + Q.where('bool_i', 'bar'), + Q.where('str_i', 'bar'), + Q.where('str', 'bar'), + ), + ]) + }) + it(`optimizes Q.on`, () => { + expect( + optimize([ + // + Q.on('tasks', Q.where('t_str', 'bar')), + Q.on('tasks', [Q.where('t_bool_i', 'bar'), Q.where('t_str_i', 'bar')]), + ]), + ).toEqual([ + Q.on('tasks', [ + // + Q.where('t_bool_i', 'bar'), + Q.where('t_str_i', 'bar'), + Q.where('t_str', 'bar'), + ]), + ]) + }) + it.only(`optimizes Q.or(Q.on)`, () => { + expect( + optimize([ + Q.or( + Q.where('foo', 'bar'), + Q.on('tasks', [ + // + Q.where('t_str', 'bar'), + Q.where('t_str_i', 'bar'), + ]), + ), + ]), + ).toEqual([ + Q.or( + Q.where('foo', 'bar'), + Q.on('tasks', [ + // + Q.where('t_str_i', 'bar'), + Q.where('t_str', 'bar'), + ]), + ), + ]) + }) + }) + it('deep freezes the query in dev', () => { + const make = () => optimize([Q.where('left_column', 'right_value')]) + const query = make() + expect(() => { + query.foo = [] + }).toThrow() + expect(() => { + query.where[0].comparison.right = {} + }).toThrow() + expect(query).toEqual(make()) + }) +})