refactor: architectural improvements for grouped expressions (demo for PR #863)

src/liquid-options.ts

@@ -84,6 +84,8 @@ export interface LiquidOptions {
   operators?: Operators;
   /** Respect parameter order when using filters like "for ... reversed limit", Defaults to `false`. */
   orderedFilterParameters?: boolean;
+  /** Allow parenthesized expressions as operands in conditions and loops, e.g. `{% if (foo | upcase) == "BAR" %}`. This is a non-standard extension to Liquid. Defaults to `false`. */
+  groupedExpressions?: boolean;
   /** For DoS handling, limit total length of templates parsed in one `parse()` call. A typical PC can handle 1e8 (100M) characters without issues. */
   parseLimit?: number;
   /** For DoS handling, limit total time (in ms) for each `render()` call. */
@@ -159,6 +161,7 @@ export interface NormalizedFullOptions extends NormalizedOptions {
   globals: object;
   keepOutputType: boolean;
   operators: Operators;
+  groupedExpressions: boolean;
   parseLimit: number;
   renderLimit: number;
   memoryLimit: number;
@@ -195,6 +198,7 @@ export const defaultOptions: NormalizedFullOptions = {
   globals: {},
   keepOutputType: false,
   operators: defaultOperators,
+  groupedExpressions: false,
   memoryLimit: Infinity,
   parseLimit: Infinity,
   renderLimit: Infinity

src/parser/parser.ts

@@ -33,7 +33,7 @@ export class Parser {
   public parse (html: string, filepath?: string): Template[] {
     html = String(html)
     this.parseLimit.use(html.length)
-    const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath)
+    const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath, undefined, this.liquid.options.groupedExpressions)
     const tokens = tokenizer.readTopLevelTokens(this.liquid.options)
     return this.parseTokens(tokens)
   }

src/parser/token-kind.ts

@@ -12,5 +12,6 @@ export enum TokenKind {
   Quoted = 1024,
   Operator = 2048,
   FilteredValue = 4096,
+  GroupedExpression = 8192,
   Delimited = Tag | Output
 }

src/parser/tokenizer.spec.ts

@@ -1,4 +1,4 @@
-import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken } from '../tokens'
+import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken, GroupedExpressionToken } from '../tokens'
 import { Tokenizer } from './tokenizer'
 import { defaultOperators } from '../render/operator'
 import { createTrie } from '../util/operator-trie'
@@ -229,22 +229,80 @@ describe('Tokenizer', function () {
   })
   describe('#readRange()', () => {
     it('should read `(1..3)`', () => {
-      const range = new Tokenizer('(1..3)').readRange()
+      const result = new Tokenizer('(1..3)').readGroupOrRange()
+      expect(result).toBeDefined()
+      expect(result!.type).toBe('range')
+      const { range } = result as { type: 'range', range: RangeToken }
       expect(range).toBeInstanceOf(RangeToken)
-      expect(range!.getText()).toEqual('(1..3)')
-      const { lhs, rhs } = range!
-      expect(lhs).toBeInstanceOf(NumberToken)
-      expect(lhs.getText()).toBe('1')
-      expect(rhs).toBeInstanceOf(NumberToken)
-      expect(rhs.getText()).toBe('3')
+      expect(range.getText()).toEqual('(1..3)')
+      expect(range.lhs).toBeInstanceOf(NumberToken)
+      expect(range.lhs.getText()).toBe('1')
+      expect(range.rhs).toBeInstanceOf(NumberToken)
+      expect(range.rhs.getText()).toBe('3')
     })
     it('should throw for `(..3)`', () => {
-      expect(() => new Tokenizer('(..3)').readRange()).toThrow('unexpected token "..3)", value expected')
+      expect(() => new Tokenizer('(..3)').readGroupOrRange()).toThrow('unexpected token "..3)", value expected')
     })
     it('should read `(a.b..c["..d"])`', () => {
-      const range = new Tokenizer('(a.b..c["..d"])').readRange()
-      expect(range).toBeInstanceOf(RangeToken)
-      expect(range!.getText()).toEqual('(a.b..c["..d"])')
+      const wrappedToken = new Tokenizer('(a.b..c["..d"])').readGroupOrRange() as { type: 'range', range: RangeToken }
+      expect(wrappedToken).toBeDefined()
+      expect(wrappedToken.type).toBe('range')
+
+      const result = wrappedToken as { type: 'range', range: RangeToken }
+
+      expect(result.range).toBeInstanceOf(RangeToken)
+      expect(result.range.getText()).toEqual('(a.b..c["..d"])')
+    })
+  })
+  describe('#readGroupedExpression()', () => {
+    function createGrouped (input: string): Tokenizer {
+      const t = new Tokenizer(input, defaultOperators)
+      t.groupedExpressions = true
+      return t
+    }
+    it('should read `(foo | upcase)` as GroupedExpressionToken', () => {
+      const token = createGrouped('(foo | upcase)').readValue()
+      expect(token).toBeInstanceOf(GroupedExpressionToken)
+      const grouped = token as GroupedExpressionToken
+      expect(grouped.getText()).toBe('(foo | upcase)')
+      expect(grouped.initial.postfix).toHaveLength(1)
+      expect(grouped.filters).toHaveLength(1)
+      expect(grouped.filters[0].name).toBe('upcase')
+    })
+    it('should read `(foo | append: "!")` with filter argument', () => {
+      const token = createGrouped('(foo | append: "!")').readValue()
+      expect(token).toBeInstanceOf(GroupedExpressionToken)
+      const grouped = token as GroupedExpressionToken
+      expect(grouped.filters).toHaveLength(1)
+      expect(grouped.filters[0].name).toBe('append')
+      expect(grouped.filters[0].args).toHaveLength(1)
+    })
+    it('should read nested `((foo | append: "!") | upcase)`', () => {
+      const token = createGrouped('((foo | append: "!") | upcase)').readValue()
+      expect(token).toBeInstanceOf(GroupedExpressionToken)
+      const grouped = token as GroupedExpressionToken
+      expect(grouped.filters).toHaveLength(1)
+      expect(grouped.filters[0].name).toBe('upcase')
+      expect(grouped.initial.postfix).toHaveLength(1)
+      expect(grouped.initial.postfix[0]).toBeInstanceOf(GroupedExpressionToken)
+    })
+    it('should parse `(a | upcase) == "BAR"` as expression', () => {
+      const exp = [...createGrouped('(a | upcase) == "BAR"').readExpressionTokens()]
+      expect(exp).toHaveLength(3)
+      expect(exp[0]).toBeInstanceOf(GroupedExpressionToken)
+      expect(exp[1]).toBeInstanceOf(OperatorToken)
+      expect(exp[1].getText()).toBe('==')
+      expect(exp[2]).toBeInstanceOf(QuotedToken)
+    })
+    it('should still parse `(1..3)` as RangeToken', () => {
+      const token = createGrouped('(1..3)').readValue()
+      expect(token).toBeInstanceOf(RangeToken)
+    })
+    it('should throw for unclosed parens', () => {
+      expect(() => createGrouped('(foo | upcase').readValue()).toThrow('unbalanced parentheses')
+    })
+    it('should fall back to readRange when flag is off', () => {
+      expect(() => new Tokenizer('(foo | upcase)', defaultOperators).readValue()).toThrow('invalid range syntax')
     })
   })
   describe('#readFilter()', () => {

src/parser/tokenizer.ts

@@ -1,4 +1,4 @@
-import { FilteredValueToken, TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken } from '../tokens'
+import { FilteredValueToken, TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken, GroupedExpressionToken } from '../tokens'
 import { OperatorHandler } from '../render/operator'
 import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, NUMBER, SIGN, isWord, isString } from '../util'
 import { Operators, Expression } from '../render'
@@ -9,6 +9,7 @@ import { whiteSpaceCtrl } from './whitespace-ctrl'
 export class Tokenizer {
   p: number
   N: number
+  public groupedExpressions: boolean
   private rawBeginAt = -1
   private opTrie: Trie<OperatorHandler>
   private literalTrie: Trie<LiteralValue>
@@ -17,12 +18,14 @@ export class Tokenizer {
     public input: string,
     operators: Operators = defaultOptions.operators,
     public file?: string,
-    range?: [number, number]
+    range?: [number, number],
+    groupedExpressions = false
   ) {
     this.p = range ? range[0] : 0
     this.N = range ? range[1] : input.length
     this.opTrie = createTrie(operators)
     this.literalTrie = createTrie(literalValues)
+    this.groupedExpressions = groupedExpressions
   }
 
   readExpression () {
@@ -80,6 +83,7 @@ export class Tokenizer {
   readFilter (): FilterToken | null {
     this.skipBlank()
     if (this.end()) return null
+    if (this.peek() === ')') return null
     this.assert(this.read() === '|', `expected "|" before filter`)
     const name = this.readIdentifier()
     if (!name.size()) {
@@ -94,9 +98,9 @@ export class Tokenizer {
         const arg = this.readFilterArg()
         arg && args.push(arg)
         this.skipBlank()
-        this.assert(this.end() || this.peek() === ',' || this.peek() === '|', () => `unexpected character ${this.snapshot()}`)
+        this.assert(this.end() || this.peek() === ',' || this.peek() === '|' || this.peek() === ')', () => `unexpected character ${this.snapshot()}`)
       } while (this.peek() === ',')
-    } else if (this.peek() === '|' || this.end()) {
+    } else if (this.peek() === '|' || this.peek() === ')' || this.end()) {
       // do nothing
     } else {
       throw this.error('expected ":" after filter name')
@@ -310,7 +314,16 @@ export class Tokenizer {
   readValue (): ValueToken | undefined {
     this.skipBlank()
     const begin = this.p
-    const variable = this.readLiteral() || this.readQuoted() || this.readRange() || this.readNumber()
+    let variable: ValueToken | undefined = this.readLiteral() || this.readQuoted()
+    if (!variable && this.peek() === '(') {
+      const rangeOrGroup = this.readGroupOrRange()
+      if (rangeOrGroup?.type === 'range') {
+        variable = rangeOrGroup.range
+      } else if (rangeOrGroup?.type === 'groupedExpression') {
+        variable = rangeOrGroup.groupedExpression
+      }
+    }
+    variable = variable || this.readNumber()
     const props = this.readProperties(!variable)
     if (!props.length) return variable
     return new PropertyAccessToken(variable, props, this.input, begin, this.p)
@@ -385,18 +398,37 @@ export class Tokenizer {
     return literal
   }
 
-  readRange (): RangeToken | undefined {
+  readGroupOrRange (): { type: 'range', range: RangeToken } | { type: 'groupedExpression', groupedExpression: GroupedExpressionToken } | undefined {
     this.skipBlank()
     const begin = this.p
     if (this.peek() !== '(') return
     ++this.p
     const lhs = this.readValueOrThrow()
     this.skipBlank()
-    this.assert(this.read() === '.' && this.read() === '.', 'invalid range syntax')
-    const rhs = this.readValueOrThrow()
-    this.skipBlank()
-    this.assert(this.read() === ')', 'invalid range syntax')
-    return new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
+
+    if (this.peek() === '.' && this.peek(1) === '.') {
+      this.p += 2
+      const rhs = this.readValueOrThrow()
+      this.skipBlank()
+      this.assert(this.read() === ')', 'invalid range syntax')
+      return {
+        type: 'range',
+        range: new RangeToken(this.input, begin, this.p, lhs, rhs, this.file)
+      }
+    }
+
+    if (this.groupedExpressions) {
+      const expression = new Expression((function * () { yield lhs })())
+      const filters = this.readFilters()
+      this.skipBlank()
+      this.assert(this.read() === ')', 'unbalanced parentheses')
+      return {
+        type: 'groupedExpression',
+        groupedExpression: new GroupedExpressionToken(expression, filters, this.input, begin, this.p, this.file)
+      }
+    }
+
+    throw this.error('invalid range syntax')
   }
 
   readValueOrThrow (): ValueToken {

src/render/expression.ts

@@ -1,5 +1,5 @@
-import { QuotedToken, RangeToken, OperatorToken, Token, PropertyAccessToken, OperatorType, operatorTypes } from '../tokens'
-import { isRangeToken, isPropertyAccessToken, UndefinedVariableError, range, isOperatorToken, assert } from '../util'
+import { QuotedToken, RangeToken, OperatorToken, Token, PropertyAccessToken, OperatorType, operatorTypes, GroupedExpressionToken } from '../tokens'
+import { isRangeToken, isPropertyAccessToken, isGroupedExpressionToken, UndefinedVariableError, range, isOperatorToken, assert } from '../util'
 import type { Context } from '../context'
 import type { UnaryOperatorHandler } from '../render'
 import { Drop } from '../drop'
@@ -40,6 +40,19 @@ export function * evalToken (token: Token | undefined, ctx: Context, lenient = f
   if ('content' in token) return token.content
   if (isPropertyAccessToken(token)) return yield evalPropertyAccessToken(token, ctx, lenient)
   if (isRangeToken(token)) return yield evalRangeToken(token, ctx)
+  if (isGroupedExpressionToken(token)) return yield evalGroupedExpressionToken(token, ctx, lenient)
+}
+
+function * evalGroupedExpressionToken (token: GroupedExpressionToken, ctx: Context, lenient: boolean): IterableIterator<unknown> {
+  assert(token.resolvedFilters, 'grouped expression filters not resolved')
+  lenient = lenient || (ctx.opts.lenientIf && token.filters.length > 0 && token.filters[0].name === 'default')
+  let val = yield token.initial.evaluate(ctx, lenient)
+
+  for (const filter of token.resolvedFilters!) {
+    val = yield filter.render(val, ctx)
+  }
+
+  return val
 }
 
 function * evalPropertyAccessToken (token: PropertyAccessToken, ctx: Context, lenient: boolean): IterableIterator<unknown> {

src/tags/case.ts

@@ -1,7 +1,7 @@
 import { ValueToken, Liquid, toValue, evalToken, Value, Emitter, TagToken, TopLevelToken, Context, Template, Tag, ParseStream } from '..'
 import { Parser } from '../parser'
 import { equals } from '../render'
-import { Arguments } from '../template'
+import { Arguments, resolveGroupedExpressionFilters } from '../template'
 
 export default class extends Tag {
   value: Value
@@ -24,7 +24,9 @@ export default class extends Tag {
 
         const values: ValueToken[] = []
         while (!token.tokenizer.end()) {
-          values.push(token.tokenizer.readValueOrThrow())
+          const val = token.tokenizer.readValueOrThrow()
+          resolveGroupedExpressionFilters(val, liquid)
+          values.push(val)
           token.tokenizer.skipBlank()
           if (token.tokenizer.peek() === ',') {
             token.tokenizer.readTo(',')

src/tags/for.ts

@@ -1,16 +1,17 @@
 import { Hash, ValueToken, Liquid, Tag, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, ParseStream } from '..'
+import { GroupedExpressionToken } from '../tokens'
 import { assertEmpty, isValueToken, toEnumerable } from '../util'
 import { ForloopDrop } from '../drop/forloop-drop'
 import { Parser } from '../parser'
-import { Arguments } from '../template'
+import { Arguments, resolveGroupedExpressionFilters } from '../template'
 
 const MODIFIERS = ['offset', 'limit', 'reversed']
 
 type valueOf<T> = T[keyof T]
 
 export default class extends Tag {
   variable: string
-  collection: ValueToken
+  collection: ValueToken | GroupedExpressionToken
   hash: Hash
   templates: Template[]
   elseTemplates: Template[]
@@ -26,6 +27,7 @@ export default class extends Tag {
 
     this.variable = variable.content
     this.collection = collection
+    resolveGroupedExpressionFilters(this.collection, liquid)
     this.hash = new Hash(this.tokenizer, liquid.options.keyValueSeparator)
     this.templates = []
     this.elseTemplates = []

src/tags/tablerow.ts

@@ -1,14 +1,15 @@
 import { isValueToken, toEnumerable } from '../util'
 import { ValueToken, Liquid, Tag, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, ParseStream } from '..'
+import { GroupedExpressionToken } from '../tokens'
 import { TablerowloopDrop } from '../drop/tablerowloop-drop'
 import { Parser } from '../parser'
-import { Arguments } from '../template'
+import { Arguments, resolveGroupedExpressionFilters } from '../template'
 
 export default class extends Tag {
   variable: string
   args: Hash
   templates: Template[]
-  collection: ValueToken
+  collection: ValueToken | GroupedExpressionToken
   constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) {
     super(tagToken, remainTokens, liquid)
     const variable = this.tokenizer.readIdentifier()
@@ -22,6 +23,7 @@ export default class extends Tag {
 
     this.variable = variable.content
     this.collection = collectionToken
+    resolveGroupedExpressionFilters(this.collection, liquid)
     this.args = new Hash(this.tokenizer, liquid.options.keyValueSeparator)
     this.templates = []
 

src/template/analysis.ts

@@ -2,6 +2,7 @@ import { Argument, Template, Value } from '.'
 import { isKeyValuePair } from '../parser/filter-arg'
 import { PropertyAccessToken, ValueToken } from '../tokens'
 import {
+  isGroupedExpressionToken,
   isNumberToken,
   isPropertyAccessToken,
   isQuotedToken,
@@ -371,11 +372,31 @@ function * extractValueTokenVariables (token: ValueToken): Generator<Variable> {
   if (isRangeToken(token)) {
     yield * extractValueTokenVariables(token.lhs)
     yield * extractValueTokenVariables(token.rhs)
+  } else if (isGroupedExpressionToken(token)) {
+    yield * extractGroupedExpressionTokenVariables(token)
   } else if (isPropertyAccessToken(token)) {
     yield extractPropertyAccessVariable(token)
   }
 }
 
+function * extractGroupedExpressionTokenVariables (token: ValueToken): Generator<Variable> {
+  if (!isGroupedExpressionToken(token)) return
+
+  for (const t of token.initial.postfix) {
+    if (isValueToken(t)) yield * extractValueTokenVariables(t)
+  }
+
+  for (const filter of token.filters) {
+    for (const arg of filter.args) {
+      if (isKeyValuePair(arg) && arg[1]) {
+        yield * extractValueTokenVariables(arg[1])
+      } else if (isValueToken(arg)) {
+        yield * extractValueTokenVariables(arg)
+      }
+    }
+  }
+}
+
 function extractPropertyAccessVariable (token: PropertyAccessToken): Variable {
   const segments: VariableSegments = []