From 4f1ba4d4b3afd04cbf2c1d50676f97745b2741c9 Mon Sep 17 00:00:00 2001 From: dplotnikov Date: Mon, 31 Aug 2026 20:46:42 -0700 Subject: [PATCH] [Pratt parser] Port changes improving compatibility with ANTLR parser from cel/c++ Ports changes from cel-cpp Pratt parser: - https://github.com/cel-expr/cel-cpp/pull/2238: Support selector/index tail after negative numeric literals in unary expressions - https://github.com/cel-expr/cel-cpp/pull/2239: Syntax error prefix on error reporting and ID allocation fixes for calls and ternaries - https://github.com/cel-expr/cel-cpp/pull/2241: Map entry ID allocation ordering and unary operator folding before ID allocation --- parser/helper.go | 15 ++++ parser/parser_test.go | 171 +++++++++++++++++++++++++++-------------- parser/pratt_parser.go | 122 ++++++++++++++--------------- 3 files changed, 189 insertions(+), 119 deletions(-) diff --git a/parser/helper.go b/parser/helper.go index ba6864cb9..8281b2d42 100644 --- a/parser/helper.go +++ b/parser/helper.go @@ -199,6 +199,21 @@ func (p *parserHelper) idFromOffsets(start, stop int32) int64 { return id } +func (p *parserHelper) allocID() int64 { + id := p.nextID + p.nextID++ + return id +} + +func (p *parserHelper) setTokenLocation(id int64, tok token) { + if tok.start >= 0 { + p.sourceInfo.SetOffsetRange(id, ast.OffsetRange{ + Start: p.computeOffset(tok.start), + Stop: p.computeOffset(tok.end), + }) + } +} + func (p *parserHelper) deleteID(id int64) { p.sourceInfo.ClearOffsetRange(id) if id == p.nextID-1 { diff --git a/parser/parser_test.go b/parser/parser_test.go index d75eed35e..85e3190c9 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -325,10 +325,6 @@ var testCases = []testInfo{ a^#3:*expr.Expr_IdentExpr#:b^#4:*expr.Expr_IdentExpr#^#2:*expr.Expr_CreateStruct_Entry#, c^#6:*expr.Expr_IdentExpr#:d^#7:*expr.Expr_IdentExpr#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, - PrattP: `{ - a^#2:*expr.Expr_IdentExpr#:b^#4:*expr.Expr_IdentExpr#^#3:*expr.Expr_CreateStruct_Entry#, - c^#5:*expr.Expr_IdentExpr#:d^#7:*expr.Expr_IdentExpr#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, }, { I: `[]`, @@ -396,18 +392,27 @@ var testCases = []testInfo{ E: `ERROR: :1:1: invalid int literal | 0xFFFFFFFFFFFFFFFFF | ^`, + PrattE: `ERROR: :1:1: Syntax error: invalid int literal + | 0xFFFFFFFFFFFFFFFFF + | ^`, }, { I: `0xFFFFFFFFFFFFFFFFFu`, E: `ERROR: :1:1: invalid uint literal | 0xFFFFFFFFFFFFFFFFFu | ^`, + PrattE: `ERROR: :1:1: Syntax error: invalid uint literal + | 0xFFFFFFFFFFFFFFFFFu + | ^`, }, { I: `1.99e90000009`, E: `ERROR: :1:1: invalid double literal | 1.99e90000009 | ^`, + PrattE: `ERROR: :1:1: Syntax error: invalid double literal + | 1.99e90000009 + | ^`, }, { I: `*@a | b`, @@ -423,13 +428,13 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: extraneous input 'b' expecting | *@a | b | ......^`, - PrattE: `ERROR: :1:1: unexpected token + PrattE: `ERROR: :1:1: Syntax error: unexpected token | *@a | b | ^ - ERROR: :1:2: unexpected character + ERROR: :1:2: Syntax error: unexpected character | *@a | b | .^ - ERROR: :1:5: unexpected single '|', expected '||' + ERROR: :1:5: Syntax error: unexpected single '|', expected '||' | *@a | b | ....^`, }, @@ -441,7 +446,7 @@ var testCases = []testInfo{ ERROR: :1:5: Syntax error: extraneous input 'b' expecting | a | b | ....^`, - PrattE: `ERROR: :1:3: unexpected single '|', expected '||' + PrattE: `ERROR: :1:3: Syntax error: unexpected single '|', expected '||' | a | b | ..^`, }, @@ -750,10 +755,6 @@ var testCases = []testInfo{ foo^#3:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry#, bar^#6:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, - PrattP: `{ - foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, }, { I: `{foo: 5, bar: "xyz", }`, @@ -761,10 +762,6 @@ var testCases = []testInfo{ foo^#3:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#2:*expr.Expr_CreateStruct_Entry#, bar^#6:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, - PrattP: `{ - foo^#2:*expr.Expr_IdentExpr#:5^#4:*expr.Constant_Int64Value#^#3:*expr.Expr_CreateStruct_Entry#, - bar^#5:*expr.Expr_IdentExpr#:"xyz"^#7:*expr.Constant_StringValue#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, }, { I: `a > 5 && a < 10`, @@ -797,7 +794,7 @@ var testCases = []testInfo{ E: `ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '}', '(', '.', ',', '-', '!', '?', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | { | .^`, - PrattE: `ERROR: :1:2: expected '}' + PrattE: `ERROR: :1:2: Syntax error: expected '}' | { | .^`, }, @@ -825,10 +822,6 @@ var testCases = []testInfo{ 1^#3:*expr.Constant_Int64Value#:2u^#4:*expr.Constant_Uint64Value#^#2:*expr.Expr_CreateStruct_Entry#, 2^#6:*expr.Constant_Int64Value#:3u^#7:*expr.Constant_Uint64Value#^#5:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, - PrattP: `{ - 1^#2:*expr.Constant_Int64Value#:2u^#4:*expr.Constant_Uint64Value#^#3:*expr.Expr_CreateStruct_Entry#, - 2^#5:*expr.Constant_Int64Value#:3u^#7:*expr.Constant_Uint64Value#^#6:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, }, { I: `TestAllTypes{single_int32: 1, single_int64: 2}`, @@ -866,7 +859,7 @@ var testCases = []testInfo{ ERROR: :1:6: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | 1 + $ | .....^`, - PrattE: `ERROR: :1:5: unexpected character + PrattE: `ERROR: :1:5: Syntax error: unexpected character | 1 + $ | ....^`, }, @@ -968,7 +961,7 @@ var testCases = []testInfo{ ERROR: :1:6: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | 1 + + | .....^`, - PrattE: `ERROR: :1:5: unexpected token + PrattE: `ERROR: :1:5: Syntax error: unexpected token | 1 + + | ....^`, }, @@ -985,7 +978,7 @@ var testCases = []testInfo{ E: `ERROR: :1:10: Syntax error: no viable alternative at input '."a"' | {"a": 1}."a" | .........^`, - PrattE: `ERROR: :1:10: expected identifier after '.' + PrattE: `ERROR: :1:10: Syntax error: expected identifier after '.' | {"a": 1}."a" | .........^`, }, @@ -1070,10 +1063,10 @@ var testCases = []testInfo{ ERROR: :2:11: Syntax error: no viable alternative at input '.' | && in.😁 | .........．^`, - PrattE: `ERROR: :2:7: unexpected token + PrattE: `ERROR: :2:7: Syntax error: unexpected token | && in.😁 | ......^ - ERROR: :2:10: unexpected character + ERROR: :2:10: Syntax error: unexpected character | && in.😁 | .........＾`, }, @@ -1139,7 +1132,7 @@ var testCases = []testInfo{ ERROR: :1:3: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | in | ..^`, - PrattE: `ERROR: :1:1: unexpected token + PrattE: `ERROR: :1:1: Syntax error: unexpected token | in | ^`, }, @@ -1226,7 +1219,7 @@ var testCases = []testInfo{ ERROR: :1:9: Syntax error: extraneous input '}' expecting | func{{a}} | ........^`, - PrattE: `ERROR: :1:6: expected struct field name + PrattE: `ERROR: :1:6: Syntax error: expected struct field name | func{{a}} | .....^ ERROR: :1:9: Syntax error: mismatched input '}' expecting @@ -1241,7 +1234,7 @@ var testCases = []testInfo{ ERROR: :1:7: Syntax error: mismatched input '}' expecting ':' | msg{:a} | ......^`, - PrattE: `ERROR: :1:5: expected struct field name + PrattE: `ERROR: :1:5: Syntax error: expected struct field name | msg{:a} | ....^`, }, @@ -1250,7 +1243,7 @@ var testCases = []testInfo{ E: `ERROR: :1:3: Syntax error: mismatched input '}' expecting ':' | {a} | ..^`, - PrattE: `ERROR: :1:3: expected ':' in map entry + PrattE: `ERROR: :1:3: Syntax error: expected ':' in map entry | {a} | ..^`, }, @@ -1262,10 +1255,10 @@ var testCases = []testInfo{ ERROR: :1:4: Syntax error: mismatched input '}' expecting ':' | {:a} | ...^`, - PrattE: `ERROR: :1:2: unexpected token + PrattE: `ERROR: :1:2: Syntax error: unexpected token | {:a} | .^ - ERROR: :1:3: expected ':' in map entry + ERROR: :1:3: Syntax error: expected ':' in map entry | {:a} | ..^`, }, @@ -1274,7 +1267,7 @@ var testCases = []testInfo{ E: `ERROR: :1:8: Syntax error: mismatched input '}' expecting ':' | ind[a{b}] | .......^`, - PrattE: `ERROR: :1:8: expected ':' in struct field + PrattE: `ERROR: :1:8: Syntax error: expected ':' in struct field | ind[a{b}] | .......^`, }, @@ -1298,7 +1291,7 @@ var testCases = []testInfo{ ERROR: :1:2: Syntax error: mismatched input '' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | ? | .^`, - PrattE: `ERROR: :1:1: unexpected token + PrattE: `ERROR: :1:1: Syntax error: unexpected token | ? | ^`, }, @@ -1313,10 +1306,10 @@ var testCases = []testInfo{ ERROR: :1:12: Syntax error: error recovery attempt limit exceeded: 4 | a ? b ((?)) | ...........^`, - PrattE: `ERROR: :1:9: unexpected token + PrattE: `ERROR: :1:9: Syntax error: unexpected token | a ? b ((?)) | ........^ - ERROR: :1:12: expected ':' in conditional expression + ERROR: :1:12: Syntax error: expected ':' in conditional expression | a ? b ((?)) | ...........^`, }, @@ -1326,7 +1319,7 @@ var testCases = []testInfo{ ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]`, E: "ERROR: :-1:0: expression recursion limit exceeded: 32", PrattE: `ERROR: :-1:0: expression recursion limit exceeded: 32 -ERROR: :1:34: expected ']' +ERROR: :1:34: Syntax error: expected ']' | [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ | .................................^`, }, @@ -1378,22 +1371,23 @@ ERROR: :1:34: expected ']' ERROR: :14:23: Syntax error: extraneous input '/' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | --1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 | ......................^`, - PrattE: `ERROR: :3:33: unexpected token + PrattE: `ERROR: :3:33: Syntax error: unexpected token | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 | ................................^ - ERROR: :3:34: expected ']' + ERROR: :3:34: Syntax error: expected ']' | --3-[-1--1--1--1---1--1--1--0-/1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1 | .................................^ - ERROR: :11:17: unexpected character + ERROR: :11:17: Syntax error: unexpected character | --1--1---1--1-À1--0--1--1--1--1--0--2--1--1--0--1--1--1--1--0--1--1--1--3-[-1--1 | ................＾ - ERROR: :34:49: expected ']' + ERROR: :34:49: Syntax error: expected ']' | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 | ................................................^ - ERROR: :34:49: expected ']' + ERROR: :34:49: Syntax error: expected ']' | --1---1--1--1--0--1--1--1--1--0--3--1--1--0--1 | ................................................^`, - }, { + }, + { I: `ó ¢ ó 0  0"""\""\"""\""\"""\""\"""\""\"""\"\"""\""\"""\""\"""\""\"""\"!\"""\""\"""\""\"`, @@ -1425,19 +1419,19 @@ ERROR: :1:34: expected ']' ERROR: :3:11: Syntax error: token recognition error at: '\' | 0"""\""\"""\""\"""\""\"""\""\"""\"\"""\""\"""\""\"""\""\"""\"!\"""\""\"""\""\" | ..........^`, - PrattE: `ERROR: :1:1: unexpected character + PrattE: `ERROR: :1:1: Syntax error: unexpected character | ó ¢ | ＾ - ERROR: :1:2: unexpected character + ERROR: :1:2: Syntax error: unexpected character | ó ¢ | ．＾ - ERROR: :1:3: unexpected character + ERROR: :1:3: Syntax error: unexpected character | ó ¢ | ．．＾ - ERROR: :2:3: unexpected character + ERROR: :2:3: Syntax error: unexpected character | ó 0  | ..＾ - ERROR: :2:4: unexpected character + ERROR: :2:4: Syntax error: unexpected character | ó 0  | ..．＾`, }, @@ -1792,7 +1786,7 @@ ERROR: :1:34: expected ']' E: `ERROR: :1:6: Syntax error: mismatched input 'true' expecting IDENTIFIER | self.true == 1 | .....^`, - PrattE: `ERROR: :1:6: expected identifier after '.' + PrattE: `ERROR: :1:6: Syntax error: expected identifier after '.' | self.true == 1 | .....^`, }, @@ -1847,9 +1841,6 @@ ERROR: :1:34: expected ']' P: `{ ?"key"^#3:*expr.Constant_StringValue#:value^#4:*expr.Expr_IdentExpr#^#2:*expr.Expr_CreateStruct_Entry# }^#1:*expr.Expr_StructExpr#`, - PrattP: `{ - ?"key"^#2:*expr.Constant_StringValue#:value^#4:*expr.Expr_IdentExpr#^#3:*expr.Expr_CreateStruct_Entry# - }^#1:*expr.Expr_StructExpr#`, }, { I: `[?a, ?b]`, @@ -2036,10 +2027,10 @@ ERROR: :1:34: expected ']' PrattE: `ERROR: :1:3: unsupported syntax '?' | x{?. | ..^ - ERROR: :1:4: expected struct field name + ERROR: :1:4: Syntax error: expected struct field name | x{?. | ...^ - ERROR: :1:5: expected '}' + ERROR: :1:5: Syntax error: expected '}' | x{?. | ....^`, }, @@ -2049,10 +2040,10 @@ ERROR: :1:34: expected ']' ERROR: :1:3: Syntax error: mismatched input '.' expecting {'}', ',', '?', IDENTIFIER, ESC_IDENTIFIER} | x{. | ..^`, - PrattE: `ERROR: :1:3: expected struct field name + PrattE: `ERROR: :1:3: Syntax error: expected struct field name | x{. | ..^ - ERROR: :1:4: expected '}' + ERROR: :1:4: Syntax error: expected '}' | x{. | ...^`, }, @@ -2070,10 +2061,10 @@ ERROR: :1:34: expected ']' | '3# < 10" '& tru ^^ | ..................^ `, - PrattE: `ERROR: :1:12: unexpected single '&', expected '&&' + PrattE: `ERROR: :1:12: Syntax error: unexpected single '&', expected '&&' | '3# < 10" '& tru ^^ | ...........^ - ERROR: :1:18: unexpected character + ERROR: :1:18: Syntax error: unexpected character | '3# < 10" '& tru ^^ | .................^`, }, @@ -2287,10 +2278,72 @@ ERROR: :1:34: expected ']' E: `ERROR: :1:9: Syntax error: mismatched input ')' expecting {'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER} | foo(a,b,) | ........^`, - PrattE: `ERROR: :1:9: unexpected token + PrattE: `ERROR: :1:9: Syntax error: unexpected token | foo(a,b,) | ........^`, }, + { + I: `-.2.V`, + P: `-0.2^#1:*expr.Constant_DoubleValue#.V^#2:*expr.Expr_SelectExpr#`, + }, + { + I: `!-.2.V`, + P: `!_( + -0.2^#2:*expr.Constant_DoubleValue#.V^#3:*expr.Expr_SelectExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `!-2.V`, + P: `!_( + -2^#2:*expr.Constant_Int64Value#.V^#3:*expr.Expr_SelectExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `!-.2[0]`, + P: `!_( + _[_]( + -0.2^#2:*expr.Constant_DoubleValue#, + 0^#4:*expr.Constant_Int64Value# + )^#3:*expr.Expr_CallExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `-x.foo[0]`, + P: `-_( + _[_]( + x^#2:*expr.Expr_IdentExpr#.foo^#3:*expr.Expr_SelectExpr#, + 0^#5:*expr.Constant_Int64Value# + )^#4:*expr.Expr_CallExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `-x[0].foo`, + P: `-_( + _[_]( + x^#2:*expr.Expr_IdentExpr#, + 0^#4:*expr.Constant_Int64Value# + )^#3:*expr.Expr_CallExpr#.foo^#5:*expr.Expr_SelectExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `!x[0].foo`, + P: `!_( + _[_]( + x^#2:*expr.Expr_IdentExpr#, + 0^#4:*expr.Constant_Int64Value# + )^#3:*expr.Expr_CallExpr#.foo^#5:*expr.Expr_SelectExpr# + )^#1:*expr.Expr_CallExpr#`, + }, + { + I: `- -1`, + P: `1^#1:*expr.Constant_Int64Value#`, + }, + { + I: `---a`, + P: `-_( + a^#2:*expr.Expr_IdentExpr# + )^#1:*expr.Expr_CallExpr#`, + }, } type testInfo struct { diff --git a/parser/pratt_parser.go b/parser/pratt_parser.go index 4d502a40d..f5f9488d3 100644 --- a/parser/pratt_parser.go +++ b/parser/pratt_parser.go @@ -175,7 +175,7 @@ func (p *prattParserWorker) nextSignificantToken(reportError bool) token { continue } if tok.kind == tokError && reportError { - p.reportError(tok, "%s", p.lexer.GetError().message) + p.reportSyntaxError(tok, "%s", p.lexer.GetError().message) if p.isRecoveryLimitExceeded() { return token{kind: tokEnd, start: p.length, end: p.length} } @@ -222,9 +222,9 @@ func (p *prattParserWorker) expect(kind tokenKind, msg string) bool { if p.peekTok.kind == tokEnd { formattedTok = "" } - msg = fmt.Sprintf("Syntax error: mismatched input %s expecting '%s'", formattedTok, kind.String()) + msg = fmt.Sprintf("mismatched input %s expecting '%s'", formattedTok, kind.String()) } - p.reportError(p.peekTok, "%s", msg) + p.reportSyntaxError(p.peekTok, "%s", msg) } p.synchronizeOnDelimiter() return false @@ -270,6 +270,10 @@ func (p *prattParserWorker) reportError(ctx any, format string, args ...any) ast return err } +func (p *prattParserWorker) reportSyntaxError(ctx any, format string, args ...any) ast.Expr { + return p.reportError(ctx, "Syntax error: "+format, args...) +} + func (p *prattParserWorker) newLogicManager(function string, term ast.Expr) *logicManager { if p.enableVariadicOperatorASTs { return newVariadicLogicManager(p.exprFactory, function, term) @@ -377,7 +381,7 @@ func (p *prattParserWorker) parse() ast.Expr { } if p.peekTok.kind != tokEnd { if p.peekTok.kind != tokError { - p.reportError(p.peekTok, "Syntax error: mismatched input '%s' expecting ", p.tokenText(p.peekTok)) + p.reportSyntaxError(p.peekTok, "mismatched input '%s' expecting ", p.tokenText(p.peekTok)) } for p.peekTok.kind != tokEnd && !p.isRecoveryLimitExceeded() { p.nextToken() @@ -471,7 +475,7 @@ func (p *prattParserWorker) parseSelectorChainTail(lhs ast.Expr) ast.Expr { fieldTok := p.nextToken() if fieldTok.kind != tokIdent && fieldTok.kind != tokReservedWord { if fieldTok.kind != tokError { - p.reportError(fieldTok, "expected identifier after '.'") + p.reportSyntaxError(fieldTok, "expected identifier after '.'") } p.synchronizeOnDelimiter() return lhs @@ -564,7 +568,7 @@ func (p *prattParserWorker) parseStruct(objID int64, structName string) ast.Expr } fieldTok := p.nextToken() if fieldTok.kind != tokIdent && fieldTok.kind != tokReservedWord { - p.reportError(fieldTok, "expected struct field name") + p.reportSyntaxError(fieldTok, "expected struct field name") p.synchronizeOnDelimiter() break } @@ -589,71 +593,63 @@ func (p *prattParserWorker) parseStruct(objID int64, structName string) ast.Expr func (p *prattParserWorker) parseUnary() ast.Expr { tok := p.peekTok.kind if tok == tokExclamation || tok == tokMinus { - return p.parseUnaryOps() + return p.parseUnaryOpsChain(p.nextToken()) } return p.parsePrimary() } -func (p *prattParserWorker) parseUnaryOps() ast.Expr { - op := p.nextToken() - if p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { - return p.parseUnaryOpsChain(op) +func (p *prattParserWorker) parseUnaryOpsChain(firstOp token) ast.Expr { + type unaryOp struct { + token token + id int64 } - - if op.kind == tokMinus { - if p.peekTok.kind == tokInt { - return p.parseNegativeIntLiteral(p.nextID(op)) - } - if p.peekTok.kind == tokFloat { - return p.parseNegativeDoubleLiteral(p.nextID(op)) - } + ops := []unaryOp{{token: firstOp}} + for p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { + ops = append(ops, unaryOp{token: p.nextToken()}) } - opID := p.nextID(op) - operand := p.parseSelectorChain() - opName := operators.LogicalNot - if op.kind == tokMinus { - opName = operators.Negate - } - return p.globalCallOrMacro(opID, opName, operand) -} + hasSolitaryTrailingMinus := len(ops) > 0 && + ops[len(ops)-1].token.kind == tokMinus && + (len(ops) == 1 || ops[len(ops)-2].token.kind != tokMinus) -func (p *prattParserWorker) parseUnaryOpsChain(firstOp token) ast.Expr { - type unaryOpInfo struct { - kind tokenKind - id int64 + write := 0 + for read := 0; read < len(ops); { + next := read + for next < len(ops) && ops[next].token.kind == ops[read].token.kind { + next++ + } + if (next-read)%2 != 0 { + ops[write] = ops[read] + write++ + } + read = next } - ops := []unaryOpInfo{{kind: firstOp.kind, id: p.nextID(firstOp)}} + ops = ops[:write] - for p.peekTok.kind == tokExclamation || p.peekTok.kind == tokMinus { - op := p.nextToken() - ops = append(ops, unaryOpInfo{kind: op.kind, id: p.nextID(op)}) + for i := range ops { + ops[i].id = p.nextID(ops[i].token) } var operand ast.Expr - if len(ops) > 0 && ops[len(ops)-1].kind == tokMinus { - switch p.peekTok.kind { - case tokInt: - lastOp := ops[len(ops)-1] - ops = ops[:len(ops)-1] + if hasSolitaryTrailingMinus && (p.peekTok.kind == tokInt || p.peekTok.kind == tokFloat) { + lastOp := ops[len(ops)-1] + ops = ops[:len(ops)-1] + if p.peekTok.kind == tokInt { operand = p.parseNegativeIntLiteral(lastOp.id) - case tokFloat: - lastOp := ops[len(ops)-1] - ops = ops[:len(ops)-1] + } else { operand = p.parseNegativeDoubleLiteral(lastOp.id) - default: - operand = p.parseSelectorChain() } + operand = p.parseSelectorChainTail(operand) } else { operand = p.parseSelectorChain() } for i := len(ops) - 1; i >= 0; i-- { opName := operators.LogicalNot - if ops[i].kind == tokMinus { + if ops[i].token.kind == tokMinus { opName = operators.Negate } - operand = p.helper.newGlobalCall(ops[i].id, opName, operand) + operand = p.globalCallOrMacro(ops[i].id, opName, operand) } return operand } @@ -747,9 +743,9 @@ func (p *prattParserWorker) parsePrimary() ast.Expr { badTok := p.nextToken() if badTok.kind != tokError { if badTok.kind == tokEnd { - p.reportError(badTok, "Syntax error: mismatched input '' expecting expression") + p.reportSyntaxError(badTok, "mismatched input '' expecting expression") } else { - p.reportError(badTok, "unexpected token") + p.reportSyntaxError(badTok, "unexpected token") } } return p.helper.newExpr(badTok) @@ -801,12 +797,13 @@ func (p *prattParserWorker) parseMap() ast.Expr { p.reportError(q, "unsupported syntax '?'") } } + entryID := p.helper.allocID() key := p.parseExpr() colonTok := p.peekTok if !p.expect(tokColon, "expected ':' in map entry") { break } - entryID := p.nextID(colonTok) + p.helper.setTokenLocation(entryID, colonTok) val := p.parseExpr() entries = append(entries, p.helper.newMapEntry(entryID, key, val, optional)) if p.peekTok.kind == tokComma { @@ -832,7 +829,7 @@ func (p *prattParserWorker) parseIdentOrCall() ast.Expr { idTok := p.nextToken() if idTok.kind != tokIdent && idTok.kind != tokReservedWord { if idTok.kind != tokError { - p.reportError(idTok, "expected identifier") + p.reportSyntaxError(idTok, "expected identifier") } return p.helper.newExpr(idTok) } @@ -846,12 +843,17 @@ func (p *prattParserWorker) parseIdentOrCall() ast.Expr { if leadingDot { name = "." + idText } - id := p.nextID(firstTok) if p.peekTok.kind == tokLeftParen { - p.nextToken() + lparen := p.nextToken() + callID := p.nextID(lparen) args := p.parseArguments(tokRightParen) - return p.globalCallOrMacro(id, name, args...) + return p.globalCallOrMacro(callID, name, args...) + } + targetTok := idTok + if leadingDot { + targetTok = firstTok } + id := p.nextID(targetTok) return p.helper.newIdent(id, name) } @@ -863,7 +865,7 @@ func (p *prattParserWorker) parseArguments(closeTok tokenKind) []ast.Expr { if p.peekTok.kind == tokComma { p.nextToken() if p.peekTok.kind == closeTok { - p.reportError(p.peekTok, "unexpected token") + p.reportSyntaxError(p.peekTok, "unexpected token") break } continue @@ -886,7 +888,7 @@ func (p *prattParserWorker) parseIntLiteral() ast.Expr { } val, err := strconv.ParseInt(text, base, 64) if err != nil { - return p.reportError(tok, "invalid int literal") + return p.reportSyntaxError(tok, "invalid int literal") } return p.helper.newLiteralInt(id, val) } @@ -901,7 +903,7 @@ func (p *prattParserWorker) parseNegativeIntLiteral(opID int64) ast.Expr { } val, err := strconv.ParseInt("-"+text, base, 64) if err != nil { - return p.reportError(tok, "invalid int literal") + return p.reportSyntaxError(tok, "invalid int literal") } return p.helper.newLiteralInt(opID, val) } @@ -918,7 +920,7 @@ func (p *prattParserWorker) parseUintLiteral() ast.Expr { } val, err := strconv.ParseUint(text, base, 64) if err != nil { - return p.reportError(tok, "invalid uint literal") + return p.reportSyntaxError(tok, "invalid uint literal") } return p.helper.newLiteralUint(id, val) } @@ -929,7 +931,7 @@ func (p *prattParserWorker) parseDoubleLiteral() ast.Expr { text := p.tokenText(tok) val, err := strconv.ParseFloat(text, 64) if err != nil { - return p.reportError(tok, "invalid double literal") + return p.reportSyntaxError(tok, "invalid double literal") } return p.helper.newLiteralDouble(id, val) } @@ -939,7 +941,7 @@ func (p *prattParserWorker) parseNegativeDoubleLiteral(opID int64) ast.Expr { text := p.tokenText(tok) val, err := strconv.ParseFloat(text, 64) if err != nil { - return p.reportError(tok, "invalid double literal") + return p.reportSyntaxError(tok, "invalid double literal") } return p.helper.newLiteralDouble(opID, -val) }