Skip to content

Optimize CEL PrattParser and Lexer performance #1216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
+233 −243
Open

Optimize CEL PrattParser and Lexer performance #1216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions parser/src/main/java/dev/cel/parser/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ java_library(
"//common:source_location",
"//common/ast",
"//common/internal",
"@maven//:com_google_errorprone_error_prone_annotations",
"@maven//:com_google_guava_guava",
"@maven//:org_jspecify_jspecify",
],
Expand Down
106 changes: 46 additions & 60 deletions parser/src/main/java/dev/cel/parser/Lexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,17 @@ static final class Token {
final TokenType type;
final int start;
final int end;
final @Nullable String text;

Token(TokenType type, int start, int end) {
this(type, start, end, null);
}

Token(TokenType type, int start, int end, @Nullable String text) {
this.type = type;
this.start = start;
this.end = end;
this.text = text;
}

@Override
Expand Down Expand Up @@ -149,35 +155,28 @@ static final class LexerError {
.buildOrThrow();

private final CelCodePointArray content;
private final int size;
private int position;
private LexerError error;

Lexer(CelCodePointArray content) {
this.content = content;
this.size = content.size();
this.position = 0;
this.error = null;
}

Token lex() {
consumeWhitespaceAndComments();
int start = position;
if (position >= content.size()) {
if (position >= size) {
return makeToken(TokenType.END, start, start);
}
int c = content.get(position);
switch (c) {
case '\f':
case '\n':
case ' ':
case '\r':
case 0x0B: // \v (vertical tab)
case '\t':
{
consumeWhitespace();
return makeToken(TokenType.WHITESPACE, start, position);
}
case '.':
{
if (position + 1 < content.size() && isDigit(content.get(position + 1))) {
if (position + 1 < size && isDigit(content.get(position + 1))) {
return consumeNumericLiteral();
}
advance(1);
Expand Down Expand Up @@ -283,10 +282,6 @@ Token lex() {
case '/':
{
advance(1);
if (consume('/')) {
consumeLine();
return makeToken(TokenType.COMMENT, start, position);
}
return makeToken(TokenType.SLASH, start, position);
}
case '&':
Expand Down Expand Up @@ -391,7 +386,7 @@ private void advance(int n) {
}

private boolean match(int c) {
return position < content.size() && content.get(position) == c;
return position < size && content.get(position) == c;
}

private boolean consume(int c) {
Expand All @@ -403,7 +398,7 @@ private boolean consume(int c) {
}

private boolean consumeIf(IntPredicate predicate) {
if (position < content.size()) {
if (position < size) {
int cp = content.get(position);
if (predicate.test(cp)) {
advance(1);
Expand All @@ -414,7 +409,7 @@ private boolean consumeIf(IntPredicate predicate) {
}

private void consumeLine() {
while (position < content.size()) {
while (position < size) {
if (content.get(position) == '\n') {
advance(1);
return;
Expand All @@ -423,8 +418,8 @@ private void consumeLine() {
}
}

private void consumeWhitespace() {
while (position < content.size()) {
private void consumeWhitespaceAndComments() {
while (position < size) {
int c = content.get(position);
switch (c) {
case '\f':
Expand All @@ -433,38 +428,35 @@ private void consumeWhitespace() {
case '\r':
case 11: // \v
case '\t':
advance(1);
position++;
break;
case '/':
if (position + 1 < size && content.get(position + 1) == '/') {
consumeLine();
break;
} else {
return;
}
default:
return;
}
}
}

private boolean consumeDigits() {
boolean advanced = false;
while (position < content.size()) {
int c = content.get(position);
if (!isDigit(c)) {
break;
}
advance(1);
advanced = true;
int start = position;
while (position < size && isDigit(content.get(position))) {
position++;
}
return advanced;
return position > start;
}

private boolean consumeHexDigits() {
boolean advanced = false;
while (position < content.size()) {
int c = content.get(position);
if (!isHexDigit(c)) {
break;
}
advance(1);
advanced = true;
int start = position;
while (position < size && isHexDigit(content.get(position))) {
position++;
}
return advanced;
return position > start;
}

private TokenType consumeIntegralSuffix() {
Expand All @@ -486,7 +478,7 @@ private Token consumeQuotedIdent() {
private boolean consumeUntilAfter(int c, boolean isRaw) {
int pos = position;
boolean escaped = false;
while (pos < content.size()) {
while (pos < size) {
int cc = content.get(pos);
if (cc == '\n' || cc == '\r') {
position = pos;
Expand All @@ -503,20 +495,20 @@ private boolean consumeUntilAfter(int c, boolean isRaw) {
}
pos++;
}
position = content.size();
position = size;
return false;
}

private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) {
int pos = position;
boolean escaped = false;
while (pos < content.size()) {
while (pos < size) {
int cc = content.get(pos);
if (!isRaw && cc == '\\') {
escaped = !escaped;
} else {
if ((isRaw || !escaped)
&& pos + 2 < content.size()
&& pos + 2 < size
&& cc == quote
&& content.get(pos + 1) == quote
&& content.get(pos + 2) == quote) {
Expand All @@ -527,16 +519,14 @@ private boolean consumeUntilAfterTripleQuote(int quote, boolean isRaw) {
}
pos++;
}
position = content.size();
position = size;
return false;
}

private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolean isRaw) {
advance(1);
boolean isTripleQuote =
position + 1 < content.size()
&& content.get(position) == quote
&& content.get(position + 1) == quote;
position + 1 < size && content.get(position) == quote && content.get(position + 1) == quote;
if (isTripleQuote) {
advance(2);
if (!consumeUntilAfterTripleQuote(quote, isRaw)) {
Expand All @@ -556,7 +546,7 @@ private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolea

private @Nullable Token consumePrefixedStringLiteral() {
int start = position;
if (position >= content.size()) {
if (position >= size) {
return null;
}
int c = content.get(position);
Expand All @@ -566,15 +556,15 @@ private Token consumeStringLiteral(int start, int quote, boolean isBytes, boolea
return null;
}
int lookahead = 1;
if (position + 1 < content.size()) {
if (position + 1 < size) {
int c2 = content.get(position + 1);
if (isBytes ? (c2 == 'r' || c2 == 'R') : (c2 == 'b' || c2 == 'B')) {
isBytes = true;
isRaw = true;
lookahead = 2;
}
}
if (position + lookahead < content.size()) {
if (position + lookahead < size) {
int quote = content.get(position + lookahead);
if (quote == '"' || quote == '\'') {
advance(lookahead);
Expand Down Expand Up @@ -612,9 +602,9 @@ private Token consumeNumericLiteral() {
return makeToken(tokenType, start, position);
}
consumeDigits();
if (position < content.size()
if (position < size
&& content.get(position) == '.'
&& position + 1 < content.size()
&& position + 1 < size
&& isDigit(content.get(position + 1))) {
floatingPoint = true;
advance(1);
Expand All @@ -639,19 +629,15 @@ && isDigit(content.get(position + 1))) {

private Token consumeIdent() {
int start = position;
while (position < content.size()) {
int c = content.get(position);
if (!isIdentTrailing(c)) {
break;
}
advance(1);
while (position < size && isIdentTrailing(content.get(position))) {
position++;
}
int end = position;
String word = content.slice(start, end).toString();
TokenType keywordType = KEYWORDS.get(word);
if (keywordType != null) {
return makeToken(keywordType, start, end);
}
return makeToken(TokenType.IDENT, start, end);
return new Token(TokenType.IDENT, start, end, word);
}
}
Loading
Loading