From e3503d411c986ff111a8f2bf3ac47e6205ccdf5f Mon Sep 17 00:00:00 2001 From: Jen Basch Date: Tue, 7 Apr 2026 12:21:16 -0700 Subject: [PATCH] Add support for multi-line string line continuations --- .../language-reference/pages/index.adoc | 9 +++++---- .../org/pkl/core/util/SyntaxHighlighter.java | 4 +++- .../input/basic/stringMultiline.pkl | 16 +++++++++++++++ .../input/errors/parser19.pkl | 5 +++++ .../input/errors/parser20.pkl | 2 ++ .../output/basic/stringMultiline.pcf | 3 +++ .../output/errors/parser19.err | 6 ++++++ .../output/errors/parser20.err | 8 ++++++++ .../main/kotlin/org/pkl/formatter/Builder.kt | 5 ++++- .../input/multi-line-strings.pkl | 20 +++++++++++++++++++ .../output/multi-line-strings.pkl | 19 ++++++++++++++++++ .../org/pkl/parser/GenericParserImpl.java | 7 ++++++- .../src/main/java/org/pkl/parser/Lexer.java | 1 + .../main/java/org/pkl/parser/ParserImpl.java | 7 ++++++- .../src/main/java/org/pkl/parser/Token.java | 1 + .../pkl/parser/syntax/generic/NodeType.java | 3 ++- .../org/pkl/parser/errorMessages.properties | 5 +++++ .../org/pkl/parser/GenericSexpRenderer.kt | 1 + 18 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err diff --git a/docs/modules/language-reference/pages/index.adoc b/docs/modules/language-reference/pages/index.adoc index 60a94e68b..192e91f61 100644 --- a/docs/modules/language-reference/pages/index.adoc +++ b/docs/modules/language-reference/pages/index.adoc @@ -255,8 +255,7 @@ String literals are enclosed in double quotes: "Hello, World!" ---- -TIP: Except for a few minor differences footnote:[Pkl's string literals have fewer character escape sequences, -have stricter rules for line indentation in multiline strings, and do not have a line continuation character.], +TIP: Except for a few minor differences footnote:[Pkl's string literals have fewer character escape sequences and stricter rules for line indentation in multiline strings.], String literals have the same syntax and semantics as in Swift 5. Learn one of them, know both of them! Inside a string literal, the following character escape sequences have special meaning: @@ -305,7 +304,8 @@ To write a string that spans multiple lines, use a multiline string literal: ---- """ Although the Dodo is extinct, -the species will be remembered. +the species \ +will be remembered. """ ---- @@ -314,9 +314,10 @@ String content and closing delimiter must each start on a new line. The content of a multiline string starts on the first line after the opening quotes and ends on the last line before the closing quotes. Line breaks are included in the string and normalized to `\n`. +Line breaks following a `\` are ignored. The previous multiline string is equivalent to this single-line string. -Notice that there is no leading or trailing whitespace. +Notice that there is no leading or trailing whitespace and the second newline is not present. [source%tested,{pkl-expr}] ---- diff --git a/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java b/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java index 88786abb7..28f60b72d 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java +++ b/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java @@ -18,6 +18,7 @@ import static org.pkl.parser.Token.FALSE; import static org.pkl.parser.Token.NULL; import static org.pkl.parser.Token.STRING_ESCAPE_BACKSLASH; +import static org.pkl.parser.Token.STRING_ESCAPE_CONTINUATION; import static org.pkl.parser.Token.STRING_ESCAPE_NEWLINE; import static org.pkl.parser.Token.STRING_ESCAPE_QUOTE; import static org.pkl.parser.Token.STRING_ESCAPE_RETURN; @@ -41,7 +42,8 @@ private SyntaxHighlighter() {} STRING_ESCAPE_RETURN, STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, - STRING_ESCAPE_UNICODE); + STRING_ESCAPE_UNICODE, + STRING_ESCAPE_CONTINUATION); private static final EnumSet constant = EnumSet.of(TRUE, FALSE, NULL); diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultiline.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultiline.pkl index 029939f76..e9f252208 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultiline.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultiline.pkl @@ -25,6 +25,22 @@ examples { """ \u{9}\u{30}\u{100}\u{1000}\u{10000}\u{010000}\u{0010000}\u{00010000} """ + + """ + hello \ + world + """ + + #""" + hello \# + world + """# + + """ + hello \ + \ + world + """ } ["dollar sign has no special meaning"] { diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl new file mode 100644 index 000000000..4adf5385b --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl @@ -0,0 +1,5 @@ +foo = """ + hello \ +\ + world + """ diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl new file mode 100644 index 000000000..85ce04fc4 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl @@ -0,0 +1,2 @@ +foo = "hello \ +world" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultiline.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultiline.pcf index 37bf2f16a..affc15c84 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultiline.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultiline.pcf @@ -14,6 +14,9 @@ examples { """ "\t0Āက𐀀𐀀𐀀𐀀" + "hello world" + "hello world" + "hello world" } ["dollar sign has no special meaning"] { "123$ $123 $&% $" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err new file mode 100644 index 000000000..1336f1541 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err @@ -0,0 +1,6 @@ +–– Pkl Error –– +Line must match or exceed indentation of the String's last line. + +x | \ + ^ +at parser19 (file:///$snippetsDir/input/errors/parser19.pkl) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err new file mode 100644 index 000000000..b6c89dfc2 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err @@ -0,0 +1,8 @@ +–– Pkl Error –– +Invalid line continuation escape sequence. + +Line continuations are only allowed in multi-line strings. + +x | foo = "hello \ + ^^ +at parser20 (file:///$snippetsDir/input/errors/parser20.pkl) diff --git a/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt index bbcb93a68..48abb3d66 100644 --- a/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt +++ b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt @@ -63,6 +63,8 @@ internal class Builder(sourceText: String, private val grammarVersion: GrammarVe NodeType.SHEBANG, NodeType.OPERATOR -> Text(node.text(source)) NodeType.STRING_NEWLINE -> mustForceLine() + NodeType.STRING_CONTINUATION -> + Nodes(listOf(Text(node.text(source).dropLast(1)), mustForceLine())) NodeType.MODULE_DECLARATION -> formatModuleDeclaration(node) NodeType.MODULE_DEFINITION -> formatModuleDefinition(node) NodeType.SINGLE_LINE_STRING_LITERAL_EXPR -> formatSingleLineString(node) @@ -887,7 +889,8 @@ internal class Builder(sourceText: String, private val grammarVersion: GrammarVe if (elem.type == NodeType.TERMINAL && elem.text().endsWith("(")) { isInStringInterpolation = true } - add(format(elem)) + val formatted = format(elem) + if (formatted is Nodes) addAll(formatted.nodes) else add(formatted) prev = elem } } diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl index feef82f9f..6068571f0 100644 --- a/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl @@ -48,6 +48,26 @@ quux { """ } +// line continuations +corge { + + """ + hello \ + world + """ + +#""" +hello \# +world +"""# + +""" +hello \ +\ +world +""" +} + obj { data { ["bar"] = """ diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl index c55737b6d..5e3d5629a 100644 --- a/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl @@ -52,6 +52,25 @@ quux { """ } +// line continuations +corge { + """ + hello \ + world + """ + + #""" + hello \# + world + """# + + """ + hello \ + \ + world + """ +} + obj { data { ["bar"] = diff --git a/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java b/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java index 8e2248e40..fa5b2658d 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java +++ b/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java @@ -978,6 +978,8 @@ private Node parseSingleLineStringLiteralExpr() { STRING_ESCAPE_RETURN, STRING_ESCAPE_UNICODE -> children.add(make(NodeType.STRING_ESCAPE, next().span)); + case STRING_ESCAPE_CONTINUATION -> + throw parserError("invalidLineContinuationEscapeSequence"); case INTERPOLATION_START -> { children.add(makeTerminal(next())); ff(children); @@ -1011,6 +1013,8 @@ private Node parseMultiLineStringLiteralExpr() { } } case STRING_NEWLINE -> children.add(make(NodeType.STRING_NEWLINE, next().span)); + case STRING_ESCAPE_CONTINUATION -> + children.add(make(NodeType.STRING_CONTINUATION, next().span)); case STRING_ESCAPE_NEWLINE, STRING_ESCAPE_TAB, STRING_ESCAPE_QUOTE, @@ -1060,7 +1064,8 @@ private void validateStringIndentation(List nodes) { throw parserError(ErrorMessages.create("stringIndentationMustMatchLastLine"), child.span); } } - previousNewline = child.type == NodeType.STRING_NEWLINE; + previousNewline = + child.type == NodeType.STRING_NEWLINE || child.type == NodeType.STRING_CONTINUATION; } } diff --git a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java index 089094710..d2cb5e3b7 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java @@ -460,6 +460,7 @@ private Token lexEscape() { yield Token.INTERPOLATION_START; } case 'u' -> lexUnicodeEscape(); + case '\n' -> Token.STRING_ESCAPE_CONTINUATION; default -> throw lexError( ErrorMessages.create("invalidCharacterEscapeSequence", "\\" + (char) ch, "\\"), diff --git a/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java b/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java index e42877705..666cdc0cd 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java +++ b/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java @@ -1122,6 +1122,8 @@ private Expr parseSingleLineStringLiteralExpr() { end = tk.span; builder.append(parseUnicodeEscape(tk)); } + case STRING_ESCAPE_CONTINUATION -> + throw parserError("invalidLineContinuationEscapeSequence"); case INTERPOLATION_START -> { var istart = next().span; if (!builder.isEmpty()) { @@ -1159,7 +1161,8 @@ private Expr parseMultiLineStringLiteralExpr() { STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, STRING_ESCAPE_RETURN, - STRING_ESCAPE_UNICODE -> + STRING_ESCAPE_UNICODE, + STRING_ESCAPE_CONTINUATION -> stringTokens.add(new TempNode(next(), null)); case INTERPOLATION_START -> { var istart = next(); @@ -1225,6 +1228,7 @@ private List renderString(List nodes, String commonIndent) builder.append('\n'); isNewLine = true; } + case STRING_ESCAPE_CONTINUATION -> isNewLine = true; case STRING_PART -> { var text = token.text(lexer); if (isNewLine) { @@ -1629,6 +1633,7 @@ private String getEscapeText(FullToken tk) { case STRING_ESCAPE_BACKSLASH -> "\\"; case STRING_ESCAPE_TAB -> "\t"; case STRING_ESCAPE_RETURN -> "\r"; + case STRING_ESCAPE_CONTINUATION -> ""; case STRING_ESCAPE_UNICODE -> parseUnicodeEscape(tk); default -> throw new RuntimeException("Unreacheable code"); }; diff --git a/pkl-parser/src/main/java/org/pkl/parser/Token.java b/pkl-parser/src/main/java/org/pkl/parser/Token.java index d80467b8b..138674260 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Token.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Token.java @@ -128,6 +128,7 @@ public enum Token { STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, STRING_ESCAPE_UNICODE, + STRING_ESCAPE_CONTINUATION, STRING_END, STRING_PART; diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java index 72c136100..34d4d1bff 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java @@ -1,5 +1,5 @@ /* - * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,6 +70,7 @@ public enum NodeType { STRING_CHARS, OPERATOR, STRING_NEWLINE, + STRING_CONTINUATION, STRING_ESCAPE, // members diff --git a/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties b/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties index bb5776577..b157504af 100644 --- a/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties +++ b/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties @@ -51,6 +51,11 @@ Invalid Unicode escape sequence `{0}`.\n\ \n\ Valid Unicode escape sequences are {1}'{'0'}' to {1}'{'10FFFF'}' (1-6 hexadecimal characters). +invalidLineContinuationEscapeSequence=\ +Invalid line continuation escape sequence.\n\ +\n\ +Line continuations are only allowed in multi-line strings. + missingDelimiter=\ Missing `{0}` delimiter. diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt index 0a1f6c550..4a3770945 100644 --- a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt @@ -197,6 +197,7 @@ class GenericSexpRenderer(code: String) { NodeType.TERMINAL, NodeType.OPERATOR, NodeType.STRING_NEWLINE, + NodeType.STRING_CONTINUATION, ) private val UNPACK_CHILDREN =