Fix a crash lexing text blocks

Follow-up to 05fee69 PiperOrigin-RevId: 577853770
google · Oct 30, 2023 · ae935f4 · ae935f4
1 parent 5b357d0
commit ae935f4
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/java/com/google/turbine/parse/StreamLexer.java b/java/com/google/turbine/parse/StreamLexer.java
@@ -502,6 +502,9 @@ private Token textBlock() {
           // that \" escapes don't count towards the closing delimiter of the text block.
           sb.appendCodePoint(ch);
           eat();
+          if (ch == ASCII_SUB && reader.done()) {
+            return Token.EOF;
+          }
           sb.appendCodePoint(ch);
           eat();
           continue;

diff --git a/javatests/com/google/turbine/parse/LexerTest.java b/javatests/com/google/turbine/parse/LexerTest.java
@@ -414,4 +414,13 @@ public void textBlockNewlineEscapes() throws Exception {
     lexerComparisonTest(input);
     assertThat(lex(input)).containsExactly("STRING_LITERAL(hellohellohello)", "EOF");
   }
+
+  // Check for EOF when skipping over escapes in text blocks
+  @Test
+  public void textBlockEOF() {
+    String input = "\"\"\"\n\\";
+    Lexer lexer = new StreamLexer(new UnicodeEscapePreprocessor(new SourceFile(null, input)));
+    assertThat(lexer.next()).isEqualTo(Token.EOF);
+    assertThat(lexer.stringValue()).isEqualTo("\\");
+  }
 }