From 84a4ff8bb76df92ec58b32f476093679cbfedd18 Mon Sep 17 00:00:00 2001
From: Fredrik Ekre <ekrefredrik@gmail.com>
Date: Sun, 7 Jul 2024 14:52:29 +0200
Subject: [PATCH] Fix emitting comments followed by `\r\n`

Before this patch the `\r` character would be part of the comment and
not the following NewlineWs (which would simply be a `\n`-style
NewlineWs instead of a `\r\n` as one would expect).
---
 src/tokenize.jl  | 4 ++--
 test/tokenize.jl | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/tokenize.jl b/src/tokenize.jl
index 9c19c040..26563d6f 100644
--- a/src/tokenize.jl
+++ b/src/tokenize.jl
@@ -747,8 +747,8 @@ function lex_comment(l::Lexer)
     if peekchar(l) != '='
         valid = true
         while true
-            pc = peekchar(l)
-            if pc == '\n' || pc == EOF_CHAR
+            pc, ppc = dpeekchar(l)
+            if pc == '\n' || (pc == '\r' && ppc == '\n') || pc == EOF_CHAR
                 return emit(l, valid ? K"Comment" : K"ErrorInvalidUTF8")
             end
             valid &= isvalid(pc)
diff --git a/test/tokenize.jl b/test/tokenize.jl
index 26ab044a..1c525a99 100644
--- a/test/tokenize.jl
+++ b/test/tokenize.jl
@@ -221,6 +221,8 @@ end
     @test toks("#=   #=   =#") == ["#=   #=   =#"=>K"ErrorEofMultiComment"]
     @test toks("#=#==#=#") == ["#=#==#=#"=>K"Comment"]
     @test toks("#=#==#=")  == ["#=#==#="=>K"ErrorEofMultiComment"]
+    # comment terminated by \r\n
+    @test toks("#\r\n") == ["#" => K"Comment", "\r\n" => K"NewlineWs"]
 end