diff --git a/src/tokenize.jl b/src/tokenize.jl index 9c19c040..26563d6f 100644 --- a/src/tokenize.jl +++ b/src/tokenize.jl @@ -747,8 +747,8 @@ function lex_comment(l::Lexer) if peekchar(l) != '=' valid = true while true - pc = peekchar(l) - if pc == '\n' || pc == EOF_CHAR + pc, ppc = dpeekchar(l) + if pc == '\n' || (pc == '\r' && ppc == '\n') || pc == EOF_CHAR return emit(l, valid ? K"Comment" : K"ErrorInvalidUTF8") end valid &= isvalid(pc) diff --git a/test/tokenize.jl b/test/tokenize.jl index 26ab044a..1c525a99 100644 --- a/test/tokenize.jl +++ b/test/tokenize.jl @@ -221,6 +221,8 @@ end @test toks("#= #= =#") == ["#= #= =#"=>K"ErrorEofMultiComment"] @test toks("#=#==#=#") == ["#=#==#=#"=>K"Comment"] @test toks("#=#==#=") == ["#=#==#="=>K"ErrorEofMultiComment"] + # comment terminated by \r\n + @test toks("#\r\n") == ["#" => K"Comment", "\r\n" => K"NewlineWs"] end