Fixes for line number node insertion in short form blocks (#279)

Copy some peculiar behaviors of the reference parser for compatibility: * Top level blocks like `a;b;c` don't have any line numbers * Blocks like `(a;b;c)` omit the first line number node As part of this, add a new flag `TOPLEVEL_SEMICOLONS_FLAG` to distinguish semicolon-delimited toplevel statements from newline-delimited.
JuliaLang · May 13, 2023 · 9e9e1a1 · 9e9e1a1
1 parent 0dcca2a
commit 9e9e1a1
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 6 deletions.
diff --git a/src/expr.jl b/src/expr.jl
@@ -215,7 +215,7 @@ function _internal_node_to_Expr(source, srcrange, head, childranges, childheads,
     elseif k == K"macrocall"
         _reorder_parameters!(args, 2)
         insert!(args, 2, loc)
-    elseif k == K"block" || k == K"toplevel"
+    elseif k == K"block" || (k == K"toplevel" && !has_flags(head, TOPLEVEL_SEMICOLONS_FLAG))
         if isempty(args)
             push!(args, loc)
         else
@@ -225,6 +225,9 @@ function _internal_node_to_Expr(source, srcrange, head, childranges, childheads,
                 args[2*i-1] = source_location(LineNumberNode, source, first(childranges[i]))
             end
         end
+        if k == K"block" && has_flags(head, PARENS_FLAG)
+            popfirst!(args)
+        end
     elseif k == K"doc"
         headsym = :macrocall
         args = [GlobalRef(Core, Symbol("@doc")), loc, args...]

diff --git a/src/parse_stream.jl b/src/parse_stream.jl
@@ -33,6 +33,8 @@ const RAW_STRING_FLAG = RawFlags(1<<6)
 const PARENS_FLAG = RawFlags(1<<5)
 # Set for K"quote" for the short form `:x` as oppsed to long form `quote x end`
 const COLON_QUOTE = RawFlags(1<<5)
+# Set for K"toplevel" which is delimited by parentheses
+const TOPLEVEL_SEMICOLONS_FLAG = RawFlags(1<<5)
 
 # Set for K"struct" when mutable
 const MUTABLE_FLAG = RawFlags(1<<5)
@@ -99,6 +101,8 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
             has_flags(head, PARENS_FLAG) && (str = str*"-p")
         elseif kind(head) == K"quote"
             has_flags(head, COLON_QUOTE) && (str = str*"-:")
+        elseif kind(head) == K"toplevel"
+            has_flags(head, TOPLEVEL_SEMICOLONS_FLAG) && (str = str*"-;")
         elseif kind(head) == K"struct"
             has_flags(head, MUTABLE_FLAG) && (str = str*"-mut")
         elseif kind(head) == K"module"

diff --git a/src/parser.jl b/src/parser.jl
@@ -495,7 +495,7 @@ function parse_stmts(ps::ParseState)
              error="extra tokens after end of expression")
     end
     if do_emit
-        emit(ps, mark, K"toplevel")
+        emit(ps, mark, K"toplevel", TOPLEVEL_SEMICOLONS_FLAG)
     end
 end
 

diff --git a/test/expr.jl b/test/expr.jl
@@ -33,10 +33,20 @@
                      LineNumberNode(4),
                      :c,
                 )
+            @test parsestmt("(a;b;c)") ==
+                Expr(:block,
+                     :a,
+                     LineNumberNode(1),
+                     :b,
+                     LineNumberNode(1),
+                     :c,
+                )
             @test parsestmt("begin end") ==
                 Expr(:block,
                      LineNumberNode(1)
                 )
+            @test parsestmt("(;;)") ==
+                Expr(:block)
 
             @test parseall("a\n\nb") ==
                 Expr(:toplevel,
@@ -45,6 +55,8 @@
                      LineNumberNode(3),
                      :b,
                 )
+            @test parsestmt("a;b") ==
+                Expr(:toplevel, :a, :b)
 
             @test parsestmt("module A\n\nbody\nend") ==
                 Expr(:module,

diff --git a/test/parser.jl b/test/parser.jl
@@ -39,7 +39,7 @@ end
 tests = [
     JuliaSyntax.parse_toplevel => [
         "a \n b"     =>  "(toplevel a b)"
-        "a;b \n c;d" =>  "(toplevel (toplevel a b) (toplevel c d))"
+        "a;b \n c;d" =>  "(toplevel (toplevel-; a b) (toplevel-; c d))"
         "a \n \n"    =>  "(toplevel a)"
         ""           =>  "(toplevel)"
     ],
@@ -51,10 +51,10 @@ tests = [
         "a\nb"    => "(block a b)"
     ],
     JuliaSyntax.parse_stmts => [
-        "a;b;c"   => "(toplevel a b c)"
-        "a;;;b;;" => "(toplevel a b)"
+        "a;b;c"   => "(toplevel-; a b c)"
+        "a;;;b;;" => "(toplevel-; a b)"
         """ "x" a ; "y" b """ =>
-            """(toplevel (doc (string "x") a) (doc (string "y") b))"""
+            """(toplevel-; (doc (string "x") a) (doc (string "y") b))"""
         "x y"  =>  "x (error-t y)"
     ],
     JuliaSyntax.parse_eq => [