Avoid macro-expand recursion into Expr(:toplevel, ...) (#53515)

Here's an example output from macroexpand:
```
Expr
  head: Symbol thunk
  args: Array{Any}((1,))
    1: Core.CodeInfo
      code: Array{Any}((2,))
        1: Expr
          head: Symbol toplevel
          args: Array{Any}((17,))
            1: Expr
              head: Symbol hygienic-scope
              args: Array{Any}((3,))
                1: LineNumberNode
                2: Module Base.Enums
                3: LineNumberNode
            2: Expr
              head: Symbol hygienic-scope
              args: Array{Any}((3,))
                1: Expr
                2: Module Base.Enums
                3: LineNumberNode
            3: Expr
              head: Symbol hygienic-scope
              args: Array{Any}((3,))
                1: LineNumberNode
                2: Module Base.Enums
                3: LineNumberNode
            4: Expr
              head: Symbol hygienic-scope
              args: Array{Any}((3,))
                1: Expr
                2: Module Base.Enums
                3: LineNumberNode
 ...
```

Currently fails during bootstrap with:
```
LoadError("sysimg.jl", 3, LoadError("Base.jl", 542, ErrorException("cannot document the following expression:\n\n#= mpfr.jl:65 =# @enum MPFRRoundingMode begin\n        #= mpfr.jl:66 =#\n        MPFRRoundNearest\n        #= mpfr.jl:67 =#\n        MPFRRoundToZero\n        #= mpfr.jl:68 =#\n        MPFRRoundUp\n        #= mpfr.jl:69 =#\n        MPFRRoundDown\n        #= mpfr.jl:70 =#\n        MPFRRoundFromZero\n        #= mpfr.jl:71 =#\n        MPFRRoundFaithful\n    end\n\n'@enum' not documentable. See 'Base.@__doc__' docs for details.\n")))
```


Perhaps we can do better than wrapping each `Expr(:toplevel, ...)` arg
individually, or I should be filtering out the LineNumberNodes?

---------

Co-authored-by: Keno Fischer <keno@juliacomputing.com>
Co-authored-by: Keno Fischer <keno@juliahub.com>

NEWS.md

@@ -17,6 +17,12 @@ Language changes
    may pave the way for inference to be able to intelligently re-use the old
    results, once the new method is deleted. ([#53415])
 
+ - Macro expansion will no longer eargerly recurse into into `Expr(:toplevel)`
+   expressions returned from macros. Instead, macro expansion of `:toplevel`
+   expressions will be delayed until evaluation time. This allows a later
+   expression within a given `:toplevel` expression to make use of macros
+   defined earlier in the same `:toplevel` expression. ([#53515])
+
 Compiler/Runtime improvements
 -----------------------------
 

base/docs/Docs.jl

@@ -446,24 +446,76 @@ more than one expression is marked then the same docstring is applied to each ex
     end
 
 `@__doc__` has no effect when a macro that uses it is not documented.
+
+!!! compat "Julia 1.12"
+
+    This section documents a very subtle corner case that is only relevant to
+    macros which themselves both define other macros and then attempt to use them
+    within the same expansion. Such macros were impossible to write prior to
+    Julia 1.12 and are still quite rare. If you are not writing such a macro,
+    you may ignore this note.
+
+    In versions prior to Julia 1.12, macroexpansion would recursively expand through
+    `Expr(:toplevel)` blocks. This behavior was changed in Julia 1.12 to allow
+    macros to recursively define other macros and use them in the same returned
+    expression. However, to preserve backwards compatibility with existing uses of
+    `@__doc__`, the doc system will still expand through `Expr(:toplevel)` blocks
+    when looking for `@__doc__` markers. As a result, macro-defining-macros will
+    have an observable behavior difference when annotated with a docstring:
+
+    ```julia
+    julia> macro macroception()
+        Expr(:toplevel, :(macro foo() 1 end), :(@foo))
+    end
+
+    julia> @macroception
+    1
+
+    julia> "Docstring" @macroception
+    ERROR: LoadError: UndefVarError: `@foo` not defined in `Main`
+    ```
+
+    The supported workaround is to manually expand the `@__doc__` macro in the
+    defining macro, which the docsystem will recognize and suppress the recursive
+    expansion:
+
+    ```julia
+    julia> macro macroception()
+        Expr(:toplevel,
+            macroexpand(__module__, :(@__doc__ macro foo() 1 end); recursive=false),
+            :(@foo))
+    end
+
+    julia> @macroception
+    1
+
+    julia> "Docstring" @macroception
+    1
+    ```
 """
 :(Core.@__doc__)
 
 function __doc__!(source, mod, meta, def, define::Bool)
     @nospecialize source mod meta def
     # Two cases must be handled here to avoid redefining all definitions contained in `def`:
     if define
-        # `def` has not been defined yet (this is the common case, i.e. when not generating
-        # the Base image). We just need to convert each `@__doc__` marker to an `@doc`.
-        finddoc(def) do each
+        function replace_meta_doc(each)
             each.head = :macrocall
             each.args = Any[Symbol("@doc"), source, mod, nothing, meta, each.args[end], define]
         end
+
+        # `def` has not been defined yet (this is the common case, i.e. when not generating
+        # the Base image). We just need to convert each `@__doc__` marker to an `@doc`.
+        found = finddoc(replace_meta_doc, mod, def; expand_toplevel = false)
+
+        if !found
+            found = finddoc(replace_meta_doc, mod, def; expand_toplevel = true)
+        end
     else
         # `def` has already been defined during Base image gen so we just need to find and
         # document any subexpressions marked with `@__doc__`.
         docs  = []
-        found = finddoc(def) do each
+        found = finddoc(mod, def; expand_toplevel = true) do each
             push!(docs, :(@doc($source, $mod, $meta, $(each.args[end]), $define)))
         end
         # If any subexpressions have been documented then replace the entire expression with
@@ -472,25 +524,30 @@ function __doc__!(source, mod, meta, def, define::Bool)
             def.head = :toplevel
             def.args = docs
         end
-        found
     end
+    return found
 end
 # Walk expression tree `def` and call `λ` when any `@__doc__` markers are found. Returns
 # `true` to signify that at least one `@__doc__` has been found, and `false` otherwise.
-function finddoc(λ, def::Expr)
+function finddoc(λ, mod::Module, def::Expr; expand_toplevel::Bool=false)
     if isexpr(def, :block, 2) && isexpr(def.args[1], :meta, 1) && (def.args[1]::Expr).args[1] === :doc
         # Found the macroexpansion of an `@__doc__` expression.
         λ(def)
         true
     else
+        if expand_toplevel && isexpr(def, :toplevel)
+            for i = 1:length(def.args)
+                def.args[i] = macroexpand(mod, def.args[i])
+            end
+        end
         found = false
         for each in def.args
-            found |= finddoc(λ, each)
+            found |= finddoc(λ, mod, each; expand_toplevel)
         end
         found
     end
 end
-finddoc(λ, @nospecialize def) = false
+finddoc(λ, mod::Module, @nospecialize def; expand_toplevel::Bool=false) = false
 
 # Predicates and helpers for `docm` expression selection:
 
@@ -528,8 +585,37 @@ iscallexpr(ex) = false
 function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true)
     @nospecialize meta ex
     # Some documented expressions may be decorated with macro calls which obscure the actual
-    # expression. Expand the macro calls and remove extra blocks.
-    x = unblock(macroexpand(mod, ex))
+    # expression. Expand the macro calls.
+    x = macroexpand(mod, ex)
+    return _docm(source, mod, meta, x, define)
+end
+
+function _docm(source::LineNumberNode, mod::Module, meta, x, define::Bool = true)
+    if isexpr(x, :var"hygienic-scope")
+        x.args[1] = _docm(source, mod, meta, x.args[1])
+        return x
+    elseif isexpr(x, :escape)
+        x.args[1] = _docm(source, mod, meta, x.args[1])
+        return x
+    elseif isexpr(x, :block)
+        docarg = 0
+        for i = 1:length(x.args)
+            isa(x.args[i], LineNumberNode) && continue
+            if docarg == 0
+                docarg = i
+                continue
+            end
+            # More than one documentable expression in the block, treat it as a whole
+            # expression, which will fall through and look for (Expr(:meta, doc))
+            docarg = 0
+            break
+        end
+        if docarg != 0
+            x.args[docarg] = _docm(source, mod, meta, x.args[docarg], define)
+            return x
+        end
+    end
+
     # Don't try to redefine expressions. This is only needed for `Base` img gen since
     # otherwise calling `loaddocs` would redefine all documented functions and types.
     def = define ? x : nothing
@@ -594,7 +680,7 @@ function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true
     # All other expressions are undocumentable and should be handled on a case-by-case basis
     # with `@__doc__`. Unbound string literals are also undocumentable since they cannot be
     # retrieved from the module's metadata `IdDict` without a reference to the string.
-    docerror(ex)
+    docerror(x)
 
     return doc
 end

src/ast.c

@@ -1154,7 +1154,7 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str
     jl_expr_t *e = (jl_expr_t*)expr;
     if (e->head == jl_inert_sym ||
         e->head == jl_module_sym ||
-        //e->head == jl_toplevel_sym || // TODO: enable this once julia-expand-macroscope is fixed / removed
+        e->head == jl_toplevel_sym ||
         e->head == jl_meta_sym) {
         return expr;
     }