Docs: manually apply hygiene to [] syntax (#27314)

Macro hygiene fails to do this automatically,
so we should do it manually (until fixed in lowering)
But first, we have to actually make macro hygiene work for `docm` at all,
so alter the `Core.@doc` to make that possible.

Also apply `@nospecialize` to many of our macro arguments,
in the hopes that this may help reduce code load times.

And try to synchronize our 5 different definitions of `isexpr`
to be merely 3 copies of the same version (in Core.CoreDocs, Core.Compiler, and Base.Meta).

base/boot.jl

@@ -452,11 +452,14 @@ import Core: CodeInfo, MethodInstance, GotoNode, LabelNode,
 end
 
 # docsystem basics
+const unescape = Symbol("hygienic-scope")
 macro doc(x...)
-    atdoc(__source__, __module__, x...)
+    docex = atdoc(__source__, __module__, x...)
+    isa(docex, Expr) && docex.head === :escape && return docex
+    return Expr(:escape, Expr(unescape, docex, typeof(atdoc).name.module))
 end
 macro __doc__(x)
-    Expr(:escape, Expr(:block, Expr(:meta, :doc), x))
+    return Expr(:escape, Expr(:block, Expr(:meta, :doc), x))
 end
 atdoc     = (source, mod, str, expr) -> Expr(:escape, expr)
 atdoc!(λ) = global atdoc = λ

base/compiler/ssair/show.jl

@@ -6,8 +6,6 @@ IOContext(io::IO, KV::Pair) = IOContext(io, Base.Pair(KV[1], KV[2]))
 length(s::String) = Base.length(s)
 ^(s::String, i::Int) = Base.:^(s, i)
 end
-isexpr(e::Expr, s::Symbol) = e.head === s
-isexpr(@nospecialize(e), s::Symbol) = false
 
 function Base.show(io::IO, cfg::CFG)
     foreach(pairs(cfg.blocks)) do (idx, block)

base/docs/Docs.jl

@@ -103,24 +103,24 @@ function signature!(tv, expr::Expr)
         for i = length(tv):-1:1
             sig = Expr(:where, sig, tv[i])
         end
-        sig
+        return sig
     elseif isexpr(expr, :where)
         append!(tv, tvar.(expr.args[2:end]))
-        signature!(tv, expr.args[1])
+        return signature!(tv, expr.args[1])
     else
-        signature!(tv, expr.args[1])
+        return signature!(tv, expr.args[1])
     end
 end
-signature!(tv, other) = :(Union{})
+signature!(tv, @nospecialize(other)) = :(Union{})
 signature(expr::Expr) = signature!([], expr)
-signature(other) = signature!([], other)
+signature(@nospecialize other) = signature!([], other)
 
 function argtype(expr::Expr)
     isexpr(expr, :(::))  && return expr.args[end]
     isexpr(expr, :(...)) && return :(Vararg{$(argtype(expr.args[1]))})
-    argtype(expr.args[1])
+    return argtype(expr.args[1])
 end
-argtype(other) = :Any
+argtype(@nospecialize other) = :Any
 
 tvar(x::Expr)   = x
 tvar(s::Symbol) = :($s <: Any)
@@ -146,7 +146,8 @@ mutable struct DocStr
     data   :: Dict{Symbol, Any}
 end
 
-function docstr(binding::Binding, @nospecialize typesig = Union{})
+function docstr(binding::Binding, typesig = Union{})
+    @nospecialize typesig
     for m in modules
         dict = meta(m)
         if haskey(dict, binding)
@@ -169,7 +170,7 @@ _docstr(doc::DocStr, data) = (doc.data = merge(data, doc.data); doc)
 macro ref(x)
     binding = bindingexpr(namify(x))
     typesig = signature(x)
-    esc(docexpr(__source__, __module__, binding, typesig))
+    return esc(docexpr(__source__, __module__, binding, typesig))
 end
 
 docexpr(__source__, __module__, args...) = Expr(:call, docstr, args...)
@@ -248,8 +249,8 @@ was found for `obj`, in which case the docsystem will fall back to searching for
 """
 function getdoc end
 
-getdoc(x, sig) = getdoc(x)
-getdoc(x) = nothing
+getdoc(@nospecialize(x), @nospecialize(sig)) = getdoc(x)
+getdoc(@nospecialize(x)) = nothing
 
 # Utilities.
 # ==========
@@ -262,18 +263,18 @@ catdoc(xs...) = vcat(xs...)
 
 const keywords = Dict{Symbol, DocStr}()
 
-function unblock(ex)
+function unblock(@nospecialize ex)
     isexpr(ex, :block) || return ex
     exs = filter(ex -> !(isa(ex, LineNumberNode) || isexpr(ex, :line)), ex.args)
     length(exs) == 1 || return ex
     return unblock(exs[1])
 end
 
-uncurly(ex) = isexpr(ex, :curly) ? ex.args[1] : ex
+uncurly(@nospecialize ex) = isexpr(ex, :curly) ? ex.args[1] : ex
 
-namify(x) = astname(x, isexpr(x, :macro))
+namify(@nospecialize x) = astname(x, isexpr(x, :macro))
 
-function astname(x::Expr, ismacro)
+function astname(x::Expr, ismacro::Bool)
     if isexpr(x, :.)
         ismacro ? macroname(x) : x
     # Call overloading, e.g. `(a::A)(b) = b` or `function (a::A)(b) b end` should document `A(b)`
@@ -284,14 +285,14 @@ function astname(x::Expr, ismacro)
         astname(x.args[n], ismacro)
     end
 end
-astname(q::QuoteNode, ismacro) = astname(q.value, ismacro)
-astname(s::Symbol, ismacro)    = ismacro ? macroname(s) : s
-astname(other, ismacro)        = other
+astname(q::QuoteNode, ismacro::Bool) = astname(q.value, ismacro)
+astname(s::Symbol, ismacro::Bool)    = ismacro ? macroname(s) : s
+astname(@nospecialize(other), ismacro::Bool) = other
 
 macroname(s::Symbol) = Symbol('@', s)
 macroname(x::Expr)   = Expr(x.head, x.args[1], macroname(x.args[end].value))
 
-isfield(x) = isexpr(x, :.) &&
+isfield(@nospecialize x) = isexpr(x, :.) &&
     (isa(x.args[1], Symbol) || isfield(x.args[1])) &&
     (isa(x.args[2], QuoteNode) || isexpr(x.args[2], :quote))
 
@@ -344,22 +345,25 @@ function metadata(__source__, __module__, expr, ismodule)
         dict = :($(Dict)($([:($(Pair)($(quot(f)), $d)) for (f, d) in fields]...)))
         push!(args, :($(Pair)(:fields, $dict)))
     end
-    :($(Dict)($(args...)))
+    return :($(Dict)($(args...)))
 end
 
-function keyworddoc(__source__, __module__, str, def)
+function keyworddoc(__source__, __module__, str, def::Base.BaseDocs.Keyword)
+    @nospecialize str
     docstr = esc(docexpr(__source__, __module__, lazy_iterpolate(str), metadata(__source__, __module__, def, false)))
-    return :($(keywords)[$(esc(quot(def.name)))] = $docstr)
+    return :($setindex!($(keywords), $docstr, $(esc(quot(def.name)))); nothing)
 end
 
 function objectdoc(__source__, __module__, str, def, expr, sig = :(Union{}))
+    @nospecialize str def expr sig
     binding = esc(bindingexpr(namify(expr)))
     docstr  = esc(docexpr(__source__, __module__, lazy_iterpolate(str), metadata(__source__, __module__, expr, false)))
     # Note: we want to avoid introducing line number nodes here (issue #24468)
-    Expr(:block, esc(def), :($(doc!)($__module__, $binding, $docstr, $(esc(sig)))))
+    return Expr(:block, esc(def), :($(doc!)($__module__, $binding, $docstr, $(esc(sig)))))
 end
 
-function calldoc(__source__, __module__, str, def)
+function calldoc(__source__, __module__, str, def::Expr)
+    @nospecialize str
     args = def.args[2:end]
     if isempty(args) || all(validcall, args)
         objectdoc(__source__, __module__, str, nothing, def, signature(def))
@@ -369,7 +373,8 @@ function calldoc(__source__, __module__, str, def)
 end
 validcall(x) = isa(x, Symbol) || isexpr(x, (:(::), :..., :kw, :parameters))
 
-function moduledoc(__source__, __module__, meta, def, def′)
+function moduledoc(__source__, __module__, meta, def, def′::Expr)
+    @nospecialize meta def
     name  = namify(def′)
     docex = Expr(:call, doc!, name, bindingexpr(name),
         docexpr(__source__, name, lazy_iterpolate(meta), metadata(__source__, __module__, name, true)))
@@ -388,15 +393,23 @@ function moduledoc(__source__, __module__, meta, def, def′)
 end
 
 # Shares a single doc, `meta`, between several expressions from the tuple expression `ex`.
-function multidoc(__source__, __module__, meta, ex, define)
-    out = Expr(:toplevel)
+function multidoc(__source__, __module__, meta, ex::Expr, define::Bool)
+    @nospecialize meta
+    out = Expr(:block)
     str = docexpr(__source__, __module__, lazy_iterpolate(meta), metadata(__source__, __module__, ex, false))
     ref = RefValue{DocStr}()
-    for (n, arg) in enumerate(ex.args)
-        # The first `arg` to be documented needs to also create the docstring for the group.
+    first = true
+    for arg in ex.args
+        # The first `arg` to be documented needs to also create the docstring for the group
+        # (after doing the action defined by the argument).
         # Subsequent `arg`s just need `ref` to be able to find the docstring without having
         # to create an entirely new one each.
-        docstr = n === 1 ? :($(ref)[] = $str) : :($(ref)[])
+        if first
+            first = false
+            docstr = :($getindex($setindex!($(ref), $str)))
+        else
+            docstr = :($getindex($(ref)))
+        end
         push!(out.args, docm(__source__, __module__, docstr, arg, define))
     end
     return out
@@ -420,7 +433,8 @@ more than one expression is marked then the same docstring is applied to each ex
 """
 :(Core.@__doc__)
 
-function __doc__!(meta, def, define)
+function __doc__!(meta, def, define::Bool)
+    @nospecialize meta def
     # Two cases must be handled here to avoid redefining all definitions contained in `def`:
     if define
         # `def` has not been defined yet (this is the common case, i.e. when not generating
@@ -460,32 +474,34 @@ function finddoc(λ, def::Expr)
         found
     end
 end
-finddoc(λ, def) = false
+finddoc(λ, @nospecialize def) = false
 
 # Predicates and helpers for `docm` expression selection:
 
 const FUNC_HEADS    = [:function, :macro, :(=)]
 const BINDING_HEADS = [:const, :global, :(=)]
 # For the special `:@mac` / `:(Base.@mac)` syntax for documenting a macro after definition.
-isquotedmacrocall(x) =
+isquotedmacrocall(@nospecialize x) =
     isexpr(x, :copyast, 1) &&
     isa(x.args[1], QuoteNode) &&
     isexpr(x.args[1].value, :macrocall, 2)
 # Simple expressions / atoms the may be documented.
-isbasicdoc(x) = isexpr(x, :.) || isa(x, Union{QuoteNode, Symbol})
-is_signature(x) = isexpr(x, :call) || (isexpr(x, :(::), 2) && isexpr(x.args[1], :call)) || isexpr(x, :where)
+isbasicdoc(@nospecialize x) = isexpr(x, :.) || isa(x, Union{QuoteNode, Symbol})
+is_signature(@nospecialize x) = isexpr(x, :call) || (isexpr(x, :(::), 2) && isexpr(x.args[1], :call)) || isexpr(x, :where)
 
 function docm(source::LineNumberNode, mod::Module, ex)
+    @nospecialize ex
     if isexpr(ex, :->) && length(ex.args) > 1
-        docm(source, mod, ex.args...)
+        return docm(source, mod, ex.args...)
     else
         # TODO: this is a shim to continue to allow `@doc` for looking up docstrings
         REPL = Base.root_module(Base, :REPL)
         return REPL.lookup_doc(ex)
     end
 end
 
-function docm(source::LineNumberNode, mod::Module, meta, ex, define = true)
+function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true)
+    @nospecialize meta ex
     # Some documented expressions may be decorated with macro calls which obscure the actual
     # expression. Expand the macro calls and remove extra blocks.
     x = unblock(macroexpand(mod, ex))
@@ -501,6 +517,7 @@ function docm(source::LineNumberNode, mod::Module, meta, ex, define = true)
     #   "..."
     #   kw"if", kw"else"
     #
+    doc =
     isa(x, Base.BaseDocs.Keyword) ? keyworddoc(source, mod, meta, x) :
 
     # Method / macro definitions and "call" syntax.
@@ -548,41 +565,35 @@ function docm(source::LineNumberNode, mod::Module, meta, ex, define = true)
     # with `@__doc__`. Unbound string literals are also undocumentable since they cannot be
     # retrieved from the module's metadata `IdDict` without a reference to the string.
     docerror(ex)
+
+    return doc
 end
 
-function docerror(ex)
+function docerror(@nospecialize ex)
     txt = """
     cannot document the following expression:
 
     $(isa(ex, AbstractString) ? repr(ex) : ex)"""
     if isexpr(ex, :macrocall)
         txt *= "\n\n'$(ex.args[1])' not documentable. See 'Base.@__doc__' docs for details."
     end
-    :($(error)($txt, "\n"))
+    return :($(error)($txt, "\n"))
 end
 
 include("utils.jl")
 
 # Swap out the bootstrap macro with the real one.
 Core.atdoc!(docm)
 
-macro local_hygiene(expr)
-    # removes `esc` Exprs relative to the module argument to expand
-    # and resolves everything else relative to this (Doc) module
-    # this allows us to get good errors and backtraces
-    # from calling docm (by not using macros),
-    # while also getting macro-expansion correct (by using the macro-expander)
-    return expr
-end
 function loaddocs(docs)
-    unescape = GlobalRef(Docs, Symbol("@local_hygiene"))
     for (mod, ex, str, file, line) in docs
         data = Dict(:path => string(file), :linenumber => line)
         doc = docstr(str, data)
         docstring = docm(LineNumberNode(line, file), mod, doc, ex, false) # expand the real @doc macro now
-        Core.eval(mod, Expr(:macrocall, unescape, nothing, docstring))
+        Core.eval(mod, Expr(Core.unescape, docstring, Docs))
     end
     empty!(docs)
+    nothing
 end
 
 function formatdoc end

base/docs/core.jl

@@ -2,22 +2,29 @@
 
 module CoreDocs
 
-import ..esc, ..push!, ..getindex, ..unsafe_load, ..Csize_t
+import ..esc, ..push!, ..getindex, ..unsafe_load, ..Csize_t, ..@nospecialize
 
 function doc!(source::LineNumberNode, mod::Module, str, ex)
-    push!(DOCS, (mod, ex, str, source.file, source.line))
+    @nospecialize str ex
+    push!(DOCS, Core.svec(mod, ex, str, source.file, source.line))
+    nothing
 end
-const DOCS = Array{Any,1}()
+const DOCS = Array{Core.SimpleVector,1}()
 
-isexpr(x, h) = isa(x, Expr) && x.head === h
+isexpr(@nospecialize(x), h::Symbol) = isa(x, Expr) && x.head === h
 
 lazy_iterpolate(s::AbstractString) = Expr(:call, Core.svec, s)
-lazy_iterpolate(x) = isexpr(x, :string) ? Expr(:call, Core.svec, x.args...) : x
+lazy_iterpolate(@nospecialize x) = isexpr(x, :string) ? Expr(:call, Core.svec, x.args...) : x
 
 function docm(source::LineNumberNode, mod::Module, str, x)
-    out = esc(Expr(:call, doc!, QuoteNode(source), mod, lazy_iterpolate(str), Expr(:quote, x)))
-    isexpr(x, :module) ? Expr(:toplevel, out, esc(x)) :
-    isexpr(x, :call) ? out : Expr(:block, esc(x), out)
+    out = Expr(:call, doc!, QuoteNode(source), mod, lazy_iterpolate(str), Expr(:quote, x))
+    if isexpr(x, :module)
+        out = Expr(:toplevel, out, x)
+    elseif isexpr(x, :call)
+    else
+        out = Expr(:block, x, out)
+    end
+    return esc(out)
 end
 docm(source::LineNumberNode, mod::Module, x) =
     isexpr(x, :->) ? docm(source, mod, x.args[1], x.args[2].args[2]) : error("invalid '@doc'.")

base/meta.jl

@@ -14,11 +14,9 @@ export quot,
 
 quot(ex) = Expr(:quote, ex)
 
-isexpr(ex::Expr, head)          = ex.head === head
-isexpr(ex::Expr, heads::Union{Set,Vector,Tuple}) = in(ex.head, heads)
-isexpr(ex,       head)          = false
-
-isexpr(ex,       head, n::Int)  = isexpr(ex, head) && length(ex.args) == n
+isexpr(@nospecialize(ex), head::Symbol) = isa(ex, Expr) && ex.head === head
+isexpr(@nospecialize(ex), heads::Union{Set,Vector,Tuple}) = isa(ex, Expr) && in(ex.head, heads)
+isexpr(@nospecialize(ex), heads, n::Int) = isexpr(ex, heads) && length(ex.args) == n
 
 
 # ---- show_sexpr: print an AST as an S-expression ----

base/show.jl

@@ -633,8 +633,9 @@ end
 module IRShow
     const Compiler = Core.Compiler
     using Core.IR
+    import ..Base
     import .Base: IdSet
-    import .Compiler: IRCode, ReturnNode, GotoIfNot, CFG, scan_ssa_use!, Argument
+    import .Compiler: IRCode, ReturnNode, GotoIfNot, CFG, scan_ssa_use!, Argument, isexpr
     Base.size(r::Compiler.StmtRange) = Compiler.size(r)
     Base.show(io::IO, r::Compiler.StmtRange) = print(io, Compiler.first(r):Compiler.last(r))
     include("compiler/ssair/show.jl")

doc/src/devdocs/ast.md

@@ -544,3 +544,10 @@ component is optional (and omitted when the current line number, but not file na
 changes).
 
 These expressions are represented as `LineNumberNode`s in Julia.
+
+### Macros
+
+Macro hygiene is represented through the expression head pair `escape` and `hygienic-scope`.
+The result of a macro expansion is automatically wrapped in `(hygienic-scope block module)`,
+to represent the result of the new scope. The user can insert `(escape block)` inside
+to interpolate code from the caller.