diff --git a/NEWS.md b/NEWS.md index d3f29ef30987d..695427d7eba8e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -17,6 +17,12 @@ Language changes may pave the way for inference to be able to intelligently re-use the old results, once the new method is deleted. ([#53415]) + - Macro expansion will no longer eargerly recurse into into `Expr(:toplevel)` + expressions returned from macros. Instead, macro expansion of `:toplevel` + expressions will be delayed until evaluation time. This allows a later + expression within a given `:toplevel` expression to make use of macros + defined earlier in the same `:toplevel` expression. ([#53515]) + Compiler/Runtime improvements ----------------------------- diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 0c83376ecfc76..0206f867b3811 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -446,6 +446,52 @@ more than one expression is marked then the same docstring is applied to each ex end `@__doc__` has no effect when a macro that uses it is not documented. + +!!! compat "Julia 1.12" + + This section documents a very subtle corner case that is only relevant to + macros which themselves both define other macros and then attempt to use them + within the same expansion. Such macros were impossible to write prior to + Julia 1.12 and are still quite rare. If you are not writing such a macro, + you may ignore this note. + + In versions prior to Julia 1.12, macroexpansion would recursively expand through + `Expr(:toplevel)` blocks. This behavior was changed in Julia 1.12 to allow + macros to recursively define other macros and use them in the same returned + expression. However, to preserve backwards compatibility with existing uses of + `@__doc__`, the doc system will still expand through `Expr(:toplevel)` blocks + when looking for `@__doc__` markers. As a result, macro-defining-macros will + have an observable behavior difference when annotated with a docstring: + + ```julia + julia> macro macroception() + Expr(:toplevel, :(macro foo() 1 end), :(@foo)) + end + + julia> @macroception + 1 + + julia> "Docstring" @macroception + ERROR: LoadError: UndefVarError: `@foo` not defined in `Main` + ``` + + The supported workaround is to manually expand the `@__doc__` macro in the + defining macro, which the docsystem will recognize and suppress the recursive + expansion: + + ```julia + julia> macro macroception() + Expr(:toplevel, + macroexpand(__module__, :(@__doc__ macro foo() 1 end); recursive=false), + :(@foo)) + end + + julia> @macroception + 1 + + julia> "Docstring" @macroception + 1 + ``` """ :(Core.@__doc__) @@ -453,17 +499,23 @@ function __doc__!(source, mod, meta, def, define::Bool) @nospecialize source mod meta def # Two cases must be handled here to avoid redefining all definitions contained in `def`: if define - # `def` has not been defined yet (this is the common case, i.e. when not generating - # the Base image). We just need to convert each `@__doc__` marker to an `@doc`. - finddoc(def) do each + function replace_meta_doc(each) each.head = :macrocall each.args = Any[Symbol("@doc"), source, mod, nothing, meta, each.args[end], define] end + + # `def` has not been defined yet (this is the common case, i.e. when not generating + # the Base image). We just need to convert each `@__doc__` marker to an `@doc`. + found = finddoc(replace_meta_doc, mod, def; expand_toplevel = false) + + if !found + found = finddoc(replace_meta_doc, mod, def; expand_toplevel = true) + end else # `def` has already been defined during Base image gen so we just need to find and # document any subexpressions marked with `@__doc__`. docs = [] - found = finddoc(def) do each + found = finddoc(mod, def; expand_toplevel = true) do each push!(docs, :(@doc($source, $mod, $meta, $(each.args[end]), $define))) end # If any subexpressions have been documented then replace the entire expression with @@ -472,25 +524,30 @@ function __doc__!(source, mod, meta, def, define::Bool) def.head = :toplevel def.args = docs end - found end + return found end # Walk expression tree `def` and call `λ` when any `@__doc__` markers are found. Returns # `true` to signify that at least one `@__doc__` has been found, and `false` otherwise. -function finddoc(λ, def::Expr) +function finddoc(λ, mod::Module, def::Expr; expand_toplevel::Bool=false) if isexpr(def, :block, 2) && isexpr(def.args[1], :meta, 1) && (def.args[1]::Expr).args[1] === :doc # Found the macroexpansion of an `@__doc__` expression. λ(def) true else + if expand_toplevel && isexpr(def, :toplevel) + for i = 1:length(def.args) + def.args[i] = macroexpand(mod, def.args[i]) + end + end found = false for each in def.args - found |= finddoc(λ, each) + found |= finddoc(λ, mod, each; expand_toplevel) end found end end -finddoc(λ, @nospecialize def) = false +finddoc(λ, mod::Module, @nospecialize def; expand_toplevel::Bool=false) = false # Predicates and helpers for `docm` expression selection: @@ -528,8 +585,37 @@ iscallexpr(ex) = false function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true) @nospecialize meta ex # Some documented expressions may be decorated with macro calls which obscure the actual - # expression. Expand the macro calls and remove extra blocks. - x = unblock(macroexpand(mod, ex)) + # expression. Expand the macro calls. + x = macroexpand(mod, ex) + return _docm(source, mod, meta, x, define) +end + +function _docm(source::LineNumberNode, mod::Module, meta, x, define::Bool = true) + if isexpr(x, :var"hygienic-scope") + x.args[1] = _docm(source, mod, meta, x.args[1]) + return x + elseif isexpr(x, :escape) + x.args[1] = _docm(source, mod, meta, x.args[1]) + return x + elseif isexpr(x, :block) + docarg = 0 + for i = 1:length(x.args) + isa(x.args[i], LineNumberNode) && continue + if docarg == 0 + docarg = i + continue + end + # More than one documentable expression in the block, treat it as a whole + # expression, which will fall through and look for (Expr(:meta, doc)) + docarg = 0 + break + end + if docarg != 0 + x.args[docarg] = _docm(source, mod, meta, x.args[docarg], define) + return x + end + end + # Don't try to redefine expressions. This is only needed for `Base` img gen since # otherwise calling `loaddocs` would redefine all documented functions and types. def = define ? x : nothing @@ -594,7 +680,7 @@ function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true # All other expressions are undocumentable and should be handled on a case-by-case basis # with `@__doc__`. Unbound string literals are also undocumentable since they cannot be # retrieved from the module's metadata `IdDict` without a reference to the string. - docerror(ex) + docerror(x) return doc end diff --git a/src/ast.c b/src/ast.c index a7c6ae3bc218c..d0ed4712c66f7 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1154,7 +1154,7 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str jl_expr_t *e = (jl_expr_t*)expr; if (e->head == jl_inert_sym || e->head == jl_module_sym || - //e->head == jl_toplevel_sym || // TODO: enable this once julia-expand-macroscope is fixed / removed + e->head == jl_toplevel_sym || e->head == jl_meta_sym) { return expr; } diff --git a/src/macroexpand.scm b/src/macroexpand.scm index ee52b97669125..403df1f3d9ef0 100644 --- a/src/macroexpand.scm +++ b/src/macroexpand.scm @@ -194,18 +194,18 @@ (unescape (cadr e)) e)) -(define (unescape-global-lhs e env m parent-scope inarg) +(define (unescape-global-lhs e env m lno parent-scope inarg) (cond ((not (pair? e)) e) - ((eq? (car e) 'escape) (unescape-global-lhs (cadr e) env m parent-scope inarg)) + ((eq? (car e) 'escape) (unescape-global-lhs (cadr e) env m lno parent-scope inarg)) ((memq (car e) '(parameters tuple)) (list* (car e) (map (lambda (e) - (unescape-global-lhs e env m parent-scope inarg)) + (unescape-global-lhs e env m lno parent-scope inarg)) (cdr e)))) ((and (memq (car e) '(|::| kw)) (length= e 3)) - (list (car e) (unescape-global-lhs (cadr e) env m parent-scope inarg) - (resolve-expansion-vars-with-new-env (caddr e) env m parent-scope inarg))) + (list (car e) (unescape-global-lhs (cadr e) env m lno parent-scope inarg) + (resolve-expansion-vars-with-new-env (caddr e) env m lno parent-scope inarg))) (else - (resolve-expansion-vars-with-new-env e env m parent-scope inarg)))) + (resolve-expansion-vars-with-new-env e env m lno parent-scope inarg)))) (define (typedef-expr-name e) (cond ((atom? e) e) @@ -290,18 +290,18 @@ ;; resolve-expansion-vars-with-new-env, but turn on `inarg` if we get inside ;; a formal argument list. `e` in general might be e.g. `(f{T}(x)::T) where T`, ;; and we want `inarg` to be true for the `(x)` part. -(define (resolve-in-lhs e env m parent-scope inarg) - (define (recur x) (resolve-in-lhs x env m parent-scope inarg)) - (define (other x) (resolve-expansion-vars-with-new-env x env m parent-scope inarg)) +(define (resolve-in-lhs e env m lno parent-scope inarg) + (define (recur x) (resolve-in-lhs x env m lno parent-scope inarg)) + (define (other x) (resolve-expansion-vars-with-new-env x env m lno parent-scope inarg)) (case (and (pair? e) (car e)) ((where) `(where ,(recur (cadr e)) ,@(map other (cddr e)))) ((|::|) `(|::| ,(recur (cadr e)) ,(other (caddr e)))) ((call) `(call ,(other (cadr e)) ,@(map (lambda (x) - (resolve-expansion-vars-with-new-env x env m parent-scope #t)) + (resolve-expansion-vars-with-new-env x env m lno parent-scope #t)) (cddr e)))) ((tuple) `(tuple ,@(map (lambda (x) - (resolve-expansion-vars-with-new-env x env m parent-scope #t)) + (resolve-expansion-vars-with-new-env x env m lno parent-scope #t)) (cdr e)))) (else (other e)))) @@ -338,7 +338,7 @@ (keywords-introduced-by x)) env))))))) -(define (resolve-expansion-vars-with-new-env x env m parent-scope inarg (outermost #f)) +(define (resolve-expansion-vars-with-new-env x env m lno parent-scope inarg (outermost #f)) (resolve-expansion-vars- x (if (and (pair? x) (eq? (car x) 'let)) @@ -346,7 +346,7 @@ ;; the same expression env (new-expansion-env-for x env outermost)) - m parent-scope inarg)) + m lno parent-scope inarg)) (define (reescape ux x) (if (and (pair? x) (eq? (car x) 'escape)) @@ -355,29 +355,29 @@ ;; type has special behavior: identifiers inside are ;; field names, not expressions. -(define (resolve-struct-field-expansion x env m parent-scope inarg) +(define (resolve-struct-field-expansion x env m lno parent-scope inarg) (let ((ux (unescape x))) (cond ((atom? ux) ux) ((and (pair? ux) (eq? (car ux) '|::|)) `(|::| ,(unescape (cadr ux)) - ,(resolve-expansion-vars- (reescape (caddr ux) x) env m parent-scope inarg))) + ,(resolve-expansion-vars- (reescape (caddr ux) x) env m lno parent-scope inarg))) ((and (pair? ux) (memq (car ux) '(const atomic))) - `(,(car ux) ,(resolve-struct-field-expansion (reescape (cadr ux) x) env m parent-scope inarg))) + `(,(car ux) ,(resolve-struct-field-expansion (reescape (cadr ux) x) env m lno parent-scope inarg))) (else - (resolve-expansion-vars-with-new-env x env m parent-scope inarg))))) + (resolve-expansion-vars-with-new-env x env m lno parent-scope inarg))))) -(define (resolve-letlike-assign bind env newenv m parent-scope inarg) +(define (resolve-letlike-assign bind env newenv m lno parent-scope inarg) (if (assignment? bind) (make-assignment ;; expand binds in newenv with dummy RHS (cadr (resolve-expansion-vars- (make-assignment (cadr bind) 0) - newenv m parent-scope inarg)) + newenv m lno parent-scope inarg)) ;; expand initial values in old env - (resolve-expansion-vars- (caddr bind) env m parent-scope inarg)) + (resolve-expansion-vars- (caddr bind) env m lno parent-scope inarg)) ;; Just expand everything else that's not an assignment. N.B.: This includes ;; assignments inside escapes, which probably need special handling (TODO). - (resolve-expansion-vars- bind newenv m parent-scope inarg))) + (resolve-expansion-vars- bind newenv m lno parent-scope inarg))) (define (for-ranges-list ranges) (if (eq? (car ranges) 'escape) @@ -386,7 +386,7 @@ (cdr ranges) (list ranges)))) -(define (resolve-expansion-vars- e env m parent-scope inarg) +(define (resolve-expansion-vars- e env m lno parent-scope inarg) (cond ((or (eq? e 'begin) (eq? e 'end) (eq? e 'ccall) (eq? e 'cglobal) (underscore-symbol? e)) e) ((symbol? e) @@ -405,23 +405,31 @@ (env (car scope)) (m (cadr scope)) (parent-scope (cdr parent-scope))) - (resolve-expansion-vars-with-new-env (cadr e) env m parent-scope inarg)))) + (resolve-expansion-vars-with-new-env (cadr e) env m lno parent-scope inarg)))) ((global) `(global ,@(map (lambda (arg) (if (assignment? arg) - `(= ,(unescape-global-lhs (cadr arg) env m parent-scope inarg) - ,(resolve-expansion-vars-with-new-env (caddr arg) env m parent-scope inarg)) - (unescape-global-lhs arg env m parent-scope inarg))) + `(= ,(unescape-global-lhs (cadr arg) env m lno parent-scope inarg) + ,(resolve-expansion-vars-with-new-env (caddr arg) env m lno parent-scope inarg)) + (unescape-global-lhs arg env m lno parent-scope inarg))) + (cdr e)))) + ((toplevel) ; re-wrap Expr(:toplevel) in the current hygienic-scope(s) + `(toplevel + ,@(map (lambda (arg) + (let loop ((parent-scope parent-scope) (m m) (lno lno) (arg arg)) + (let ((wrapped `(hygienic-scope ,arg ,m ,@lno))) + (if (null? parent-scope) wrapped + (loop (cdr parent-scope) (cadar parent-scope) (caddar parent-scope) wrapped))))) (cdr e)))) ((using import export meta line inbounds boundscheck loopinfo inline noinline purity) (map unescape e)) ((macrocall) e) ; invalid syntax anyways, so just act like it's quoted. ((symboliclabel) e) ((symbolicgoto) e) ((struct) - `(struct ,(cadr e) ,(resolve-expansion-vars- (caddr e) env m parent-scope inarg) + `(struct ,(cadr e) ,(resolve-expansion-vars- (caddr e) env m lno parent-scope inarg) ,(map (lambda (x) - (resolve-struct-field-expansion x env m parent-scope inarg)) + (resolve-struct-field-expansion x env m lno parent-scope inarg)) (cadddr e)))) ((parameters) @@ -432,17 +440,17 @@ (x (if (and (not inarg) (symbol? ux)) `(kw ,ux ,x) x))) - (resolve-expansion-vars- x env m parent-scope #f))) + (resolve-expansion-vars- x env m lno parent-scope #f))) (cdr e)))) ((->) - `(-> ,(resolve-in-lhs (tuple-wrap-arrow-sig (cadr e)) env m parent-scope inarg) - ,(resolve-expansion-vars-with-new-env (caddr e) env m parent-scope inarg))) + `(-> ,(resolve-in-lhs (tuple-wrap-arrow-sig (cadr e)) env m lno parent-scope inarg) + ,(resolve-expansion-vars-with-new-env (caddr e) env m lno parent-scope inarg))) ((= function) - `(,(car e) ,(resolve-in-lhs (cadr e) env m parent-scope inarg) + `(,(car e) ,(resolve-in-lhs (cadr e) env m lno parent-scope inarg) ,@(map (lambda (x) - (resolve-expansion-vars-with-new-env x env m parent-scope inarg)) + (resolve-expansion-vars-with-new-env x env m lno parent-scope inarg)) (cddr e)))) ((kw) @@ -456,67 +464,67 @@ `(kw (|::| ,@(if argname (list (if inarg - (resolve-expansion-vars- argname env m parent-scope inarg) + (resolve-expansion-vars- argname env m lno parent-scope inarg) ;; in keyword arg A=B, don't transform "A" (unescape argname))) '()) - ,(resolve-expansion-vars- type env m parent-scope inarg)) - ,(resolve-expansion-vars-with-new-env (caddr e) env m parent-scope inarg)))) + ,(resolve-expansion-vars- type env m lno parent-scope inarg)) + ,(resolve-expansion-vars-with-new-env (caddr e) env m lno parent-scope inarg)))) (else `(kw ,(if inarg - (resolve-expansion-vars- (cadr e) env m parent-scope inarg) + (resolve-expansion-vars- (cadr e) env m lno parent-scope inarg) (unescape (cadr e))) - ,(resolve-expansion-vars-with-new-env (caddr e) env m parent-scope inarg))))) + ,(resolve-expansion-vars-with-new-env (caddr e) env m lno parent-scope inarg))))) ((let) (let* ((newenv (new-expansion-env-for e env)) - (body (resolve-expansion-vars- (caddr e) newenv m parent-scope inarg)) + (body (resolve-expansion-vars- (caddr e) newenv m lno parent-scope inarg)) (binds (let-binds e))) `(let (block ,@(map (lambda (bind) - (resolve-letlike-assign bind env newenv m parent-scope inarg)) + (resolve-letlike-assign bind env newenv m lno parent-scope inarg)) binds)) ,body))) ((for) (let* ((newenv (new-expansion-env-for e env)) - (body (resolve-expansion-vars- (caddr e) newenv m parent-scope inarg)) + (body (resolve-expansion-vars- (caddr e) newenv m lno parent-scope inarg)) (expanded-ranges (map (lambda (range) - (resolve-letlike-assign range env newenv m parent-scope inarg)) (for-ranges-list (cadr e))))) + (resolve-letlike-assign range env newenv m lno parent-scope inarg)) (for-ranges-list (cadr e))))) (if (length= expanded-ranges 1) `(for ,@expanded-ranges ,body)) `(for (block ,@expanded-ranges) ,body))) ((generator) (let* ((newenv (new-expansion-env-for e env)) - (body (resolve-expansion-vars- (cadr e) newenv m parent-scope inarg)) + (body (resolve-expansion-vars- (cadr e) newenv m lno parent-scope inarg)) (filt? (eq? (car (caddr e)) 'filter)) (range-exprs (if filt? (cddr (caddr e)) (cddr e))) - (filt (if filt? (resolve-expansion-vars- (cadr (caddr e)) newenv m parent-scope inarg))) + (filt (if filt? (resolve-expansion-vars- (cadr (caddr e)) newenv m lno parent-scope inarg))) (expanded-ranges (map (lambda (range) - (resolve-letlike-assign range env newenv m parent-scope inarg)) range-exprs))) + (resolve-letlike-assign range env newenv m lno parent-scope inarg)) range-exprs))) (if filt? `(generator ,body (filter ,filt ,@expanded-ranges)) `(generator ,body ,@expanded-ranges)))) ((hygienic-scope) ; TODO: move this lowering to resolve-scopes, instead of reimplementing it here badly - (let ((parent-scope (cons (list env m) parent-scope)) + (let ((parent-scope (cons (list env m lno) parent-scope)) (body (cadr e)) (m (caddr e)) (lno (cdddr e))) - (resolve-expansion-vars-with-new-env body env m parent-scope inarg #t))) + (resolve-expansion-vars-with-new-env body env m lno parent-scope inarg #t))) ((tuple) (cons (car e) (map (lambda (x) (if (assignment? x) `(= ,(unescape (cadr x)) - ,(resolve-expansion-vars-with-new-env (caddr x) env m parent-scope inarg)) - (resolve-expansion-vars-with-new-env x env m parent-scope inarg))) + ,(resolve-expansion-vars-with-new-env (caddr x) env m lno parent-scope inarg)) + (resolve-expansion-vars-with-new-env x env m lno parent-scope inarg))) (cdr e)))) ;; todo: trycatch (else (cons (car e) (map (lambda (x) - (resolve-expansion-vars-with-new-env x env m parent-scope inarg)) + (resolve-expansion-vars-with-new-env x env m lno parent-scope inarg)) (cdr e)))))))) ;; decl-var that also identifies f in f()=... @@ -617,11 +625,11 @@ (cdr v) '()))) -(define (resolve-expansion-vars e m) +(define (resolve-expansion-vars e m lno) ;; expand binding form patterns ;; keep track of environment, rename locals to gensyms ;; and wrap globals in (globalref module var) for macro's home module - (resolve-expansion-vars-with-new-env e '() m '() #f #t)) + (resolve-expansion-vars-with-new-env e '() m lno '() #f #t)) (define (julia-expand-quotes e) (cond ((not (pair? e)) e) @@ -637,11 +645,12 @@ (cond ((not (pair? e)) e) ((eq? (car e) 'inert) e) ((eq? (car e) 'module) e) + ((eq? (car e) 'toplevel) e) ((eq? (car e) 'hygienic-scope) (let ((form (cadr e)) ;; form is the expression returned from expand-macros (modu (caddr e)) ;; m is the macro's def module (lno (cdddr e))) ;; lno is (optionally) the line number node - (resolve-expansion-vars form modu))) + (resolve-expansion-vars form modu lno))) (else (map julia-expand-macroscopes- e)))) diff --git a/test/core.jl b/test/core.jl index 3a9945f152f5f..173526d3b2128 100644 --- a/test/core.jl +++ b/test/core.jl @@ -8125,3 +8125,10 @@ let M = @__MODULE__ end @test Base.unsafe_convert(Ptr{Int}, [1]) !== C_NULL + +# Test that new macros are allowed to be defined inside Expr(:toplevel) returned by macros +macro macroception() + Expr(:toplevel, :(macro foo() 1 end), :(@foo)) +end + +@test (@macroception()) === 1 diff --git a/test/docs.jl b/test/docs.jl index 36893b8614170..61d0271c25811 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -1562,3 +1562,15 @@ Base.@ccallable c51586_long()::Int = 3 @test_broken isempty(undoc) @test undoc == [Symbol("@var")] end + +# Docing the macroception macro +macro docmacroception() + Expr(:toplevel, macroexpand(__module__, :(@Base.__doc__ macro docmacrofoo() 1 end); recursive=false), :(@docmacrofoo)) +end + +""" +This docmacroception has a docstring +""" +@docmacroception() + +@test Docs.hasdoc(@__MODULE__, :var"@docmacrofoo")