diff --git a/JuliaLowering/src/JuliaLowering.jl b/JuliaLowering/src/JuliaLowering.jl index 911654dadb5e1..ec18b4d1d9fff 100644 --- a/JuliaLowering/src/JuliaLowering.jl +++ b/JuliaLowering/src/JuliaLowering.jl @@ -18,7 +18,10 @@ using .JuliaSyntax: @KSet_str, @stm, Kind, NodeId, SourceAttrType, SourceRef, Sy flattened_provenance, hasattr, head, highlight, is_compatible_graph, is_leaf, is_literal, kind, last_byte, mapchildren, mapsyntax, mkleaf, mknode, newleaf, newnode, node_string, numchildren, provenance, reparent, setattr, setattr!, - source_location, sourcefile, sourceref, syntax_graph, tree_ids, mapindex, mktree + source_location, sourcefile, sourceref, syntax_graph, tree_ids, mapindex, mktree, + ScopeLayer, SyntaxContext, is_base_layer, base_layer, escape_layer, + syntax_module, is_flisp_compat, adopt_scope, remove_context, fill_context!, + fill_context, JL_NEW_SYNTAX_VERSION, JL_OLD_SYNTAX_VERSION const DEBUG = true diff --git a/JuliaLowering/src/ast.jl b/JuliaLowering/src/ast.jl index 5ff0ddf086f04..b66b0901f94e9 100644 --- a/JuliaLowering/src/ast.jl +++ b/JuliaLowering/src/ast.jl @@ -50,36 +50,6 @@ Unique symbolic identity for a variable, constant, label, or other entity """ const IdTag = Int -""" -Id for hygienic scope layers in macro expansion -""" -const LayerId = Int - -""" -A `ScopeLayer` is a mechanism for automatic hygienic macros; every identifier -is assigned to a particular layer and can only match against bindings which are -themselves part of that layer. - -Normal code contains a single scope layer, whereas each macro expansion -generates a new layer. -""" -struct ScopeLayer - id::LayerId - mod::Module - # Index of parent layer in a macro expansion. Equal to 0 for no parent - parent_id::LayerId - # True if generated by an old macro expansion - hygiene_compat::Bool - is_internal::Bool -end - -is_base_layer(sl::ScopeLayer) = sl.parent_id === 0 - -base_layer(ctx, sl::ScopeLayer) = while true - is_base_layer(sl) && return sl - sl = ctx.scope_layers[sl.parent_id] -end - """ Lexical scope ID """ @@ -359,48 +329,6 @@ macro ast(ctx, srcref, tree) end #------------------------------------------------------------------------------- -function set_scope_layer(ctx, ex, layer_id, force) - k = kind(ex) - new_layer = force ? layer_id : get(ex, :scope_layer, layer_id) - - ex2 = if k == K"module" || k == K"toplevel" || k == K"inert" || k == K"inert_syntaxtree" - mknode(ex, children(ex)) - elseif k == K"." && numchildren(ex) == 2 - cs = tree_ids(set_scope_layer(ctx, ex[1], layer_id, force), ex[2]) - mknode(ex, cs) - elseif !is_leaf(ex) - mapchildren(e->set_scope_layer(ctx, e, layer_id, force), ctx, ex) - else - mkleaf(ex) - end - setattr!(ex2, :scope_layer, new_layer) -end - -""" - adopt_scope(ex, ref) - -Copy `ex`, adopting the scope layer of `ref`. -""" -function adopt_scope(ex::SyntaxTree, scope_layer::LayerId) - set_scope_layer(ex._graph, ex, scope_layer, true) -end - -function adopt_scope(ex::SyntaxTree, layer::ScopeLayer) - adopt_scope(ex, layer.id) -end - -function adopt_scope(ex::SyntaxTree, ref::SyntaxTree) - adopt_scope(ex, ref.scope_layer) -end - -function adopt_scope(exs::SyntaxList, ref) - out = SyntaxList(syntax_graph(exs)) - for e in exs - push!(out, adopt_scope(e, ref)) - end - return out -end - # Type for `meta` attribute, to replace `Expr(:meta)`. # It's unclear how much flexibility we need here - is a dict good, or could we # just use a struct? Likely this will be sparse. Alternatively we could just @@ -432,7 +360,7 @@ name_hint(name) = CompileHints(:name_hint, name) function is_quoted(ex) kind(ex) in KSet"Symbol quote top core globalref break inert - inert_syntaxtree meta inbounds inline noinline loopinfo" + syntaxinert meta inbounds inline noinline loopinfo" end function extension_type(ex) @@ -541,33 +469,6 @@ function to_symbol(ctx, ex) @ast ctx ex ex=>K"Symbol" end -# Internal scope layers may also have parents: desugaring generates a new -# internal scope layer to hold the global struct name (since any user references -# should hit the local instead), but the global struct name should still follow -# hygiene_compat rules as if it were on its original layer. -function new_internal_escapable_scope_layer(ctx, ident::SyntaxTree) - @jl_assert kind(ident) == K"Identifier" ident - old_sl = ctx.scope_layers[ident.scope_layer::LayerId] - sl = ScopeLayer( - length(ctx.scope_layers)+1, old_sl.mod, old_sl.id, - old_sl.hygiene_compat, true) - push!(ctx.scope_layers, sl) - sl -end -function new_internal_scope_layer(ctx, mod_ref::Module) - sl = ScopeLayer(length(ctx.scope_layers)+1, mod_ref, 0, false, true) - push!(ctx.scope_layers, sl) - sl -end - -function new_macro_scope_layer(ctx, parent::ScopeLayer, mod::Module, - hygiene_compat::Bool) - sl = ScopeLayer(length(ctx.scope_layers)+1, - mod, parent.id, hygiene_compat, false) - push!(ctx.scope_layers, sl) - sl -end - #------------------------------------------------------------------------------- # Context wrapper which helps to construct a list of statements to be executed # prior to some expression. Useful when we need to use subexpressions multiple diff --git a/JuliaLowering/src/closure_conversion.jl b/JuliaLowering/src/closure_conversion.jl index 3095ccfbadf51..2f92290d44603 100644 --- a/JuliaLowering/src/closure_conversion.jl +++ b/JuliaLowering/src/closure_conversion.jl @@ -343,7 +343,7 @@ function _convert_closures(ctx::ClosureConversionCtx, ex) else access end - elseif is_leaf(ex) || k == K"inert" || k == K"inert_syntaxtree" || k == K"static_eval" + elseif is_leaf(ex) || k == K"inert" || k == K"syntaxinert" || k == K"static_eval" ex elseif k == K"=" convert_assignment(ctx, ex) @@ -376,7 +376,7 @@ function _convert_closures(ctx::ClosureConversionCtx, ex) binfo = get_binding(ctx, ex[1]) if binfo.kind == :global # flisp has this, but our K"assert" handling is in a previous pass - # [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex]] + # [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] make_globaldecl(ctx, ex, binfo.mod, binfo.name, true, _convert_closures(ctx, ex[2])) else newleaf(ctx, ex, K"TOMBSTONE") @@ -635,7 +635,10 @@ Invariants: @fzone "JL: closures" function convert_closures( ctx::VariableAnalysisContext, ex::SyntaxTree{Attrs} ) where Attrs - ctx_out = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod, + # TODO: ctx.mod is used instead of syntax_module(ex) beyond this point, + # which is dubious + ctx_out = ClosureConversionCtx(ctx.graph, ctx.bindings, + ctx.layer.mod, ctx.closure_bindings, nothing, ex.lambda_bindings, false, true, SyntaxList(ctx.graph), diff --git a/JuliaLowering/src/compat.jl b/JuliaLowering/src/compat.jl index 39ffc4f7c3cec..8086f41bf1139 100644 --- a/JuliaLowering/src/compat.jl +++ b/JuliaLowering/src/compat.jl @@ -31,8 +31,8 @@ function expr_to_est(@nospecialize(e), expr_to_est(graph, e, lnn) end -function expr_to_est(graph::SyntaxGraph, @nospecialize(e), lnn::LineNumberNode) - SyntaxTree(graph, _expr_to_est(graph, e, lnn)[1]) +function expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::SourceAttrType) + SyntaxTree(graph, _expr_to_est(graph, e, src)[1]) end function _get_inner_lnn(e::Expr, default::LineNumberNode) @@ -57,17 +57,16 @@ function is_expr_value(st::SyntaxTree) return JuliaSyntax.is_literal(k) || k === K"Value" end -function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::LineNumberNode) +# Most of the complexity here is LineNumberNode absorption logic: linenodes are +# always considered provenance if unquoted, then removed in certain forms. If +# `src` is not an linenode, it is assumed to be a better provenance source, so +# linenodes in `e` are not used for provenance (but still removed). +function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::SourceAttrType) st = if e isa Symbol setattr!(newleaf(graph, src, K"Identifier"), :name_val, String(e)) elseif e isa QuoteNode cid, _ = _expr_to_est(graph, e.value, src) newnode(graph, src, K"inert", NodeId[cid]) - elseif e isa Expr && e.head === :scope_layer - @assert length(e.args) === 2 && e.args[1] isa Symbol - ident = newleaf(graph, src, K"Identifier") - setattr!(ident, :name_val, String(e.args[1]::Symbol)) - setattr!(ident, :scope_layer, e.args[2]) elseif e isa Expr && e.head === :lambda && length(e.args) == 2 argnames = e.args[1]::Vector{Any} arg_cs = NodeId[] @@ -86,12 +85,12 @@ function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::LineNumberNode) elseif e isa Expr head_s = string(e.head) st_k = find_kind(head_s) - src = old_src = _get_inner_lnn(e, src) + src = old_src = src isa LineNumberNode ? _get_inner_lnn(e, src) : src cs = NodeId[] rm_linenodes = e.head in (:block, :toplevel) for arg in e.args if rm_linenodes && arg isa LineNumberNode - src = arg + src isa LineNumberNode && (src = arg) else cid, src = _expr_to_est(graph, arg, src) push!(cs, cid) @@ -109,7 +108,7 @@ function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::LineNumberNode) # We may want additional special cases for other types where # `Base.isa_ast_node(e)`, but `K"Value"` should be fine for most, since # most are produced in or after lowering - if e isa LineNumberNode + if e isa LineNumberNode && src isa LineNumberNode # linenode outside of block or toplevel src = e end @@ -126,11 +125,10 @@ end function est_to_expr(st::SyntaxTree, suppress_linenodes=false) k = kind(st) if kind(st) === K"Identifier" + # @jl_assert scope layer is base n = Symbol(st.name_val::String) mod = get(st, :mod, nothing) - !isnothing(mod) ? GlobalRef(mod, n) : - hasattr(st, :scope_layer) ? Expr(:scope_layer, n, st.scope_layer) : - n + !isnothing(mod) ? GlobalRef(mod, n) : n elseif is_leaf(st) && is_expr_value(st) v = st.value # Let `st.value isa Symbol` (or other AST node). Since we enforce that @@ -318,7 +316,7 @@ end split_generated(st::SyntaxTree, gen_part) = @stm st begin (_, when=is_leaf(st)||is_quoted(st)) -> st [K"if" [K"generated"] gen nongen] -> if gen_part - @ast(st._graph, st, [K"$" gen]) + @ast(st._graph, st, [K"syntaxunquote" gen]) else nongen end @@ -419,7 +417,6 @@ We can assume `st` has passed `valid_st1`. Errors arising from invalid AST function est_to_dst(st::SyntaxTree) g = ensure_macro_attributes!(st._graph) rec = var"#self#" - return @stm st begin [K"Identifier"] -> _est_to_dst_ident(st) [K"Value"] -> st.value === nothing ? newleaf(g, st, K"nothing") : st @@ -432,7 +429,6 @@ function est_to_dst(st::SyntaxTree) op_leaf = newleaf(g, st, K"Identifier") setattr!(op_leaf, :name_val, op_s) - setattr!(op_leaf, :scope_layer, st.scope_layer) @ast g st [out_k rec(l) op_leaf rec(r)] end [K"comparison" cs0...] -> let cs = copy(cs0) @@ -442,15 +438,13 @@ function est_to_dst(st::SyntaxTree) mknode(st, cs) end [K"'" x] -> - @ast g st [K"call" "'"::K"Identifier"(scope_layer=st.scope_layer) rec(x)] + @ast g st [K"call" "'"::K"Identifier"(st) rec(x)] [K"." f [K"tuple" args...]] -> _expand_literal_pow( @ast g st [K"dotcall" rec(f) _dst_sink_parameters(args)...]) ([K"inert" [K"Identifier"]], when=!hasattr(st[1], :mod)) -> @ast g st st[1]=>K"Symbol" - ([K"inert_syntaxtree" [K"Identifier"]], when=!hasattr(st[1], :mod)) -> - @ast g st st[1]=>K"Symbol" + [K"syntaxinert" _] -> st [K"inert" _] -> st - [K"inert_syntaxtree" _] -> st [K"module" _...] -> st [K"toplevel" _...] -> st [K"for" [K"=" _ _] body] -> @@ -521,7 +515,7 @@ function est_to_dst(st::SyntaxTree) l = apply_arglist_meta(l, collect_body_arg_meta(r)) if has_if_generated(r) gen, nongen = split_generated(r, true), split_generated(r, false) - r2 = @ast g st [K"_generated_body" [K"quote" gen] rec(nongen)] + r2 = @ast g st [K"_generated_body" [K"syntaxquote" gen] rec(nongen)] else r2 = rec(r) end @@ -531,7 +525,7 @@ function est_to_dst(st::SyntaxTree) l = apply_arglist_meta(_dst_fix_arglist(l), collect_body_arg_meta(r)) if has_if_generated(r) gen, nongen = split_generated(r, true), split_generated(r, false) - r2 = @ast g st [K"_generated_body" [K"quote" gen] rec(nongen)] + r2 = @ast g st [K"_generated_body" [K"syntaxquote" gen] rec(nongen)] else r2 = rec(r) end @@ -541,7 +535,7 @@ function est_to_dst(st::SyntaxTree) l = apply_arglist_meta(_dst_fix_arglist(l), collect_body_arg_meta(r)) if has_if_generated(r) gen, nongen = split_generated(r, true), split_generated(r, false) - r2 = @ast g st [K"_generated_body" [K"quote" gen] rec(nongen)] + r2 = @ast g st [K"_generated_body" [K"syntaxquote" gen] rec(nongen)] else r2 = rec(r) end @@ -591,8 +585,7 @@ function est_to_dst(st::SyntaxTree) [K"top" x] -> setattr!(mkleaf(st), :name_val, x.name_val) [K"static_parameter" x] -> setattr!(mkleaf(st), :var_id, x.value::IdTag) [K"copyast" [K"inert" ex]] -> @ast g st [K"call" - interpolate_ast::K"Value" - Expr::K"Value" + interpolate_expr::K"Value" [K"inert"(st[1]) ex] ] [K"symbolicgoto" lab] -> setattr!(mkleaf(st), :name_val, lab.name_val) @@ -623,7 +616,8 @@ function est_to_dst(st::SyntaxTree) # forms (e.g. function definitions) stay inert. out_fptr = if kind(fptr) == K"inert" && numchildren(fptr) == 1 && kind(fptr[1]) == K"Identifier" - ident = setattr!(mkleaf(fptr[1]), :scope_layer, 1) + ident = mkleaf(fptr[1]) + # TODO: relayer if unhygienic @ast g fptr [K"static_eval"(fptr) ident] else rec(fptr) diff --git a/JuliaLowering/src/desugaring.jl b/JuliaLowering/src/desugaring.jl index 2412ace104ca8..6fd1ef57dc616 100644 --- a/JuliaLowering/src/desugaring.jl +++ b/JuliaLowering/src/desugaring.jl @@ -2,10 +2,8 @@ struct DesugaringContext{Attrs} <: AbstractLoweringContext graph::SyntaxGraph{Attrs} + layer::ScopeLayer bindings::Bindings - scope_layers::Vector{ScopeLayer} - mod::Module - expr_compat_mode::Bool ssa_mapping::Dict{Int, IdTag} world::UInt end @@ -49,7 +47,7 @@ end function contains_unquoted(f::Function, ex::SyntaxTree) if f(ex) return true - elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert inert_syntaxtree meta") + elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert syntaxinert meta") return any(contains_unquoted(f, e) for e in children(ex)) else return false @@ -63,7 +61,7 @@ function is_effect_free(ex) k = kind(ex) # TODO: metas is_literal(k) || is_identifier_like(ex) || k == K"Symbol" || - k == K"inert" || k == K"inert_syntaxtree" || k == K"top" || + k == K"inert" || k == K"syntaxinert" || k == K"top" || k == K"core" || k == K"Value" # flisp also includes `a.b` with simple `a`, but this seems like a bug # because this calls the user-defined getproperty? @@ -83,13 +81,55 @@ function check_no_assignment(exs, msg="misplaced assignment statement in `[ ... end end +function new_internal_context(st::SyntaxTree) + sc_orig = st.context::SyntaxContext + SyntaxContext( + ScopeLayer(syntax_module(st), nothing), + # macro provenance: could use nothing, but this is easier for consumers + sc_orig.unexpanded, + # version: internal bindings are only used in syntax we create, so the + # version should be the latest one + JL_NEW_SYNTAX_VERSION, + true) +end + # Generating a new_local_binding or ssaval should only be done if we can # guarantee there's some scope it's declared in, and that it's not declared or # used outside of that scope (binding capture is OK). This is the alternative. function newsym(ctx, src::SyntaxTree, name::String; unused=false) out = newleaf(ctx, src, unused ? K"Placeholder" : K"Identifier", name) hasattr(src, :meta) && setattr!(out, :meta, src.meta) - setattr!(out, :scope_layer, new_internal_scope_layer(ctx, ctx.mod).id) + setattr!(out, :context, new_internal_context(src)) +end + +# In an flisp-compatible expansion, some explicit global declarations (and any +# initialization in the same expression) are unhygienic; they are declared in +# the macrocall module (unless wrapped in a top-level form). This is buggy +# (references in the same scope don't resolve to it, op-equal assignments don't +# work, etc.), but compatible. flisp: `unescape`, `unescape-global-lhs`. TODO: +# It would be cleaner to do this in compat.jl. +function relayer_global_if_unhygienic(ctx, st::SyntaxTree) + sc = st.context::SyntaxContext + relayered = SyntaxList(st._graph) + # TODO: is_base_layer(sc) or sc.layer == ctx.layer? + (!is_flisp_compat(sc) || is_base_layer(sc)) && return st, relayered + sc2 = escape_layer(sc, true) + return _relayer_global_if_unhygienic(relayered, st, sc2), relayered +end +function _relayer_global_if_unhygienic(done::SyntaxList, st::SyntaxTree, sc::SyntaxContext) + k = kind(st) + if k === K"Identifier" && is_flisp_compat(st) && st.context::SyntaxContext !== sc + push!(done, st) + setattr(st, :context, sc) + elseif k === K"::" || k === K"kw" + n_done = length(done) + lhs = _relayer_global_if_unhygienic(done, st[1], sc) + n_done == length(done) ? st : (@ast st._graph st [k lhs st[2]]) + elseif k === K"tuple" || k === K"parameters" + mapchildren(e->_relayer_global_if_unhygienic(done, e, sc), st._graph, st) + else + st + end end #------------------------------------------------------------------------------- @@ -141,7 +181,7 @@ function tuple_to_assignments(ctx, ex, is_const) end k = kind(r) if is_literal(k) || k == K"Symbol" || k == K"inert" || - k == K"inert_syntaxtree" || k == K"top" || k == K"core" || + k == K"syntaxinert" || k == K"top" || k == K"core" || k == K"Value" # Effect-free and nothrow right hand sides do not need a temporary # (we require nothrow because the order of rhs terms is observable @@ -766,7 +806,7 @@ end function find_return(ex::SyntaxTree) if kind(ex) == K"return" return ex - elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert inert_syntaxtree meta function ->") + elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert syntaxinert meta function ->") for e in children(ex) r = find_return(e) if !isnothing(r) @@ -1953,8 +1993,10 @@ function expand_dot(ctx, ex) # of DOTOP_FLAG? This way, `K"."` will be exclusively used for # getproperty. [K"." op] -> @ast ctx ex [K"call" "BroadcastFunction"::K"top" op] + [K"." l [K"syntaxinert" r]] -> + @ast ctx ex [K"call" "getproperty"::K"top" l [K"inert" r]] [K"." l r] -> begin - @jl_assert is_leaf(r) || kind(r) in KSet"inert inert_syntaxtree" ex + @jl_assert is_leaf(r) || kind(r) in KSet"inert syntaxinert" ex @ast ctx ex [K"call" "getproperty"::K"top" l r] end end @@ -2153,10 +2195,6 @@ end function make_lhs_decls(ctx, stmts, declkind, declmeta, ex, type_decls=true) declname = @stm ex begin [K"Identifier"] -> ex - # TODO: consider removing support for Expr(:global, GlobalRef(...)) and - # other Exprs that cannot be produced by the parser (tested by - # test/precompile.jl #50538). - ([K"Value"], when=ex.value isa GlobalRef) -> ex [K"Placeholder"] -> nothing ([K"::" [K"Identifier"] t], when=type_decls) -> let x = ex[1] t2 = expand_forms_2(ctx, t) @@ -2199,7 +2237,18 @@ function expand_decls(ctx, ex) @jl_assert declkind in KSet"local global" ex stmts = SyntaxList(ctx) for c in children(ex) - simple = kind(c) in KSet"Identifier :: Value Placeholder" + simple = kind(c) in KSet"Identifier :: Placeholder" + if declkind === K"global" + if kind(c) === K"=" + (lhs, relayered) = relayer_global_if_unhygienic(ctx, c[1]); + !isempty(relayered) && (c = @ast ctx c [K"=" lhs c[2]]) + elseif simple + (c, relayered) = relayer_global_if_unhygienic(ctx, c); + end + @isdefined(relayered) && for x in relayered + push!(stmts, @ast ctx x [K"relayered_global" x]) + end + end lhs = @stm c begin (_, when=simple) -> c [K"=" x _] -> x @@ -2245,10 +2294,14 @@ function expand_const_decl(ctx, ex) [K"global" x] -> let decls = SyntaxList(ctx) @jl_assert kind(x) === K"=" ex + (lhs, relayered) = relayer_global_if_unhygienic(ctx, x[1]) make_lhs_decls( - ctx, decls, K"global", get(ex[1], :meta, nothing), x[1], false) - ex2 = @ast ctx ex [K"const" x] - @ast ctx ex [K"block" decls... expand_const_decl(ctx, ex2)] + ctx, decls, K"global", get(ex[1], :meta, nothing), lhs, false) + for x in relayered + push!(decls, @ast ctx x [K"relayered_global" x]) + end + x2 = @ast ctx x [K"=" lhs x[2]] + @ast ctx ex [K"block" decls... expand_assignment(ctx, x2, true)] end [K"=" _ _] -> expand_assignment(ctx, ex[1], true) # Expr(:const, v) where v is a Symbol or a GlobalRef is an unfortunate @@ -2379,7 +2432,7 @@ end # nkw, and destructuring arg assignments) we do this, so handle them manually. function prepend_function_body(ctx, body, ex) @stm body begin - [K"_generated_body" [K"quote" gen] nongen] -> begin + [K"_generated_body" [K"syntaxquote" gen] nongen] -> begin ex_est = @stm ex begin [K"meta" [K"Symbol"] n] -> @ast ctx ex [K"meta" "nkw"::K"Identifier" n] @@ -2389,7 +2442,7 @@ function prepend_function_body(ctx, body, ex) _ -> @jl_assert false (ex, "unexpected prepend_function_body") end @ast ctx body [K"_generated_body" - [K"quote" [K"block" ex_est gen]] [K"block" ex nongen]] + [K"syntaxquote" [K"block" ex_est gen]] [K"block" ex nongen]] end _ -> @ast ctx body [K"block" ex body] end @@ -2448,25 +2501,27 @@ end # The Julia runtime associates the code generator with the non-generated method # by adding (meta generated ...) to the non-generated body +# May need hygiene/provenance adjustments function generated_method_defs(ctx, src, mtable, sparams, argl, body, rett) @jl_assert kind(body) === K"_generated_body" && numchildren(body) == 2 body gen_name = let mangled = reserve_module_binding_i( - ctx.mod, string("#", kind(mtable) === K"nothing" ? "_" : mtable, "@generator#")) - new_global_binding(ctx, src, mangled, ctx.mod) + ctx.layer.mod, + string("#", kind(mtable) === K"nothing" ? "_" : mtable, "@generator#")) + new_global_binding(ctx, src, mangled, ctx.layer.mod) end + sc = src.context::SyntaxContext gen_mdef = let arg1_name = newsym(ctx, argl[1], "#self#"), gen_argl = SyntaxList( - @ast(ctx, src, [K"::" arg1_name [K"function_type" gen_name]]), - @ast(ctx, src, [K"::" - # TODO: correct scope layer? - "__context__"::K"Identifier"(scope_layer=get(mtable, :scope_layer, 1)) - MacroContext::K"Value" - ]), - mapsyntax(_untyped_arg, sparams)..., + @ast(ctx, src, [K"::" arg1_name [K"function_type" gen_name]]), + @ast(ctx, src, [K"::" + "__context__"::K"Identifier"(context=sc) + SyntaxContext::K"Value" + ]), + mapsyntax(_untyped_arg, sparams)..., mapsyntax(_untyped_arg, argl)...) - @jl_assert kind(body[1]) === K"quote" body - gen_body = est_to_dst(expand_quote(ctx, body[1][1])) + @jl_assert kind(body[1]) === K"syntaxquote" body + gen_body = est_to_dst(expand_syntaxquote(ctx, body[1][1])) method_def_expr(ctx, src, gen_name, SyntaxList(ctx), gen_argl, gen_body, @ast(ctx, src, "Any"::K"core")) @@ -2476,7 +2531,7 @@ function generated_method_defs(ctx, src, mtable, sparams, argl, body, rett) nongen_body = @ast ctx body[2] [K"block" [K"meta" "generated"::K"Symbol" [K"new" GeneratedFunctionStub::K"Value" # Use stub type from JuliaLowering - ctx.expr_compat_mode::K"Value" + SyntaxContext(ctx.layer.mod, sc.version)::K"Value" gen_name # Truncate provenance to just the source file range, as this # will live permanently in the IR and we probably don't want @@ -2619,7 +2674,10 @@ function keywords_method_def_expr(ctx, src, mtable, sparams, argl, body, rett, p m1_name = let n = kind(mtable) === K"nothing" ? "_" : mtable.name_val, mangled = string(startswith(n, '#') ? "" : "#kw_body#", n, "#") - newsym(ctx, argl[1], reserve_module_binding_i(ctx.mod, mangled)) + # probably not desirable, but fixes eval-into-closed-module + a1 = setattr(mtable, :context, + escape_layer(mtable.context::SyntaxContext, true)) + newsym(ctx, a1, reserve_module_binding_i(syntax_module(a1), mangled)) end # (1) Body method. This contains the actual function body, and requires # every possible default to be filled. `rett` is only passed here since it @@ -2947,18 +3005,17 @@ function expand_macro_def(ctx, ex) _ -> @jl_assert false ex end - scope_ref = kind(name) == K"." ? name[1] : name - if ctx.expr_compat_mode + sc_ref = (kind(name) == K"." ? name[1] : name) + if is_flisp_compat(ex) @ast ctx ex [K"function" [K"call"(sig) _make_macro_name(ctx, name) [K"::" - # TODO: should we be adopting the scope of the K"macro" expression itself? - adopt_scope(@ast(ctx, sig, "__source__"::K"Identifier"), scope_ref) + adopt_scope(sc_ref, @ast(ctx, sig, "__source__"::K"Identifier")) "LineNumberNode"::K"core" ] [K"::" - adopt_scope(@ast(ctx, sig, "__module__"::K"Identifier"), scope_ref) + adopt_scope(sc_ref, @ast(ctx, sig, "__module__"::K"Identifier")) "Module"::K"core" ] mapsyntax(e->apply_arg_meta(e, :nospecialize), args)... @@ -2970,10 +3027,10 @@ function expand_macro_def(ctx, ex) [K"call"(sig) _make_macro_name(ctx, name) [K"::" - adopt_scope(@ast(ctx, sig, "__context__"::K"Identifier"), scope_ref) + adopt_scope(sc_ref, @ast(ctx, sig, "__context__"::K"Identifier")) MacroContext::K"Value" ] - # flisp: We don't mark these @nospecialize because all arguments to + # We don't mark these @nospecialize because all arguments to # new macros will be of type SyntaxTree args... ] @@ -3083,6 +3140,7 @@ function expand_abstract_or_primitive_type(ctx, ex) end nbits = is_abstract ? nothing : ex[2] name, type_params, supertype = analyze_type_sig(ctx, ex[1]) + name, _ = relayer_global_if_unhygienic(ctx, name) typevar_names, typevar_stmts = expand_typevars(ctx, type_params) newtype_var = ssavar(ctx, ex, "new_type") @ast ctx ex [K"block" @@ -3095,7 +3153,7 @@ function expand_abstract_or_primitive_type(ctx, ex) newtype_var [K"call" (is_abstract ? "_abstracttype" : "_primitivetype")::K"core" - ctx.mod::K"Value" + syntax_module(name)::K"Value" name=>K"Symbol" [K"call" "svec"::K"core" typevar_names...] if !is_abstract @@ -3108,13 +3166,13 @@ function expand_abstract_or_primitive_type(ctx, ex) [K"call" "_typebody!"::K"core" false::K"Bool" name] ] ] - [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex] ] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex] ] [K"global" name] [K"if" [K"&&" [K"call" "isdefinedglobal"::K"core" - ctx.mod::K"Value" + syntax_module(name)::K"Value" name=>K"Symbol" false::K"Bool"] [K"call" "_equiv_typedef"::K"core" name newtype_var] @@ -3464,7 +3522,7 @@ function _insert_fieldtype_struct_shim(ctx, name, ex) numchildren(ex) == 2 && kind(ex[2]) == K"Symbol" && ex[2].name_val == name.name_val - @ast ctx ex [K"call" "struct_name_shim"::K"core" ex[1] ex[2] ctx.mod::K"Value" name] + @ast ctx ex [K"call" "struct_name_shim"::K"core" ex[1] ex[2] syntax_module(ex)::K"Value" name] elseif numchildren(ex) > 0 mapchildren(e->_insert_fieldtype_struct_shim(ctx, name, e), ctx, ex) else @@ -3525,6 +3583,7 @@ function expand_typegroup_def(ctx, ex) struct_names = SyntaxList(ctx) # local name bindings (splatted into AST) global_names = SyntaxList(ctx) # global name bindings (splatted into AST) info_vars = SyntaxList(ctx) # SSA vars for struct info svecs (splatted into AST) + struct_mod_prev = nothing for child in children(body) if kind(child) == K"struct" @@ -3566,14 +3625,20 @@ function expand_typegroup_def(ctx, ex) supertype, is_mutable, min_initialized, inner_defs, field_docs)) push!(struct_names, struct_name) - layer = new_internal_escapable_scope_layer(ctx, struct_name).id - push!(global_names, adopt_scope(struct_name, layer)) + global_struct_name, _ = relayer_global_if_unhygienic(ctx, struct_name) + struct_mod = syntax_module(global_struct_name) + isnothing(struct_mod_prev) || struct_mod == struct_mod_prev || throw( + LoweringError(ex, "typegroup of types from multiple modules")) + struct_mod_prev = struct_mod + struct_globalref = setattr!(mkleaf(global_struct_name), :mod, struct_mod) + push!(global_names, struct_globalref) push!(info_vars, ssavar(ctx, sdef, "struct_info")) end n = length(entries) if n == 0 return nothing_(ctx, ex) end + typegroup_mod = syntax_module(global_names[1]) # Build the lowered code # @@ -3613,7 +3678,7 @@ function expand_typegroup_def(ctx, ex) push!(inner_stmts, @ast ctx e.sdef [K"local" tv_name]) end append!(inner_stmts, typevar_stmts) - push!(inner_stmts, @ast ctx e.sdef [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" e.sdef]]) + push!(inner_stmts, @ast ctx e.sdef [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" e.sdef]]) push!(inner_stmts, @ast ctx e.sdef [K"=" info_var [K"call" "svec"::K"core" @@ -3636,7 +3701,7 @@ function expand_typegroup_def(ctx, ex) push!(stmts, @ast ctx ex [K"=" [K"tuple" struct_names...] [K"call" "resolve_typegroup"::K"core" - ctx.mod::K"Value" + typegroup_mod::K"Value" [K"call" "svec"::K"core" struct_names...] [K"call" "svec"::K"core" info_vars...] ] @@ -3698,13 +3763,10 @@ function expand_typegroup_def(ctx, ex) # Build the toplevel assertion + scope block, then do the expand and replace scope_block_stmts = SyntaxList(ctx) - for name in global_names - push!(scope_block_stmts, @ast ctx ex [K"global" name]) - end push!(scope_block_stmts, @ast ctx ex [K"block" stmts...]) result = @ast ctx ex [K"block" - [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex]] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] [K"scope_block"(scope_type=:hard) scope_block_stmts... ] @@ -3739,11 +3801,12 @@ function expand_struct_def(ctx, ex, docs) hasprev = ssavar(ctx, ex, "hasprev") prev = ssavar(ctx, ex, "prev") newdef = ssavar(ctx, ex, "newdef") - layer = new_internal_escapable_scope_layer(ctx, struct_name).id - global_struct_name = adopt_scope(struct_name, layer) + global_struct_name, _ = relayer_global_if_unhygienic(ctx, struct_name) + struct_mod = syntax_module(global_struct_name) + struct_globalref = setattr!(mkleaf(global_struct_name), :mod, struct_mod) if !isempty(typevar_names) # Generate expression like `prev_struct.body.body.parameters` - prev_typevars = global_struct_name + prev_typevars = struct_globalref for _ in 1:length(typevar_names) prev_typevars = @ast ctx type_sig [K"." prev_typevars "body"::K"Symbol"] end @@ -3793,7 +3856,7 @@ function expand_struct_def(ctx, ex, docs) # constructor-like signatures for (def_i, def) in enumerate(inner_defs) inner_defs[def_i] = - rewrite_ctor(ctx, def, struct_name, global_struct_name, + rewrite_ctor(ctx, def, struct_name, struct_globalref, typevar_names, field_types) end @@ -3801,9 +3864,9 @@ function expand_struct_def(ctx, ex, docs) # typevars when "redefining" structs. # See https://github.com/JuliaLang/julia/pull/36121 @ast ctx ex [K"block" - [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex] ] - [K"scope_block"(scope_type=:hard) - [K"global" global_struct_name] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex] ] + [K"block" + [K"global" struct_globalref] [K"scope_block"(scope_type=:hard) [K"local" struct_name] [K"always_defined" struct_name] @@ -3812,7 +3875,7 @@ function expand_struct_def(ctx, ex, docs) newtype_var [K"call" "_structtype"::K"core" - ctx.mod::K"Value" + struct_mod::K"Value" struct_name=>K"Symbol" [K"call"(type_sig) "svec"::K"core" typevar_names...] [K"call"(type_body) "svec"::K"core" [n=>K"Symbol" for n in field_names]...] @@ -3825,12 +3888,12 @@ function expand_struct_def(ctx, ex, docs) [K"call"(supertype) "_setsuper!"::K"core" newtype_var supertype] [K"=" hasprev [K"&&" [K"call" "isdefinedglobal"::K"core" - ctx.mod::K"Value" + struct_mod::K"Value" struct_name=>K"Symbol" false::K"Bool"] - [K"call" "_equiv_typedef"::K"core" global_struct_name newtype_var] + [K"call" "_equiv_typedef"::K"core" struct_globalref newtype_var] ]] - [K"=" prev [K"if" hasprev global_struct_name false::K"Bool"]] + [K"=" prev [K"if" hasprev struct_globalref false::K"Bool"]] [K"if" hasprev [K"block" # if this is compatible with an old definition, use the old parameters, but the @@ -3852,7 +3915,7 @@ function expand_struct_def(ctx, ex, docs) [K"call" "svec"::K"core" insert_struct_shim(ctx, field_types, struct_name)...] ]] [K"constdecl" - global_struct_name + struct_globalref newdef ] ] @@ -3862,7 +3925,7 @@ function expand_struct_def(ctx, ex, docs) [K"block" [K"call" "_defaultctors"::K"top" - global_struct_name + struct_globalref ::K"SourceLocation"(ex) ] (::K"latestworld") @@ -3969,6 +4032,8 @@ function expand_importpath(ctx, path) setattr(path, :kind, K".") end +# importer does not obey hygiene. Doesn't bother with relayering any imported +# items, as the runtime functions don't see hygiene anyway function expand_import_or_using(ctx, ex) if kind(ex[1]) == K":" # import M: x.y as z, w @@ -4010,7 +4075,7 @@ function expand_import_or_using(ctx, ex) push!(stmts, @ast ctx spec [K"call" eval_using ::K"Value" - ctx.mod ::K"Value" + ctx.layer.mod::K"Value" spec ] ) @@ -4019,7 +4084,7 @@ function expand_import_or_using(ctx, ex) @ast ctx spec [K"call" eval_import ::K"Value" (!is_using) ::K"Bool" - ctx.mod ::K"Value" + ctx.layer.mod::K"Value" (::K"nothing") spec ] @@ -4033,29 +4098,34 @@ function expand_import_or_using(ctx, ex) push!(stmts, @ast ctx ex [K"call" eval_import ::K"Value" (!is_using) ::K"Bool" - ctx.mod ::K"Value" + ctx.layer.mod::K"Value" from_path path_specs... ]) push!(stmts, @ast ctx ex (::K"latestworld")) end @ast ctx ex [K"block" - [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex]] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] stmts... [K"removable" (::K"nothing")] ] end -# Expand `public` or `export` +# flisp: export is relayered, and no-esc public is a syntax error (we relayer) function expand_public(ctx, ex) identifiers = String[] + numchildren(ex) == 0 && return @ast ctx ex (::K"nothing") + mod = syntax_module(relayer_global_if_unhygienic(ctx, ex[1])[1]) for e in children(ex) @jl_assert kind(e) == K"Identifier" (ex, "Expected identifier") + syntax_module(relayer_global_if_unhygienic(ctx, e)[1]) !== mod && + throw(LoweringError( + ex, "unexpected public/export with names from multiple modules")) push!(identifiers, e.name_val) end @ast ctx ex [K"call" eval_public::K"Value" - ctx.mod::K"Value" + mod::K"Value" (kind(ex) == K"export")::K"Bool" identifiers::K"Value" ] @@ -4070,7 +4140,7 @@ function isquotedmacrocall(ex) let (f, ex) = (ex[1], ex[3]) kind(f) == K"Value" || return false kind(ex) == K"inert" || return false - f.value === interpolate_ast || return false + f.value === interpolate_expr || return false kind(ex[1]) == K"macrocall" || return false return true end @@ -4080,7 +4150,7 @@ function expand_doc(ctx, ex, docex) if kind(ex) in (K"Identifier", K".") expand_forms_2(ctx, @ast ctx docex [K"call" bind_static_docs!::K"Value" - (kind(ex) === K"." ? ex[1] : ctx.mod::K"Value") + (kind(ex) === K"." ? ex[1] : syntax_module(ex)::K"Value") (kind(ex) === K"." ? ex[2] : ex).name_val::K"Symbol" docex[1] ::K"SourceLocation"(ex) @@ -4147,7 +4217,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) ] elseif k == K"<:" || k == K">:" || k == K"-->" expand_forms_2(ctx, @ast ctx ex [K"call" - adopt_scope(string(k)::K"Identifier", ex) + adopt_scope(ex, string(k)::K"Identifier") children(ex)... ]) elseif k == K"op=" || k == K".op=" @@ -4239,7 +4309,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) @ast ctx ex [K"block" [K"assert" "global_toplevel_only"::K"Symbol" - [K"inert_syntaxtree" ex] + [K"syntaxinert" ex] ] expand_forms_2(ctx, expand_macro_def(ctx, ex)) ] @@ -4312,20 +4382,17 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) elseif k == K"curly" expand_forms_2(ctx, expand_curly(ctx, ex)) elseif k == K"toplevel" - # The toplevel form can't be lowered here - it needs to just be quoted - # and passed through to a call to eval. + # Temporary: It would make more sense to return this unchanged once + # toplevel iteration over SyntaxTree exists, but for now, a call to + # `eval` lets JuliaLowering retain provenance and hygiene here. ex2 = @ast ctx ex [K"block" - [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex]] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] [K"call" - eval ::K"Value" - ctx.mod ::K"Value" - [K"inert_syntaxtree" ex] - [K"parameters" - [K"kw" - "expr_compat_mode"::K"Identifier" - ctx.expr_compat_mode::K"Bool" - ] - ] + eval::K"Value" + # a macro expanding to toplevel does not change the eval module, + # but does change the name resolution module + ctx.layer.mod::K"Value" + [K"syntaxinert" ex] ] ] expand_forms_2(ctx, ex2) @@ -4354,7 +4421,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) ] ] ] - elseif k == K"inert" || k == K"inert_syntaxtree" || k == K"foreignsymbol" + elseif k == K"inert" || k == K"syntaxinert" || k == K"foreignsymbol" ex elseif k == K"foreignglobal" @ast ctx ex [K"foreignglobal" expand_csymbol(ctx, ex[1])] @@ -4417,12 +4484,11 @@ ensure_desugaring_attributes!(graph) = ensure_attributes!( toplevel_pure=Bool, scope_type=Symbol) -@fzone "JL: desugar" function expand_forms_2(ctx::MacroExpansionContext, ex::SyntaxTree) - graph = ensure_desugaring_attributes!(copy_attrs(ctx.graph)) +@fzone "JL: desugar" function expand_forms_2(ex::SyntaxTree, world::UInt) + graph = ensure_desugaring_attributes!(copy_attrs(ex._graph)) ex = reparent(graph, ex) - ctx_out = DesugaringContext(graph, ctx.bindings, ctx.scope_layers, - ctx.scope_layers[1].mod, ctx.expr_compat_mode, - Dict{Int, IdTag}(), ctx.macro_world) + sl = base_layer(ex.context::SyntaxContext) + ctx_out = DesugaringContext(graph, sl, Bindings(), Dict{Int, IdTag}(), world) vr = valid_st1(ex) # surface only one error until we have pretty-printing for multiple if !vr.ok diff --git a/JuliaLowering/src/eval.jl b/JuliaLowering/src/eval.jl index b923bee845466..b1aa122f2a6d4 100644 --- a/JuliaLowering/src/eval.jl +++ b/JuliaLowering/src/eval.jl @@ -1,19 +1,80 @@ # Non-incremental lowering API for non-toplevel non-module expressions. # May be removed? -function lower(mod::Module, ex0::SyntaxTree; expr_compat_mode::Bool=false, world::UInt=Base.get_world_counter(), +function lower(mod::Module, ex_in::SyntaxTree; expr_compat_mode::Bool=false, soft_scope::Union{Nothing,Bool}=nothing) - ctx1, ex1 = expand_forms_1( mod, ex0, expr_compat_mode, world) - ctx2, ex2 = expand_forms_2( ctx1, ex1) - ctx3, ex3 = resolve_scopes( ctx2, ex2; soft_scope) - ctx4, ex4 = convert_closures(ctx3, ex3) - _ctx5, ex5 = linearize_ir( ctx4, ex4) + ver = expr_compat_mode ? JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION + ex0 = rebase_layers(ex_in, mod, ver) + world = Base.get_world_counter() + ex1 = expand_forms_1(ex0, world, true) + ctx2, ex2 = expand_forms_2(ex1, world) + ctx3, ex3 = resolve_scopes(ctx2, ex2; soft_scope) + ctx4, ex4 = convert_closures(ctx3, ex3) + _ctx5, ex5 = linearize_ir(ctx4, ex4) ex5 end -function macroexpand(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false, world::UInt=Base.get_world_counter()) - _ctx1, ex1 = expand_forms_1(mod, ex, expr_compat_mode, world) - ex1 +function macroexpand(mod::Module, ex_in::SyntaxTree; + expr_compat_mode::Bool=false, + ver::VersionNumber=expr_compat_mode ? + JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION, + recursive::Bool=true) + ex0 = rebase_layers(ex_in, mod, ver) + expand_forms_1(ex0, Base.get_world_counter(), recursive) +end + +"May be used in macros or from any module" +function macroexpand(st::SyntaxTree) + DEBUG && assert_expandable(st) + ctx = MacroExpansionContext(st, Base.get_world_counter(), true) + expand_forms_1(ctx, st) +end + +# If a top-level thunk has existing context, we can assume all syntax has the +# same base layer: either it was produced by a macro expansion and went through +# `apply_expansion_layer`, or it was produced by parsing (which we assume either +# adds zero or uniform context to the tree). + +# We ignore old the base layer's module, which should usually be the same as the +# current lowering module. (counterexample: macroexpand in mod A producing +# escaped :toplevel st, then eval st in mod B, but flisp does the same thing by +# spamming globalrefs to mod A throughout st). +function rebase_layers(st, mod::Module, ver::VersionNumber) + out = if !hasattr(st, :context) + # assert zero context + sc = SyntaxContext(mod, ver) + fill_context!(st, sc) + else + base = base_layer(st.context::SyntaxContext) + newbase = ScopeLayer(mod, nothing) + _rebase_layers( + st, Dict{ScopeLayer, ScopeLayer}(base=>newbase), + Dict{SyntaxContext, SyntaxContext}()) + end + DEBUG && assert_expandable(out) + out +end + +function _rebase_layers(st, slmap, scmap) + sc = st.context::SyntaxContext + sc2 = get(scmap, sc, nothing) + if isnothing(sc2) + sl2 = _get_sl!(slmap, sc.layer) + sc2 = scmap[sc] = SyntaxContext(sl2, sc.unexpanded, sc.version, sc.internal) + end + if is_leaf(st) || numchildren(st) == 0 + setattr(st, :context, sc2) + else + setattr!(mapchildren(c->_rebase_layers(c, slmap, scmap), st._graph, st), + :context, sc2) + end +end + +function _get_sl!(slmap, sl::ScopeLayer) + out = get(slmap, sl, nothing) + out isa ScopeLayer && return out + slmap[sl] = ScopeLayer( + sl.mod, isnothing(sl.escaped) ? nothing : _get_sl!(slmap, sl.escaped)) end # Incremental lowering API which can manage toplevel and module expressions. @@ -34,12 +95,12 @@ end # how we end up putting this into Base. struct LoweringIterator{Attrs} - expr_compat_mode::Bool # later stored in module? + ver::VersionNumber # later stored in module? todo::Vector{Tuple{SyntaxTree{Attrs}, Bool, Int}} end -function lower_init(ex::SyntaxTree{T}; expr_compat_mode::Bool=false) where {T} - LoweringIterator{T}(expr_compat_mode, [(ex, false, 0)]) +function lower_init(ex::SyntaxTree{T}, ver) where {T} + LoweringIterator{T}(ver, [(ex, false, 0)]) end function lower_step(iter::LoweringIterator, mod::Module, world::UInt; @@ -64,7 +125,8 @@ function lower_step(iter::LoweringIterator, mod::Module, world::UInt; k = kind(ex) if !(k in KSet"toplevel module") - ctx1, ex = expand_forms_1(mod, ex, iter.expr_compat_mode, world) + ex = rebase_layers(ex, mod, iter.ver) + ex = expand_forms_1(ex, world, true) k = kind(ex) end if k == K"toplevel" @@ -85,9 +147,7 @@ function lower_step(iter::LoweringIterator, mod::Module, world::UInt; push!(iter.todo, (body, true, 1)) return Core.svec(:begin_module, version, newmod_name, notbare, loc) else - # Non macro expansion parts of lowering - @assert @isdefined(ctx1) "Assertion to tell the compiler about the definedness of this variable" - ctx2, ex2 = expand_forms_2(ctx1, ex) + ctx2, ex2 = expand_forms_2(ex, world) ctx3, ex3 = resolve_scopes(ctx2, ex2; soft_scope) ctx4, ex4 = convert_closures(ctx3, ex3) _ctx5, ex5 = linearize_ir(ctx4, ex4) @@ -312,10 +372,13 @@ function _di_pos(st::SyntaxTree) end # TODO sourcefile(::LNN) should return Symbol, not LNN -_di_sourcefile(st) = - let x = JuliaSyntax.unexpanded_sourceref(st) - x isa LineNumberNode ? x.file : x.file[]::SourceFile - end +function _di_sourcefile(st) + # if st.context.unexpanded isa SyntaxTree + # @jl_assert st.context.unexpanded._graph === st._graph (st, "bad unexpanded: different graph") (st.context.unexpanded, "this is the unexpanded tree") + # end + x = JuliaSyntax.unexpanded_sourceref(st) + x isa LineNumberNode ? x.file : x.file[]::SourceFile +end # A single pass over all IR to collect unique byte/line positions and CodeInfos function collect_locs!(node_sources, codeinfos, top_sf, st) @@ -578,10 +641,12 @@ function _to_lowered_expr(ex::SyntaxTree) elseif k == K"SSAValue" Core.SSAValue(ex.var_id::IdTag) elseif k == K"return" - Core.ReturnNode(_to_lowered_expr(ex[1])) + v = _to_lowered_expr(ex[1]) + @jl_assert Base.Compiler.is_valid_return(v) ex + Core.ReturnNode(v) elseif k == K"inert" - est_to_expr(remove_scope_layer!(ex)) - elseif k == K"inert_syntaxtree" + est_to_expr(ex) + elseif k == K"syntaxinert" ex[1] elseif k == K"code_info" ir = to_code_info(ex, ex.slots, ex.meta) @@ -591,10 +656,9 @@ function _to_lowered_expr(ex::SyntaxTree) ir end elseif k == K"Value" - # TODO: we still do this in some interpolation, genfunc situations - # @jl_assert !isa_lowering_ast_node(ex.value) ( - # ex, string("smuggling AST through Value is asking for trouble; ", - # "find a SyntaxTree representation")) + @jl_assert !isa_lowering_ast_node(ex.value) ( + ex, string("smuggling AST through Value is asking for trouble; ", + "find a SyntaxTree representation")) ex.value isa LineNumberNode ? QuoteNode(ex.value) : ex.value elseif k == K"goto" Core.GotoNode(ex[1].id) @@ -702,10 +766,10 @@ end #------------------------------------------------------------------------------- # Our version of eval - should be upstreamed though? @fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; - macro_world::UInt=Base.get_world_counter(), soft_scope::Union{Nothing,Bool}=nothing, - opts...) - iter = lower_init(ex; opts...) + expr_compat_mode::Bool=false) + ver = expr_compat_mode ? JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION + iter = lower_init(ex, ver) _eval(mod, iter; soft_scope) end @@ -775,7 +839,7 @@ Like `include`, except reads code from the given string rather than from a file. """ function include_string(mod::Module, code::AbstractString, filename::AbstractString="string"; expr_compat_mode=false, version::VersionNumber=VERSION) - eval(mod, parseall(SyntaxTree, code; filename=filename, version=version); expr_compat_mode) + eval(mod, parseall(SyntaxTree, code; filename, version); expr_compat_mode) end include(path::AbstractString) = include(JuliaLowering, path) diff --git a/JuliaLowering/src/hooks.jl b/JuliaLowering/src/hooks.jl index 0dc95be474dd7..57c81ad10468a 100644 --- a/JuliaLowering/src/hooks.jl +++ b/JuliaLowering/src/hooks.jl @@ -26,11 +26,12 @@ function core_lowering_hook(@nospecialize(code), mod::Module, file::Union{String # TODO: this ignores module docstrings for now return Core.svec(est_to_expr(st0[2])) end - ctx1, st1 = expand_forms_1( mod, st0, true, world) - ctx2, st2 = expand_forms_2( ctx1, st1) - ctx3, st3 = resolve_scopes( ctx2, st2) + st0 = rebase_layers(st0, mod, JL_OLD_SYNTAX_VERSION) + st1 = expand_forms_1(st0, world, true) + ctx2, st2 = expand_forms_2(st1, world) + ctx3, st3 = resolve_scopes(ctx2, st2) ctx4, st4 = convert_closures(ctx3, st3) - ctx5, st5 = linearize_ir( ctx4, st4) + ctx5, st5 = linearize_ir(ctx4, st4) ex = to_lowered_expr(st5) return Core.svec(ex, st5, ctx5) catch exc diff --git a/JuliaLowering/src/kinds.jl b/JuliaLowering/src/kinds.jl index ddff3d68e18c1..63b065384eed0 100644 --- a/JuliaLowering/src/kinds.jl +++ b/JuliaLowering/src/kinds.jl @@ -72,6 +72,8 @@ function _register_kinds() "locals" "thisfunction" "overlay" + "syntaxquote" + "syntaxunquote" "END_EXTENSION_KINDS" # The following kinds are internal to lowering @@ -93,6 +95,10 @@ function _register_kinds() # [K"always_defined" x] is an assertion that variable `x` is assigned before use # ('local-def in flisp implementation is K"local" plus K"always_defined" "always_defined" + # `(relayered_global old::Identifier)` is used to tell scope + # resolution that any declaration conflicting with `(global old)` + # should fail, even though `old` was never actually declared + "relayered_global" "_while" "_do_while" # (_typevar name lb ub). flisp usually uses 3-long lists for these, diff --git a/JuliaLowering/src/linear_ir.jl b/JuliaLowering/src/linear_ir.jl index 9444b48624fdf..ce76f3d919f15 100644 --- a/JuliaLowering/src/linear_ir.jl +++ b/JuliaLowering/src/linear_ir.jl @@ -4,7 +4,7 @@ # Must outline anything that can throw, e.g. globalrefs, static params function is_valid_ir_argument(ctx, ex) k = kind(ex) - if is_simple_atom(ctx, ex) || k in KSet"inert inert_syntaxtree top core quote static_eval foreignsymbol" + if is_simple_atom(ctx, ex) || k in KSet"inert syntaxinert top core quote static_eval foreignsymbol" true elseif k == K"BindingId" binfo = get_binding(ctx, ex) @@ -115,7 +115,7 @@ end function is_simple_arg(ctx, ex) k = kind(ex) return is_simple_atom(ctx, ex) || k == K"BindingId" || k == K"quote" || - k == K"inert" || k == K"inert_syntaxtree" || k == K"top" || + k == K"inert" || k == K"syntaxinert" || k == K"top" || k == K"core" || k == K"globalref" || k == K"static_eval" || k == K"foreignsymbol" end @@ -133,7 +133,7 @@ function is_const_read_arg(ctx, ex) # Even if we have side effects, we know that singly-assigned # locals cannot be affected by them so we can inline them anyway. # TODO from flisp: "We could also allow const globals here" - return k == K"inert" || k == K"inert_syntaxtree" || k == K"top" || + return k == K"inert" || k == K"syntaxinert" || k == K"top" || k == K"core" || k == K"static_eval" || k == K"foreignsymbol" || is_simple_atom(ctx, ex) || is_single_assign_var(ctx, ex) end @@ -620,7 +620,7 @@ end function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos) k = kind(ex) if k == K"BindingId" || is_literal(k) || k == K"nothing" || - k == K"inert" || k == K"inert_syntaxtree" || k == K"top" || + k == K"inert" || k == K"syntaxinert" || k == K"top" || k == K"core" || k == K"Value" || k == K"Symbol" || k == K"SourceLocation" || k == K"static_eval" || k == K"foreignsymbol" || k == K"static_parameter" diff --git a/JuliaLowering/src/macro_expansion.jl b/JuliaLowering/src/macro_expansion.jl index 24c9e95db066c..ad6def02382d9 100644 --- a/JuliaLowering/src/macro_expansion.jl +++ b/JuliaLowering/src/macro_expansion.jl @@ -1,28 +1,19 @@ -# Lowering pass 1: Macro expansion, simple normalizations and quote expansion - -# One per pass. Current scope layer is managed separately. +# One per pass struct MacroExpansionContext{Attrs} <: AbstractLoweringContext graph::SyntaxGraph{Attrs} - bindings::Bindings - scope_layers::Vector{ScopeLayer} - expr_compat_mode::Bool - macro_world::UInt + syntax_context::SyntaxContext + known_layers::Dict{ScopeLayer, Bool} + world::UInt + recursive::Bool end -parent_layer(ctx::MacroExpansionContext, sl, err_ex) = - is_base_layer(sl) ? - throw(MacroExpansionError(err_ex, "`escape` node in outer context")) : - ctx.scope_layers[sl.parent_id] - -function MacroExpansionContext(graph::SyntaxGraph, mod::Module, - expr_compat_mode::Bool, world::UInt) - base_layer = ScopeLayer(1, mod, 0, false, false) +function MacroExpansionContext(st, world, recursive) + sc = st.context::SyntaxContext MacroExpansionContext( - graph, Bindings(), ScopeLayer[base_layer], expr_compat_mode, world) + st._graph, sc, Dict{ScopeLayer, Bool}(base_layer(sc)=>true), + world, recursive) end -#-------------------------------------------------- -# Expansion of quoted expressions function collect_unquoted!(ctx, unquoted, ex, depth) if kind(ex) == K"$" && depth == 0 # children(ex) is usually length 1, but for double interpolation it may @@ -40,56 +31,62 @@ function collect_unquoted!(ctx, unquoted, ex, depth) return unquoted end -function expand_quote(ctx, ex) +# TODO: Implementing interpolations with a macro could give us better provenance +function expand_quote(ctx, st) unquoted = SyntaxList(ctx) - collect_unquoted!(ctx, unquoted, ex, 0) - # Unlike user-defined macro expansion, we don't call append_sourceref! for - # the entire expression produced by `quote` expansion. We could, but it - # seems unnecessary for `quote` because the surface syntax is a transparent - # representation of the expansion process. However, it's useful to add the - # extra srcref in a more targeted way for $ interpolations inside - # interpolate_ast, so we do that there. - # - # In principle, particular user-defined macros could opt into a similar - # mechanism. - # - # TODO: Should we try adding a srcref to the `quote` node only for the - # extra syntax generated by expand_quote so srcref essentially becomes - # (ex, @HERE) ? - if is_expr_value(ex) - @jl_assert isempty(unquoted) ex - ex - elseif kind(ex) === K"Identifier" && !hasattr(ex, :mod) - @jl_assert isempty(unquoted) ex - # not just an optimization; expected e.g. in (. mod (quote field)) - @ast ctx ex [(ctx.expr_compat_mode ? K"inert" : K"inert_syntaxtree") ex] - elseif ctx.expr_compat_mode - @ast ctx ex [K"call" - interpolate_ast::K"Value" - Expr::K"Value" - [K"inert" ex] - unquoted... - ] + collect_unquoted!(ctx, unquoted, st, 0) + # not just optimizations; expected e.g. in `(. mod (quote field))` + if is_expr_value(st) + @jl_assert isempty(unquoted) st + st + elseif kind(st) === K"$" + numchildren(st) != 1 && throw(LoweringError( + st, raw"More than one value in bare `$` expression")) + kind(st[1]) === K"..." && throw(LoweringError( + st, raw"unexpected `...` in bare `$` expression")) + @ast ctx st st[1] + elseif kind(st) === K"Identifier" && !hasattr(st, :mod) + @jl_assert isempty(unquoted) st + @ast ctx st [K"inert" st] else - @ast ctx ex [K"call" - interpolate_ast::K"Value" - SyntaxTree::K"Value" - [K"inert_syntaxtree" ex] - unquoted... - ] + @ast ctx st [K"call" interpolate_expr::K"Value" [K"inert" st] unquoted...] end end -#-------------------------------------------------- -struct MacroContext <: AbstractLoweringContext - graph::SyntaxGraph - macrocall::Union{SyntaxTree,LineNumberNode,SourceRef} - scope_layer::ScopeLayer # of outer context - expr_compat_mode::Bool +function collect_syntaxunquote!(ctx, unquoted, st, depth) + if kind(st) === K"syntaxunquote" && depth == 0 + numchildren(st) !== 1 && throw(LoweringError(st, "malformed syntaxunquote")) + push!(unquoted, @ast ctx st[1] [K"tuple" st[1]]) + else + inner_depth = kind(st) == K"syntaxquote" ? depth + 1 : + kind(st) == K"syntaxunquote" ? depth - 1 : depth + for c in children(st) + collect_syntaxunquote!(ctx, unquoted, c, inner_depth) + end + end + unquoted end -function adopt_scope(ex, ctx::MacroContext) - adopt_scope(ex, ctx.scope_layer.id) +# compared to quote: (1) no need for a copy when no unquotes, since output +# should be immutable. (2) we do not optimize (syntaxquote (syntaxunquote x)) +# -> x, since x may not be a SyntaxTree +function expand_syntaxquote(ctx, st) + if kind(st) === K"syntaxunquote" + numchildren(st) != 1 && throw(LoweringError( + st, raw"More than one value in bare `syntaxunquote` expression")) + kind(st[1]) === K"..." && throw(LoweringError( + st, raw"unexpected `...` in bare `syntaxunquote` expression")) + end + unquoted = collect_syntaxunquote!(ctx, SyntaxList(ctx), st, 0) + length(unquoted) == 0 ? @ast(ctx, st, [K"syntaxinert" st]) : + @ast ctx st [K"call" interpolate_syntax::K"Value" + [K"syntaxinert" st] unquoted...] +end + +# Passed to the user as an implicit macro argument +struct MacroContext{Attrs} <: AbstractLoweringContext + graph::SyntaxGraph + macrocall::SyntaxTree{Attrs} end struct MacroExpansionError <: Exception @@ -121,7 +118,7 @@ function Base.showerror(io::IO, exc::MacroExpansionError) macname_str = string(Expr( :macrocall, est_to_expr(ctx.macrocall[1]), nothing)) print(io, " while expanding ", macname_str, - " in module ", ctx.scope_layer.mod) + " in module ", syntax_module(ctx.macrocall)) end print(io, ":\n") # TODO: Display niceties: @@ -163,118 +160,43 @@ function _eval_dot(world::UInt, mod, ex::SyntaxTree) end # If macroexpand(ex[1]) is an identifier or dot-expression, we can simply grab -# it from the scope layer's module in ctx.macro_world. Otherwise, we need to -# eval arbitrary code (which, TODO: does not use the correct world age, and it -# isn't clear the language is meant to support this). -function eval_macro_name(ctx, mctx::MacroContext, ex0::SyntaxTree, sl) - mod = sl.mod - ex = expand_forms_1(ctx, ex0, sl) +# it from the correct module in ctx.world. Otherwise, we need to eval arbitrary +# code (which, TODO: does not use the correct world age, and it isn't clear the +# language is meant to support this). +function eval_macro_name(ctx, mctx::MacroContext, st0::SyntaxTree) + sc = st0.context::SyntaxContext + mod = syntax_module(sc) + st = expand_forms_1(ctx, st0) try - if kind(ex) === K"Value" - ex.value - elseif kind(ex) === K"Identifier" - # module from globalref, or some expansion, or default - if hasattr(ex, :mod) - mod = ex.mod::Module - elseif hasattr(ex, :scope_layer) - mod = ctx.scope_layers[ex.scope_layer::LayerId].mod - end - Base.invoke_in_world(ctx.macro_world, getproperty, - mod, Symbol(ex.name_val)) - elseif kind(ex) === K"." && - (ed = _eval_dot(ctx.macro_world, mod, ex); !isnothing(ed)) + if kind(st) === K"Value" + st.value + elseif kind(st) === K"Identifier" + Base.invoke_in_world(ctx.world, getproperty, + syntax_module(st), Symbol(st.name_val)) + elseif kind(st) === K"." && + # TODO: correct mod? + (ed = _eval_dot(ctx.world, mod, st); !isnothing(ed)) ed else - # `ex` might contain a nontrivial mix of scope layers so we can't - # just `eval()` it, as it's already been partially lowered by this - # point. Instead, we repeat the latter parts of `lower()` here. - ctx2, ex2 = expand_forms_2(ctx, ex) - ctx3, ex3 = resolve_scopes(ctx2, ex2) - ctx4, ex4 = convert_closures(ctx3, ex3) - _ctx5, ex5 = linearize_ir(ctx4, ex4) - expr_form = to_lowered_expr(ex5) + # `ex` might contain a nontrivial mix of scopes so we can't just + # `eval()` it, as it's already been partially lowered by this point. + # Instead, we repeat the latter parts of `lower()` here. + ctx2, st2 = expand_forms_2(st, ctx.world) + ctx3, st3 = resolve_scopes(ctx2, st2) + ctx4, st4 = convert_closures(ctx3, st3) + _ctx5, st5 = linearize_ir(ctx4, st4) + expr_form = to_lowered_expr(st5) ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form) end catch err - throw(MacroExpansionError(mctx, ex, "Macro not found", :all, err)) - end -end - -# Record scope layer information for symbols passed to a macro by setting -# scope_layer for each expression and also processing any K"escape" arising -# from previous expansion of old-style macros. -# -# See also set_scope_layer() -function set_macro_arg_hygiene(ctx, ex, sl) - k = kind(ex) - sl = ctx.scope_layers[get(ex, :scope_layer, sl.id)] - if is_leaf(ex) - setattr!(mkleaf(ex), :scope_layer, sl.id) - elseif k === K"hygienic-scope" - set_macro_arg_hygiene( - ctx, ex[1], new_macro_scope_layer( - ctx, sl, ex[2].value::Module, sl.hygiene_compat)) - elseif k === K"escape" - set_macro_arg_hygiene(ctx, ex[1], parent_layer(ctx, sl, ex)) - else - node = mapchildren(e->set_macro_arg_hygiene(ctx, e, sl), ctx, ex) - setattr!(node, :scope_layer, sl.id) - end -end - -function prepare_macro_args(ctx, sl, mctx, raw_args) - macro_args = Any[mctx] - for arg in raw_args - # Add hygiene information to be carried along with macro arguments. - # - # Macro call arguments may be either - # * Unprocessed by the macro expansion pass - # * Previously processed, but spliced into a further macro call emitted by - # a macro expansion. - # In either case, we need to set scope layers before passing the - # arguments to the macro call. - push!(macro_args, set_macro_arg_hygiene(ctx, arg, sl)) - end - return macro_args -end - -# TODO: Do we need to handle :scope_layer or multiple escapes here? -# See https://github.com/c42f/JuliaLowering.jl/issues/39 -""" -Insert a hygienic-scope around each arg of K"toplevel" returned from a macro. - -It isn't correct for macro expansion to recurse into a K"toplevel" expression -since one child may define a macro and the next may use it. However, not -recursing now means we lose some important context: the module of the macro we -just expanded, which is necessary for resolving the identifiers in the -K"toplevel" AST. The solution implemented in JuliaLang/julia#53515 was to save -our place and expand later using `Expr(:hygienic-scope toplevel_child mod)`. - -Of course, these hygienic-scopes are also necessary because existing user code -contains the corresponding escaping, which would otherwise cause errors. We -already consumed the hygienic-scope that comes with every expansion, but won't -be looking for escapes under :toplevel, so push hygienic-scope under toplevel -""" -function fix_toplevel_expansion(ctx, ex::SyntaxTree, mod::Module, lnn::LineNumberNode) - if kind(ex) === K"toplevel" - mapchildren(ctx, ex) do e - @ast ctx ex [K"hygienic-scope" e mod::K"Value" lnn::K"Value"] - end - else - mapchildren(e->fix_toplevel_expansion(ctx, e, mod, lnn), ctx, ex) + throw(MacroExpansionError(mctx, st, "Macro not found", :all, err)) end end -function expand_macro(ctx, ex, outer_sl) - @jl_assert kind(ex) == K"macrocall" ex - - macname = ex[1] - # TODO: should this be base_layer(ctx, outer_sl)? - mctx = MacroContext(ctx.graph, ex, outer_sl, ctx.expr_compat_mode) - macfunc = eval_macro_name(ctx, mctx, macname, outer_sl) - raw_args = ex[3:end] - macro_loc = if kind(ex[2]) === K"Value" - loc = ex[2].value +function _macrocall_expr_location(st::SyntaxTree) + @jl_assert kind(st) === K"macrocall" st + if kind(st[2]) === K"Value" + loc = st[2].value if loc isa MacroSource loc elseif loc isa LineNumberNode @@ -283,182 +205,192 @@ function expand_macro(ctx, ex, outer_sl) else LineNumberNode(0, :none) end - elseif kind(ex[2]) === K"VERSION" - loc = source_location(LineNumberNode, ex) - isdefined(Core, :MacroSource) ? Core.MacroSource(loc, ex[2].value) : loc + elseif kind(st[2]) === K"VERSION" + loc = source_location(LineNumberNode, st) + isdefined(Core, :MacroSource) ? Core.MacroSource(loc, st[2].value) : loc else LineNumberNode(0, :none) end - # We use a specific well defined world age for the next checks and macro - # expansion invocations. This avoids inconsistencies if the latest world - # age changes concurrently. - # - # TODO: Allow this to be passed in +end + +function expand_macro(ctx::MacroExpansionContext, st::SyntaxTree) + @jl_assert kind(st) === K"macrocall" st + numchildren(st) >= 2 || throw(LoweringError( + st, "`macrocall` requires a macro name and source location")) + sc_in = st.context::SyntaxContext + macname = st[1] + mctx = MacroContext(ctx.graph, st) + macfunc = eval_macro_name(ctx, mctx, macname) + raw_args = st[3:end] + # TODO: hasmethod always returns false for our `typemax(UInt)` meaning # "latest world," which we shouldn't be using. - has_new_macro = ctx.macro_world === typemax(UInt) ? + has_new_macro = ctx.world === typemax(UInt) ? hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}) : - hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=ctx.macro_world) + hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=ctx.world) if has_new_macro - macro_args = prepare_macro_args(ctx, outer_sl, mctx, raw_args) + macro_args = [mctx, raw_args...] expanded = try - Base.invoke_in_world(ctx.macro_world, macfunc, macro_args...) + Base.invoke_in_world(ctx.world, macfunc, macro_args...) catch exc newexc = exc isa MacroExpansionError ? MacroExpansionError(mctx, exc.ex, exc.msg, exc.position, exc.err) : - MacroExpansionError(mctx, ex, "Error expanding macro", :all, exc) - # TODO: We can delete this rethrow when we move to AST-based error propagation. + MacroExpansionError(mctx, st, "Error expanding macro", :all, exc) rethrow(newexc) end - if expanded isa SyntaxTree - if expanded._graph !== ctx.graph - # If the macro has produced syntax outside the macro context, - # copy it over. TODO: Do we expect this always to happen? What - # is the API for access to the macro expansion context? - expanded = copy_ast(ctx, expanded) - end + st_out = if expanded isa SyntaxTree + expanded._graph !== ctx.graph ? copy_ast(ctx, expanded) : expanded else - expanded = @ast ctx ex expanded::K"Value" + expanded isa Expr && throw(LoweringError( + st, "implicit expr->syntaxtree: may later be allowed, but is probably a mistake today")) + expr_to_est(st._graph, expanded, st._id) end else - # Compat: attempt to invoke an old-style macro if there's no applicable - # method for new-style macro arguments. - macro_args = Any[macro_loc, base_layer(ctx, outer_sl).mod] - + macro_loc = _macrocall_expr_location(st) + macro_args = Any[macro_loc, base_layer(ctx.syntax_context).mod] for arg in raw_args @jl_assert kind(arg) !== K"VERSION" arg # handled in EST conversion - # For hygiene in old-style macros, we omit any additional scope - # layer information from macro arguments. Old-style macros will - # handle that using manual escaping in the macro itself. - # - # Note that there's one slight incompatibility here for identifiers - # interpolated into the `raw_args` from outer macro expansions of - # new-style macros which call old-style macros. Instead of seeing - # `Expr(:escape)` in such situations, old-style macros will now see - # `Expr(:scope_layer)` inside `macro_args`. push!(macro_args, est_to_expr(arg)) end - expanded = try - Base.invoke_in_world(ctx.macro_world, macfunc, macro_args...) + st_out = try + Base.invoke_in_world(ctx.world, macfunc, macro_args...) catch exc - if exc isa MethodError && exc.f === macfunc - if !isempty(methods_in_world( - macfunc, Tuple{typeof(mctx), Vararg{Any}}, - ctx.macro_world, ex)) - # If the macro has at least some methods implemented in the - # new style, assume the user meant to call one of those - # rather than any old-style macro methods which might exist - exc = MethodError( - macfunc, - (prepare_macro_args(ctx, outer_sl, mctx, raw_args)..., ), - ctx.macro_world) - end + if exc isa MethodError && exc.f === macfunc && !isempty( + methods_in_world(macfunc, Tuple{typeof(mctx), Vararg{Any}}, ctx.world, st)) + # If the macro has at least some methods implemented in the + # new style, assume the user meant to call one of those + # rather than any old-style macro methods which might exist + exc = MethodError(macfunc, (mctx, raw_args...,), ctx.world) end - rethrow(MacroExpansionError(mctx, ex, "Error expanding macro", :all, exc)) + rethrow(MacroExpansionError(mctx, st, "Error expanding macro", :all, exc)) end macro_lnn = macro_loc isa MacroSource ? macro_loc.lno : macro_loc - expanded = expr_to_est(ex._graph, expanded, macro_lnn) + st_out = expr_to_est(st._graph, st_out, macro_lnn) end - - expanded = append_sourceref!(ctx, expanded, ex._id) - if kind(expanded) != K"Value" - # Module scope for the returned AST is the module where this particular - # method was defined (may be different from `parentmodule(macfunc)`) - mod_for_ast = lookup_method_instance(macfunc, macro_args, - ctx.macro_world).def.module - expanded = expand_forms_1( - ctx, expanded, - new_macro_scope_layer(ctx, outer_sl, mod_for_ast, !has_new_macro)) - end - return expanded + # Module scope for the returned AST is the module where this particular + # method was defined (may be different from `parentmodule(macfunc)`) + mod_for_ast = lookup_method_instance(macfunc, macro_args, ctx.world).def.module + sc2 = SyntaxContext( + ScopeLayer(mod_for_ast, sc_in.layer), st, + (has_new_macro ? JL_NEW_SYNTAX_VERSION : JL_OLD_SYNTAX_VERSION), false) + st_out2 = apply_expansion_layer(ctx, st_out, sc2, true, 0, 0) + !ctx.recursive ? st_out2 : expand_forms_1(ctx, st_out2) end -function append_sourceref!(ctx, ex, id::NodeId) - @jl_assert ex._id != id ex - setattr!(ex, :macro_source, id) - for c in children(ex) - append_sourceref!(ctx, c, id) +function known_layer(ctx, sl::Union{Nothing, ScopeLayer}) + isnothing(sl) && return false + get!(ctx.known_layers, sl) do + known_layer(ctx, sl.escaped) end - ex end -function remove_scope_layer!(ex) - if !is_leaf(ex) - for c in children(ex) - remove_scope_layer!(c) +""" +When a macro expands, we add a fresh layer to all new syntax in the expansion. +Any syntax that doesn't share a base layer with the top-level thunk is +considered "new". This is similar to racket's flip-scope operation, but +simpler and less powerful (we lose any layer we overwrite, but we gain the +invariant that all layers have the same root after every expansion, so +`escape` is well-defined). + +Implementation notes: + +- `escape` can never be resolved to a layer inside a macrocall, since we must + expand to know whether (old) more escapes will surround it or (new) the macro + moves the escape to another layer. + +- `escape` nodes coming in usually have no layer (old expansion). New + expansions can't create escapes, but can pass an argument containing `escape` + through, and arguments must have full context. Thus, if we see `escape` with + any context, we know the layer is uniform, and is the layer we want to escape + from, so we remove inner context. (if there's an an old macro requiring + caller-side `esc(arg)`, the new macro must also bump `esc(arg)`'s layer.) + +- We could try some shortcuts in module/toplevel, but note that macrocall/quote + need full context (a macro may extract an arbitrary child), and even when + `done`, module/toplevel may contain syntax with arbitrary layers that we must + clean up now (later, we lose the base layer used to detect new syntax) +""" +function apply_expansion_layer(ctx, st::SyntaxTree, sc_in::SyntaxContext, done, + qdepth, sqdepth) + @jl_assert known_layer(ctx, base_layer(sc_in)) st + sc0 = get(st, :context, nothing)::Union{Nothing, SyntaxContext} + sc = (isnothing(sc0) || !known_layer(ctx, sc0.layer)) ? sc_in : sc0 + k = kind(st) + absorb_esc = done && qdepth == 0 && sqdepth == 0 + out = if is_leaf(st) || numchildren(st) == 0 + setattr(st, :context, sc) + elseif k === K"escape" && absorb_esc + if numchildren(st) !== 1 + throw(LoweringError(st, "`escape` requires one argument")) + elseif is_base_layer(sc) + throw(LoweringError(st, "`escape` node in outer context")) + elseif !is_flisp_compat(sc) + throw(LoweringError(st, "new macros should not use `escape`")) end + st1 = isnothing(sc0) ? st[1] : remove_context(st[1]) + apply_expansion_layer( + ctx, st1, escape_layer(sc, false), true, qdepth, sqdepth) + elseif k === K"hygienic-scope" && absorb_esc + if !(2 <= numchildren(st) <= 3) + throw(LoweringError(st, "`hygienic-scope` requires 2-3 children")) + elseif kind(st[2]) !== K"Value" || !(st[2].value isa Module) + throw(LoweringError(st, "`hygienic-scope` arg 2: expected Module")) + elseif !is_flisp_compat(sc) + throw(LoweringError(st, "new macros should not use `hygienic-scope`")) + end + new_sl = ScopeLayer(st[2].value::Module, sc.layer) + st1 = isnothing(sc0) ? st[1] : remove_context(st[1]) + sc2 = SyntaxContext(new_sl, sc.unexpanded, sc.version, sc.internal) + apply_expansion_layer(ctx, st1, sc2, true, qdepth, sqdepth) + else + done2 = done && !(k in KSet"macrocall inert syntaxinert") + qdepth2 = qdepth + (k === K"quote" ? 1 : k === K"$" ? -1 : 0) + sqdepth2 = sqdepth + (k === K"syntaxquote" ? 1 : k === K"syntaxunquote" ? -1 : 0) + out = mapchildren(c->apply_expansion_layer( + ctx, c, sc_in, done2, qdepth2, sqdepth2), ctx, st) + setattr!(out, :context, sc) end - JuliaSyntax.deleteattr!(ex, :scope_layer) - ex -end - -function remove_scope_layer(ctx, ex) - remove_scope_layer!(mktree(ex)) + out end """ -Lowering pass 1 - -This pass expands macros and quote (interpolations). It also annotates every -identifier with the expansion it came from. +Expands macros and quote/interpolation forms. """ -function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree, sl::ScopeLayer) - k = kind(ex) - expr_sl = ctx.scope_layers[get(ex, :scope_layer, sl.id)] - if k == K"Identifier" - hasattr(ex, :scope_layer) ? - ex : setattr!(mkleaf(ex), :scope_layer, sl.id) - elseif k == K"escape" - if numchildren(ex) !== 1 - throw(LoweringError(ex, "`escape` requires one argument")) +function expand_forms_1(ctx::MacroExpansionContext, st::SyntaxTree) + k = kind(st) + if is_leaf(st) + st + elseif k === K"macrocall" + expand_macro(ctx, st) + elseif (k === K"do" && numchildren(st) == 2 && kind(st[1]) === K"macrocall" && + kind(st[2]) === K"->") + mac_ex = @ast ctx st [ + K"macrocall" + st[1][1] # mac name + st[1][2] # loc + st[2] # do-lambda + children(st[1])[3:end]... + ] + expand_macro(ctx, mac_ex) + elseif k in KSet"inert syntaxinert toplevel module" + st + elseif k === K"quote" + if numchildren(st) !== 1 + throw(LoweringError(st, "`quote` requires one argument")) end - expand_forms_1(ctx, ex[1], parent_layer(ctx, expr_sl, ex)) - elseif k == K"hygienic-scope" - if !(2 <= numchildren(ex) <= 3 && ex[2].value isa Module) - throw(LoweringError(ex,"`hygienic-scope` requires an AST and a module")) + expand_forms_1(ctx, expand_quote(ctx, st[1])) + elseif k === K"syntaxquote" + if numchildren(st) !== 1 + throw(LoweringError(st, "`syntaxquote` requires one argument")) end + expand_forms_1(ctx, expand_syntaxquote(ctx, st[1])) + elseif k === K"escape" || k === K"hygienic-scope" expand_forms_1( - ctx, ex[1], - new_macro_scope_layer( - ctx, expr_sl, ex[2].value::Module, expr_sl.hygiene_compat)) - elseif k == K"quote" - if numchildren(ex) !== 1 - throw(LoweringError(ex,"`quote` expects one argument")) - end - expand_forms_1(ctx, expand_quote(ctx, ex[1]), sl) - elseif k == K"macrocall" - expand_macro(ctx, ex, expr_sl) - elseif k == K"toplevel" && !is_base_layer(expr_sl) - fix_toplevel_expansion( - ctx, ex, expr_sl.mod, source_location(LineNumberNode, ex)) - elseif k == K"module" || k == K"toplevel" || k == K"inert" || k == K"inert_syntaxtree" - # Remove scope layer information from any inert syntax which survives - # macro expansion so that it doesn't contaminate lowering passes which - # are later run against the quoted code. TODO: This works as a first - # approximation but is incorrect in general. We need to revisit such - # "deferred hygiene" situations (see https://github.com/c42f/JuliaLowering.jl/issues/111) - remove_scope_layer(ctx, ex) - elseif k in KSet"<: >: --> ' unknown_head" - # if this is a form that requires resolving some identifier, add the - # scope layer. if this is an unknown form to be cleaned up by AST - # conversion, save the scope layer just in case. - setattr(mapchildren(e->expand_forms_1(ctx,e,sl), ctx, ex), - :scope_layer, expr_sl.id) - elseif is_leaf(ex) - ex - elseif (k == K"do" && numchildren(ex) == 2 && kind(ex[1]) == K"macrocall" && - kind(ex[2]) == K"->") - mac_ex = @ast ctx ex [ - K"macrocall" - ex[1][1] # mac name - ex[1][2] # loc - ex[2] # do-lambda - children(ex[1])[3:end]... - ] - expand_macro(ctx, mac_ex, expr_sl) + ctx, apply_expansion_layer( + ctx, st, st.context::SyntaxContext, true, 0, 0)) else - mapchildren(e->expand_forms_1(ctx,e,sl), ctx, ex) + mapchildren(c->expand_forms_1(ctx, c), ctx, st) end end @@ -466,23 +398,25 @@ function ensure_macro_attributes!(graph) g2 = ensure_attributes!( graph; var_id=IdTag, - scope_layer=LayerId, - macro_source=NodeId, - __macro_ctx__=Nothing, meta=CompileHints) DEBUG ? ensure_attributes!(g2; jl_source=LineNumberNode) : g2 end -@fzone "JL: macroexpand" function expand_forms_1(mod::Module, ex::SyntaxTree, expr_compat_mode::Bool, macro_world::UInt) - graph = ensure_macro_attributes!(copy_attrs(syntax_graph(ex))) - ex = reparent(graph, ex) - ctx_out = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world) - ex_out = expand_forms_1(ctx_out, ex, only(ctx_out.scope_layers)) - graph2 = delete_attributes(ex._graph, :__macro_ctx__) - # TODO: Returning the context with pass-specific mutable data is a bad way - # to carry state into the next pass. We might fix this by attaching such - # data to the graph itself as global attributes? - ctx2 = MacroExpansionContext(graph2, ctx_out.bindings, ctx_out.scope_layers, - expr_compat_mode, macro_world) - return ctx2, reparent(ctx2.graph, ex_out) +function assert_expandable(st, l=base_layer(st.context::SyntaxContext)) + @jl_assert hasattr(st, :context) (st, "expected syntax context") + @jl_assert base_layer(st.context::SyntaxContext) == l (st, "expected consistent layer") + for c in children(st) + assert_expandable(c, l) + end +end + +@fzone "JL: macroexpand" function expand_forms_1( + st::SyntaxTree, world::UInt, recursive::Bool) + + graph = ensure_macro_attributes!(copy_attrs(syntax_graph(st))) + st = reparent(graph, st) + DEBUG && assert_expandable(st) + ctx = MacroExpansionContext(st, world, recursive) + st_out = expand_forms_1(ctx, st) + return st_out end diff --git a/JuliaLowering/src/runtime.jl b/JuliaLowering/src/runtime.jl index 2aa167fd6bd7d..ee2ddcf81cc1a 100644 --- a/JuliaLowering/src/runtime.jl +++ b/JuliaLowering/src/runtime.jl @@ -13,140 +13,63 @@ Base.@assume_effects :removable function current_exception() @ccall jl_current_exception(current_task()::Any)::Any end -#-------------------------------------------------- -# Supporting functions for AST interpolation (`quote`) -struct InterpolationContext{Graph} <: AbstractLoweringContext - graph::Graph - values::Tuple - current_index::Ref{Int} -end - -# Context for `Expr`-based AST interpolation in compat mode -struct ExprInterpolationContext <: AbstractLoweringContext - values::Tuple - current_index::Ref{Int} -end - -# Helper functions to make shared interpolation code which works with both -# SyntaxTree and Expr data structures. -_interp_kind(ex::SyntaxTree) = kind(ex) -function _interp_kind(@nospecialize(ex)) - return (ex isa Expr && ex.head === :quote) ? K"quote" : - (ex isa Expr && ex.head === :$) ? K"$" : - K"None" # Other cases irrelevant to interpolation -end - -_children(ex::SyntaxTree) = children(ex) -_children(@nospecialize(ex)) = ex isa Expr ? ex.args : () - -_numchildren(ex::SyntaxTree) = numchildren(ex) -_numchildren(@nospecialize(ex)) = ex isa Expr ? length(ex.args) : 0 - -_syntax_list(ctx::InterpolationContext) = SyntaxList(ctx) -_syntax_list(::ExprInterpolationContext) = Any[] - -_interp_makenode(::InterpolationContext, ex, args) = mknode(ex, args) -_interp_makenode(::ExprInterpolationContext, ex, args) = Expr((ex::Expr).head, args...) - -_is_leaf(ex::SyntaxTree) = is_leaf(ex) -_is_leaf(::Expr) = false -_is_leaf(@nospecialize(_)) = true - -# Produce interpolated node for `$x` syntax -function _interpolated_value(ctx::InterpolationContext, srcref, @nospecialize(ex)) - if ex isa SyntaxTree - if !is_compatible_graph(ctx, ex) - ex = copy_ast(ctx, ex) - end - append_sourceref!(ctx, ex, srcref._id) - elseif ex isa Symbol - # Plain symbols become identifiers. This is an accommodation for - # compatibility to allow `:x` (a Symbol) and `:(x)` (a SyntaxTree) to - # be used interchangeably in macros. - newleaf(ctx, srcref, K"Identifier", string(ex)) +function _interpolate_expr(@nospecialize(ex), depth, @nospecialize(vals::Tuple), val_i) + if ex isa QuoteNode + out = _interpolate_expr(Expr(:inert, ex.value), depth, vals, val_i) + QuoteNode(only(out.args)) + elseif !(ex isa Expr) + ex else - newleaf(ctx, srcref, K"Value", ex) - end -end - -function _interpolated_value(::ExprInterpolationContext, _, @nospecialize(ex)) - ex -end - -function _interpolate_ast(ctx::ExprInterpolationContext, ex::QuoteNode, depth) - out = _interpolate_ast(ctx, Expr(:inert, ex.value), depth) - QuoteNode(only(out.args)) -end - -function _interpolate_ast(ctx, @nospecialize(ex), depth) - _is_leaf(ex) && return ex - k = _interp_kind(ex) - inner_depth = k == K"quote" ? depth + 1 : - k == K"$" ? depth - 1 : - depth - expanded_children = _syntax_list(ctx) - - for e in _children(ex) - if _interp_kind(e) == K"$" && inner_depth == 0 - vals = ctx.values[ctx.current_index[]]::Tuple - ctx.current_index[] += 1 - for (i,v) in enumerate(vals) - srcref = _numchildren(e) == 1 ? e : _children(e)[i] - push!(expanded_children, _interpolated_value(ctx, srcref, v)) + inner_depth = ex.head == :quote ? depth + 1 : + ex.head == :$ ? depth - 1 : depth + cs_out = Any[] + for e in ex.args + if e isa Expr && e.head == :$ && inner_depth == 0 + tup = vals[val_i[] += 1]::Tuple + for v in tup + push!(cs_out, v) + end + else + push!(cs_out, _interpolate_expr(e, inner_depth, vals, val_i)) end - else - push!(expanded_children, _interpolate_ast(ctx, e, inner_depth)) end + Expr(ex.head, cs_out...) end - - _interp_makenode(ctx, ex, expanded_children) +end +function interpolate_expr(@nospecialize(ex), @nospecialize(values...)) + @jl_assert !Meta.isexpr(ex, :$) (expr_to_est(ex), "expand_quote should handle this") + _interpolate_expr(ex, 0, values, Ref(0)) end -# Produced by expanding K"quote". Must create a copy of the AST. Note that -# wrapping `ex` in an extra node handles the edge case where the root `ex` is -# `$` (our recursion is one step removed due to forms like `($ a b)`.) -function interpolate_ast(::Type{SyntaxTree}, ex::SyntaxTree, values...) - # Construct graph for interpolation context. We inherit this from the macro - # context where possible by detecting it using __macro_ctx__. This feels - # hacky though. - # - # Perhaps we should use a ScopedValue for this instead or get it from - # the macro __context__? None of the options feel great here. - graph = nothing - for vals in values - for v in vals - if v isa SyntaxTree && hasattr(syntax_graph(v), :__macro_ctx__) - graph = syntax_graph(v) - break +function _interpolate_syntax(st::SyntaxTree, depth, @nospecialize(vals), val_i) + is_leaf(st) && return mkleaf(st) + k = kind(st) + inner_depth = k == K"syntaxquote" ? depth + 1 : + k == K"syntaxunquote" ? depth - 1 : depth + cs_out = SyntaxList(st._graph) + for c in children(st) + if kind(c) == K"syntaxunquote" && inner_depth == 0 + tup = vals[val_i[] += 1]::Tuple + @jl_assert numchildren(c) == 1 st + @jl_assert kind(c[1]) === K"..." || length(tup) == 1 st + for v in tup + v2 = !(v isa SyntaxTree) ? expr_to_est(st._graph, v, c._id) : + copy_ast(st._graph, v) + push!(cs_out, v2) end + else + push!(cs_out, _interpolate_syntax(c, inner_depth, vals, val_i)) end end - if isnothing(graph) - graph = ensure_macro_attributes!(SyntaxGraph()) - end - ctx = InterpolationContext(graph, values, Ref(1)) - - # We must copy the AST into our context to use it as the source reference of - # generated expressions. - ex1 = copy_ast(ctx, ex) - out = _interpolate_ast(ctx, @ast(ctx, ex1, [K"None" ex1]), 0) - length(children(out)) === 1 || throw( - LoweringError(ex1, "More than one value in bare `\$` expression")) - return only(children(out)) + mknode(st, cs_out) end - -function interpolate_ast(::Type{Expr}, @nospecialize(ex), values...) - ctx = ExprInterpolationContext(values, Ref(1)) - if ex isa Expr && ex.head === :$ - @assert length(values) === 1 - if length(ex.args) !== 1 - throw(LoweringError( - expr_to_est(ex), "More than one value in bare `\$` expression")) - end - only(values[1]) - else - _interpolate_ast(ctx, ex, 0) - end +function interpolate_syntax(st::SyntaxTree, @nospecialize(vals...)) + st = copy_ast(ensure_macro_attributes!(SyntaxGraph()), st) + val_i = Ref(0) + out = _interpolate_syntax((@ast st._graph st [K"None" st]), 0, vals, val_i) + @jl_assert val_i[] == length(vals) st + @jl_assert numchildren(out) == 1 st + out[1] end #-------------------------------------------------- @@ -288,19 +211,19 @@ end # An alternative to Core.GeneratedFunctionStub which works on SyntaxTree rather # than Expr. struct GeneratedFunctionStub - expr_compat_mode::Bool + syntax_context::SyntaxContext gen::Function srcref::Union{LineNumberNode,SourceRef} argnames::Core.SimpleVector spnames::Core.SimpleVector end -function _gen_args_from_syms(ctx, src, layer, args) +function _gen_args_from_syms(ctx, src, args) out = SyntaxList(ctx.graph) for a in args id = newleaf(syntax_graph(ctx), src, K"Identifier", string(a)) id = _est_to_dst_ident(id) # support placeholders - push!(out, adopt_scope(id, layer)) + push!(out, id) end out end @@ -319,45 +242,37 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a graph = ensure_desugaring_attributes!(SyntaxGraph()) __module__ = source.module - # Macro expansion. Note that we expand in `tls_world_age()` (see - # Core.GeneratedFunctionStub) - macro_world = Base.tls_world_age() - ctx1 = MacroExpansionContext(graph, __module__, g.expr_compat_mode, macro_world) - - layer = only(ctx1.scope_layers) + sc = g.syntax_context - # Run code generator - this acts like a macro expander and like a macro - # expander it gets a MacroContext. - mctx = MacroContext(syntax_graph(ctx1), g.srcref, layer, g.expr_compat_mode) - ex0 = g.gen(mctx, args...) + # Run code generator - this acts like a macro expander + ex0 = g.gen(sc, args...) if ex0 isa Expr ex0 = expr_to_est( - syntax_graph(ctx1), ex0, source_location(LineNumberNode, g.srcref)) + graph, ex0, source_location(LineNumberNode, g.srcref)) end if ex0 isa SyntaxTree - if !is_compatible_graph(ctx1, ex0) - # If the macro has produced syntax outside the macro context, copy it over. - # TODO: Do we expect this always to happen? What is the API for access - # to the macro expansion context? - ex0 = copy_ast(ctx1, ex0) + if !is_compatible_graph(graph, ex0) + ex0 = copy_ast(graph, ex0) end else - ex0 = newleaf(syntax_graph(ctx1), g.srcref, K"Value", ex0) + ex0 isa Expr && throw(LoweringError( + ex0, "implicit expr->syntaxtree: may later be allowed, but is probably a mistake today")) + ex0 = expr_to_est(graph, ex0, g.srcref) end - # Expand any macros emitted by the generator - ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0), layer) - ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__), - ctx1.bindings, ctx1.scope_layers, - g.expr_compat_mode, macro_world) - ex1 = reparent(ctx1, ex1) + @jl_assert base_layer(sc).mod == __module__ ex0 + # Note that we expand in `tls_world_age()` (see Core.GeneratedFunctionStub) + world = Base.tls_world_age() + ex0 = JuliaSyntax.fill_context!(ex0, sc) + ctx1 = MacroExpansionContext(ex0, world, true) + ex1 = expand_forms_1(ctx1, ex0) # Desugaring - ctx2, ex2 = expand_forms_2(ctx1, ex1) + ctx2, ex2 = expand_forms_2(ex1, world) # Wrap expansion in a non-toplevel lambda and run scope resolution ex2 = @ast ctx2 ex0 [K"lambda"(is_toplevel_thunk=false, toplevel_pure=true) - [K"block" _gen_args_from_syms(ctx2, ex0, layer, g.argnames)...] - [K"block" _gen_args_from_syms(ctx2, ex0, layer, g.spnames)...] + [K"block" _gen_args_from_syms(ctx2, ex1, g.argnames)...] + [K"block" _gen_args_from_syms(ctx2, ex1, g.spnames)...] ex2 ] ctx3, ex3 = resolve_scopes(ctx2, ex2) diff --git a/JuliaLowering/src/scope_analysis.jl b/JuliaLowering/src/scope_analysis.jl index 9f14be6f3a2a5..ed0e4a6cf9f05 100644 --- a/JuliaLowering/src/scope_analysis.jl +++ b/JuliaLowering/src/scope_analysis.jl @@ -5,16 +5,12 @@ Key to use when transforming names into bindings """ struct NameKey name::String - layer::LayerId -end - -function Base.isless(a::NameKey, b::NameKey) - (a.name, a.layer) < (b.name, b.layer) + layer::ScopeLayer end function NameKey(ex::SyntaxTree) @jl_assert kind(ex) === K"Identifier" ex - NameKey(ex.name_val, ex.scope_layer) + NameKey(ex.name_val, (ex.context::SyntaxContext).layer) end struct ScopeInfo @@ -43,12 +39,6 @@ struct ScopeInfo # See `LambdaBindings`. Nothing if not a lambda scope. This is the final # collecting place for locals going in to closure conversion. locals_capt::Union{Nothing, Dict{IdTag,Bool}} - # Globals declared at the base scope layer instead of the apparent scope - # layer due to hygiene_compat rules. Macros generally an expect explicit - # global declaration (and any initialization in the same expression) to be - # unhygienic, but JL is more generous than flisp in resolving references to - # the rescoped global. - rescoped_globals::Set{NameKey} end function ScopeInfo(ctx, parent_id, ex::SyntaxTree) @@ -69,27 +59,26 @@ function ScopeInfo(ctx, parent_id, ex::SyntaxTree) s = ScopeInfo( id, parent_id, lambda_id, ex._id, is_permeable, is_lifted, Dict{IdTag, NodeId}(), Dict{NameKey, NodeId}(), Dict{NameKey,IdTag}(), - kind(ex) === K"lambda" ? Dict{IdTag,Bool}() : nothing, Set{NameKey}()) + kind(ex) === K"lambda" ? Dict{IdTag,Bool}() : nothing) push!(ctx.scopes, s) return s end struct ScopeResolutionContext{Attrs} <: AbstractLoweringContext graph::SyntaxGraph{Attrs} + layer::ScopeLayer bindings::Bindings - mod::Module + # Purely for display and deterministic ordering of scope layers + layer_ids::Dict{ScopeLayer, Int} # Every lexical scope, indexed by ScopeId scopes::Vector{ScopeInfo} # Current stack of scopes to look for names in, innermost scope last scope_stack::Vector{ScopeId} - # Macro hygienic scopes (confusing name here) - scope_layers::Vector{ScopeLayer} # Usually, globals in the top scope are ignored. This is a subset that may # be assigned to without the `global` keyword in soft scopes due to being # assigned to at top level, or passing the defined-and-owned-global check. soft_assignable_globals::Set{NameKey} enable_soft_scopes::Bool - expr_compat_mode::Bool world::UInt end @@ -129,19 +118,6 @@ function explicit_declare_in_scope!(ctx, scope::ScopeInfo, ex, new_k::Symbol) elseif kind(ex) === K"Placeholder" return nothing end - if new_k === :global && (sl = ctx.scope_layers[ex.scope_layer]; sl.hygiene_compat) - # Check no conflict in the original scope before rescoping - bid_conflict = get(scope.vars, NameKey(ex), nothing) - !isnothing(bid_conflict) && throw(LoweringError(ex, """ - unhygienic global name `$(NameKey(ex).name)` conflicts with an \ - existing $(_var_str(get_binding(ctx, bid_conflict).kind))""")) - push!(scope.rescoped_globals, NameKey(ex)) - ex = setattr(ex, :scope_layer, base_layer(ctx, sl).id) - elseif NameKey(ex) in scope.rescoped_globals - throw(LoweringError(ex, """ - $(_var_str(new_k)) name `$(NameKey(ex).name)` conflicts with an \ - existing unhygienic global""")) - end bid = get(scope.vars, NameKey(ex), nothing) old_k = isnothing(bid) ? nothing : get_binding(ctx, bid).kind if isnothing(old_k) @@ -166,20 +142,18 @@ function explicit_declare_in_scope!(ctx, scope::ScopeInfo, ex, new_k::Symbol) end # globals are added to both `scope` and the top scope (mainly so we can get the -# same binding for many unrelated global references, which doesn't apply to -# rescoped globals). +# same binding for many unrelated global references). function declare_in_scope!(ctx, scope::ScopeInfo, ex, bk::Symbol; kws...) nk = NameKey(ex) if bk === :global - mod = hasattr(ex, :mod) ? ex.mod::Module : - ctx.scope_layers[ex.scope_layer::LayerId].mod - declaration_scope = nk in scope.rescoped_globals ? scope : top_scope(ctx) + mod = syntax_module(ex) + declaration_scope = top_scope(ctx) else declaration_scope = scope mod = hasattr(ex, :mod) ? throw(LoweringError(ex, "cannot use GlobalRef as local identifier")) : nothing end - is_internal = ctx.scope_layers[nk.layer].is_internal || + is_internal = (ex.context::SyntaxContext).internal || getmeta(ex, :is_internal, false)::Bool b = _new_binding(ctx, ex, nk.name, bk; mod, is_internal, kws...) declaration_scope.vars[nk] = b.id @@ -233,25 +207,18 @@ function resolve_name(ctx, ex; exclude_toplevel_globals=false) return b end end - # hygiene compat: retry at the base layer for rescoped globals only - if (sl = ctx.scope_layers[nk.layer]; sl.hygiene_compat && !is_base_layer(sl)) - for sid in Iterators.reverse(ctx.scope_stack) - scope = ctx.scopes[sid] - if nk in scope.rescoped_globals - nk2 = NameKey(nk.name, base_layer(ctx, sl).id) - bid = get(scope.vars, nk2, nothing) - @jl_assert !isnothing(bid) && get_binding(ctx, bid).kind === :global ex - if (!exclude_toplevel_globals || sid !== top_scope(ctx).id) - return get_binding(ctx, bid) - end - end - end - end return nothing end +function _record_layer!(ctx, ex) + !hasattr(ex, :context) && return + sl = (ex.context::SyntaxContext).layer + get!(ctx.layer_ids, sl, length(ctx.layer_ids)+1) +end + function _find_scope_decls!(ctx, scope, ex) k = kind(ex) + _record_layer!(ctx, ex) if k === K"local" && kind(ex[1]) === K"Identifier" var_k = getmeta(ex, :is_destructured_arg, false) ? :destructured_arg : :local @@ -260,6 +227,7 @@ function _find_scope_decls!(ctx, scope, ex) explicit_declare_in_scope!(ctx, scope, ex[1], :global) elseif k === K"function_decl" k1 = kind(ex[1]) + _record_layer!(ctx, ex[1]) if k1 === K"BindingId" b = get_binding(ctx, ex[1]) @jl_assert b.is_ssa || b.kind === :global ( @@ -268,16 +236,25 @@ function _find_scope_decls!(ctx, scope, ex) elseif k1 === K"Identifier" hasattr(ex[1], :mod) && explicit_declare_in_scope!(ctx, scope, ex[1], :global) get!(scope.assignments, NameKey(ex[1]), ex[1]._id) + get!(ctx.layer_ids, (ex[1].context::SyntaxContext).layer, + length(ctx.layer_ids)+1) else @jl_assert false (ex, "unknown kind in assignment") end elseif k in KSet"= constdecl assign_or_constdecl_if_global" k1 = kind(ex[1]) - if k1 === K"BindingId" + _record_layer!(ctx, ex[1]) + sc = ex[1].context::SyntaxContext + if k === K"constdecl" && is_flisp_compat(ex[1]) && + is_top_scope(scope) && sc.layer !== ctx.layer + # hack: flisp declares a mangled global in expansion; we must not error + explicit_declare_in_scope!(ctx, scope, ex[1], :global) + elseif k1 === K"BindingId" b = get_binding(ctx, ex[1]) get!(scope.binding_assignments, b.id, ex[1]._id) elseif k1 === K"Identifier" - get!(scope.assignments, NameKey(ex[1]), ex[1]._id) + !hasattr(ex[1], :mod) && + get!(scope.assignments, NameKey(ex[1]), ex[1]._id) elseif k1 === K"Placeholder" # nothing to declare else @@ -291,6 +268,7 @@ function _find_scope_decls!(ctx, scope, ex) _find_scope_decls!(ctx, scope, e) end end + nothing end # Produce a complete ScopeInfo and add it to the stack of active scopes. This @@ -334,17 +312,22 @@ function enter_scope!(ctx, ex) b = get_binding(ctx, bid) b.lambda_id != 0 || add_lambda_local!(ctx, scope, b) end - for (vk, node_id) in sort!(collect(scope.assignments)) + for (vk, node_id) in sort!(collect(scope.assignments); + by=x->let nk=x[1]; (nk.name, ctx.layer_ids[nk.layer]); end) local ex = SyntaxTree(ctx.graph, node_id) b = resolve_name(ctx, ex) if b === nothing - sl = ctx.scope_layers[vk.layer] - if is_toplevel_thunk && is_base_layer(sl) + sc = ex.context::SyntaxContext + # Top-level assignments are locals in hygienic expansions. We may + # need to adjust this, as flisp makes them name-mangled globals. + hygienic_toplevel = !is_base_layer(sc) && sc.layer !== ctx.layer + if is_toplevel_thunk && !hygienic_toplevel # top-level assignments in no scope and no expansion push!(ctx.soft_assignable_globals, vk) declare_in_scope!(ctx, top_scope(ctx), ex, :global) - elseif scope.is_permeable && is_base_layer(sl) && - is_defined_and_owned_global(sl.mod, Symbol(vk.name), ctx.world) + elseif scope.is_permeable && !hygienic_toplevel && + is_defined_and_owned_global( + syntax_module(sc), Symbol(vk.name), ctx.world) # special soft scope rules: existing global variables are assigned to if ctx.enable_soft_scopes push!(ctx.soft_assignable_globals, vk) @@ -572,6 +555,14 @@ function _resolve_scopes(ctx, ex::SyntaxTree, @jl_assert false (ex, "unknown syntax assertion") end newleaf(ctx, ex, K"TOMBSTONE") + elseif k === K"relayered_global" + bid = get(scope.vars, NameKey(ex[1]), nothing) + !isnothing(bid) && let b = get_binding(ctx, bid) + b.kind !== :global && throw(LoweringError(ex, string( + "unhygienic global name `$(NameKey(ex[1]).name)` conflicts ", + "with an existing $(_var_str(b.kind))"))) + end + newleaf(ctx, ex, K"TOMBSTONE") elseif k == K"function_decl" resolved = mapchildren(e->_resolve_scopes(ctx, e, scope), ctx, ex) name = resolved[1] @@ -630,8 +621,8 @@ ClosureBindings(name_stack) = ClosureBindings(name_stack, Vector{LambdaBindings} struct VariableAnalysisContext{Attrs} <: AbstractLoweringContext graph::SyntaxGraph{Attrs} + layer::ScopeLayer bindings::Bindings - mod::Module scopes::Vector{ScopeInfo} lambda_bindings::LambdaBindings # Stack of method definitions for closure naming @@ -777,8 +768,8 @@ function analyze_variables!(ctx, ex) end end ctx2 = VariableAnalysisContext( - ctx.graph, ctx.bindings, ctx.mod, ctx.scopes, lambda_bindings, - ctx.method_def_stack, ctx.closure_bindings) + ctx.graph, ctx.layer, ctx.bindings, ctx.scopes, + lambda_bindings, ctx.method_def_stack, ctx.closure_bindings) foreach(e->analyze_variables!(ctx2, e), ex[3:end]) # body & return type else foreach(e->analyze_variables!(ctx, e), children(ex)) @@ -819,14 +810,14 @@ enclosing lambda form and information about variables captured by closures. graph = ensure_scope_attributes!(copy_attrs(ctx.graph)) ex = reparent(graph, ex) enable_soft_scopes = soft_scope !== nothing ? soft_scope : contains_softscope_marker(ex) - ctx2 = ScopeResolutionContext(graph, ctx.bindings, ctx.mod, + ctx2 = ScopeResolutionContext(graph, ctx.layer, ctx.bindings, + Dict{ScopeLayer, Int}(), Vector{ScopeInfo}(), Vector{ScopeId}(), - ctx.scope_layers, Set{NameKey}(), + Set{NameKey}(), enable_soft_scopes, - ctx.expr_compat_mode, world) ex2 = resolve_scopes(ctx2, ex) - ctx3 = VariableAnalysisContext(graph, ctx2.bindings, ctx2.mod, + ctx3 = VariableAnalysisContext(graph, ctx2.layer, ctx2.bindings, ctx2.scopes, ex2.lambda_bindings, SyntaxList(graph), Dict{IdTag,ClosureBindings}()) diff --git a/JuliaLowering/src/syntax_macros.jl b/JuliaLowering/src/syntax_macros.jl index 342323c9d79ca..a8ebbfd56a4a1 100644 --- a/JuliaLowering/src/syntax_macros.jl +++ b/JuliaLowering/src/syntax_macros.jl @@ -15,6 +15,14 @@ # `JuliaLowering.include()` or something. Then we'll be in the fun little world # of bootstrapping but it shouldn't be too painful :) +# Note that `@ast __context__ __context__.macrocall [K"foo" ...]` is unhygienic, +# since `@ast` is meant for internal lowering use (it requires an explicit +# provenance argument, and then copies any syntax context from the provenance to +# any created syntax). A real user-facing macro to replace it should use the +# provenance of the literal K"foo" expression in the file instead, and should +# not copy context (this is not hard to implement, but the provenance requires +# it and callers to be JL-lowered, which this file currently isn't.) + function Base.var"@nospecialize"(__context__::MacroContext, exs::SyntaxTree...) if length(exs) == 0 @ast __context__ __context__.macrocall [K"meta" @@ -237,37 +245,35 @@ function Base.Experimental.var"@opaque"(__context__::MacroContext, ex) ] end -function _at_eval_code(ctx, srcref, mod, ex) - @ast ctx srcref [K"block" +# @eval should mostly ignore hygiene against our system's best wishes. Still +# attempt to preserve provenance. +function _at_eval_code(mc::MacroContext, mod_st::SyntaxTree, ex) + sc = mc.macrocall.context::SyntaxContext + val = remove_context(@ast mc mc.macrocall ("eval_result"::K"Identifier")) + q = _legacy_quote_to_syntax((@ast mc mc.macrocall [K"quote" ex]), 0, true) + new_sc = SyntaxContext(base_layer(sc).mod, sc.version) + @ast mc mc.macrocall [K"block" [K"local" [K"=" - "eval_result"::K"Identifier" - [K"call" - # TODO: Call "eval"::K"core" here - JuliaLowering.eval::K"Value" - [K"parameters" - [K"kw" - "expr_compat_mode"::K"Identifier" - ctx.expr_compat_mode::K"Bool" - ] - ] - mod - [K"quote" ex] + val + [K"call" JuliaLowering.eval::K"Value" + mod_st + [K"call" JuliaSyntax.fill_context::K"Value" q new_sc::K"Value"] ] ] ] [K"unknown_head"(name_val="latestworld-if-toplevel")] - "eval_result"::K"Identifier" + val ] end - function Base.var"@eval"(__context__::MacroContext, ex) - mod = @ast __context__ __context__.macrocall __context__.scope_layer.mod::K"Value" - _at_eval_code(__context__, __context__.macrocall, mod, ex) + sc = __context__.macrocall.context::SyntaxContext + mod = @ast __context__ __context__.macrocall base_layer(sc).mod::K"Value" + _at_eval_code(__context__, mod, ex) end function Base.var"@eval"(__context__::MacroContext, mod, ex) - _at_eval_code(__context__, __context__.macrocall, mod, ex) + _at_eval_code(__context__, mod, ex) end #-------------------------------------------------------------------------------- @@ -337,3 +343,100 @@ function var"@inert"(__context__::MacroContext, ex) @jl_assert kind(ex) == K"quote" ex @ast __context__ __context__.macrocall [K"inert" ex] end + +# `quote`/`inert` for syntaxtree +function var"@syntaxinert"(__context__::MacroContext, st) + @ast __context__ __context__.macrocall [K"syntaxinert" st] +end +function var"@syntaxquote"(__context__::MacroContext, st) + @ast __context__ __context__.macrocall [K"syntaxquote" st] +end +# not particularly good or useful, as @syntaxquote must expand first +function var"@syntaxunquote"(__context__::MacroContext, st) + @ast __context__ __context__.macrocall [K"syntaxunquote" st] +end + +# If the syntax version allows, convert quote/$ to syntaxquote/syntaxunquote. +# This is just a convenient way to create SyntaxTree with full provenance +# without dedicated surface syntax, mainly for testing metaprogramming in JL. +# It is insufficient in many ways, e.g. not all forms can be expressed (need +# surface syntax) +function var"@legacy_quote_to_syntax"(__context__::MacroContext, st) + @jl_assert kind(st) === K"quote" || kind(st) === K"inert" st + if is_flisp_compat(__context__.macrocall) + st + elseif kind(st) === K"inert" + setattr(st, :kind, K"syntaxinert") # parser simplifies quote to inert + else + _legacy_quote_to_syntax(st, 0, false) + end +end +function _legacy_quote_to_syntax(st::SyntaxTree, depth, force::Bool) + k = kind(st) + if k === K"quote" && depth == 0 && (force || !is_flisp_compat(st)) + @jl_assert numchildren(st) == 1 st + cs = mapsyntax(c->_legacy_quote_to_syntax(c, depth+1, force), children(st)) + setattr(mknode(st, cs), :kind, K"syntaxquote") + elseif k === K"$" && depth == 1 && (force || !is_flisp_compat(st)) + @jl_assert numchildren(st) == 1 (st, "bad multi-syntaxunquote") + setattr(mknode(st, children(st)), :kind, K"syntaxunquote") + else + depth2 = k === K"quote" ? depth + 1 : k === K"$" ? depth - 1 : depth + cs = SyntaxList(st._graph) + for c in children(st) + # Convert multi-unquote to single unquote + if depth2 == 1 && kind(c) === K"$" && numchildren(c) > 1 + for c2 in children(c) + push!(cs, @ast st._graph c [K"$" c2]) + end + else + push!(cs, c) + end + end + cs_out = mapsyntax(c->_legacy_quote_to_syntax(c, depth2, force), cs) + cs_out == children(st) ? st : mknode(st, cs_out) + end +end +macro legacy_quote_to_syntax(x) + esc(x) +end + +""" +Retrieve the syntax version of the macrocall +""" +function var"@syntax_version"(__context__::MacroContext) + (__context__.macrocall.context::SyntaxContext).version +end +macro syntax_version() + JL_OLD_SYNTAX_VERSION +end + +""" +Set the syntax version of some syntax. This can be used to define macros +producing older syntax than the current version. +""" +function var"@syntax_version"(__context__::MacroContext, ver_st, st) + kind(st) === K"macro" || throw(LoweringError( + st, "`@syntax_version version macro` only supports macro definitions")) + ver = JuliaLowering.eval(syntax_module(ver_st), ver_st) + ver isa VersionNumber || throw(LoweringError( + ver_st, "version argument should be literal `v\"...\" call`")) + _ensure_syntax_version(st, ver) +end +macro syntax_version(_, x) + throw(ArgumentError("@syntax_version can't set version when lowering with flisp")) +end + +function _ensure_syntax_version(st, ver::VersionNumber) + st_sc = st.context::SyntaxContext + sc = st_sc.version == ver ? st_sc : + SyntaxContext(st_sc.layer, st_sc.unexpanded, ver, st_sc.internal) + + if is_leaf(st) || numchildren(st) == 0 + st_sc == sc ? st : setattr(st, :context, sc) + else + out = mapchildren(c->_ensure_syntax_version(c, ver), st._graph, st) + st_sc == sc ? out : + out !== st ? setattr!(out, :context, sc) : setattr(out, :context, sc) + end +end diff --git a/JuliaLowering/src/utils.jl b/JuliaLowering/src/utils.jl index 800243492ff08..28732053c542c 100644 --- a/JuliaLowering/src/utils.jl +++ b/JuliaLowering/src/utils.jl @@ -1,6 +1,7 @@ attrsummary(name, _value) = string(name) attrsummary(name, value::Number) = "$name=$value" attrsummary(name, value::LineNumberNode) = "$name=L$(value.line)" +attrsummary(name, value::Module) = "$name=$value" function _value_string(ex) k = kind(ex) @@ -22,9 +23,11 @@ function _value_string(ex) k == K"oldsymbolicgoto" ? "goto:$(ex.name_val)" : k == K"SourceLocation" ? "SourceLocation:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : - k == K"Value" && ex.value isa SourceRef ? + k == K"Value" ? + (ex.value isa SourceRef ? "SourceRef:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : - hasattr(ex, :value) ? repr(ex.value) : "::K\"$(untokenize(k))\"" + ex.value isa SyntaxContext ? "SyntaxContext(#=omitted=#)" : repr(ex.value)) : + hasattr(ex, :value) ? repr(ex.value) : "::K\"$(untokenize(k))\"" id = get(ex, :var_id, nothing) if isnothing(id) id = get(ex, :id, nothing) @@ -48,24 +51,32 @@ end # symbol is used in the IR (its write-only properties are enforced in codegen). const UNUSED = "#unused#" -function _show_syntax_tree(io, ex, indent, show_kinds) - nodestr = !is_leaf(ex) ? "[$(untokenize(head(ex)))]" : _value_string(ex) +function _show_syntax_tree(io, ex, indent, show_kinds, @nospecialize(parent_sc)) + nodestr = kind(ex) === K"unknown_head" ? ("unknown_head:"*ex.name_val::String) : + !is_leaf(ex) ? "[$(untokenize(head(ex)))]" : _value_string(ex) treestr = rpad(string(indent, nodestr), 40) if show_kinds && is_leaf(ex) treestr = treestr*" :: "*string(kind(ex)) end - std_attrs = Set([:name_val,:value,:kind,:syntax_flags,:source,:var_id]) + std_attrs = Set([:name_val,:value,:kind,:syntax_flags,:source,:var_id,:context]) attrstr = join([attrsummary(n, getproperty(ex, n)) for n in JuliaSyntax.attrnames(ex) if n ∉ std_attrs], ",") - treestr = string(rpad(treestr, 60), " │ $attrstr") + print(io, rpad(treestr, 60)) + print(io, " | ($(ex._id)) ") + sc = get(ex, :context, nothing) + if hasattr(ex, :context) && sc !== parent_sc + print(io, sc) + print(io, ",") + end + print(io, attrstr) + println(io) - println(io, treestr) if !is_leaf(ex) new_indent = indent*" " for n in children(ex) - _show_syntax_tree(io, n, new_indent, show_kinds) + _show_syntax_tree(io, n, new_indent, show_kinds, sc) end end end @@ -74,7 +85,7 @@ function Base.show(io::IO, ::MIME"text/plain", ex::SyntaxTree, show_kinds=true) anames = join(string.(JuliaSyntax.attrnames(syntax_graph(ex))), ",") println(io, "SyntaxTree with attributes $anames") assert_syntaxtree(ex) - _show_syntax_tree(io, ex, "", show_kinds) + _show_syntax_tree(io, ex, "", show_kinds, nothing) end function _show_syntax_tree_sexpr(io, ex) if is_leaf(ex) @@ -183,7 +194,7 @@ function _show_provtree(io::IO, ex::SyntaxTree, indent) print(io, "\n") src = ex.source - msrc = get(ex, :macro_source, nothing) + msrc = JuliaSyntax.macro_prov(ex) printstyled(io, string( indent, msrc === nothing ? "└─ " : "├─ "); color=:light_black) if src isa NodeId @@ -196,9 +207,9 @@ function _show_provtree(io::IO, ex::SyntaxTree, indent) line, _ = source_location(src) printstyled(io, "@ $fn:$line\n", color=:light_black) end - if msrc isa NodeId + if msrc isa SyntaxTree printstyled(io, string(indent, "└─ "); color=:light_black) - _show_provtree(io, SyntaxTree(ex._graph, msrc), indent*" ") + _show_provtree(io, msrc, indent*" ") end end @@ -341,106 +352,6 @@ else end end -#------------------------------------------------------------------------------- -# @SyntaxTree(::Expr) - -function _find_SyntaxTree_macro(ex, line) - @jl_assert !is_leaf(ex) ex - for c in children(ex) - rng = byte_range(c) - firstline = JuliaSyntax.source_line(sourcefile(c), first(rng)) - lastline = JuliaSyntax.source_line(sourcefile(c), last(rng)) - if line < firstline || lastline < line - continue - end - # We're in the line range. Either - if firstline == line && kind(c) == K"macrocall" && begin - name = c[1] - if kind(name) == K"." - name = name[2] - end - @jl_assert kind(name) == K"Identifier" name - name.name_val == "@SyntaxTree" - end - # We find the node we're looking for. NB: Currently assuming a max - # of one @SyntaxTree invocation per line. Though we could relax - # this with more heuristic matching of the Expr-AST... - @jl_assert numchildren(c) == 2 c - return c[2] - elseif !is_leaf(c) - # Recurse - ex1 = _find_SyntaxTree_macro(c, line) - if !isnothing(ex1) - return ex1 - end - end - end - return nothing # Will get here if multiple children are on the same line. -end - -# Translate JuliaLowering hygiene to esc() for use in @SyntaxTree -function _scope_layer_1_to_esc!(ex) - if ex isa Expr - if ex.head == :scope_layer - @assert ex.args[2] === 1 - return esc(_scope_layer_1_to_esc!(ex.args[1])) - else - map!(_scope_layer_1_to_esc!, ex.args, ex.args) - return ex - end - else - return ex - end -end - -""" -Macro to construct quoted SyntaxTree literals (instead of quoted Expr literals) -in normal Julia source code. - -Example: - -```julia -tree1 = @SyntaxTree :(some_unique_identifier) -tree2 = @SyntaxTree quote - x = 1 - \$tree1 = x -end -``` -""" -macro SyntaxTree(ex_old) - # The implementation here is hilarious and arguably very janky: we - # 1. Briefly check but throw away the Expr-AST - if !(Meta.isexpr(ex_old, :quote) || ex_old isa QuoteNode) - throw(ArgumentError("@SyntaxTree expects a `quote` block or `:`-quoted expression")) - end - # 2. Re-parse the current source file as SyntaxTree instead - fname = isnothing(__source__.file) ? error("No current file") : String(__source__.file) - if occursin(r"REPL\[\d+\]", fname) - # Assume we should look at last history entry in REPL - text = try - # Wow digging in like this is an awful hack but `@SyntaxTree` is - # already a hack so let's go for it I guess 😆 - Base.active_repl.mistate.interface.modes[1].hist.history[end] - catch - error("Text not found in REPL history") - end - occursin("@SyntaxTree", text) || error("Text not found in last REPL history line") - else - text = read(fname, String) - end - full_ex = parseall(SyntaxTree, text) - # 3. Using the current file and line number, dig into the re-parsed tree and - # discover the piece of AST which should be returned. - ex = _find_SyntaxTree_macro(full_ex, __source__.line) - isnothing(ex) && error("_find_SyntaxTree_macro failed") - # 4. Do the first step of JuliaLowering's syntax lowering to get - # syntax interpolations to work - _, ex1 = expand_forms_1(__module__, ex, false, Base.tls_world_age()) - @jl_assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast ex1 - Expr(:call, :interpolate_ast, SyntaxTree, ex1[3][1], - map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[4:end])...) -end - function _flatten_blocks(st::SyntaxTree) if kind(st) === K"block" out = SyntaxList(st._graph) diff --git a/JuliaLowering/src/validation.jl b/JuliaLowering/src/validation.jl index 8961194aeba35..88cb541774a52 100644 --- a/JuliaLowering/src/validation.jl +++ b/JuliaLowering/src/validation.jl @@ -251,7 +251,7 @@ vst1(vcx::Validation1Context, st::SyntaxTree)::ValidationResult = @stm st begin [K"ssavalue" [K"Value"]] -> pass() [K"static_parameter" [K"Value"]] -> pass() [K"inert" _] -> pass() - [K"inert_syntaxtree" _] -> pass() + [K"syntaxinert" _] -> pass() [K"core" [K"Identifier"]] -> pass() [K"top" [K"Identifier"]] -> pass() [K"meta" _...] -> pass() # TODO @@ -497,7 +497,7 @@ end # syntax TODO: all-underscore variables may be read from with dot syntax vst1_dot_getproperty_rhs(vcx, st) = @stm st begin [K"inert" x] -> pass() - [K"inert_syntaxtree" x] -> pass() + [K"syntaxinert" x] -> pass() [K"Identifier"] -> pass() (_, when=is_expr_value(st)) -> pass() _ -> @fail(st, "invalid `.` syntax") @@ -516,7 +516,7 @@ end vst1_calldecl_dot_name_rhs(vcx, st) = @stm st begin [K"inert" x] -> vst1_calldecl_dot_name_rhs(vcx, x) - [K"inert_syntaxtree" x] -> vst1_calldecl_dot_name_rhs(vcx, x) + [K"syntaxinert" x] -> vst1_calldecl_dot_name_rhs(vcx, x) [K"Identifier"] -> vst1_ident(vcx, st; lhs=true) ([K"Value"], when=st.value isa String) -> _ident_str(vcx, st, st.value; lhs=true) [K"String"] -> _ident_str(vcx, st, st.value; lhs=true) @@ -596,7 +596,7 @@ vst1_call_kwarg(vcx, st) = @stm st begin [K"=" id val] -> vst1_ident(vcx, id; lhs=true) & vst1(vcx, val) [K"..." x] -> vst1(vcx, x) [K"." x [K"inert" id]] -> vst1(vcx, x) & vst1_ident(vcx, id; lhs=true) - [K"." x [K"inert_syntaxtree" id]] -> vst1(vcx, x) & vst1_ident(vcx, id; lhs=true) + [K"." x [K"syntaxinert" id]] -> vst1(vcx, x) & vst1_ident(vcx, id; lhs=true) ([K"call" [K"Identifier"] symval v], when=(st[1].name_val==="=>")) -> vst1(vcx, symval) & vst1(vcx, v) _ -> @fail(st, "expected identifier, `=`, or `...` after semicolon") @@ -1150,7 +1150,7 @@ function _assert_syntaxtree(st::SyntaxTree, parents::Vector{NodeId}, vr) end return vr & @fail(st, err*"]") end - for a in (:kind, :source) + for a in (:kind, :source) # TODO: context vr &= hasattr(st, a) ? pass() : @fail(st, string("needs attribute ", a)) end if is_leaf(st) @@ -1179,7 +1179,7 @@ function _assert_syntaxtree(st::SyntaxTree, parents::Vector{NodeId}, vr) (_, when=JuliaSyntax.is_literal(st)) -> (:value,) (_, when=JuliaSyntax.is_trivia(st)) -> () # green tree only (_, when=JuliaSyntax.is_operator(st)) -> (:name_val,) # TODO: remove - _ -> return vr & @fail(st, "unrecognized leaf kind") + _ -> return vr & @fail(st, "unrecognized leaf kind $(kind(st))") end else required_attrs = @stm st begin @@ -1196,8 +1196,9 @@ function _assert_syntaxtree(st::SyntaxTree, parents::Vector{NodeId}, vr) # results to avoid exponential repeated lookups, and figure out how these # edges may form cycles with child edges) st.source === st._id && (vr &= @fail(st, ".source equal to self ID")) - get(st, :macro_source, nothing) === st._id && - (vr &= @fail(st, ".macro_source equal to self ID")) + sc = get(st, :context, nothing) + sc isa SyntaxContext && + sc.unexpanded === st && (vr &= @fail(st, "unexpanded equal to self")) push!(parents, st._id) for c in children(st) @@ -1233,7 +1234,7 @@ vst2(vcx::Validation2Context, st::SyntaxTree) = @stm st begin Bool Char Float Float32 BinInt OctInt HexInt Integer SourceLocation String Symbol Value core top latestworld latestworld_if_toplevel symbolicgoto oldsymbolicgoto symboliclabel TOMBSTONE - """ ? pass() : @fail(st, "unrecognized leaf kind") + """ ? pass() : @fail(st, "unrecognized leaf kind $(kind(st))") [K"call" [K"static_eval" cg] xs...] -> get(cg, :name_val, nothing) == "cglobal" ? all(vst2, vcx, xs) : @fail(st, "expected (call (static_eval cglobal) _...)") @@ -1266,7 +1267,7 @@ vst2(vcx::Validation2Context, st::SyntaxTree) = @stm st begin [K"_do_while" body cond] -> vst2(vcx, body) & vst2(vcx, cond) [K"_while" cond body] -> vst2(vcx, cond) & vst2(vcx, body) [K"inert" _] -> pass() - [K"inert_syntaxtree" _] -> pass() + [K"syntaxinert" _] -> pass() [K"lambda" _...] -> vst2_lam(vcx, st) [K"function_decl" x] -> vst2_ident(vcx, x) [K"function_type" x] -> vst2(vcx, x) @@ -1301,6 +1302,7 @@ vst2(vcx::Validation2Context, st::SyntaxTree) = @stm st begin [K"always_defined" x] -> vst2_ident(vcx, x) [K"assert" [K"Symbol"] x] -> vst2(vcx, x) [K"removable" x] -> vst2(vcx, x) + [K"relayered_global" [K"Identifier"]] -> pass() # Could be made stricter [K"foreigncall" _ [K"static_eval" rt] [K"static_eval" at] cconv roots_args...] -> diff --git a/JuliaLowering/test/decls.jl b/JuliaLowering/test/decls.jl index ee28fc795c7ad..ad820c7e2c2da 100644 --- a/JuliaLowering/test/decls.jl +++ b/JuliaLowering/test/decls.jl @@ -871,11 +871,11 @@ gr_mod = Module() # error: try/catch with GlobalRef catch var @gensym sym - @test_throws "cannot use GlobalRef as local identifier" jl_eval( + @test_throws ErrorException fl_eval( test_mod, Expr(:try, Expr(:block, Expr(:call, :error, "oops")), GlobalRef(gr_mod, sym), - Expr(:block, GlobalRef(gr_mod, sym)))) + Expr(:block, 1))) broken=true end @testset "All possible `let` forms" for run in [fl_eval, jl_eval], diff --git a/JuliaLowering/test/demo.jl b/JuliaLowering/test/demo.jl deleted file mode 100644 index 030e856c63209..0000000000000 --- a/JuliaLowering/test/demo.jl +++ /dev/null @@ -1,913 +0,0 @@ -# Just some hacking - -using JuliaSyntax -using JuliaLowering - -using JuliaSyntax: ensure_attributes -using JuliaLowering: @ast, SyntaxGraph, SyntaxTree, child, children, is_leaf, - lookup_binding, makenode, newnode!, numchildren, setattr!, setchildren!, showprov, - sourceref - -using JuliaSyntaxFormatter - -# Extract variable kind for highlighting purposes -function var_kind(ctx, ex) - id = get(ex, :var_id, nothing) - if isnothing(id) - return nothing - end - binfo = get_binding(ctx, id) - return binfo.kind == :local ? - (binfo.is_captured ? :local_captured : :local) : - binfo.kind -end - -# Extract module of globals for highlighting -function var_mod(ctx, ex) - id = get(ex, :var_id, nothing) - if isnothing(id) - return nothing - end - return get_binding(ctx, id).mod -end - -function formatsrc(ex; kws...) - Text(JuliaSyntaxFormatter.formatsrc(ex; kws...)) -end - -function debug_lower(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false, verbose::Bool=false, do_eval::Bool=false) - ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex, expr_compat_mode, Base.get_world_counter()) - - verbose && @info "Macro expanded" formatsrc(ex_macroexpand, color_by=:scope_layer) - - ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) - verbose && @info "Desugared" formatsrc(ex_desugar, color_by=:scope_layer) - - ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) - verbose && @info "Resolved scopes" formatsrc(ex_scoped, color_by=e->var_kind(ctx2,e)) - - ctx4, ex_converted = JuliaLowering.convert_closures(ctx3, ex_scoped) - verbose && @info "Closure converted" formatsrc(ex_converted, color_by=:var_id) - - ctx5, ex_compiled = JuliaLowering.linearize_ir(ctx4, ex_converted) - verbose && @info "Linear IR" formatsrc(ex_compiled, color_by=:var_id) Text(sprint(JuliaLowering.print_ir, ex_compiled)) - - ex_expr = JuliaLowering.to_lowered_expr(ex_compiled) - verbose && @info "CodeInfo" ex_expr - - if do_eval - eval_result = Base.eval(mod, ex_expr) - verbose && @info "Eval" eval_result - else - eval_result = nothing - end - - (ctx1, ex_macroexpand, ctx2, ex_desugar, ctx3, ex_scoped, ctx4, ex_converted, ctx5, ex_compiled, ex_expr, eval_result) -end - - -# Currently broken - need to push info back onto src -# function annotate_scopes(mod, ex) -# ex = ensure_attributes(ex, var_id=Int) -# ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex, false) -# ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand) -# ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar) -# ex -# end - -#------------------------------------------------------------------------------- -# Module containing macros used in the demo. -define_macros = false -if !define_macros - eval(:(module M end)) -else -eval(JuliaLowering.@SyntaxTree :(baremodule M - using Base - - using JuliaLowering: @ast, @chk, JuliaLowering, MacroExpansionError, adopt_scope, makenode - using JuliaSyntax - using JuliaLowering: @goto, @inert, @islocal, @label - using Base: @locals - - macro K_str(str) - JuliaSyntax.Kind(str) - end - - # Introspection - macro __MODULE__() - __context__.scope_layer.mod - end - - macro __FILE__() - JuliaLowering.filename(__context__.macrocall) - end - - macro __LINE__() - JuliaLowering.source_location(__context__.macrocall)[1] - end - - # Macro with local variables - module A - another_global = "global in A" - - macro bar(ex) - quote - x = "`x` in @bar" - (x, another_global, $ex) - end - end - end - - someglobal = "global in module M" - - # Macro with local variables - macro foo(ex) - quote - x = "`x` from @foo" - (x, someglobal, A.@bar $ex) - #(x, someglobal, $ex, A.@bar($ex), A.@bar(x)) - end - end - - macro call_show(x) - quote - z = "z in @call_show" - @show z $x - end - end - - macro call_info(x) - quote - z = "z in @call_info" - @info "hi" z $x - end - end - - macro call_oldstyle_macro(y) - quote - x = "x in call_oldstyle_macro" - @oldstyle $y x - end - end - - macro newstyle(x, y, z) - quote - x = "x in @newstyle" - ($x, $y, $z, x) - end - end - - macro set_a_global(val) - quote - global a_global = $val - end - end - - macro set_global_in_parent(ex) - e1 = adopt_scope(:(sym_introduced_from_M), __context__) - quote - $e1 = $ex - end - end - - macro baz(ex) - quote - let $ex = 10 - $ex - end - end - end - - macro make_module() - :(module X - blah = 10 - end) - end - - macro return_a_value() - 42 - end - - macro nested_return_a_value() - :( - @return_a_value - ) - end - - macro inner() - :(2) - end - - macro outer() - :((1, @inner)) - end - - macro K_str(str) - JuliaSyntax.Kind(str[1].value) - end - - # Recursive macro call - macro recursive(N) - Nval = if kind(N) == K"Integer" || kind(N) == K"Value" - N.value - end - if !(Nval isa Integer) - throw(MacroExpansionError(N, "argument must be an integer")) - end - if Nval < 1 - return N - end - quote - x = $N - (@recursive($(Nval-1)), x) - end - end - - xx = "xx in M" - - macro test_inert_quote() - println(xx) - @inert quote - ($xx, xx) - end - end - - macro mmm(ex) - :(let - local x - function f() - (x, $ex) - end - f() - end) - end - -end)) -end - -Base.eval(M, :( -macro oldstyle(a, b) - quote - x = "x in @oldstyle" - @newstyle $(esc(a)) $(esc(b)) x - end -end -)) - -# -#------------------------------------------------------------------------------- -# Demos of the prototype - -# src = """ -# let -# local x, (y = 2), (w::T = ww), q::S -# end -# """ - -# src = """ -# function foo(x::f(T), y::w(let ; S end)) -# "a \$("b \$("c")")" -# end -# """ - -src = """ -begin - function f(x) - nothing - end - - f(1) -end -""" - -# src = """ -# x + y -# """ - -# src = """ -# module A -# function f(x)::Int -# x + 1 -# end -# -# b = f(2) -# end -# """ - -# src = """ -# function f() -# end -# """ -# -# src = """ -# # import A.B: C.c as d, E.e as f -# # import JuliaLowering -# using JuliaLowering -# """ -# -# src = """ -# module A -# z = 1 + 1 -# end -# """ - -src = raw""" -begin - x = 10 - y = :(g(z)) - quote - f($(x+1), $y) - end -end -""" - -function wrapscope(ex, scope_type) - makenode(ex, ex, K"scope_block", ex; scope_type=scope_type) -end - -function softscope_test(ex) - g = ensure_attributes(ex._graph, scope_type=Symbol) - wrapscope(wrapscope(JuliaLowering.reparent(g, ex), :neutral), :soft) -end - -# src = """ -# M.@test_inert_quote() -# """ - -# src = """ -# macro mmm(a; b=2) -# end -# macro A.b(ex) -# end -# """ - -# src = """ -# M.@set_global_in_parent "bent hygiene!" -# """ - -# src = """ -# begin -# M.@__LINE__ -# end -# """ - -# src = """@foo z""" - -src = """ -M.@recursive 3 -""" - -# src = """ -# M.@set_global_in_parent "bent hygiene!" -# """ - -# src = """ -# begin -# x = 10 -# y = 20 -# let x = y + x -# z = "some string \$x \$y" -# -# function f(y) -# a = M.@foo z -# "\$z \$y \$a \$x" -# end -# print(x) -# end -# print(x) -# end -# """ - -# src = """ -# begin -# x = -1 -# M.@baz x -# end -# """ - -# src = """ -# _ = -1 -# """ - -# src = """ -# M.@make_module -# """ - -# src = """ -# M.@nested_return_a_value -# """ - -# src = """ -# function f(y) -# x = 42 + y -# M.@foo error(x) -# end -# """ - -src = """ -let - y = 0 - x = 1 - let x = x + 1 - y = x - end - (x, y) -end -""" - -#src = """M.@outer""" - -src = """ -begin - local a, b, c - if a - b - else - c - end -end -""" - -src = """ -begin - local i = 0 - while i < 10 - i = i + 1 - if isodd(i) - continue - end - println(i) - end -end -""" - -src = """ -for i in [3,1,2] - println("i = ", i, ", j = ", j) -end -""" - -# src = """ -# @ccall f()::T -# """ -# -# src = """ -# begin -# a = 1 -# xs = [:(a),] -# x = :(:(\$(\$(xs...)))) -# end -# """ - -# src = """ -# try -# a -# catch exc -# b -# end -# """ - -src = """ -let - a = [] - for i = 1:2, j = 3:4 - push!(a, (i,j)) - i = 100 - end - a -end -""" - -src = """ -begin - function f(x) - y = x + 1 - "hi", x, y - end - - f(1) -end -""" - -src = """ -let - x = try - error("hi") - 1 - catch exc - current_exceptions() - else - 3 - end - x -end -""" - -src = """ -function f(y) - x = - try - try - error("hi") - 1 - catch exc - if y - return 2 - end - 3 - else - 4 - end - catch - 5 - end - x -end -""" - -src = """ -function f(x)::Int - if x - 42.0 - end - 0xff -end -""" - -src = """ -let x = 10 - global a = [] - try - try - return 100 - finally - push!(a, 1) - end - finally - push!(a, 2) - end - x -end -""" - -src = """ -let - for outer i = 1:2 - body - end -end -""" - -src = """ -let - i = "hi" - j = 1 - M.@label foo - try - println("i = ", i) - i = i + 1 - if i <= 2 - M.@goto foo - end - catch exc - println("Caught exception ", exc) - j = j + 1 - if j <= 2 - println("Trying again ", exc) - M.@goto foo - end - end -end -""" - -src = """ -let - M.@goto foo - M.@label foo -end -""" - -src = """ -x = M.@label foo -""" - -src = """ -begin - local x::T = 1 - local x::S = 1 -end -""" - -src = """ -begin - local a, b - if a - b - end -end -""" - -src = """ -let - A{S} = B{S} -end -""" - -src = """ -let - a = b = c = sin(1) - (a,b,c) -end -""" - -src = """ -a.b = c -""" - -src = """ -a[i j] = c -""" - -src = """ -let - as = [1,2,3,4] - (x,ys...,z) = as - (x,ys,z) -end -""" - -src = """ -let - x = (1,2) - (y,x) = x - (x,y) -end -""" - -src = """ -let - a = b = c = sin(1) - (a,b,c) -end -""" - -src = """ -begin - as = [(1,2), (3,4)] - ((x,y), (z,w)) = as -end -""" - -src = """ -let -(x, y) = (y,x) -end -""" - -src = """ -let x = 1 - M.@islocal x -end -""" - -src = """ -let x = 1 - local y - M.@locals -end -""" - -src = """ -let - (a, bs...,) = (1,2,3) - bs -end -""" - -src = """ -(; a=1, a=2) -""" - -src = """ -begin - kws = (c=3, d=4) - xs = 1:3 - f(xs...; kws..., a=1, b=2) -end -""" - -src = """ -"some docs" -function f() - println("hi") -end -""" - -src = """ -function f(::T, ::U, ::S) where T where {U,S} - println(T) - println(U) - println(S) -end -""" - -src = """ -function (x::XXX)(y) - println("hi", " ", x, " ", y) -end -""" - -src = """ -struct X - x - y::String -end -""" - -src = """ -struct X{U,V} - x::U - y::V -end -""" - -src = """ -struct S9{T} - x - y - - "Docs for S9" - S9{Int}(xs) = new(xs...) -end -""" - -# Default positional args with missing arg names -src = """ -function f(::Int, y=1, z=2) - (y, z) -end -""" - -# Default positional args with placeholders -src = """ -function f(_::Int, x=1) - x -end -""" - -# Positional args and type parameters with transitive dependencies -# Bug in flisp lowering - see https://github.com/JuliaLang/julia/issues/49275 -src = """ -function f(x, y::S=[1], z) where {T, S<:AbstractVector{T}} - (x, y, z, T) -end -""" - -# Default positional args before trailing slurp are allowed -src = """ -function f(x=1, ys...) - ys -end -""" - -# Default positional args after a slurp is an error -src = """ -function f(x=1, ys..., z=2) - ys -end -""" - -# Positional arg with slurp and default -src = """ -function f(x=1, ys...="hi") - ys -end -""" - -# Positional arg with slurp and splat -src = """ -function f(x=1, ys...=(1,2)...) - ys -end -""" - -src = """ -let - x = 10 - function f(y) - x + y - end -end -""" - -src = """ -begin - local f, set_x - local x = 10 - local y = 100 - function f() - z = 1 + y - x - z - end - function set_x() - x = 1 - end - println("f = ", f()) - set_x() - y = 10 - println("f = ", f()) -end -""" - -# TODO: fix this - it's interpreted in a bizarre way as a kw call. -# src = """ -# function f(x=y=1) -# x -# end -# """ - -function gen_stuff(ctx, N, x) - JuliaLowering.@ast ctx ctx.macrocall [K"tuple" - (i::K"Integer" for i in 1:N)... - ] -end - -src = raw""" -function gen(x::NTuple{N}) where {N} - nongen_stuff = :nongen - if @generated - quote - maybe_gen_stuff = ($N, $x) - end - else - maybe_gen_stuff = :nongen_2 - end - (nongen_stuff, maybe_gen_stuff) -end -""" - -src = raw""" -begin - function partially_gen(x::NTuple{N,T}) where {N,T} - shared = :shared_stuff - if @generated - quote - unshared = ($x, $N, $T) - end - else - # Uuuum. How do we test both sides of this branch?? - unshared = :nongen # (typeof(x), N, T) - end - (shared, unshared) - end - - partially_gen((1,2,3,4,5)) -end -""" - -src = """ -let - z = "z in outer ctx" - @call_show z -end -""" - -src = """ -let - x = "x in outer ctx" - @call_oldstyle_macro x -end -""" - -src = """ -let - z = "z in outer ctx" - @call_info z -end -""" - -ex = parsestmt(SyntaxTree, src, filename="foo.jl") -#ex = ensure_attributes(ex, var_id=Int) -#ex = softscope_test(ex) -@info "Input code" formatsrc(ex) - -(ctx1, ex_macroexpand, - ctx2, ex_desugar, - ctx3, ex_scoped, - ctx4, ex_converted, - ctx5, ex_compiled, - ex_expr, eval_result) = debug_lower(M, ex; verbose=true, do_eval=true) - -# Automatic test reduction -# bad = reduce_any_failing_toplevel(JuliaLowering, joinpath(@__DIR__, "../src/desugaring.jl")) -# if !isnothing(bad) -# @error "Reduced expression as code" formatsrc(bad) -# write("bad.jl", JuliaSyntaxFormatter.formatsrc(bad)) -# end - -# Old lowering -# text = read(joinpath(@__DIR__, "../src/desugaring.jl"), String) -# ex = parseall(SyntaxTree, text, filename="desugaring.jl") -# for e in Meta.parseall(text).args -# Meta.lower(JuliaLowering, e) -# end diff --git a/JuliaLowering/test/functions.jl b/JuliaLowering/test/functions.jl index e5b85850847b6..7e02ff0af022d 100644 --- a/JuliaLowering/test/functions.jl +++ b/JuliaLowering/test/functions.jl @@ -636,10 +636,7 @@ end @testset for arg0 in [:x, :(x::Type), :(::Type), :(_), :(_::Type)], arg1 in [arg0, Expr(:..., arg0)], arg2 in [arg1, Expr(:kw, arg1, :Int)], - expander in [fl_macroexpand, (_,x)->x] - # TODO: would use jl_macroexpand, but that cannot be done as a separate - # step due to MacroExpansionContext, and we must use jl_eval. - @test_broken false + expander in [fl_macroexpand, jl_macroexpand] @testset let expanded = expander( test_mod, :(function (specialized, @nospecialize($arg2)) @@ -1583,6 +1580,7 @@ end @testset "Generated functions" begin; for expr_compat_mode in (false, true) local genfunc_s, genfunc_f + @eval test_mod import JuliaLowering.@legacy_quote_to_syntax @testset "returning special syntax forms" begin @test JuliaLowering.include_string(test_mod, raw""" @@ -1802,12 +1800,22 @@ end f_gen_calls_macros(1) end """; expr_compat_mode) === "foo" + @test JuliaLowering.include_string(test_mod, raw"""begin + @generated function calls_versioned_macro(::Type{T}, ::Val{i}) where {T, i} + i isa Integer || @goto err + return i + @label err + return 0 + end + + calls_versioned_macro(Tuple{Int}, Val(1)) + end """; expr_compat_mode) == 1 end genfunc_quote_s = """ begin function f_gen_quote_1(::Tuple{T}) where {T} - out = :(:x1,first) + out = @legacy_quote_to_syntax :(:x1,first) if @generated else end @@ -1828,7 +1836,7 @@ end function f_gen_quote_2(::Tuple{T}) where {T} out = nothing if @generated - :(out = :(:x2,generated)) + @legacy_quote_to_syntax :(out = @legacy_quote_to_syntax :(:x2,generated)) else out = (:x2,nongen) end @@ -1850,7 +1858,7 @@ end if @generated else end - return :(:x4,after) + return @legacy_quote_to_syntax :(:x4,after) end f_gen_quote_3((1,)) @@ -1867,9 +1875,9 @@ end function f_gen_interpolate(::Tuple{T}) where {T} out = :(:x1,first) if @generated - out = :($out, generated) + out = @legacy_quote_to_syntax :($out, generated) else - out = :($out, nongen) + out = @legacy_quote_to_syntax :($out, nongen) end return out end @@ -1888,19 +1896,22 @@ end genfunc_quote_s = raw""" begin - @eval function f_gen_quote_1(::Tuple{T}) where {T} + @eval function f_gen_eval_quote_1(::Tuple{T}) where {T} out = $(Expr(:quote, Expr(:call, :+, 1, Expr(:if, Expr(:generated), 1, 2)))) if @generated else end return out end - f_gen_quote_1((1,)) + f_gen_eval_quote_1((1,)) end """ @test JuliaLowering.include_string( test_mod, genfunc_quote_s; expr_compat_mode=true) == :(1 + $(Expr(:if, Expr(:generated), 1, 2))) + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=false) == + :(1 + $(Expr(:if, Expr(:generated), 1, 2))) # Test generated function edges to bindings # (see also https://github.com/JuliaLang/julia/pull/57230) diff --git a/JuliaLowering/test/functions_ir.jl b/JuliaLowering/test/functions_ir.jl index 3b1f9025da64d..17bbbd4b47ca7 100644 --- a/JuliaLowering/test/functions_ir.jl +++ b/JuliaLowering/test/functions_ir.jl @@ -2075,7 +2075,7 @@ end 6 latestworld 7 TestMod.#f_only_generated@generator#0 8 (call core.TypeEqOf %₇) -9 (call core.svec %₈ JuliaLowering.MacroContext core.Any core.Any core.Any) +9 (call core.svec %₈ JuliaSyntax.SyntaxContext core.Any core.Any core.Any) 10 (call core.svec) 11 SourceLocation::1:21 12 (call core.svec %₉ %₁₀ %₁₁) @@ -2085,7 +2085,7 @@ end 2 TestMod.generator_code 3 (call %₂ slot₄/x slot₅/y) 4 (call core.tuple %₃) - 5 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block ($ (block (call generator_code x y))))) %₄) + 5 (call JuliaLowering.interpolate_syntax (syntaxinert (block (syntaxunquote (block (call generator_code x y))))) %₄) 6 (return %₅) 14 latestworld 15 TestMod.f_only_generated @@ -2096,7 +2096,7 @@ end 20 (call core.svec %₁₇ %₁₈ %₁₉) 21 --- method TestMod.f_only_generated %₂₀ slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/y(!read)] - 1 (meta :generated (new JuliaLowering.GeneratedFunctionStub false TestMod.#f_only_generated@generator#0 SourceRef::1:1 (call core.svec :#self# :x :y) (call core.svec))) + 1 (meta :generated (new JuliaLowering.GeneratedFunctionStub SyntaxContext(#=omitted=#) TestMod.#f_only_generated@generator#0 SourceRef::1:1 (call core.svec :#self# :x :y) (call core.svec))) 2 (meta :generated_only) 3 (return core.nothing) 22 latestworld @@ -2125,16 +2125,16 @@ end 6 latestworld 7 TestMod.#f_partially_generated@generator#0 8 (call core.TypeEqOf %₇) -9 (call core.svec %₈ JuliaLowering.MacroContext core.Any core.Any core.Any) +9 (call core.svec %₈ JuliaSyntax.SyntaxContext core.Any core.Any core.Any) 10 (call core.svec) 11 SourceLocation::1:10 12 (call core.svec %₉ %₁₀ %₁₁) 13 --- method TestMod.#f_partially_generated@generator#0 %₁₂ slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/#self#(nospecialize,!read) slot₄/x(nospecialize,!read) slot₅/y(nospecialize,!read)] 1 (meta :nospecialize slot₃/#self# slot₄/x slot₅/y) - 2 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block (= maybe_gen_stuff (call some_gen_stuff x y))))) + 2 (call JuliaLowering.interpolate_expr (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))) 3 (call core.tuple %₂) - 4 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block (= nongen_stuff (call bothgen x y)) ($ (block (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple-p nongen_stuff maybe_gen_stuff))) %₃) + 4 (call JuliaLowering.interpolate_syntax (syntaxinert (block (= nongen_stuff (call bothgen x y)) (syntaxunquote (block (call JuliaLowering.interpolate_expr (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple-p nongen_stuff maybe_gen_stuff))) %₃) 5 (return %₄) 14 latestworld 15 TestMod.f_partially_generated @@ -2145,7 +2145,7 @@ end 20 (call core.svec %₁₇ %₁₈ %₁₉) 21 --- method TestMod.f_partially_generated %₂₀ slots: [slot₁/#self#(!read) slot₂/x slot₃/y slot₄/maybe_gen_stuff(single_assign) slot₅/nongen_stuff(single_assign)] - 1 (meta :generated (new JuliaLowering.GeneratedFunctionStub false TestMod.#f_partially_generated@generator#0 SourceRef::1:37 (call core.svec :#self# :x :y) (call core.svec))) + 1 (meta :generated (new JuliaLowering.GeneratedFunctionStub SyntaxContext(#=omitted=#) TestMod.#f_partially_generated@generator#0 SourceRef::1:37 (call core.svec :#self# :x :y) (call core.svec))) 2 TestMod.bothgen 3 (= slot₅/nongen_stuff (call %₂ slot₂/x slot₃/y)) 4 TestMod.some_nongen_stuff diff --git a/JuliaLowering/test/generators_ir.jl b/JuliaLowering/test/generators_ir.jl index 011761908ac44..e7651f2a1a853 100644 --- a/JuliaLowering/test/generators_ir.jl +++ b/JuliaLowering/test/generators_ir.jl @@ -208,7 +208,7 @@ LoweringError: 9 (call core.svec %₆ %₇ %₈) 10 --- method core.nothing %₉ slots: [slot₁/#self#(!read) slot₂/#unused#(!read)] - 1 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (return x))) + 1 (call JuliaLowering.interpolate_expr (inert (return x))) 2 (return %₁) 11 latestworld 12 TestMod.##->###6 diff --git a/JuliaLowering/test/import.jl b/JuliaLowering/test/import.jl index 874879ea0b559..07c2470cf0680 100644 --- a/JuliaLowering/test/import.jl +++ b/JuliaLowering/test/import.jl @@ -77,6 +77,7 @@ import .H.I, .I.J @testset "(AI) from macro expansion" for expr_compat_mode in (true, false) macrocall_mod = Module() + @eval macrocall_mod import JuliaLowering, JuliaLowering.@legacy_quote_to_syntax JuliaLowering.include_string(macrocall_mod, raw""" module Exporter export val @@ -84,10 +85,10 @@ import .H.I, .I.J other = [456] end macro imp_names() - :(import .Exporter: val, other as o) + @legacy_quote_to_syntax :(import .Exporter: val, other as o) end macro use_target() - :(using .Exporter) + @legacy_quote_to_syntax :(using .Exporter) end """; expr_compat_mode) Core.@latestworld @@ -152,3 +153,147 @@ fl_eval(test_mod, :( @test !isdefined(test_mod, :neither_var) @test test_mod.mod_p_e_n isa Module end + +@testset "(AI) public/export module resolution from macros" for (is_new, run) in [ + (false, (mod, x)->fl_eval(mod,JuliaSyntax.parsestmt(Expr, x))), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=true)), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=false)) + ] + + defs_mod = Module(:Defs) + call_mod = Module(:CallSite) + Core.eval(call_mod, :(const Defs = $defs_mod)) + Core.eval(defs_mod, :(import JuliaLowering, JuliaLowering.@legacy_quote_to_syntax)) + Core.eval(defs_mod, :(const var"@ast" = $(JuliaLowering.var"@ast"))) + Core.eval(defs_mod, :(const var"@K_str" = $(JuliaSyntax.var"@K_str"))) + + # old-style macros: hygienic plain name vs escaped argument + fl_eval(defs_mod, :(macro old_pub_plain(); Expr(:public, :op_hyg); end)) + fl_eval(defs_mod, :(macro old_exp_plain(); Expr(:export, :oe_hyg); end)) + fl_eval(defs_mod, :(macro old_pub_arg(name); Expr(:public, esc(name)); end)) + fl_eval(defs_mod, :(macro old_exp_arg(name); Expr(:export, esc(name)); end)) + + # new-style macros: hygienic name (from syntaxquote) vs argument + JuliaLowering.include_string(defs_mod, raw""" + macro new_exp_plain(); @legacy_quote_to_syntax quote export ne_hyg end; end + macro new_pub_arg(name); @ast __context__ __context__.macrocall [K"public" name]; end + macro new_exp_arg(name); @ast __context__ __context__.macrocall [K"export" name]; end + """) + Core.@latestworld + + # Define the names being marked so `isexported`/`ispublic` are meaningful. + # Hygienic names live in defs_mod; argument/escaped names live in call_mod. + Core.eval(defs_mod, :(global op_hyg=1; global oe_hyg=1; global ne_hyg=1)) + Core.eval(call_mod, :(global cp=1; global ce=1; global np=1; global ne=1)) + + # old macros, hygienic plain name -> macro-definition module (defs_mod) + # flisp error: globalref means malformed public + # @test run(call_mod, "Defs.@old_pub_plain()") + # @test Base.ispublic(defs_mod, :op_hyg) + # @test !Base.ispublic(call_mod, :op_hyg) + run(call_mod, "Defs.@old_exp_plain()"); Core.@latestworld + @test !Base.isexported(defs_mod, :oe_hyg) + @test Base.isexported(call_mod, :oe_hyg) + + # old macros, escaped argument -> call-site module (call_mod) + run(call_mod, "Defs.@old_pub_arg(cp)"); Core.@latestworld + @test Base.ispublic(call_mod, :cp) + @test !Base.ispublic(defs_mod, :cp) + run(call_mod, "Defs.@old_exp_arg(ce)"); Core.@latestworld + @test Base.isexported(call_mod, :ce) + @test !Base.isexported(defs_mod, :ce) + + if is_new + # new macros, hygienic name -> macro-definition module (defs_mod) + run(call_mod, "Defs.@new_exp_plain()"); Core.@latestworld + @test Base.isexported(defs_mod, :ne_hyg) + @test !Base.isexported(call_mod, :ne_hyg) + + # new macros, argument -> call-site module (call_mod) + run(call_mod, "Defs.@new_pub_arg(np)"); Core.@latestworld + @test Base.ispublic(call_mod, :np) + @test !Base.ispublic(defs_mod, :np) + run(call_mod, "Defs.@new_exp_arg(ne)"); Core.@latestworld + @test Base.isexported(call_mod, :ne) + @test !Base.isexported(defs_mod, :ne) + end +end + +@testset "(AI) using/import module resolution from macros" for (is_new, run) in [ + (false, (mod, x)->fl_eval(mod, JuliaSyntax.parsestmt(Expr, x))), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=true)), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=false)) + ] + + # Unlike `public`/`export`, `using`/`import` are never hygienic + defs_mod = Module(:Defs) + call_mod = Module(:CallSite) + Core.eval(call_mod, :(const Defs = $defs_mod)) + Core.eval(defs_mod, :(import JuliaLowering, JuliaLowering.@legacy_quote_to_syntax)) + + # Distinguishable `Exporter` submodules in *both* modules. The imported + # values reveal which `.Exporter` a relative path resolved to. + Core.eval(defs_mod, :( + module Exporter + export val + val = [:defs_val] + other = [:defs_other] + end)) + Core.eval(call_mod, :( + module Exporter + export val + val = [:call_val] + other = [:call_other] + end)) + # A submodule reachable only from defs_mod: if a path were resolved against + # the macro-definition module, `.OnlyInDefs` would be found there. + Core.eval(defs_mod, :( + module OnlyInDefs + secret = [:secret] + end)) + + fl_eval(defs_mod, :(macro old_use(); :(using .Exporter); end)) + fl_eval(defs_mod, :(macro old_imp(); :(import .Exporter: other as o); end)) + fl_eval(defs_mod, :(macro old_priv(); :(using .OnlyInDefs); end)) + + JuliaLowering.include_string(defs_mod, raw""" + macro new_use(); @legacy_quote_to_syntax quote using .Exporter end; end + macro new_imp(); @legacy_quote_to_syntax quote import .Exporter: other as o end; end + macro new_priv(); @legacy_quote_to_syntax quote using .OnlyInDefs end; end + """) + Core.@latestworld + + # `using .Exporter` -> exported `val` becomes visible in call_mod, resolved + # against call_mod.Exporter, not defs_mod.Exporter; defs_mod is untouched. + run(call_mod, "Defs.@old_use()"); Core.@latestworld + @test call_mod.val === call_mod.Exporter.val + @test call_mod.val !== defs_mod.Exporter.val + @test !isdefined(defs_mod, :val) + + # `import .Exporter: other as o` -> the (plain, non-hygienic) rename target + # `o` is bound in call_mod, again resolved against call_mod.Exporter. + run(call_mod, "Defs.@old_imp()"); Core.@latestworld + @test call_mod.o === call_mod.Exporter.other + @test call_mod.o !== defs_mod.Exporter.other + @test !isdefined(defs_mod, :o) + + # The path is resolved in call_mod, where `.OnlyInDefs` does not exist, so + # the statement errors rather than reaching defs_mod.OnlyInDefs. + @test_throws UndefVarError run(call_mod, "Defs.@old_priv()") + @test !isdefined(call_mod, :OnlyInDefs) + + if is_new + run(call_mod, "Defs.@new_use()"); Core.@latestworld + @test call_mod.val === call_mod.Exporter.val + @test call_mod.val !== defs_mod.Exporter.val + @test !isdefined(defs_mod, :val) + + run(call_mod, "Defs.@new_imp()"); Core.@latestworld + @test call_mod.o === call_mod.Exporter.other + @test call_mod.o !== defs_mod.Exporter.other + @test !isdefined(defs_mod, :o) + + @test_throws UndefVarError run(call_mod, "Defs.@new_priv()") + @test !isdefined(call_mod, :OnlyInDefs) + end +end diff --git a/JuliaLowering/test/macros.jl b/JuliaLowering/test/macros.jl index f514e7cd174a1..8ccc20dd2dbdc 100644 --- a/JuliaLowering/test/macros.jl +++ b/JuliaLowering/test/macros.jl @@ -1,11 +1,6 @@ -test_mod = Module(:macro_test) +test_mod = @newmod(macro_test) +@eval test_mod import JuliaLowering Base.eval(test_mod, :(const var"@ast" = $(JuliaLowering.var"@ast"))) -Base.eval(test_mod, :(const var"@K_str" = $(JuliaLowering.var"@K_str"))) - -# These libraries may either be packages or vendored into Base - need to pull -# them in via relative paths in the `using` statements below. -Base.eval(test_mod, :(const JuliaLowering = $(JuliaLowering))) -Base.eval(test_mod, :(const JuliaSyntax = $(JuliaSyntax))) # Set up identity macros for use in this file # - `old_e`, escaping its whole output, should do nothing to an expression @@ -17,7 +12,21 @@ fl_eval(test_mod, :(macro old_h(x); x; end)) JuliaLowering.include_string(test_mod, "macro new_m(x); x; end") fl_eval(test_mod, :(global mvar = "global mvar")) -@testset "basic hygiene and escaping: old macros" for run in [ +@testset "syntax versioning sanity-check" begin + @test JuliaLowering.include_string( + test_mod, "JuliaLowering.@syntax_version") == + JuliaSyntax.JL_NEW_SYNTAX_VERSION + @test JuliaLowering.include_string( + test_mod, "JuliaLowering.@syntax_version"; expr_compat_mode=false) == + JuliaSyntax.JL_NEW_SYNTAX_VERSION + @test JuliaLowering.include_string( + test_mod, "JuliaLowering.@syntax_version"; expr_compat_mode=true) == + JuliaSyntax.JL_OLD_SYNTAX_VERSION +end + +# Basic checks that arbitrary nesting of transparent macros (no new syntax in new +# macros, escaped/unhygienic in old macros) doesn't introduce opaque layers +@testset "basic transparent macros: old macros" for run in [ (x::String)->Base.include_string( test_mod, "#=FLISP SANITY-CHECK=# "*x), (x::String)->JuliaLowering.include_string( @@ -52,8 +61,7 @@ fl_eval(test_mod, :(global mvar = "global mvar")) @test run("@old_h @old_h let mvar = 0; @old_e(@old_e(mvar)); end") == 0 @test run("@old_h @old_h let @old_e(@old_e(mvar) = 0); @old_e(mvar); end") == 0 end - -@testset "basic hygiene and escaping: new macros only" for expr_compat_mode in [true, false] +@testset "basic transparent macros: new macros only" for expr_compat_mode in [true, false] local run = (x::String)->JuliaLowering.include_string(test_mod, x; expr_compat_mode) @test run("@new_m let mvar = 0; mvar; end") == 0 @@ -65,8 +73,7 @@ end @test run("@new_m let mvar = 0; @new_m(@new_m(mvar)); end") == 0 @test run("@new_m let @new_m(@new_m(mvar) = 0); @new_m(mvar); end") == 0 end - -@testset "basic hygiene and escaping: new+old interop" for expr_compat_mode in [true, false], +@testset "basic transparent macros: new+old interop" for expr_compat_mode in [true, false], mcall in ["@old_e ", "@new_m ", "@old_e @new_m ", "@new_m @old_e "], old_h in ["", "@old_h "] @@ -84,14 +91,490 @@ end end end +# More simple checks with no difference between macro module and macrocall module +isdefined(test_mod, :x) && Base.delete_binding(test_mod, :x) +fl_eval(test_mod, :(macro old_read_x(); :x; end)) +fl_eval(test_mod, :(macro old_suggest_x(arg) + quote + let x = "suggested (old)" + $(esc(arg)) + end + end + end)) +JuliaLowering.include_string(test_mod, raw""" + macro new_read_x(); @legacy_quote_to_syntax :x; end +""") +JuliaLowering.include_string(test_mod, raw""" + macro new_suggest_x(arg) + @legacy_quote_to_syntax quote + let x = "suggested (new)" + $arg + end + end + end +""") +@testset "basic hygiene: check that name resolution fails where it should (flisp)" for run in [ + (x::String)->fl_eval(test_mod,JuliaSyntax.parsestmt(Expr, "#=FLISP SANITY-CHECK=# "*x)), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + @test_throws UndefVarError run("@old_read_x()") + @test_throws UndefVarError run("let x = 0; @old_read_x(); end") + @test_throws UndefVarError run("@old_suggest_x(x)") + @test_throws UndefVarError run("@old_suggest_x(@old_read_x())") + @test run("let x = 1; @old_suggest_x(x); end") == 1 + @test run("@old_suggest_x(let x = 1; x; end)") == 1 + @test_throws UndefVarError run("@old_suggest_x(let x = 1; @old_read_x(); end)") == 1 +end +@testset "basic hygiene: check that name resolution fails where it should (new)" for run in [ + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + @test_throws UndefVarError run("@new_read_x()") + @test_throws UndefVarError run("let x = 0; @new_read_x(); end") + @test_throws UndefVarError run("@new_suggest_x(x)") + @test_throws UndefVarError run("@new_suggest_x(@new_read_x())") + @test run("let x = 1; @new_suggest_x(x); end") == 1 + @test run("@new_suggest_x(let x = 1; x; end)") == 1 + @test_throws UndefVarError run("@new_suggest_x(let x = 1; @new_read_x(); end)") == 1 + + @testset "old/new interop" begin + @testset for wrapper in ["", "@old_e ", "@old_h ", "@new_m "] + @test_throws UndefVarError run(wrapper*"@old_suggest_x(@new_read_x())") + @test_throws UndefVarError run(wrapper*"@new_suggest_x(@old_read_x())") + @test run(wrapper*"let x = 1; @old_suggest_x(x); end") == 1 + @test run(wrapper*"let x = 1; @new_suggest_x(x); end") == 1 + @test run(wrapper*"@old_suggest_x(let x = 1; x; end)") == 1 + @test run(wrapper*"@new_suggest_x(let x = 1; x; end)") == 1 + end + end +end + +@eval test_mod (global test_mod_global = "test_mod_global") +@newmod(EvalMod, test_mod) +@testset "@eval" for run in [ + (x::String)->fl_eval(test_mod,JuliaSyntax.parsestmt(Expr, "#=FLISP SANITY-CHECK=# "*x)), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + has_syntax = run(raw"@legacy_quote_to_syntax :x") isa SyntaxTree + treetype = has_syntax ? SyntaxTree : Expr + symtype = has_syntax ? SyntaxTree : Symbol + valtype = has_syntax ? SyntaxTree : Any + + @test run(raw"@eval nothing") == nothing + @test run(raw"@eval :sym") == :sym + @test run(raw"@eval QuoteNode(:sym)") == QuoteNode(:sym) + @test run(raw"@eval Expr(:call, :identity, 1)") == Expr(:call, :identity, 1) + @test run(raw"@eval :(identity(1))") == Expr(:call, :identity, 1) + # syntax version of the caller should be propagated to JL.eval + @test run(raw"@eval @legacy_quote_to_syntax(:sym)") isa symtype + @test run(raw"@eval @legacy_quote_to_syntax(:(identity(1)))") isa treetype + @test run(raw"@eval @eval @legacy_quote_to_syntax(:(identity(1)))") isa treetype + + # quoting behaves the same as outside of eval + @test run(raw"@eval(:(1 + 2))") == Expr(:call, :+, 1, 2) + @test run(raw"@eval(:true)") == true + @test run(raw"@eval(:x)") == :x + + # interpolation + @test run(raw"let x = nothing; @eval $x; end") == nothing + @test run(raw"let x = :identity; @eval $x; end") == Base.identity + @test run(raw"let x = QuoteNode(:sym); @eval $x; end") == :sym + @test run(raw"let x = Expr(:call, :identity, 1); @eval $x; end") == 1 + @test run(raw"let x = :(identity(1)); @eval $x; end") == 1 + @test run(raw"let x = @legacy_quote_to_syntax(:identity); @eval $x; end") == Base.identity + @test run(raw"let x = @legacy_quote_to_syntax(:(identity(1))); @eval $x; end") == 1 + + # interpolate into quote + @test run(raw"let test_mod_global = 0xbad + @eval (@legacy_quote_to_syntax :($test_mod_global)) + end") isa valtype + @test run(raw"let test_mod_global = 0xbad + @eval @legacy_quote_to_syntax(:(1,$test_mod_global)) + end") isa treetype + + # interpolate into quote, double-unquote (mixes of syntax and expr may not + # need to work) + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval (:($($x))) + end") == Base.identity + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval (@legacy_quote_to_syntax :($($x))) + end") isa valtype + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval @legacy_quote_to_syntax(:(1,$$x)) + end") isa treetype + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval $(@eval (:(1,$$x))) + end") == (1, Base.identity) + + # module eval-ed into + @test run(raw"@eval @__MODULE__") == test_mod + @test run(raw"@eval @eval @__MODULE__") == test_mod + # two-arg eval should not obey typical hygiene: decls go to specified module + @test run(raw"@eval EvalMod @__MODULE__") == test_mod.EvalMod + run(raw"@eval EvalMod global eval_mod_global = 1"); Core.@latestworld + @test test_mod.EvalMod.eval_mod_global == 1 + run(raw"@eval EvalMod eval_mod_global_implicit = 1"); Core.@latestworld + @test test_mod.EvalMod.eval_mod_global_implicit == 1 + # standard hygiene atop two-arg eval + fl_eval(test_mod, :(module MacroMod + module MacroModInner; end + macro m_setglobal(); esc(:(mmglobal0 = 0)); end + macro m_eval_inner(x); :(@eval $MacroModInner $x) ; end + end)) + Core.@latestworld + @eval test_mod.EvalMod (const MacroMod2 = $(test_mod.MacroMod)) + @eval test_mod.EvalMod (const var"@legacy_quote_to_syntax" = + $(test_mod.var"@legacy_quote_to_syntax")) + @test run(raw"@eval EvalMod @legacy_quote_to_syntax(:(1 + 1))") isa treetype + @test run(raw"@eval @eval EvalMod @legacy_quote_to_syntax(:(1 + 1))") isa treetype + + @test run(raw"@eval EvalMod MacroMod2.@m_setglobal") == 0 + Core.@latestworld + @test isdefined(test_mod.EvalMod, :mmglobal0) + @test !isdefined(test_mod, :mmglobal0) + @test !isdefined(test_mod.MacroMod, :mmglobal0) + + @test run(raw"@eval EvalMod MacroMod2.@m_eval_inner(global mmglobal1 = 1)") == 1 + Core.@latestworld + @test isdefined(test_mod.MacroMod.MacroModInner, :mmglobal1) + @test !isdefined(test_mod, :mmglobal1) + @test !isdefined(test_mod.MacroMod, :mmglobal1) + @test !isdefined(test_mod.EvalMod, :mmglobal1) + + # interpolation into top-level: symbol declared in the new module + run(raw"let x = @legacy_quote_to_syntax(:sym) + @eval EvalMod module tmp; module inner_eval_mod; global $x = 123; end; end + end") isa Module + Core.@latestworld + @test test_mod.EvalMod.tmp.inner_eval_mod isa Module + @test test_mod.EvalMod.tmp.inner_eval_mod.sym == 123 + + # hygiene + run("let eval_result = 0; @eval 1+1; eval_result; end") == 0 + + @testset "(AI) single-arg @eval does not over-preserve hygiene" for expr_compat_mode in (true, false) + root = @newmod(root) + JuliaLowering.include_string(root, raw""" + module MacB + import JuliaLowering.@legacy_quote_to_syntax + macro do_eval() + @legacy_quote_to_syntax quote + @eval (@__MODULE__) + end + end + end + module MacA + import JuliaLowering.@legacy_quote_to_syntax + import ..MacB + macro wrap() + @legacy_quote_to_syntax quote + @eval (@__MODULE__) + end + end + macro via_b() + @legacy_quote_to_syntax quote + MacB.@do_eval() + end + end + macro wrap_ee() + @legacy_quote_to_syntax quote + @eval @eval (@__MODULE__) + end + end + macro wrap_two_arg() + # two-arg control: explicit target module; the payload's + # `@__MODULE__` must still see the *target* module + @legacy_quote_to_syntax quote + @eval MacB (@__MODULE__) + end + end + macro wrap_arg(ex) + # caller-provided payload (caller's hygiene layer) + @legacy_quote_to_syntax quote + @eval $ex + end + end + macro wrap_fn() + # `@eval` captures the module current when the enclosing function + # *definition* is expanded, like flisp + @legacy_quote_to_syntax quote + () -> @eval (@__MODULE__) + end + end + macro mkmod() + mod = gensym("EvalMod") + @legacy_quote_to_syntax quote + @eval module $mod + const inside = (@__MODULE__) + end + end + end + macro mkmod_payload(ex) + mod = gensym("EvalMod2") + @legacy_quote_to_syntax quote + @eval module $mod + $ex + end + end + end + end + module Sub + import ..MacA + end + """; expr_compat_mode) + Core.@latestworld + + run(str) = JuliaLowering.include_string(root, str; expr_compat_mode) + + # `@eval` inside another macro's unescaped expansion evaluates in the + # caller's module, not the macro's + @test run("MacA.@wrap()") === root + # ... even when the `@eval`-ing macro is called by another macro's expansion + # (flisp: still the dynamic module, not either macro's module) + @test run("MacA.@via_b()") === root + # `@eval` nested in `@eval` re-expands against the outer target + @test run("MacA.@wrap_ee()") === root + # two-arg control: explicit module wins; payload `@__MODULE__` follows it + @test run("MacA.@wrap_two_arg()") === root.MacB + # macro-generated closure: `@eval` binds the definition-time module + @test Base.invokelatest(run("MacA.@wrap_fn()")) === root + # the same macro evaluated into a different module follows the live module + @test JuliaLowering.include_string( + root.Sub, "MacA.@wrap()"; expr_compat_mode) === root.Sub + + # Caller-provided payloads evaluate in the caller's module + @test run("MacA.@wrap_arg(arg_marker = (@__MODULE__))") === root + if !expr_compat_mode + # With SyntaxTree-passed arguments the payload keeps the caller's + # hygiene: the global lands in `root` and is visible there. (In + # expr_compat_mode the old-style Expr round-trip re-layers the payload + # with the macro's hygiene and the assignment becomes a hygienic + # toplevel local -- a pre-existing divergence from flisp tracked by + # the "hygienic toplevel assignments" TODO in scope_analysis.jl.) + @test Base.invokelatest(isdefined, root, :arg_marker) + @test Base.invokelatest(getfield, root, :arg_marker) === root + end + + # The SafeTestsets shape: a macro-generated `@eval module $mod ... end` + # creates the module under the dynamic (caller) module + m = run("MacA.@mkmod()") + @test m isa Module + @test parentmodule(m) === root + @test Base.invokelatest(getfield, m, :inside) === m + # ... and user payload interpolated into the module body sees the fresh + # module as its dynamic module (a user's own `@eval` inside a + # `@safetestset` acts on the anonymous test module) + m2 = run("MacA.@mkmod_payload(@eval user_marker = (@__MODULE__))") + @test m2 isa Module + @test parentmodule(m2) === root + @test Base.invokelatest(getfield, m2, :user_marker) === m2 + + if expr_compat_mode + # Escaped expansions (old-style macros only): same dynamic target + JuliaLowering.include_string(root, raw""" + module MacEsc + macro wrap_esc() + esc(quote + @eval esc_marker = (@__MODULE__) + end) + end + end + """; expr_compat_mode) + Core.@latestworld + @test run("MacEsc.@wrap_esc()") === root + @test Base.invokelatest(isdefined, root, :esc_marker) + @test Base.invokelatest(getfield, root, :esc_marker) === root + end + + # An old-style (flisp-defined and -lowered) macro whose expansion calls + # `@eval` gets the same treatment when invoked under JuliaLowering + fl_eval(root, :(module MacFl + macro flwrap() + quote + @eval (@__MODULE__) + end + end + end)) + Core.@latestworld + @test run("MacFl.@flwrap()") === root + end + + @testset "(AI) const shows up in caller mod" begin + Core.eval(test_mod, :(module MacHome2 + macro make_const() + :( @eval const CMARKER = 42 ) + end + end)) + Core.eval(test_mod, :(MacHome2.@make_const())) + @test isdefined(test_mod, :CMARKER) + + JuliaLowering.eval(test_mod, :(module MacHome2 + macro make_const() + :( @eval const CMARKER = 42 ) + end + end); expr_compat_mode=true) + JuliaLowering.eval(test_mod, :(MacHome2.@make_const())) + @test isdefined(test_mod, :CMARKER) + end +end + +@eval test_mod module hscope_mod; global hscope_g = 123; end +@eval test_mod module nothing_mod; end +@eval test_mod global hscope_g = 234 +@testset "hygienic scope should be usable without macros" begin + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), + 1, test_mod); expr_compat_mode=true) == 1 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), + :hscope_g, + test_mod.hscope_mod); expr_compat_mode=true) == 123 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), + Expr(:escape, :hscope_g), + test_mod.nothing_mod); expr_compat_mode=true) == 234 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), Expr( + Symbol("hygienic-scope"), + Expr(:escape, Expr(:escape, :hscope_g)), + test_mod.nothing_mod), + test_mod.nothing_mod); expr_compat_mode=true) == 234 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), Expr( + Symbol("hygienic-scope"), + Expr(:escape, :hscope_g), + test_mod.nothing_mod), + test_mod.hscope_mod); expr_compat_mode=true) == 123 +end + +Base.eval(test_mod, :( + test_hscope(x, mod=$test_mod) = Expr(Symbol("hygienic-scope"), x, mod) +)) +Base.eval(test_mod, :( + # +3 new scopes and -4 escapes = normal unhygienic macro + macro oldstyle_silly_scopes(x, y) + stmt1 = test_hscope(test_hscope(test_hscope(esc(esc(esc(esc(:($x = 123)))))))) + stmt2 = esc(test_hscope(esc(test_hscope(esc(test_hscope(esc(:($y = 456)))))))) + Expr(:block, stmt1, stmt2) + end)) +@testset "escape and hygienic-scope forms" for run in [ + (x::String)->Base.include_string( + test_mod, "#=FLISP SANITY-CHECK=# "*x), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + @test run(raw""" + let (x, y) = (0, 0); @oldstyle_silly_scopes(x, y); (x, y); end + """) === (123, 456) + @test run(raw"""begin + global_x, global_y = 0, 0 + @oldstyle_silly_scopes(global_x, global_y) + global_x, global_y + end""") === (123, 456) +end + +@testset "apply_expansion_layer mutation testing" begin + local test_mod = @newmod(apply_expansion_layer) + # recursion can't stop at module/toplevel/inert without tweaks, because a + # macro can pull random stuff out of it. This also tests calling into macro + # expansion from macros, mostly because re-using macros I've already written + # is the easiest way to create non-surface-syntax SyntaxTree as of writing. + JuliaLowering.include_string(test_mod, raw""" + macro undo_inert(x) + x2 = JuliaLowering.macroexpand(x) + x2[1] + end + """) + @test JuliaLowering.include_string(test_mod, raw""" + let foo = 1; @undo_inert(:foo); end + """) == 1 + @test JuliaLowering.include_string(test_mod, raw""" + let foo = 1; @undo_inert(@legacy_quote_to_syntax(:foo)); end + """) == 1 + + JuliaLowering.include_string(test_mod, raw""" + macro mk_toplevel(x, y, z) + JuliaSyntax.newnode( + x._graph, __context__.macrocall, K"toplevel", + JuliaSyntax.SyntaxList(x, y, z)) + end + macro toplevel_first_child(x) + x2 = JuliaLowering.macroexpand(x) + x2[1] + end + """) + JuliaLowering.include_string(test_mod, raw""" + macro mk_module(x, y, z) + @legacy_quote_to_syntax :(module mk_module_mod; $x; $y; $z; end) + end + macro module_first_child(x) + x2 = JuliaLowering.macroexpand(x) + x2[end][1] + end + """) + # sanity + @test JuliaLowering.include_string(test_mod, """ + @mk_toplevel(1, :y, "z") + """) == "z" + @test JuliaLowering.include_string(test_mod, """ + @mk_module(1, :y, "z") + """) isa Module + @test JuliaLowering.include_string(test_mod, """ + let (x, y, z) = (1, :y, "z") + @toplevel_first_child(@mk_toplevel(x, y, z)) + end + """) == 1 + @test JuliaLowering.include_string(test_mod, """ + let (x, y, z) = (1, :y, "z") + @toplevel_first_child(@mk_toplevel(x, y, z)) + end + """) == 1 + + # escape should obey quote/unquote + JuliaLowering.include_string(test_mod, raw""" + macro esc_in_quote(); Expr(:quote, Expr(:escape, :x)); end + """; expr_compat_mode=true) + @test JuliaLowering.include_string(test_mod, raw""" + @esc_in_quote + """; expr_compat_mode=true) == Expr(:escape, :x) + @test JuliaLowering.include_string(test_mod, raw""" + @esc_in_quote + """) == Expr(:escape, :x) + + JuliaLowering.include_string(test_mod, raw""" + macro esc_in_unquote(); Expr(:quote, Expr(:$, Expr(:escape, :x))); end + """; expr_compat_mode=true) + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1; @esc_in_unquote(); end + """; expr_compat_mode=true) == 1 + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1; @esc_in_unquote(); end + """) == 1 +end + JuliaLowering.include_string(test_mod, raw""" module M - using ..JuliaLowering: JuliaLowering, adopt_scope + using ..JuliaLowering: JuliaLowering, adopt_scope, @legacy_quote_to_syntax using ..JuliaSyntax # Introspection macro __MODULE__() - __context__.scope_layer.mod + JuliaLowering.syntax_module(__context__.macrocall) end macro __FILE__() @@ -106,7 +589,7 @@ module M # Macro with local variables macro foo(ex) - :(begin + @legacy_quote_to_syntax :(begin x = "`x` from @foo" (x, someglobal, $ex) end) @@ -114,32 +597,32 @@ module M # Set `a_global` in M macro set_a_global(val) - :(begin + @legacy_quote_to_syntax :(begin global a_global = $val end) end macro set_other_global(ex, val) - :(begin + @legacy_quote_to_syntax :(begin global $ex = $val end) end macro set_global_in_parent(ex) - sym_ex = quote; sym_introduced_from_M; end - e1 = adopt_scope(sym_ex[1], __context__) - quote + sym_ex = @legacy_quote_to_syntax quote; sym_introduced_from_M; end + e1 = adopt_scope(__context__.macrocall, sym_ex[1]) + @legacy_quote_to_syntax quote $e1 = $ex nothing end end macro inner() - :(y, z) + @legacy_quote_to_syntax :(y, z) end macro outer() - :((x, @inner)) + @legacy_quote_to_syntax :((x, @inner)) end macro recursive(N) @@ -147,7 +630,7 @@ module M if Nval < 1 return N end - quote + @legacy_quote_to_syntax quote x = $N (x, @recursive $(Nval-1)) end @@ -191,7 +674,7 @@ M.@recursive 3 """) == (3, (2, (1, 0))) ex = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "M.@outer()", filename="foo.jl") -ctx, expanded = JuliaLowering.expand_forms_1(test_mod, ex, false, Base.get_world_counter()) +expanded = JuliaLowering.macroexpand(test_mod, ex) @test JuliaSyntax.sourcetext.(JuliaLowering.flattened_provenance(expanded[2])) == [ "M.@outer()" "@inner" @@ -201,7 +684,7 @@ ctx, expanded = JuliaLowering.expand_forms_1(test_mod, ex, false, Base.get_world @testset "expansion special case: macrocall in do expression" for expr_compat_mode in [true, false] @test JuliaLowering.include_string(test_mod, raw""" macro mac_called_in_do_expression(dofunc, arg) - :($dofunc($arg)) + @legacy_quote_to_syntax :($dofunc($arg)) end """; expr_compat_mode) isa Function @test JuliaLowering.include_string(test_mod, raw""" @@ -245,17 +728,12 @@ end """) world2 = Base.get_world_counter() -call_world_arg_test = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "@world_age_test()") - @test JuliaLowering.expand_forms_1(test_mod, call_world_arg_test, false, world1)[2] ≈ +call_world_arg_test = JuliaLowering.rebase_layers(JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "@world_age_test()"), test_mod, JuliaSyntax.JL_NEW_SYNTAX_VERSION) + @test JuliaLowering.expand_forms_1(call_world_arg_test, world1, true) ≈ @ast_ 1::K"Value" - @test JuliaLowering.expand_forms_1(test_mod, call_world_arg_test, false, world2)[2] ≈ + @test JuliaLowering.expand_forms_1(call_world_arg_test, world2, true) ≈ @ast_ 2::K"Value" -# Layer parenting -@test expanded[1].scope_layer == 2 -@test expanded[2][1].scope_layer == 3 -@test getfield.(ctx.scope_layers, :parent_id) == [0,1,2] - JuliaLowering.include_string(test_mod, """ f_throw(x) = throw(x) macro m_throw(x) @@ -310,17 +788,18 @@ end # Tests for interop between old and new-style macros -# Hygiene interop +# Hygiene interop: +# call_oldstyle_macro -> oldstyle -> newstyle3 JuliaLowering.include_string(test_mod, raw""" macro call_oldstyle_macro(a) - quote + @legacy_quote_to_syntax quote x = "x in call_oldstyle_macro" @oldstyle $a x end end macro newstyle3(a, b, c) - quote + @legacy_quote_to_syntax quote x = "x in @newstyle3" ($a, $b, $c, x) end @@ -340,10 +819,15 @@ end let x = "x in outer scope" @call_oldstyle_macro x end -""") == ("x in outer scope", +""") == ("x in call_oldstyle_macro", "x in call_oldstyle_macro", "x in @oldstyle", "x in @newstyle3") +# # would be ideal, but we can't get hygiene through oldstyle +# ("x in outer scope", +# "x in call_oldstyle_macro", +# "x in @oldstyle", +# "x in @newstyle3") # Old style unhygienic escaping with esc() Base.eval(test_mod, :( @@ -376,13 +860,6 @@ MacroExpansionError while expanding @oldstyle_error in module Main.macro_test: Caused by: Some error in old style macro""" -@test sprint( - showerror, - JuliaLowering.MacroExpansionError( - JuliaLowering.expr_to_est(:(foo), LineNumberNode(1)), - "fake error")) == - "MacroExpansionError:\n#= line 1 =# - fake error" - # Old-style macros returning non-Expr values Base.eval(test_mod, :( macro oldstyle_non_Expr() @@ -437,13 +914,13 @@ end MethodError: no method matching var"@sig_mismatch"(""") end -@testset "old macros producing exotic expr heads" begin +@testset "old macros producing exotic expr heads (or are otherwise complex)" for expr_compat_mode in [true, false] @test JuliaLowering.include_string(test_mod, """ let # example from @preserve docstring x = Ref{Int}(101) p = Base.unsafe_convert(Ptr{Int}, x) GC.@preserve x unsafe_load(p) - end""") === 101 # Expr(:gc_preserve) + end"""; expr_compat_mode) === 101 # Expr(:gc_preserve) # JuliaLowering.jl/issues/121 @test JuliaLowering.include_string(test_mod, """ @@ -451,7 +928,7 @@ end """) isa Module @test JuliaLowering.include_string(test_mod, """ GC.@preserve @static if true v"1.14" else end - """) isa VersionNumber + """; expr_compat_mode) isa VersionNumber # JuliaLowering.jl/issues/144 @test JuliaLowering.include_string(test_mod, """ @@ -460,7 +937,7 @@ end GC.@preserve val begin; end end f_preserve144() - """) == nothing + """; expr_compat_mode) == nothing # JuliaLowering.jl/issues/145 @test JuliaLowering.include_string(test_mod, """ @@ -470,7 +947,7 @@ end GC.@preserve debug_buffer 1 end f_preserve145() - """) == 1 + """; expr_compat_mode) == 1 # only invokelatest produces :isglobal now, so MWE here Base.eval(test_mod, :(macro isglobal(x); esc(Expr(:isglobal, x)); end)) @@ -481,7 +958,7 @@ end (@isglobal(some_undefined), @isglobal(some_global), @isglobal(some_arg), @isglobal(some_local)) end isglobal_chk(1) - """) === (true, true, false, false) + """; expr_compat_mode) === (true, true, false, false) # with K"Placeholder"s @test JuliaLowering.include_string(test_mod, """ __ = 1 @@ -490,22 +967,39 @@ end (@isglobal(_), @isglobal(__), @isglobal(___), @isglobal(____)) end isglobal_chk(1) - """) === (false, false, false, false) + """; expr_compat_mode) === (false, false, false, false) # @test appears to be the only macro in base to use :inert test_result = JuliaLowering.include_string(test_mod, """ using Test @test identity(123) === 123 - """; expr_compat_mode=true) + """; expr_compat_mode) @test test_result.value === true # @enum produces Expr(:toplevel) JuliaLowering.include_string(test_mod, """ @enum SOME_ENUM X1 X2 X3 - """; expr_compat_mode=true) + """; expr_compat_mode) + Core.@latestworld @test test_mod.SOME_ENUM <: Enum @test test_mod.X1 isa Enum + # @deprecate also produces Expr(:toplevel), and :public with expression + # hygiene different from the contained names. + @testset "@deprecate" begin + @test JuliaLowering.include_string(test_mod, """ + module DeprecateMod + d2(x) = x+1 + @deprecate d1(x) d2(0) + end + """; expr_compat_mode) isa Module + Core.@latestworld + @test isdefined(test_mod.DeprecateMod, :d2) + @test isdefined(test_mod.DeprecateMod, :d1) + @test Base.isexported(test_mod.DeprecateMod, :d1) + @test !Base.isexported(test_mod, :d1) + end + # @testset produces :tryfinally with secret third arg @eval test_mod :(using Test) @test JuliaLowering.include_string(test_mod, "@test true") isa Test.Pass @@ -513,7 +1007,7 @@ end @testset begin @test true end - """; expr_compat_mode=true) + """; expr_compat_mode) @test jltestset isa Test.AbstractTestSet @test jltestset.n_passed == 1 end @@ -526,12 +1020,12 @@ end A[I] = Base.Experimental.Const(B)[I] end return 0 - end)) isa Function + end); expr_compat_mode) isa Function @test jl_eval( test_mod, :(let A = [1,2,3], B = [4,5,6] simple_aliasscope(A,B), A, B - end)) == (0, [4,5,6], [4,5,6]) + end); expr_compat_mode) == (0, [4,5,6], [4,5,6]) end @testset "empty meta" begin @@ -825,8 +1319,8 @@ end @testset "scope layers for normally-inert ASTs" begin # Right hand side of `.` @test JuliaLowering.include_string(test_mod, raw""" - let x = :(hi) - :(A.$x) + let x = @legacy_quote_to_syntax :(hi) + @legacy_quote_to_syntax :(A.$x) end """) ≈ @ast_ [K"." "A"::K"Identifier" @@ -834,10 +1328,8 @@ end ] # module @test JuliaLowering.include_string(test_mod, raw""" - let x = :(AA) - :(module $x - end - ) + let x = @legacy_quote_to_syntax :(AA) + @legacy_quote_to_syntax :(module $x end) end """) ≈ @ast_ [K"module" v"1.14.0"::K"Value" @@ -850,24 +1342,37 @@ end # *arguments* get the lexical scope of the calling context, even for the # `x` in `M.$x` where the right hand side of `.` is normally quoted. @test JuliaLowering.include_string(test_mod, raw""" - let x = :(someglobal) + let x = @legacy_quote_to_syntax :(someglobal) @eval M.$x end - """) == "global in module M" + """; expr_compat_mode=false) == "global in module M" + @test JuliaLowering.include_string(test_mod, raw""" + let x = @legacy_quote_to_syntax :(someglobal) + @eval M.$x + end + """; expr_compat_mode=true) == "global in module M" - JuliaLowering.include_string(test_mod, raw""" - let y = 101 + # @eval quoting should embed the value, not the syntax + @test JuliaLowering.include_string(test_mod, raw""" + let some_local = 101 @eval module AA - x = $y + x = $some_local end end - """) + """; expr_compat_mode=false) isa Module + @test test_mod.AA.x == 101 + @test JuliaLowering.include_string(test_mod, raw""" + let some_local = 101 + @eval module AA + x = $some_local + end + end + """; expr_compat_mode=true) isa Module @test test_mod.AA.x == 101 - # "Deferred hygiene" in macros which emit quoted code currently doesn't - # work as might be expected. + # "Deferred hygiene" in macros which emit quoted code. OK to break # - # The old macro system also doesn't handle this - here's the equivalent + # The old macro system doesn't handle this - here's the equivalent # implementation # macro make_quoted_code(init, y) # QuoteNode(:(let @@ -876,24 +1381,22 @@ end # ($(esc(y)), x) # end)) # end - # - # TODO: The following should throw an error rather than producing a - # surprising value, or work "as expected" whatever that is! JuliaLowering.include_string(test_mod, raw""" macro make_quoted_code(init, y) - q = :(let + q = @legacy_quote_to_syntax :(let x = "inner x" $init ($y, x) end) - @ast q._graph q [K"inert_syntaxtree" q] + @ast q._graph q [K"syntaxinert" q] end """) code = JuliaLowering.include_string(test_mod, """@make_quoted_code(x="outer x", x)""") - @test_broken JuliaLowering.eval(test_mod, code) == ("outer x", "inner x") + @test JuliaLowering.eval(test_mod, code) == ("outer x", "inner x") end -@testset "toplevel macro hygiene" begin +@testset "toplevel macro hygiene" for run in [JuliaLowering.include_string, + Base.include_string] @eval test_mod global mod = $test_mod @eval test_mod module MacroMod global mod = MacroMod @@ -907,9 +1410,37 @@ end Expr(:toplevel, :(mod)) end end - @test JuliaLowering.include_string(test_mod, "MacroMod.@escaped_toplevel") === test_mod - @test JuliaLowering.include_string(test_mod, "MacroMod.@inner_escaped_toplevel") === test_mod - @test JuliaLowering.include_string(test_mod, "MacroMod.@unescaped_toplevel") === test_mod.MacroMod + Core.@latestworld + @test run(test_mod, "MacroMod.@escaped_toplevel") === test_mod + @test run(test_mod, "MacroMod.@inner_escaped_toplevel") === test_mod + @test run(test_mod, "MacroMod.@unescaped_toplevel") === test_mod.MacroMod + + unrelated = @newmod(unrelated) + @eval unrelated const MacroMod = $(test_mod.MacroMod) + @eval unrelated global mod = 123 + + @test run(unrelated, "MacroMod.@escaped_toplevel") == 123 + @test run(unrelated, "MacroMod.@inner_escaped_toplevel") == 123 + @test run(unrelated, "MacroMod.@unescaped_toplevel") === test_mod.MacroMod +end + +@testset "toplevel macro hygiene: @__MODULE__" for run in [JuliaLowering.include_string, + Base.include_string] + @eval test_mod module MacroMod + macro atmodule_in_toplevel() + Expr(:toplevel, :(@__MODULE__)) + end + macro atmodule_in_module() + Expr(:toplevel, Expr(:module, true, esc(:atmod_mod), Expr( + :block, :(global global_mod = @__MODULE__)))) + end + end + Core.@latestworld + @test run(test_mod, "MacroMod.@atmodule_in_toplevel") === test_mod + @test run(test_mod, "MacroMod.@atmodule_in_module") isa Module + Core.@latestworld + @test isdefined(test_mod, :atmod_mod) + @test test_mod.atmod_mod.global_mod == test_mod.atmod_mod end # JuliaLang/JuliaLowering.jl#120 @@ -918,8 +1449,9 @@ end # code, not the module corresponding to the current hygienic scope JuliaLowering.include_string(test_mod, raw""" module Mod1 +import ..JuliaLowering.@legacy_quote_to_syntax macro indirect_MODULE() - return :(@__MODULE__()) + return @legacy_quote_to_syntax :(@__MODULE__()) end end """) @@ -928,6 +1460,56 @@ code = JuliaLowering.include_string(test_mod, """Mod1.@indirect_MODULE()""") # the lowering/eval iterator needs to expand in the correct world age (currently # the only way to hit this from user code is macros producing toplevel) +@testset "old macros defining modules" begin + # escaped module nested in tmpmod_1 + jl_eval(test_mod, :( + module MacMod + macro makemod(name) + Expr(:toplevel, + esc(Expr(:module, false, :tmpmod_1, + Expr(:block, + Expr(:module, false, name, + Expr(:block, Expr(:const, Expr(:(=), :c, 1)))))))) + end + end); expr_compat_mode=true) + + @testset for expr_compat_mode in [true, false] + @test JuliaLowering.include_string( + test_mod, "MacMod.@makemod(newmod)") isa Module + Core.@latestworld + # module name should escape macmod->test_mod + @test test_mod.tmpmod_1.newmod isa Module + @test !isdefined(test_mod.MacMod, :newmod) + @test !isdefined(test_mod.MacMod, :tmpmod_1) + # const in mod body should work + @test test_mod.tmpmod_1.newmod.c == 1 + end + + # escaped module name + jl_eval(test_mod, :( + module MacMod + macro makemod(name) + Expr(:toplevel, + Expr(:module, false, esc(name), + Expr(:block, + # TODO: escape node in outer context + # Expr(:const, Expr(:(=), esc(:c), 1)) + ))) + end + end); expr_compat_mode=true) + + @testset for expr_compat_mode in [true, false] + @test JuliaLowering.include_string( + test_mod, "MacMod.@makemod(newmod)") isa Module + Core.@latestworld + # module name should escape macmod->test_mod + @test test_mod.newmod isa Module + @test !isdefined(test_mod.MacMod, :newmod) + # const in mod body should + @test_broken test_mod.newmod.c == 1 + end +end + @testset "macros defining macros" begin @eval test_mod macro make_and_use_macro_toplevel() Expr(:toplevel, @@ -1254,3 +1836,28 @@ end @test _version.syntax == version end end + +# produces import/using in module that is `@eval`ed. +@testset "safetestset" begin + macro_mod = @newmod(macro_mod, test_mod) + JuliaLowering.include_string(macro_mod, raw""" + macro safetestset(testname, expr) + quote + @eval module $(gensym("safetestset_mod")) + using Test + @testset $testname $expr + end + nothing + end + end + """; expr_compat_mode=true) + + JuliaLowering.include_string(test_mod, """ + macro_mod.@safetestset "Tests" begin + a = 1; b = 2; c = a + b; @test c == 3 + @isdefined(a) == true + end + """; expr_compat_mode=true) + @test !isdefined(test_mod, :a) + @test !isdefined(macro_mod, :a) +end diff --git a/JuliaLowering/test/macros_ir.jl b/JuliaLowering/test/macros_ir.jl index c561f45e171ec..aaae8ec5d5e7c 100644 --- a/JuliaLowering/test/macros_ir.jl +++ b/JuliaLowering/test/macros_ir.jl @@ -1,14 +1,16 @@ module MacroMethods + using ..JuliaLowering macro some_macro() - quote + JuliaLowering.@legacy_quote_to_syntax quote some_global end end module ExtraMacroMethods using ..MacroMethods + using ..JuliaLowering macro MacroMethods.some_macro(ex) - quote + JuliaLowering.@legacy_quote_to_syntax quote some_global end end @@ -35,7 +37,7 @@ end ######################################## # Simple macro macro add_one(ex) - quote + JuliaLowering.@legacy_quote_to_syntax quote $ex + 1 end end @@ -51,7 +53,7 @@ end 9 --- method TestMod.@add_one %₈ slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/ex] 1 (call core.tuple slot₃/ex) - 2 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block (call-i + ($ ex) 1))) %₁) + 2 (call JuliaLowering.interpolate_syntax (syntaxinert (block (call-i + (syntaxunquote ex) 1))) %₁) 3 (return %₂) 10 latestworld 11 TestMod.@add_one diff --git a/JuliaLowering/test/misc.jl b/JuliaLowering/test/misc.jl index 00930beb2026e..418df860c3049 100644 --- a/JuliaLowering/test/misc.jl +++ b/JuliaLowering/test/misc.jl @@ -1,4 +1,4 @@ -test_mod = Module() +test_mod = @newmod(misc) # Blocks @test JuliaLowering.include_string(test_mod, """ @@ -27,13 +27,22 @@ end @test JuliaLowering.include_string(test_mod, raw""" module EvalTest - _some_var = 2 + _some_var = 1 end let x = 10 @eval EvalTest $x + _some_var end -""") == 12 +""") == 11 +@test JuliaLowering.include_string(test_mod, raw""" +module EvalTest2 + _some_var = 2 +end +let + x = 10 + @eval EvalTest2 $x + _some_var +end +"""; expr_compat_mode=true) == 12 @test JuliaLowering.include_string(test_mod, """ let x=11 @@ -123,7 +132,8 @@ end Expr(:tuple, Symbol(""), :a)))) == (1, 2) # quote of empty - @test jl_eval(test_mod, Expr(:quote, Symbol(""))) === Symbol("") + @test jl_eval(test_mod, Expr(:quote, Symbol("")); + expr_compat_mode=true) === Symbol("") end @eval test_mod libccalltest_var = "libccalltest" @@ -220,13 +230,14 @@ end ccall((:ctest, libccalltest_var), Complex{Int}, (Complex{Int},), 10 + 20im) """) === 11 + 18im -@testset "(robot-generated) ccall (sym, lib) tuple: globals and hygiene" begin +@testset "(robot-generated) ccall (sym, lib) tuple: globals and hygiene" for expr_compat_mode in [true, false] # library is a module-qualified global JuliaLowering.include_string(test_mod, """ module CCallLibMod const the_lib = "libccalltest" end - """) + """; expr_compat_mode) + Core.@latestworld @test JuliaLowering.include_string(test_mod, """ ccall((:ctest, CCallLibMod.the_lib), Complex{Int}, (Complex{Int},), 10 + 20im) """) === 11 + 18im @@ -235,11 +246,14 @@ end JuliaLowering.include_string(test_mod, raw""" module CCallHygieneMod const mylib = "libccalltest" + import ..JuliaLowering.@legacy_quote_to_syntax macro do_ccall() - :(ccall((:ctest, mylib), Complex{Int}, (Complex{Int},), 10 + 20im)) + @legacy_quote_to_syntax( + :(ccall((:ctest, mylib), Complex{Int}, (Complex{Int},), 10 + 20im))) end end - """) + """; expr_compat_mode) + Core.@latestworld @test JuliaLowering.include_string(test_mod, """ CCallHygieneMod.@do_ccall() """) === 11 + 18im @@ -249,28 +263,32 @@ end @test JuliaLowering.include_string(test_mod, """ mylib = "this_lib_does_not_exist" CCallHygieneMod.@do_ccall() - """) === 11 + 18im + """; expr_compat_mode) === 11 + 18im # macro that interpolates the lib value at expansion time JuliaLowering.include_string(test_mod, raw""" module CCallHygieneMod2 + import ..JuliaLowering.@legacy_quote_to_syntax const mylib2 = "libccalltest" macro do_ccall_interp() lib = mylib2 - :(ccall((:ctest, $lib), Complex{Int}, (Complex{Int},), 10 + 20im)) + @legacy_quote_to_syntax( + :(ccall((:ctest, $lib), Complex{Int}, (Complex{Int},), 10 + 20im))) end end - """) + """; expr_compat_mode) + Core.@latestworld @test JuliaLowering.include_string(test_mod, """ CCallHygieneMod2.@do_ccall_interp() - """) === 11 + 18im + """; expr_compat_mode) === 11 + 18im # ccall with plain symbol name still works inside a function @test JuliaLowering.include_string(test_mod, """ function ccall_plain_sym() ccall(:strlen, Csize_t, (Cstring,), "abc") end - """) isa Function + """; expr_compat_mode) isa Function + Core.@latestworld @test test_mod.ccall_plain_sym() == 3 # ccall with (sym, lib) tuple where lib is a global, inside a function @@ -278,7 +296,8 @@ end function ccall_global_lib() ccall((:ctest, libccalltest_var), Complex{Int}, (Complex{Int},), 10 + 20im) end - """) isa Function + """; expr_compat_mode) isa Function + Core.@latestworld @test test_mod.ccall_global_lib() === 11 + 18im # ccall with module-qualified lib inside a function @@ -286,7 +305,8 @@ end function ccall_qualified_lib() ccall((:ctest, CCallLibMod.the_lib), Complex{Int}, (Complex{Int},), 10 + 20im) end - """) isa Function + """; expr_compat_mode) isa Function + Core.@latestworld @test test_mod.ccall_qualified_lib() === 11 + 18im end @@ -310,9 +330,10 @@ JuliaLowering.include_string(test_mod, raw""" f_ccallable_hygiene() = 1 module Nested + import ..JuliaLowering.@legacy_quote_to_syntax f_ccallable_hygiene() = 2 macro cfunction_hygiene() - :(@cfunction($f_ccallable_hygiene, Int, ())) + @legacy_quote_to_syntax :(@cfunction($f_ccallable_hygiene, Int, ())) end end """) @@ -327,9 +348,10 @@ JuliaLowering.include_string(test_mod, raw""" f_ccallable_hygiene() = 10 module Nested + import ..JuliaLowering.@legacy_quote_to_syntax f_ccallable_hygiene() = 20 macro cfunction_hygiene() - :(@cfunction(f_ccallable_hygiene, Int, ())) + @legacy_quote_to_syntax :(@cfunction(f_ccallable_hygiene, Int, ())) end end """) @@ -432,7 +454,7 @@ ccallable_sptest_name(::Type{String}) = :strlen @generated function ccall_with_sparams_in_name(s::T) where {T} name = QuoteNode(ccallable_sptest_name(T)) - :(ccall($name, Csize_t, (Cstring,), s)) + @legacy_quote_to_syntax :(ccall($name, Csize_t, (Cstring,), s)) end """) @test test_mod.ccall_with_sparams_in_name("hii") == 3 @@ -551,15 +573,16 @@ end end # SyntaxTree @eval should pass along expr_compat_mode -@test JuliaLowering.include_string(test_mod, "@eval quote x end"; - expr_compat_mode=false) isa SyntaxTree -@test JuliaLowering.include_string(test_mod, "@eval quote x end"; - expr_compat_mode=true) isa Expr @test JuliaLowering.include_string(test_mod, raw""" - let T = :foo + let T = gensym("documented_sym_no_logspam") @eval @doc $"This is a $T" $T = 1 end """; expr_compat_mode=true) === 1 +@test JuliaLowering.include_string(test_mod, raw""" + let T = gensym("documented_sym_no_logspam") + @eval @doc $"This is a $T" $T = 1 + end +"""; expr_compat_mode=false) === 1 @testset "tryfinally with scopedvalues" begin @eval test_mod scopedval = Base.ScopedValues.ScopedValue(1) diff --git a/JuliaLowering/test/misc_ir.jl b/JuliaLowering/test/misc_ir.jl index bcdbb02fb4dd0..db2d1bb2c6e5c 100644 --- a/JuliaLowering/test/misc_ir.jl +++ b/JuliaLowering/test/misc_ir.jl @@ -1,6 +1,7 @@ module JuxtuposeTest + using ..JuliaLowering macro emit_juxtupose() - :(10x) + JuliaLowering.@legacy_quote_to_syntax :(10x) end end @@ -307,36 +308,28 @@ end # @eval without module @eval $f(x, y) #--------------------- -1 JuliaLowering.eval -2 (call core.tuple :expr_compat_mode) -3 (call core.apply_type core.NamedTuple %₂) -4 (call core.tuple false) -5 (call %₃ %₄) -6 TestMod.f -7 (call core.tuple %₆) -8 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (call ($ f) x y)) %₇) -9 (= slot₁/eval_result (call core.kwcall %₅ %₁ TestMod %₈)) -10 latestworld -11 slot₁/eval_result -12 (return %₁₁) +1 TestMod.f +2 (call core.tuple %₁) +3 (call JuliaLowering.interpolate_syntax (syntaxinert (call (syntaxunquote f) x y)) %₂) +4 (call JuliaSyntax.fill_context %₃ SyntaxContext(#=omitted=#)) +5 (= slot₁/eval_result (call JuliaLowering.eval TestMod %₄)) +6 latestworld +7 slot₁/eval_result +8 (return %₇) ######################################## # @eval with module @eval mod $f(x, y) #--------------------- -1 JuliaLowering.eval -2 (call core.tuple :expr_compat_mode) -3 (call core.apply_type core.NamedTuple %₂) -4 (call core.tuple false) -5 (call %₃ %₄) -6 TestMod.mod -7 TestMod.f -8 (call core.tuple %₇) -9 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (call ($ f) x y)) %₈) -10 (= slot₁/eval_result (call core.kwcall %₅ %₁ %₆ %₉)) -11 latestworld -12 slot₁/eval_result -13 (return %₁₂) +1 TestMod.mod +2 TestMod.f +3 (call core.tuple %₂) +4 (call JuliaLowering.interpolate_syntax (syntaxinert (call (syntaxunquote f) x y)) %₃) +5 (call JuliaSyntax.fill_context %₄ SyntaxContext(#=omitted=#)) +6 (= slot₁/eval_result (call JuliaLowering.eval %₁ %₅)) +7 latestworld +8 slot₁/eval_result +9 (return %₈) ######################################## # Juxtaposition @@ -348,14 +341,7 @@ end 4 (return %₃) ######################################## -# Juxtaposition - check the juxtapose multiply is resolved to `JuxtuposeTest.*` when -# emitted by the macro in the JuxtuposeTest module. -# -# This is consistent with Julia's existing system but it's not entirely clear -# this is good - perhaps we should resolve to Base.* instead? Resolving to the -# module-local version makes it exactly equivalent to `*`. But one might argue -# this is confusing because the symbol `*` appears nowhere in the user's source -# code. +# Juxtaposition - resolve to macro's mod's `JuxtuposeTest.*` JuxtuposeTest.@emit_juxtupose #--------------------- 1 TestMod.JuxtuposeTest.* diff --git a/JuliaLowering/test/quoting.jl b/JuliaLowering/test/quoting.jl index 2e377d3bce766..3618604979b00 100644 --- a/JuliaLowering/test/quoting.jl +++ b/JuliaLowering/test/quoting.jl @@ -1,10 +1,203 @@ -test_mod = Module() +test_mod = @newmod(quoting) + +@testset "basic quoting and dollar-interpolation" begin + @eval test_mod global interpolated_var + + @testset for run in [ + (x::String)->fl_eval(test_mod, Expr(:block, JuliaSyntax.parsestmt(Expr, x))), + (x::String)->jl_eval(test_mod, JuliaSyntax.parsestmt(SyntaxTree, x); expr_compat_mode=true), + (x::String)->jl_eval(test_mod, JuliaSyntax.parsestmt(SyntaxTree, x); expr_compat_mode=false), + ] + @test run(raw":x") == :x + @test run(raw":(:x)") == QuoteNode(:x) + @test run(raw":(:(:x))") == Expr(:quote, (QuoteNode(:x))) + @test run(raw":(:($x))") == Expr(:quote, Expr(:$, :x)) + @test run(raw":($(:($(:x))))") == :x + @test run(raw":($(:(:($x))))") == Expr(:quote, Expr(:$, :x)) + @test run(raw":(:($(:($x))))") == Expr(:quote, Expr(:$, Expr(:quote, Expr(:$, :x)))) + + @testset for ivar_val in [:y, Symbol(""), GlobalRef(Base, :push!), Expr(:call, :identity, 2), 1, nothing], + ivar in [ivar_val, Expr(:quote, ivar_val), Expr(:inert, ivar_val), QuoteNode(ivar_val)] + + Base.setglobal!(test_mod, :interpolated_var, ivar) + + @test run(raw"interpolated_var") == ivar + @test run(raw":($interpolated_var)") == ivar + @test run(raw":($(:($interpolated_var)))") == ivar + @test run(raw":(:($$interpolated_var))") == Expr(:quote, Expr(:$, ivar)) + @test run(raw":(:($($interpolated_var)))") == Expr(:quote, Expr(:$, ivar)) + @test run(raw":(identity($interpolated_var))") == Expr(:call, :identity, ivar) + end + end +end + +@testset "self-quoting forms" for + form in [1, true, "string", [], nothing,], + quoted in [Expr(:quote, form), Expr(:inert, form), QuoteNode(form)] + + @test fl_eval(test_mod, Expr(:block, quoted)) == form + @test jl_eval(test_mod, Expr(:block, quoted); expr_compat_mode=true) == form + @test jl_eval(test_mod, Expr(:block, quoted); expr_compat_mode=false) == form +end +@testset "self-quoting forms, interpolated into quote" for + form in [1, true, "string", [], nothing,], + quoted in [form, Expr(:quote, form), Expr(:inert, form), QuoteNode(form)] + + @test fl_eval(test_mod, Expr(:quote, Expr(:$, quoted))) == form + @test jl_eval(test_mod, Expr(:quote, Expr(:$, quoted)); expr_compat_mode=true) == form + @test jl_eval(test_mod, Expr(:quote, Expr(:$, quoted)); expr_compat_mode=false) == form +end + +@eval test_mod global quotesplatvar = [1,[2,[3,[4]]]] +@testset "unquote-splicing `...`" for run in [ + (x)->fl_eval(test_mod, x), + (x)->jl_eval(test_mod, x; expr_compat_mode=true), + (x)->jl_eval(test_mod, x; expr_compat_mode=false), + ] + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, + Expr(:..., :quotesplatvar))))), + Expr(:call, Base.vect, 1, [2, [3, [4]]])) + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:quote, + Expr(:$, + Expr(:..., # A quoted `...` is left unchanged + Expr(:$, + Expr(:..., :quotesplatvar))))))), + Expr(:quote, Expr(:$, Expr(:..., 1, [2, [3, [4]]])))) + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:quote, + Expr(:$, + Expr(:$, + Expr(:..., + Expr(:..., :quotesplatvar))))))), + Expr(:quote, Expr(:$, 1, 2, [3, [4]]))) + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:quote, + Expr(:$, + Expr(:$, + Expr(:..., + Expr(:..., + Expr(:..., :quotesplatvar)))))))), + Expr(:quote, Expr(:$, 1, 2, 3, [4]))) +end + +@testset "@legacy_quote_to_syntax" begin + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :x") isa SyntaxTree + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :x") |> kind === K"Identifier" + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :($1)") isa SyntaxTree + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :($1)") |> kind === K"Value" + + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :(x+1)") isa SyntaxTree + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :(x+1)") |> kind === K"call" + + # compat mode makes standard quote + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :x"; expr_compat_mode=true) == :x + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :(x+1)"; expr_compat_mode=true) == + Expr(:call, :+, :x, 1) + + # syntaxunquote does not support the equivalent of Expr(:$, :a, :b), but + # legacy_quote_to_syntax can convert it + @test expr_structure_eq( + jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, :b))))) + ; expr_compat_mode=true), + Expr(:call, Base.vect, 1, [2, [3, [4]]])) + # splat :b + @test expr_structure_eq( + jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., :b)))))) + ; expr_compat_mode=true), + Expr(:call, Base.vect, 1, 2, [3, [4]])) + # double-splat :b + @test expr_structure_eq( + jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., Expr(:..., :b))))))) + ; expr_compat_mode=true), + Expr(:call, Base.vect, 1, 2, 3, [4])) + + # with compat=false + st = jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, :b))))) + ; expr_compat_mode=false) + @test st isa SyntaxTree + @test JuliaSyntax.numchildren(st) == 3 + st = jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., :b)))))) + ; expr_compat_mode=false) + @test st isa SyntaxTree + @test JuliaSyntax.numchildren(st) == 4 + st = jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., Expr(:..., :b))))))) + ; expr_compat_mode=false) + @test st isa SyntaxTree + @test JuliaSyntax.numchildren(st) == 5 +end ex = JuliaLowering.include_string(test_mod, """ begin x = 10 - y = :(g(z)) - quote + y = @legacy_quote_to_syntax :(g(z)) + @legacy_quote_to_syntax quote f(\$(x+1), \$y) end end @@ -21,81 +214,21 @@ end ] @test sourcetext(ex[1]) == "f(\$(x+1), \$y)" @test sourcetext(ex[1][2]) == "\$(x+1)" -@test sourcetext.(flattened_provenance(ex[1][3])) == ["\$y", "g(z)"] -@test sprint(io->JuliaLowering._show_provtree(io, ex[1][3], "")) == raw""" -(call g z) -├─ (call g z) -│ └─ (call g z) -│ └─ (call g ✘ z ✘) -│ └─ @ string:3 -└─ ($ y) - └─ ($ y) - └─ ($ ::K"$" y) - └─ @ string:5 -""" -@test sprint(io->showprov(io, ex[1][3])) == raw""" - begin - x = 10 - y = :(g(z)) - # └──┘ ── in source - quote - f($(x+1), $y) - # @ string:3 - - y = :(g(z)) - quote - f($(x+1), $y) - # └┘ ── interpolated here - end - end - # @ string:5""" -@test sprint(io->showprov(io, ex[1][3]; note="foo")) == raw""" - begin - x = 10 - y = :(g(z)) - # └──┘ ── foo - quote - f($(x+1), $y) - # @ string:3 - - y = :(g(z)) - quote - f($(x+1), $y) - # └┘ ── foo - end - end - # @ string:5""" - - -# Test expression flags are preserved during interpolation -@test JuliaSyntax.is_infix_op_call(JuliaLowering.include_string(test_mod, """ -let - x = 1 - :(\$x + \$x) -end -""")) - -# Test that trivial interpolation without any nesting works. -ex = JuliaLowering.include_string(test_mod, """ -let - x = 123 - :(\$x) -end -""") -@test kind(ex) == K"Value" -@test ex.value == 123 # Test that interpolation with field access works # (the field name can be interpolated after the dot). -ex = JuliaLowering.include_string(test_mod, """ +@test JuliaLowering.include_string(test_mod, """ let - field_name = :(a) - :(x.\$field_name) + field_name = @legacy_quote_to_syntax :(a) + @legacy_quote_to_syntax :(x.\$field_name) end -""") -@test kind(ex[2]) == K"inert" -@test kind(ex[2][1]) == K"Identifier" -@test ex[2][1].name_val == "a" +""") ≈ @ast_ [K"." "x"::K"Identifier" [K"inert" "a"::K"Identifier"]] +@test JuliaLowering.include_string(test_mod, """ +let + field_name = @legacy_quote_to_syntax :(a) + @legacy_quote_to_syntax :(x.\$field_name) +end +"""; expr_compat_mode=true) == Expr(:., :x, QuoteNode(:a)) # Test quoted property access syntax like `Core.:(foo)` and `Core.:(!==)` @test JuliaLowering.include_string(test_mod, """ @@ -137,37 +270,14 @@ let end end """) -@test ex ≈ @ast_ [K"block" - [K"=" - "x"::K"Identifier" - 1::K"Integer" - ] - [K"=" - "y"::K"Identifier" - 2::K"Integer" - ] - [K"quote" - [K"block" - [K"call" - "f"::K"Identifier" - [K"$" - [K"tuple" "x"::K"Identifier" "x"::K"Identifier"] - [K"tuple" "y"::K"Identifier" "y"::K"Identifier"] - ] - ] - ] - ] -] -@test sourcetext(ex[3][1][1][2]) == "\$\$(args...)" -@test sourcetext(ex[3][1][1][2][1]) == "(x,x)" -@test sourcetext(ex[3][1][1][2][2]) == "(y,y)" - -ex2 = JuliaLowering.eval(test_mod, ex) -@test sourcetext(ex2[1][2]) == "(x,x)" -@test sourcetext(ex2[1][3]) == "(y,y)" - -@test JuliaLowering.include_string(test_mod, ":x") isa Symbol -@test JuliaLowering.include_string(test_mod, ":(x)") isa Symbol +@test Base.remove_linenums!(ex) == + Expr(:block, + Expr(:(=), :x, 1), + Expr(:(=), :y, 2), + Expr(:quote, + Expr(:block, + Expr(:call, :f, Expr(:$, Expr(:tuple, :x, :x), + Expr(:tuple, :y, :y)))))) # Double interpolation double_interp_ex = JuliaLowering.include_string(test_mod, raw""" @@ -178,8 +288,7 @@ end """) Base.eval(test_mod, :(xxx = 111)) dinterp_eval = JuliaLowering.eval(test_mod, double_interp_ex) -@test kind(dinterp_eval) == K"Value" -@test dinterp_eval.value == 111 +@test dinterp_eval == 111 multi_interp_ex = JuliaLowering.include_string(test_mod, raw""" let @@ -206,13 +315,13 @@ catch exc end @test contains(err, raw"More than one value in bare `$` expression") -# Interpolation of SyntaxTree Identifier vs plain Symbol +# Symbol should be interpolated (converted from expr) @eval test_mod using JuliaLowering symbol_interp = JuliaLowering.include_string(test_mod, """ let - x = :xx # Plain Symbol - y = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "yy") # SyntaxTree K"Identifier" - :(f(\$x, \$y, z)) + x = :xx + y = @legacy_quote_to_syntax :yy + @legacy_quote_to_syntax :(f(\$x, \$y, z)) end """) @test symbol_interp ≈ @ast_ [K"call" @@ -221,26 +330,16 @@ end "yy"::K"Identifier" "z"::K"Identifier" ] -@test sourcetext(symbol_interp[2]) == "\$x" # No provenance for plain Symbol +@test sourcetext(symbol_interp[2]) == raw"$x" @test sourcetext(symbol_interp[3]) == "yy" -# Mixing Expr into a SyntaxTree doesn't graft it onto the SyntaxTree AST but -# treats it as a plain old value. (This is the conservative API choice and also -# encourages ASTs to be written in the new form. However we may choose to -# change this if necessary for compatibility.) -expr_interp_is_value = JuliaLowering.include_string(test_mod, raw""" +# (may change) Expr interpolated into SyntaxTree +@test_throws LoweringError JuliaLowering.include_string(test_mod, raw""" let x = Expr(:call, :f, :x) - :(g($x)) + @legacy_quote_to_syntax :(g($x)) end -""") -@test expr_interp_is_value ≈ @ast_ [K"call" - "g"::K"Identifier" - Expr(:call, :f, :x)::K"Value" - # ^^ NB not [K"call" "f"::K"Identifier" "x"::K"Identifier"] -] -@test JuliaLowering.est_to_expr(expr_interp_is_value) == - Expr(:call, :g, QuoteNode(Expr(:call, :f, :x))) +""") broken=true @testset "Interpolation in Expr compat mode" begin expr_interp = JuliaLowering.include_string(test_mod, raw""" @@ -277,42 +376,27 @@ end """; expr_compat_mode=true) == Base.push! end -@testset "self-quoting forms" for expr_compat_mode in [true, false], - form in [1, true, "string", [], nothing, :symbol], - quoted in [Expr(:quote, form), Expr(:inert, form), QuoteNode(form)] - - @test fl_eval(test_mod, Expr(:block, quoted)) == form - @test jl_eval(test_mod, Expr(:block, quoted); expr_compat_mode) == form - -end - -@testset "self-quoting forms, interpolated into quote, expr compat" for - form in [1, true, "string", [], nothing, :symbol], - quoted in [Expr(:quote, form), Expr(:inert, form), QuoteNode(form)] - - @test fl_eval(test_mod, Expr(:quote, Expr(:$, quoted))) == form - @test jl_eval(test_mod, Expr(:quote, Expr(:$, quoted)); expr_compat_mode=true) == form - - # Just to track behaviour - @test jl_eval(test_mod, Expr(:quote, Expr(:$, quoted)); expr_compat_mode=false) isa SyntaxTree -end - # (. l r) should pass lowering only when r is one of: # - simple identifier (resolved variable) # - any simple atom, bare, inert, or in quote # - anything else if inert (not evaluated) +# - any valid `r` wrapped in unquote, then quote # # note Expr(:block) is to avoid the special top-level evaluation of Expr(:.) in # flisp, which skips handling :quote -@testset "getproperty quoting" begin - @eval test_mod begin - struct GetProperty; gs_field; end - Base.getproperty(::GetProperty, x) = ("got", x) - Base.getproperty(::GetProperty, x::Symbol) = ("got", x) # avoid ambiguity - - global gs = GetProperty([]) - global outer_field = :gs_field - end +@eval test_mod begin + struct GetProperty; gs_field; end + Base.getproperty(::GetProperty, x) = ("got", x) + Base.getproperty(::GetProperty, x::Symbol) = ("got", x) # avoid ambiguity + + global gs = GetProperty([]) + global outer_field = :gs_field +end +@testset "getproperty quoting" for wrap_quote in [ + identity, + x->Expr(:quote, Expr(:$, x)), + x->Expr(:quote, Expr(:$, Expr(:quote, Expr(:$, x))))] + @testset "arg2 unquoted identifier" for expr_compat_mode in [true, false] local field = :outer_field @@ -376,3 +460,57 @@ end test_mod, Expr(:block, Expr(:., test_mod.gs, field))) end end + +@testset "syntax context in macro body should be discarded" begin + # Both `x`s should have argument context in the macrocall, not new-syntax + # context (resulting in (99, 2)) + @test JuliaLowering.include_string(test_mod, raw""" + macro set_value(name, body) + @legacy_quote_to_syntax(quote + $name = 1 + $body + end) + end + (function () + x = 99 + r = @set_value x x + 1 + (x, r) + end)() + """) == (1,2) + + @test JuliaLowering.include_string(test_mod, raw""" + macro addone_value(name, body) + @legacy_quote_to_syntax(quote + x = 99 # hygienic + $name += 1 + $body + end) + end + (function () + x = 0 + out = [] + push!(out, (x, @addone_value x x + 1)) + push!(out, (x, @addone_value x x + 1)) + push!(out, (x, @addone_value x x + 1)) + out + end)() + """) == [(0, 2), (1, 3), (2, 4)] + + @test JuliaLowering.include_string(test_mod, raw""" + macro addone_value_quote2(name, body) + @legacy_quote_to_syntax(quote + x = 99 # hygienic + $(:($name)) += 1 + $(:($body)) + end) + end + (function () + x = 0 + out = [] + push!(out, (x, @addone_value_quote2 x x + 1)) + push!(out, (x, @addone_value_quote2 x x + 1)) + push!(out, (x, @addone_value_quote2 x x + 1)) + out + end)() + """) == [(0, 2), (1, 3), (2, 4)] +end diff --git a/JuliaLowering/test/quoting_ir.jl b/JuliaLowering/test/quoting_ir.jl index cfcf19739293f..b5687cd31f3a9 100644 --- a/JuliaLowering/test/quoting_ir.jl +++ b/JuliaLowering/test/quoting_ir.jl @@ -6,7 +6,7 @@ end #--------------------- 1 TestMod.x 2 (call core.tuple %₁) -3 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block (call-i + ($ x) 1))) %₂) +3 (call JuliaLowering.interpolate_expr (inert (block (call-i + ($ x) 1))) %₂) 4 (return %₃) ######################################## @@ -14,9 +14,7 @@ end :($x) #--------------------- 1 TestMod.x -2 (call core.tuple %₁) -3 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree ($ x)) %₂) -4 (return %₃) +2 (return %₁) ######################################## # Double escape @@ -28,7 +26,7 @@ end #--------------------- 1 TestMod.x 2 (call core.tuple %₁) -3 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (block (quote (block (call-i + ($ ($ x)) 1))))) %₂) +3 (call JuliaLowering.interpolate_expr (inert (block (quote (block (call-i + ($ ($ x)) 1))))) %₂) 4 (return %₃) ######################################## @@ -40,7 +38,7 @@ end 1 1 2 (= slot₁/x %₁) 3 (call core.tuple slot₁/x) -4 (call JuliaLowering.interpolate_ast SyntaxTree (inert_syntaxtree (. A (inert ($ x)))) %₃) +4 (call JuliaLowering.interpolate_expr (inert (. A (inert ($ x)))) %₃) 5 (return %₄) ######################################## diff --git a/JuliaLowering/test/runtests.jl b/JuliaLowering/test/runtests.jl index 35131c9ec8257..b0fd4817cd5b2 100644 --- a/JuliaLowering/test/runtests.jl +++ b/JuliaLowering/test/runtests.jl @@ -18,7 +18,6 @@ include("utils.jl") @testset "decls" include("decls.jl") @testset "destructuring" include("destructuring.jl") @testset "exceptions" include("exceptions.jl") - @testset "functions" include("functions.jl") @testset "generators" include("generators.jl") @testset "import" include("import.jl") @testset "loops" include("loops.jl") @@ -28,6 +27,7 @@ include("utils.jl") @testset "quoting" include("quoting.jl") @testset "scopes" include("scopes.jl") @testset "typedefs" include("typedefs.jl") + @testset "functions" include("functions.jl") @testset "provenance" include("provenance.jl") @testset "compat" include("compat.jl") diff --git a/JuliaLowering/test/scopes.jl b/JuliaLowering/test/scopes.jl index 5dca74e8ee290..013bccb5624f1 100644 --- a/JuliaLowering/test/scopes.jl +++ b/JuliaLowering/test/scopes.jl @@ -418,8 +418,9 @@ function resolve_and_get_bindings( soft_scope::Union{Nothing,Bool} = nothing, ) est = JuliaLowering.expr_to_est(ex) - ctx1, ex1 = JuliaLowering.expand_forms_1(mod, est, false, world) - ctx2, ex2 = JuliaLowering.expand_forms_2(ctx1, ex1) + ex0 = JuliaLowering.rebase_layers(est, mod, JuliaLowering.JL_NEW_SYNTAX_VERSION) + ex1 = JuliaLowering.expand_forms_1(ex0, world, true) + ctx2, ex2 = JuliaLowering.expand_forms_2(ex1, world) ctx3, _ = JuliaLowering.resolve_scopes(ctx2, ex2; soft_scope) return ctx3.bindings.info end @@ -560,8 +561,8 @@ end @test !isdefined(test_mod.macro_mod, :f_local_1) JuliaLowering.include_string(test_mod, "macro_mod.@mesc function f_nonlocal_2(); 1; end") @test isdefined(test_mod, :f_nonlocal_2) - # An unescaped const is local to a macro expansion - @test_throws LoweringError JuliaLowering.include_string(test_mod, "macro_mod.@m const c_local_1 = 1") + # An unescaped const should not error coming from an old-style macro + @test JuliaLowering.include_string(test_mod, "macro_mod.@m const c_local_1 = 1") == 1 # The const may be escaped into test_mod JuliaLowering.include_string(test_mod, "macro_mod.@mesc const c_nonlocal_2 = 1") @test isdefined(test_mod, :c_nonlocal_2) @@ -941,7 +942,7 @@ end end)) run(:(import ..test_mod.@old_hyg_g_rescope_conflict_new3)) Core.@latestworld - @test_throws "unhygienic global" run(:(@old_hyg_g_rescope_conflict_new3())) context=ctx + @test_throws "conflicts with an existing local variable" run(:(@old_hyg_g_rescope_conflict_new3())) context=ctx fl_eval(test_mod, :(macro old_hyg_g_rescope_conflict_new4(); :(let @@ -951,7 +952,7 @@ end end)) run(:(import ..test_mod.@old_hyg_g_rescope_conflict_new4)) Core.@latestworld - @test_throws "unhygienic global" run(:(@old_hyg_g_rescope_conflict_new4())) context=ctx + @test_throws "conflicts with an existing local variable" run(:(@old_hyg_g_rescope_conflict_new4())) context=ctx end # A reference in the same scope as the declaration: flisp keeps the @@ -960,8 +961,8 @@ end # created in the calling module, which is more consistent. @testset "references in the declaring scope" for (ctx, mod, run, ref_resolves) in [ ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x), false), - ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true), true), - ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false), true)] + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true), false), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false), false)] fl_eval(test_mod, :(macro old_hyg_g_ref_top(); quote diff --git a/JuliaLowering/test/typedefs_ir.jl b/JuliaLowering/test/typedefs_ir.jl index ca0dd7bf1896d..62b41fed94a06 100644 --- a/JuliaLowering/test/typedefs_ir.jl +++ b/JuliaLowering/test/typedefs_ir.jl @@ -1264,49 +1264,45 @@ typegroup end end #--------------------- -1 (call core.declare_global TestMod :A false) -2 latestworld -3 (call core.declare_global TestMod :B false) -4 latestworld -5 (= slot₂/A (call core.TypeVar :A)) -6 (= slot₃/B (call core.TypeVar :B)) -7 (call core.svec) -8 (call core.svec :b) +1 (= slot₂/A (call core.TypeVar :A)) +2 (= slot₃/B (call core.TypeVar :B)) +3 (call core.svec) +4 (call core.svec :b) +5 (call core.svec) +6 slot₃/B +7 (call core.svec %₆) +8 (call core.svec %₃ %₄ %₅ false 1 core.Any %₇) 9 (call core.svec) -10 slot₃/B -11 (call core.svec %₁₀) -12 (call core.svec %₇ %₈ %₉ false 1 core.Any %₁₁) -13 (call core.svec) -14 (call core.svec :a) -15 (call core.svec) -16 slot₂/A -17 (call core.svec %₁₆) -18 (call core.svec %₁₃ %₁₄ %₁₅ false 1 core.Any %₁₇) -19 slot₂/A -20 slot₃/B -21 (call core.svec %₁₉ %₂₀) -22 (call core.svec %₁₂ %₁₈) -23 (call core.resolve_typegroup TestMod %₂₁ %₂₂) -24 (call top.indexed_iterate %₂₃ 1) -25 (= slot₂/A (call core.getfield %₂₄ 1)) -26 (= slot₁/iterstate (call core.getfield %₂₄ 2)) -27 slot₁/iterstate -28 (call top.indexed_iterate %₂₃ 2 %₂₇) -29 (= slot₃/B (call core.getfield %₂₈ 1)) -30 slot₂/A -31 (call core.declare_const TestMod :A %₃₀) -32 latestworld -33 slot₃/B -34 (call core.declare_const TestMod :B %₃₃) -35 latestworld -36 TestMod.A -37 SourceLocation::2:5 -38 (call top._defaultctors %₃₆ %₃₇) -39 TestMod.B -40 SourceLocation::5:5 -41 (call top._defaultctors %₃₉ %₄₀) -42 latestworld -43 (return core.nothing) +10 (call core.svec :a) +11 (call core.svec) +12 slot₂/A +13 (call core.svec %₁₂) +14 (call core.svec %₉ %₁₀ %₁₁ false 1 core.Any %₁₃) +15 slot₂/A +16 slot₃/B +17 (call core.svec %₁₅ %₁₆) +18 (call core.svec %₈ %₁₄) +19 (call core.resolve_typegroup TestMod %₁₇ %₁₈) +20 (call top.indexed_iterate %₁₉ 1) +21 (= slot₂/A (call core.getfield %₂₀ 1)) +22 (= slot₁/iterstate (call core.getfield %₂₀ 2)) +23 slot₁/iterstate +24 (call top.indexed_iterate %₁₉ 2 %₂₃) +25 (= slot₃/B (call core.getfield %₂₄ 1)) +26 slot₂/A +27 (call core.declare_const TestMod :A %₂₆) +28 latestworld +29 slot₃/B +30 (call core.declare_const TestMod :B %₂₉) +31 latestworld +32 TestMod.A +33 SourceLocation::2:5 +34 (call top._defaultctors %₃₂ %₃₃) +35 TestMod.B +36 SourceLocation::5:5 +37 (call top._defaultctors %₃₅ %₃₆) +38 latestworld +39 (return core.nothing) ######################################## # Typegroup with supertype and apply_type_or_typeapp replacement @@ -1319,55 +1315,51 @@ typegroup end end #--------------------- -1 (call core.declare_global TestMod :A false) -2 latestworld -3 (call core.declare_global TestMod :B false) -4 latestworld -5 (= slot₂/A (call core.TypeVar :A)) -6 (= slot₃/B (call core.TypeVar :B)) -7 (call core.svec) -8 (call core.svec :b) -9 (call core.svec) -10 TestMod.AbstractVector +1 (= slot₂/A (call core.TypeVar :A)) +2 (= slot₃/B (call core.TypeVar :B)) +3 (call core.svec) +4 (call core.svec :b) +5 (call core.svec) +6 TestMod.AbstractVector +7 slot₃/B +8 (call core.apply_type_or_typeapp %₆ %₇) +9 TestMod.Union +10 TestMod.Nothing 11 slot₃/B -12 (call core.apply_type_or_typeapp %₁₀ %₁₁) -13 TestMod.Union -14 TestMod.Nothing -15 slot₃/B -16 (call core.apply_type_or_typeapp %₁₃ %₁₄ %₁₅) -17 (call core.svec %₁₆) -18 (call core.svec %₇ %₈ %₉ false 1 %₁₂ %₁₇) -19 (call core.svec) -20 (call core.svec :a) -21 (call core.svec) -22 slot₂/A -23 (call core.svec %₂₂) -24 (call core.svec %₁₉ %₂₀ %₂₁ false 1 core.Any %₂₃) -25 slot₂/A -26 slot₃/B -27 (call core.svec %₂₅ %₂₆) -28 (call core.svec %₁₈ %₂₄) -29 (call core.resolve_typegroup TestMod %₂₇ %₂₈) -30 (call top.indexed_iterate %₂₉ 1) -31 (= slot₂/A (call core.getfield %₃₀ 1)) -32 (= slot₁/iterstate (call core.getfield %₃₀ 2)) -33 slot₁/iterstate -34 (call top.indexed_iterate %₂₉ 2 %₃₃) -35 (= slot₃/B (call core.getfield %₃₄ 1)) -36 slot₂/A -37 (call core.declare_const TestMod :A %₃₆) -38 latestworld -39 slot₃/B -40 (call core.declare_const TestMod :B %₃₉) -41 latestworld -42 TestMod.A -43 SourceLocation::2:5 -44 (call top._defaultctors %₄₂ %₄₃) -45 TestMod.B -46 SourceLocation::5:5 -47 (call top._defaultctors %₄₅ %₄₆) -48 latestworld -49 (return core.nothing) +12 (call core.apply_type_or_typeapp %₉ %₁₀ %₁₁) +13 (call core.svec %₁₂) +14 (call core.svec %₃ %₄ %₅ false 1 %₈ %₁₃) +15 (call core.svec) +16 (call core.svec :a) +17 (call core.svec) +18 slot₂/A +19 (call core.svec %₁₈) +20 (call core.svec %₁₅ %₁₆ %₁₇ false 1 core.Any %₁₉) +21 slot₂/A +22 slot₃/B +23 (call core.svec %₂₁ %₂₂) +24 (call core.svec %₁₄ %₂₀) +25 (call core.resolve_typegroup TestMod %₂₃ %₂₄) +26 (call top.indexed_iterate %₂₅ 1) +27 (= slot₂/A (call core.getfield %₂₆ 1)) +28 (= slot₁/iterstate (call core.getfield %₂₆ 2)) +29 slot₁/iterstate +30 (call top.indexed_iterate %₂₅ 2 %₂₉) +31 (= slot₃/B (call core.getfield %₃₀ 1)) +32 slot₂/A +33 (call core.declare_const TestMod :A %₃₂) +34 latestworld +35 slot₃/B +36 (call core.declare_const TestMod :B %₃₅) +37 latestworld +38 TestMod.A +39 SourceLocation::2:5 +40 (call top._defaultctors %₃₈ %₃₉) +41 TestMod.B +42 SourceLocation::5:5 +43 (call top._defaultctors %₄₁ %₄₂) +44 latestworld +45 (return core.nothing) ######################################## # Error: Duplicate field name in struct diff --git a/JuliaLowering/test/utils.jl b/JuliaLowering/test/utils.jl index 42f30220f7e27..dd87116a81033 100644 --- a/JuliaLowering/test/utils.jl +++ b/JuliaLowering/test/utils.jl @@ -364,6 +364,32 @@ function reduce_any_failing_toplevel(mod::Module, filename::AbstractString; do_e nothing end +function expr_structure_eq(e1,e2) + @nospecialize + typeof(e1) == typeof(e2) || return false + e1 isa Expr || return e1 == e2 + e1.head === e2.head || return false + length(e1.args) == length(e2.args) || return false + for (a, b) in Iterators.zip(e1.args, e2.args) + expr_structure_eq(a,b) || return false + end + true +end + +macro newmod(name="newmod_$(string(__source__))", parentmod=__module__, body...) + mod_ex = :( + module $(Symbol(name)) + const JuliaLowering = $(JuliaLowering) + const JuliaSyntax = $(JuliaSyntax) + const var"@K_str" = JuliaSyntax.var"@K_str" + const var"@legacy_quote_to_syntax" = JuliaLowering.var"@legacy_quote_to_syntax" + $(body...) + end) + Expr(:block, + :(mod = $(Expr(:escape, :(Core.eval($parentmod, $(QuoteNode(mod_ex))))))), + Expr(Symbol("latestworld-if-toplevel")), :mod) +end + function fl_macroexpand(mod::Module, x::Expr) ccall(:jl_macroexpand, Any, (Any, Any, Cint, Cint, Cint), x, mod, true, false, true) end @@ -377,7 +403,7 @@ function fl_eval(mod::Module, x::Expr) end function jl_macroexpand(mod::Module, x::SyntaxTree; expr_compat_mode=false) - JuliaLowering.expand_forms_1(mod, x, expr_compat_mode, Base.get_world_counter())[2] + JuliaLowering.macroexpand(mod, x; expr_compat_mode) end function jl_lower(mod::Module, st::SyntaxTree; expr_compat_mode=false) diff --git a/JuliaLowering/test/validation.jl b/JuliaLowering/test/validation.jl index 976922e3d2733..9a58174906ce3 100644 --- a/JuliaLowering/test/validation.jl +++ b/JuliaLowering/test/validation.jl @@ -39,7 +39,7 @@ let "macrocall", "quote", "inert", - "inert_syntaxtree", + "syntaxinert", "top", "opaque_closure", "symboliclabel", diff --git a/JuliaSyntax/src/integration/expr.jl b/JuliaSyntax/src/integration/expr.jl index 1381c5f98707c..39edcd10bbb09 100644 --- a/JuliaSyntax/src/integration/expr.jl +++ b/JuliaSyntax/src/integration/expr.jl @@ -244,8 +244,7 @@ function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt elseif k == K"VERSION" return version_to_expr(nodehead) else - scoped_val = _expr_leaf_val(cursor, txtbuf, txtbuf_offset) - val = @isexpr(scoped_val, :scope_layer) ? scoped_val.args[1] : scoped_val + val = _expr_leaf_val(cursor, txtbuf, txtbuf_offset) if val isa Union{Int128,UInt128,BigInt} # Ignore the values of large integers and convert them back to # symbolic/textual form for compatibility with the Expr @@ -256,11 +255,9 @@ function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt Symbol("@big_str") return Expr(:macrocall, GlobalRef(Core, macname), nothing, str) elseif is_identifier(k) - val2 = lower_identifier_name(val, k) - return @isexpr(scoped_val, :scope_layer) ? - Expr(:scope_layer, val2, scoped_val.args[2]) : val2 + return lower_identifier_name(val, k) else - return scoped_val + return val end end end diff --git a/JuliaSyntax/src/julia/kinds.jl b/JuliaSyntax/src/julia/kinds.jl index 02aa0af407865..bd3c5acb76a38 100644 --- a/JuliaSyntax/src/julia/kinds.jl +++ b/JuliaSyntax/src/julia/kinds.jl @@ -406,7 +406,7 @@ register_kinds!(JuliaSyntax, 0, [ "flatten" # QuoteNode; not quasiquote "inert" - "inert_syntaxtree" + "syntaxinert" "END_SYNTAXTREE_KINDS" # Special tokens diff --git a/JuliaSyntax/src/porcelain/syntax_graph.jl b/JuliaSyntax/src/porcelain/syntax_graph.jl index 4962fe5abda93..c4efa7c7d4d7b 100644 --- a/JuliaSyntax/src/porcelain/syntax_graph.jl +++ b/JuliaSyntax/src/porcelain/syntax_graph.jl @@ -77,6 +77,7 @@ ensure_required_attributes!(g::SyntaxGraph) = ensure_attributes!( g, kind=Kind, source=SourceAttrType, + context=SyntaxContext, syntax_flags=UInt16, value=Any, name_val=String, @@ -294,6 +295,7 @@ setattr(ex::SyntaxTree, name::Symbol, @nospecialize(val)) = function deleteattr!(ex::SyntaxTree, name::Symbol) deleteattr!(ex._graph, ex._id, name) + ex end # JuliaSyntax tree API @@ -322,6 +324,114 @@ function flags(ex::SyntaxTree) get(ex, :syntax_flags, 0x0000)::UInt16 end +mutable struct ScopeLayer + const mod::Module + const escaped::Union{Nothing, ScopeLayer} +end +Base.var"=="(si1::ScopeLayer, si2::ScopeLayer) = si1 === si2 + +""" +Each node has a SyntaxContext describing its macro expansion and syntax version. +`SyntaxContext` is shared between all nodes of a single macro expansion, and is +one-to-one with ScopeLayer, with a few exceptions (contexts sharing same layer): +- `escape` and adopt_scope +- Desugaring creates internal contexts in its better version of `gensym` + +We may want to move layer out of this struct for easier adopt_scope and +rebase_layer operations, but assuming mostly hygienic macros and few +scope-changing functions, this is most compact. + +(TODO) GlobalRef (attr :mod) could probably also go through this system +""" +mutable struct SyntaxContext + const layer::ScopeLayer + # For provenance; is not affected by escaping + const unexpanded::Union{SyntaxTree, Nothing} + const version::VersionNumber + const internal::Bool +end + +# A default context corresponding to no expansion +function SyntaxContext(mod::Module, version::VersionNumber) + SyntaxContext(ScopeLayer(mod, nothing), nothing, version, false) +end + +# TODO: switch from bool-based `expr_compat_mode` to `version` +const JL_NEW_SYNTAX_VERSION = v"1.14" +const JL_OLD_SYNTAX_VERSION = v"1.13" + +is_base_layer(sc::SyntaxContext) = sc.layer.escaped === nothing + +# The scope corresponding to no macro expansion. Use with caution: macros may +# expand to top-level forms, so "base layer" !== "this top-level thunk's +# pre-expansion context" (usually ctx.syntax_context) +function base_layer(sc::SyntaxContext) + l = sc.layer + while l.escaped !== nothing + l = l.escaped + end + return l +end + +function escape_layer(sc::SyntaxContext, recursive) + l2 = recursive ? base_layer(sc) : sc.layer.escaped + SyntaxContext(l2, sc.unexpanded, sc.version, sc.internal) +end + +syntax_module(sc::SyntaxContext) = sc.layer.mod +function syntax_module(st::SyntaxTree) + st_mod = get(st, :mod, nothing) + st_mod === nothing || return st_mod::Module + syntax_module(st.context::SyntaxContext) +end + +is_flisp_compat(sc::SyntaxContext) = sc.version < JL_NEW_SYNTAX_VERSION +is_flisp_compat(st::SyntaxTree) = is_flisp_compat(st.context) + +# Unconditional; tramples existing scope, and includes quoted forms. Only +# changes layer where it needs changing. +function adopt_scope(sc_in::SyntaxContext, st::SyntaxTree, scmap) + st_sc = get(st, :context, nothing) + sc2 = st_sc isa SyntaxContext ? get(scmap, st_sc, nothing) : nothing + if isnothing(sc2) && st_sc isa SyntaxContext + sc2 = scmap[st_sc] = st_sc.layer === sc_in.layer ? st_sc : + SyntaxContext( + sc_in.layer, st_sc.unexpanded, st_sc.version, st_sc.internal) + elseif isnothing(sc2) + sc2 = sc_in + end + if is_leaf(st) || numchildren(st) == 0 + sc2 === st_sc ? st : setattr(st, :context, sc2) + else + out = mapchildren(c->adopt_scope(sc_in, c, scmap), st._graph, st) + sc2 === st_sc ? out : + out !== st ? setattr!(out, :context, sc2) : + setattr(out, :context, sc2) + end +end +function adopt_scope(reference::SyntaxTree, st::SyntaxTree) + adopt_scope(reference.context::SyntaxContext, st, + Dict{SyntaxContext, SyntaxContext}()) +end + +function fill_context!(st::SyntaxTree, sc::SyntaxContext) + setattr!(st, :context, sc) + for c in children(st) + fill_context!(c, sc) + end + st +end +fill_context(st, sc) = fill_context!(mktree(st), sc) + +function remove_context!(st::SyntaxTree) + sc = get(st, :context, nothing) + isnothing(sc) || JuliaSyntax.deleteattr!(st, :context) + for c in children(st) + remove_context!(c) + end + st +end +remove_context(st) = remove_context!(mktree(st)) # Reference to bytes within a source file struct SourceRef @@ -330,6 +440,38 @@ struct SourceRef last_byte::UInt32 end +function Base.show(io::IO, ::MIME"text/plain", sl::ScopeLayer) + color = isnothing(sl.escaped) ? :normal : :cyan + printstyled(io, "SL("; color) + print(io, string(sl.mod)) + print(io, ",") + !isnothing(sl.escaped) && print(io, sl.escaped) + print(io, ",") + printstyled(io, string(objectid(sl);base=62); color) + printstyled(io, ")"; color) +end +Base.show(io::IO, sl::ScopeLayer) = Base.show(io::IO, MIME"text/plain"(), sl) + +function Base.show(io::IO, ::MIME"text/plain", sc::SyntaxContext) + color = sc.internal ? :light_black : + sc.version == JL_NEW_SYNTAX_VERSION ? :normal : :blue + printstyled(io, "["; color) + if sc.version != JL_NEW_SYNTAX_VERSION + printstyled(io, "old,"; color) + end + if sc.internal + printstyled(io, "internal,"; color) + end + print(io, sc.layer) + print(io, ",") + if sc.unexpanded isa SyntaxTree + k = kind(sc.unexpanded) + k === K"macrocall" ? print(io, sc.unexpanded[1]) : print(io, k) + end + printstyled(io, "]"; color) +end +Base.show(io::IO, sc::SyntaxContext) = Base.show(io::IO, MIME"text/plain"(), sc) + sourcefile(src::SourceRef) = src.file[] first_byte(src::SourceRef) = Int(src.first_byte) last_byte(src::SourceRef) = Int(src.last_byte) @@ -370,9 +512,6 @@ Provenance notes: A SyntaxTree `st` has `.source` equal to one of: Let "textref" refer to a SyntaxTree with non-NodeId `.source`. Every SyntaxTree is either a textref or has one at the end of its `.source` chain. -`st` may also have `.macro_source`, which is the NodeId of a macrocall if `st` -was returned from its expansion. - All invariants noted in this section are awaiting the design of the "new macro" API. As of writing this, the user has more freedom than they should have. """ @@ -405,18 +544,17 @@ function prov_end(st::SyntaxTree) return out end -"`st`'s textref's `.source`, ignoring all `.macro_source`" +"`st`'s textref's `.source`, ignoring all expansions" function sourceref(st::SyntaxTree) prov_end(st).source::Union{LineNumberNode, SourceRef} end "The last macro expansion `st` was involved in, or nothing" function macro_prov(st::SyntaxTree) - while !hasattr(st, :macro_source) && st.source isa NodeId - st = prov(st) - end - hasattr(st, :macro_source) && return SyntaxTree(st._graph, st.macro_source) - return nothing + sc = get(st, :context, nothing) + isnothing(sc) && return nothing + msrc = sc.unexpanded + isnothing(msrc) ? nothing : msrc::SyntaxTree end "The first macro expansion `st` was involved in (chronologically), or nothing" @@ -453,11 +591,11 @@ function flattened_provenance(st::SyntaxTree) _flattened_provenance(st, SyntaxList(st._graph)) end -# Only recurse on the first .macro_source in any source chain +# Only recurse on the first macro source in any source chain function _flattened_provenance(st::SyntaxTree, out) msrc = macro_prov(st) - # macro_source === source means `st` is from the `msrc` macro body - !isnothing(msrc) && msrc._id !== st.source && + # macro source === source means `st` is from the `msrc` macro body + !isnothing(msrc) && msrc != prov(st) && _flattened_provenance(msrc, out) push!(out, prov_end(st)) out @@ -665,6 +803,10 @@ end function newleaf(graph::SyntaxGraph, prov::SourceAttrType, k::Kind) st = SyntaxTree(graph, new_id!(graph)) setattr!(st, :kind, k) + let prov_st = prov isa NodeId ? SyntaxTree(graph, prov) : nothing + !isnothing(prov_st) && hasattr(prov_st, :context) && + setattr!(st, :context, prov_st.context) + end setattr!(st, :source, prov) end @@ -690,6 +832,7 @@ function mkleaf(old::SyntaxTree) graph = syntax_graph(old) st = SyntaxTree(graph, new_id!(graph)) copy_attrs!(st, old) + hasattr(old, :context) && setattr!(st, :context, old.context) setattr!(st, :source, old._id) end function mktree(old::SyntaxTree) @@ -706,7 +849,7 @@ end function copy_attrs!(dest, src) # TODO: Make this faster? for (name, attr) in pairs(src._graph.attributes) - if (name !== :source && name !== :macro_source) && haskey(attr, src._id) + if (name !== :source) && haskey(attr, src._id) setattr!(dest, name, attr[src._id]) end end @@ -765,14 +908,12 @@ function _copy_ast(graph2::SyntaxGraph, graph1::SyntaxGraph, id1::NodeId, seen) end setchildren!(graph2, id2, cs) end - for src_attr in (:source, :macro_source) - src1 = get(SyntaxTree(graph1, id1), src_attr, nothing) - if src1 isa NodeId - src2 = _copy_ast(graph2, graph1, src1, seen) - setattr!(graph2, id2, src_attr, src2) - elseif src_attr == :source - setattr!(graph2, id2, src_attr, src1) - end + src1 = get(SyntaxTree(graph1, id1), :source, nothing) + if src1 isa NodeId + src2 = _copy_ast(graph2, graph1, src1, seen) + setattr!(graph2, id2, :source, src2) + else + setattr!(graph2, id2, :source, src1) end copy_attrs!(SyntaxTree(graph2, id2), SyntaxTree(graph1, id1)) return id2 @@ -811,7 +952,6 @@ function _unalias_copy_tree(old::SyntaxTree) mknode(old, cs) end # difference from mktree: don't add to provenance chain - hasattr(old, :macro_source) && setattr!(out, :macro_source, old.macro_source) setattr!(out, :source, old.source) end @@ -885,7 +1025,7 @@ function prune(graph1_a::SyntaxGraph, entrypoints_a::Vector{NodeId}) append!(graph2.edges, 1:length(nodes1)) # our reward for unaliasing for attr in attrnames(graph1) - (attr === :source || attr === :macro_source) && continue + (attr === :source) && continue for (n2, n1) in enumerate(nodes1) if haskey(graph1.attributes[attr], n1) graph2.attributes[attr][n2] = graph1.attributes[attr][n1] @@ -899,12 +1039,6 @@ function prune(graph1_a::SyntaxGraph, entrypoints_a::Vector{NodeId}) for (n2, n1) in enumerate(nodes1) graph2.source[n2] = _prune_get_resolved!(n1, graph1, map12, resolved_sources, :source) - if hasattr(graph1, :macro_source) && haskey(graph1.macro_source, n1) - msrc1 = graph1.macro_source[n1] - if haskey(map12, msrc1) - graph2.macro_source[n2] = map12[msrc1] - end - end end # The first n entries in nodes1 were our entrypoints, unique from unaliasing diff --git a/JuliaSyntax/test/syntax_graph.jl b/JuliaSyntax/test/syntax_graph.jl index 900cf260c9447..b3895a45bd66e 100644 --- a/JuliaSyntax/test/syntax_graph.jl +++ b/JuliaSyntax/test/syntax_graph.jl @@ -1,4 +1,4 @@ -using .JuliaSyntax: SyntaxGraph, SyntaxTree, SyntaxList, ensure_attributes, ensure_attributes!, delete_attributes, copy_ast, attrdefs, @stm, NodeId, SourceRef, SourceAttrType, Kind, syntax_graph, prov, prov_end, provenance, macro_prov, macro_prov_end, flattened_provenance, sourceref, unexpanded_sourceref, newleaf, mkleaf, mknode, mktree, setattr!, hasattr +using .JuliaSyntax: SyntaxGraph, SyntaxTree, SyntaxList, ensure_attributes, ensure_attributes!, delete_attributes, copy_ast, attrdefs, @stm, NodeId, SourceRef, SourceAttrType, Kind, syntax_graph, prov, prov_end, provenance, macro_prov, macro_prov_end, flattened_provenance, sourceref, unexpanded_sourceref, newleaf, mkleaf, mknode, mktree, setattr!, hasattr, SyntaxContext, ScopeLayer "For filling required attrs in graphs created by hand" function testgraph(edge_ranges, edges, more_attrs...) @@ -20,6 +20,7 @@ end SyntaxGraph(Vector{UnitRange{Int}}(), Vector{NodeId}(), (;)), kind=Kind, source=SourceAttrType, + context=SyntaxContext, syntax_flags=UInt16, value=Any, name_val=String, @@ -103,8 +104,13 @@ end end @testset "flattened_provenance" begin + ctx_with_unexpanded(u) = SyntaxContext( + ScopeLayer(JuliaSyntax, nothing), + u, + v"0.0", + false) + g = SyntaxGraph() - ensure_attributes!(g; macro_source=NodeId) st1 = setattr!(newleaf(g, LineNumberNode(1), K"Identifier"), :name_val, "st1") st2 = setattr!(mkleaf(st1), :name_val, "st2") st3 = setattr!(mkleaf(st2), :name_val, "st3") @@ -118,10 +124,10 @@ end stmm2 = setattr!(mkleaf(stmm1), :name_val, "stmm2") stmm3 = setattr!(mkleaf(stmm2), :name_val, "stmm3") - setattr!(st1, :macro_source, stm_unused._id) - setattr!(st2, :macro_source, stm_unused._id) - setattr!(st3, :macro_source, stm3._id) - setattr!(stm3, :macro_source, stmm3._id) + setattr!(st1, :context, ctx_with_unexpanded(stm_unused)) + setattr!(st2, :context, ctx_with_unexpanded(stm_unused)) + setattr!(st3, :context, ctx_with_unexpanded(stm3)) + setattr!(stm3, :context, ctx_with_unexpanded(stmm3)) # julia> JL._show_provtree(stdout, st3, "") # st3 @@ -180,9 +186,6 @@ end end @testset "SyntaxTree utils" begin - mprov(st::SyntaxTree) = get(st, :macro_source, nothing) isa NodeId ? - SyntaxTree(st._graph, st.macro_source) : nothing - @testset "copy_ast, mktree" begin # 1 --> 2 --> 3 src(7-9) = line 7-9 # 4 --> 5 --> 6 src(i) = i + 3 @@ -364,79 +367,25 @@ end # 3 --> 6 g = testgraph([1:1, 2:2, 3:3, 0:-1, 0:-1, 0:-1], [4, 5, 6], :source => Dict{Int, SourceAttrType}( - 1=>2,map(i->(i=>LineNumberNode(i)), 2:6)...), - :macro_source => Dict{Int, SourceAttrType}(1=>3)) + 1=>2,map(i->(i=>LineNumberNode(i)), 2:6)...)) st = SyntaxTree(g, 1) let stp = JuliaSyntax.prune(st) @test st ≈ stp @test length(stp._graph.edge_ranges) === 2 @test stp.orig == 1 @test stp[1].orig == 4 - @test isempty(stp._graph.macro_source) end let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[2])) @test st ≈ stp @test length(stp._graph.edge_ranges) === 4 @test stp.orig == 1 @test stp[1].orig == 4 - @test isempty(stp._graph.macro_source) @test prov(stp).orig == 2 end let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[2, 3])) @test st ≈ stp @test prov(st) ≈ prov(stp) @test length(stp._graph.edge_ranges) === 6 - @test hasattr(stp, :macro_source) - @test SyntaxTree(stp._graph, stp.macro_source).orig == 3 - end - - # 1 - # 10 -->[2]--> 6 src(2) = 3 - # 11 --> 3 --> 7 src(3) = 4, msrc(3) = 4 - # 12 --> 4 --> 8 src(4) = 5 - # 13 --> 5 --> 9 else src(i) = line(i) - g = testgraph([0:-1, 1:1, 2:2, 3:3, 4:4, 0:-1, 0:-1, 0:-1, 0:-1, 5:5, 6:6, 7:7, 8:8], - [6, 7, 8, 9, 2, 3, 4, 5], - :source => Dict{Int, SourceAttrType}( - 2=>3, 3=>4, 4=>5, 1=>LineNumberNode(1), - map(i->(i=>LineNumberNode(i)), 4:12)...), - :macro_source => Dict{Int, SourceAttrType}(3=>4)) - st = SyntaxTree(g, 2) - stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[5])) - @test st ≈ stp - # 1, 5, 4, 8 should remain; macro_source is removed - @test length(stp._graph.edge_ranges) === 4 - @test isempty(stp._graph.macro_source) - - # need both 3 and 4 preserved to keep the macro_source link (though we - # may want it to be a stronger prune-preserved reference in the future.) - let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[5, 3])) - @test st ≈ stp - @test length(stp._graph.edge_ranges) === 6 - @test isempty(stp._graph.macro_source) - end - let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[5, 4])) - @test st ≈ stp - @test length(stp._graph.edge_ranges) === 6 - @test isempty(stp._graph.macro_source) - end - let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[3])) - @test st ≈ stp - @test length(stp._graph.edge_ranges) === 4 - @test isempty(stp._graph.macro_source) - end - let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[4])) - @test st ≈ stp - @test length(stp._graph.edge_ranges) === 4 - @test isempty(stp._graph.macro_source) - end - let stp = JuliaSyntax.prune(st; keep=SyntaxList(g, NodeId[3, 4])) - @test st ≈ stp - @test length(stp._graph.edge_ranges) === 6 - @test hasattr(prov(stp), :macro_source) - new_4 = SyntaxTree(stp._graph, prov(stp).macro_source) - @test new_4.orig == 4 - @test new_4[1].source == LineNumberNode(8) end # test with real parsed, then copied output---not many properties we can