Skip to content

[JuliaLowering] Syntax versioning and macro-expansion improvements #62221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
topolarity merged 6 commits into from
Jul 10, 2026
Merged

[JuliaLowering] Syntax versioning and macro-expansion improvements #62221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b5d21e5
[JuliaLowering] Syntax versioning and macro-expansion improvements
mlechu Jun 24, 2026
d894910
[JuliaLowering] Ensure `import`,`using`,`@eval` disobey scope layer
mlechu Jul 8, 2026
97d32fa
Add AI tests for `@eval`
mlechu Jul 8, 2026
10b5170
Fix module passed to old macros
mlechu Jul 8, 2026
183fb38
Stop calling it an "ABI"
mlechu Jul 8, 2026
2c87d08
Misc fixes
mlechu Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion JuliaLowering/src/JuliaLowering.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ using .JuliaSyntax: @KSet_str, @stm, Kind, NodeId, SourceAttrType, SourceRef, Sy
flattened_provenance, hasattr, head, highlight, is_compatible_graph,
is_leaf, is_literal, kind, last_byte, mapchildren, mapsyntax, mkleaf, mknode, newleaf,
newnode, node_string, numchildren, provenance, reparent, setattr, setattr!,
source_location, sourcefile, sourceref, syntax_graph, tree_ids, mapindex, mktree
source_location, sourcefile, sourceref, syntax_graph, tree_ids, mapindex, mktree,
ScopeLayer, SyntaxContext, is_base_layer, base_layer, escape_layer,
syntax_module, is_flisp_compat, adopt_scope, remove_context, fill_context!,
fill_context, JL_NEW_SYNTAX_VERSION, JL_OLD_SYNTAX_VERSION

const DEBUG = true

Expand Down
101 changes: 1 addition & 100 deletions JuliaLowering/src/ast.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,36 +50,6 @@ Unique symbolic identity for a variable, constant, label, or other entity
"""
const IdTag = Int

"""
Id for hygienic scope layers in macro expansion
"""
const LayerId = Int

"""
A `ScopeLayer` is a mechanism for automatic hygienic macros; every identifier
is assigned to a particular layer and can only match against bindings which are
themselves part of that layer.

Normal code contains a single scope layer, whereas each macro expansion
generates a new layer.
"""
struct ScopeLayer
id::LayerId
mod::Module
# Index of parent layer in a macro expansion. Equal to 0 for no parent
parent_id::LayerId
# True if generated by an old macro expansion
hygiene_compat::Bool
is_internal::Bool
end

is_base_layer(sl::ScopeLayer) = sl.parent_id === 0

base_layer(ctx, sl::ScopeLayer) = while true
is_base_layer(sl) && return sl
sl = ctx.scope_layers[sl.parent_id]
end

"""
Lexical scope ID
"""
Expand Down Expand Up @@ -359,48 +329,6 @@ macro ast(ctx, srcref, tree)
end

#-------------------------------------------------------------------------------
function set_scope_layer(ctx, ex, layer_id, force)
k = kind(ex)
new_layer = force ? layer_id : get(ex, :scope_layer, layer_id)

ex2 = if k == K"module" || k == K"toplevel" || k == K"inert" || k == K"inert_syntaxtree"
mknode(ex, children(ex))
elseif k == K"." && numchildren(ex) == 2
cs = tree_ids(set_scope_layer(ctx, ex[1], layer_id, force), ex[2])
mknode(ex, cs)
elseif !is_leaf(ex)
mapchildren(e->set_scope_layer(ctx, e, layer_id, force), ctx, ex)
else
mkleaf(ex)
end
setattr!(ex2, :scope_layer, new_layer)
end

"""
adopt_scope(ex, ref)

Copy `ex`, adopting the scope layer of `ref`.
"""
function adopt_scope(ex::SyntaxTree, scope_layer::LayerId)
set_scope_layer(ex._graph, ex, scope_layer, true)
end

function adopt_scope(ex::SyntaxTree, layer::ScopeLayer)
adopt_scope(ex, layer.id)
end

function adopt_scope(ex::SyntaxTree, ref::SyntaxTree)
adopt_scope(ex, ref.scope_layer)
end

function adopt_scope(exs::SyntaxList, ref)
out = SyntaxList(syntax_graph(exs))
for e in exs
push!(out, adopt_scope(e, ref))
end
return out
end

# Type for `meta` attribute, to replace `Expr(:meta)`.
# It's unclear how much flexibility we need here - is a dict good, or could we
# just use a struct? Likely this will be sparse. Alternatively we could just
Expand Down Expand Up @@ -432,7 +360,7 @@ name_hint(name) = CompileHints(:name_hint, name)

function is_quoted(ex)
kind(ex) in KSet"Symbol quote top core globalref break inert
inert_syntaxtree meta inbounds inline noinline loopinfo"
syntaxinert meta inbounds inline noinline loopinfo"
end

function extension_type(ex)
Expand Down Expand Up @@ -541,33 +469,6 @@ function to_symbol(ctx, ex)
@ast ctx ex ex=>K"Symbol"
end

# Internal scope layers may also have parents: desugaring generates a new
# internal scope layer to hold the global struct name (since any user references
# should hit the local instead), but the global struct name should still follow
# hygiene_compat rules as if it were on its original layer.
function new_internal_escapable_scope_layer(ctx, ident::SyntaxTree)
@jl_assert kind(ident) == K"Identifier" ident
old_sl = ctx.scope_layers[ident.scope_layer::LayerId]
sl = ScopeLayer(
length(ctx.scope_layers)+1, old_sl.mod, old_sl.id,
old_sl.hygiene_compat, true)
push!(ctx.scope_layers, sl)
sl
end
function new_internal_scope_layer(ctx, mod_ref::Module)
sl = ScopeLayer(length(ctx.scope_layers)+1, mod_ref, 0, false, true)
push!(ctx.scope_layers, sl)
sl
end

function new_macro_scope_layer(ctx, parent::ScopeLayer, mod::Module,
hygiene_compat::Bool)
sl = ScopeLayer(length(ctx.scope_layers)+1,
mod, parent.id, hygiene_compat, false)
push!(ctx.scope_layers, sl)
sl
end

#-------------------------------------------------------------------------------
# Context wrapper which helps to construct a list of statements to be executed
# prior to some expression. Useful when we need to use subexpressions multiple
Expand Down
9 changes: 6 additions & 3 deletions JuliaLowering/src/closure_conversion.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ function _convert_closures(ctx::ClosureConversionCtx, ex)
else
access
end
elseif is_leaf(ex) || k == K"inert" || k == K"inert_syntaxtree" || k == K"static_eval"
elseif is_leaf(ex) || k == K"inert" || k == K"syntaxinert" || k == K"static_eval"
ex
elseif k == K"="
convert_assignment(ctx, ex)
Expand Down Expand Up @@ -376,7 +376,7 @@ function _convert_closures(ctx::ClosureConversionCtx, ex)
binfo = get_binding(ctx, ex[1])
if binfo.kind == :global
# flisp has this, but our K"assert" handling is in a previous pass
# [K"assert" "toplevel_only"::K"Symbol" [K"inert_syntaxtree" ex]]
# [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]]
make_globaldecl(ctx, ex, binfo.mod, binfo.name, true, _convert_closures(ctx, ex[2]))
else
newleaf(ctx, ex, K"TOMBSTONE")
Expand Down Expand Up @@ -635,7 +635,10 @@ Invariants:
@fzone "JL: closures" function convert_closures(
ctx::VariableAnalysisContext, ex::SyntaxTree{Attrs}
) where Attrs
ctx_out = ClosureConversionCtx(ctx.graph, ctx.bindings, ctx.mod,
# TODO: ctx.mod is used instead of syntax_module(ex) beyond this point,
# which is dubious
ctx_out = ClosureConversionCtx(ctx.graph, ctx.bindings,
ctx.layer.mod,
ctx.closure_bindings, nothing,
ex.lambda_bindings,
false, true, SyntaxList(ctx.graph),
Expand Down
48 changes: 21 additions & 27 deletions JuliaLowering/src/compat.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ function expr_to_est(@nospecialize(e),
expr_to_est(graph, e, lnn)
end

function expr_to_est(graph::SyntaxGraph, @nospecialize(e), lnn::LineNumberNode)
SyntaxTree(graph, _expr_to_est(graph, e, lnn)[1])
function expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::SourceAttrType)
SyntaxTree(graph, _expr_to_est(graph, e, src)[1])
end

function _get_inner_lnn(e::Expr, default::LineNumberNode)
Expand All @@ -57,17 +57,16 @@ function is_expr_value(st::SyntaxTree)
return JuliaSyntax.is_literal(k) || k === K"Value"
end

function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::LineNumberNode)
# Most of the complexity here is LineNumberNode absorption logic: linenodes are
# always considered provenance if unquoted, then removed in certain forms. If
# `src` is not an linenode, it is assumed to be a better provenance source, so
# linenodes in `e` are not used for provenance (but still removed).
function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::SourceAttrType)
st = if e isa Symbol
setattr!(newleaf(graph, src, K"Identifier"), :name_val, String(e))
elseif e isa QuoteNode
cid, _ = _expr_to_est(graph, e.value, src)
newnode(graph, src, K"inert", NodeId[cid])
elseif e isa Expr && e.head === :scope_layer
@assert length(e.args) === 2 && e.args[1] isa Symbol
ident = newleaf(graph, src, K"Identifier")
setattr!(ident, :name_val, String(e.args[1]::Symbol))
setattr!(ident, :scope_layer, e.args[2])
elseif e isa Expr && e.head === :lambda && length(e.args) == 2
argnames = e.args[1]::Vector{Any}
arg_cs = NodeId[]
Expand All @@ -86,12 +85,12 @@ function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::LineNumberNode)
elseif e isa Expr
head_s = string(e.head)
st_k = find_kind(head_s)
src = old_src = _get_inner_lnn(e, src)
src = old_src = src isa LineNumberNode ? _get_inner_lnn(e, src) : src
cs = NodeId[]
rm_linenodes = e.head in (:block, :toplevel)
for arg in e.args
if rm_linenodes && arg isa LineNumberNode
src = arg
src isa LineNumberNode && (src = arg)
else
cid, src = _expr_to_est(graph, arg, src)
push!(cs, cid)
Expand All @@ -109,7 +108,7 @@ function _expr_to_est(graph::SyntaxGraph, @nospecialize(e), src::LineNumberNode)
# We may want additional special cases for other types where
# `Base.isa_ast_node(e)`, but `K"Value"` should be fine for most, since
# most are produced in or after lowering
if e isa LineNumberNode
if e isa LineNumberNode && src isa LineNumberNode
# linenode outside of block or toplevel
src = e
end
Expand All @@ -126,11 +125,10 @@ end
function est_to_expr(st::SyntaxTree, suppress_linenodes=false)
k = kind(st)
if kind(st) === K"Identifier"
# @jl_assert scope layer is base
n = Symbol(st.name_val::String)
mod = get(st, :mod, nothing)
!isnothing(mod) ? GlobalRef(mod, n) :
hasattr(st, :scope_layer) ? Expr(:scope_layer, n, st.scope_layer) :
n
!isnothing(mod) ? GlobalRef(mod, n) : n
elseif is_leaf(st) && is_expr_value(st)
v = st.value
# Let `st.value isa Symbol` (or other AST node). Since we enforce that
Expand Down Expand Up @@ -318,7 +316,7 @@ end
split_generated(st::SyntaxTree, gen_part) = @stm st begin
(_, when=is_leaf(st)||is_quoted(st)) -> st
[K"if" [K"generated"] gen nongen] -> if gen_part
@ast(st._graph, st, [K"$" gen])
@ast(st._graph, st, [K"syntaxunquote" gen])
else
nongen
end
Expand Down Expand Up @@ -419,7 +417,6 @@ We can assume `st` has passed `valid_st1`. Errors arising from invalid AST
function est_to_dst(st::SyntaxTree)
g = ensure_macro_attributes!(st._graph)
rec = var"#self#"

return @stm st begin
[K"Identifier"] -> _est_to_dst_ident(st)
[K"Value"] -> st.value === nothing ? newleaf(g, st, K"nothing") : st
Expand All @@ -432,7 +429,6 @@ function est_to_dst(st::SyntaxTree)

op_leaf = newleaf(g, st, K"Identifier")
setattr!(op_leaf, :name_val, op_s)
setattr!(op_leaf, :scope_layer, st.scope_layer)
@ast g st [out_k rec(l) op_leaf rec(r)]
end
[K"comparison" cs0...] -> let cs = copy(cs0)
Expand All @@ -442,15 +438,13 @@ function est_to_dst(st::SyntaxTree)
mknode(st, cs)
end
[K"'" x] ->
@ast g st [K"call" "'"::K"Identifier"(scope_layer=st.scope_layer) rec(x)]
@ast g st [K"call" "'"::K"Identifier"(st) rec(x)]
[K"." f [K"tuple" args...]] -> _expand_literal_pow(
@ast g st [K"dotcall" rec(f) _dst_sink_parameters(args)...])
([K"inert" [K"Identifier"]], when=!hasattr(st[1], :mod)) ->
@ast g st st[1]=>K"Symbol"
([K"inert_syntaxtree" [K"Identifier"]], when=!hasattr(st[1], :mod)) ->
@ast g st st[1]=>K"Symbol"
[K"syntaxinert" _] -> st
[K"inert" _] -> st
[K"inert_syntaxtree" _] -> st
[K"module" _...] -> st
[K"toplevel" _...] -> st
[K"for" [K"=" _ _] body] ->
Expand Down Expand Up @@ -521,7 +515,7 @@ function est_to_dst(st::SyntaxTree)
l = apply_arglist_meta(l, collect_body_arg_meta(r))
if has_if_generated(r)
gen, nongen = split_generated(r, true), split_generated(r, false)
r2 = @ast g st [K"_generated_body" [K"quote" gen] rec(nongen)]
r2 = @ast g st [K"_generated_body" [K"syntaxquote" gen] rec(nongen)]
else
r2 = rec(r)
end
Expand All @@ -531,7 +525,7 @@ function est_to_dst(st::SyntaxTree)
l = apply_arglist_meta(_dst_fix_arglist(l), collect_body_arg_meta(r))
if has_if_generated(r)
gen, nongen = split_generated(r, true), split_generated(r, false)
r2 = @ast g st [K"_generated_body" [K"quote" gen] rec(nongen)]
r2 = @ast g st [K"_generated_body" [K"syntaxquote" gen] rec(nongen)]
else
r2 = rec(r)
end
Expand All @@ -541,7 +535,7 @@ function est_to_dst(st::SyntaxTree)
l = apply_arglist_meta(_dst_fix_arglist(l), collect_body_arg_meta(r))
if has_if_generated(r)
gen, nongen = split_generated(r, true), split_generated(r, false)
r2 = @ast g st [K"_generated_body" [K"quote" gen] rec(nongen)]
r2 = @ast g st [K"_generated_body" [K"syntaxquote" gen] rec(nongen)]
else
r2 = rec(r)
end
Expand Down Expand Up @@ -591,8 +585,7 @@ function est_to_dst(st::SyntaxTree)
[K"top" x] -> setattr!(mkleaf(st), :name_val, x.name_val)
[K"static_parameter" x] -> setattr!(mkleaf(st), :var_id, x.value::IdTag)
[K"copyast" [K"inert" ex]] -> @ast g st [K"call"
interpolate_ast::K"Value"
Expr::K"Value"
interpolate_expr::K"Value"
[K"inert"(st[1]) ex]
]
[K"symbolicgoto" lab] -> setattr!(mkleaf(st), :name_val, lab.name_val)
Expand Down Expand Up @@ -623,7 +616,8 @@ function est_to_dst(st::SyntaxTree)
# forms (e.g. function definitions) stay inert.
out_fptr = if kind(fptr) == K"inert" && numchildren(fptr) == 1 &&
kind(fptr[1]) == K"Identifier"
ident = setattr!(mkleaf(fptr[1]), :scope_layer, 1)
ident = mkleaf(fptr[1])
# TODO: relayer if unhygienic
@ast g fptr [K"static_eval"(fptr) ident]
else
rec(fptr)
Expand Down
Loading