From f27158e7e8e54a6a3cdb73ab3a5579bd78f243a5 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Mon, 8 Dec 2025 12:48:38 -0800 Subject: [PATCH 1/6] Clarify tree types in docstrings, tweak test --- JuliaLowering/src/syntax_graph.jl | 11 +++++++---- JuliaLowering/src/utils.jl | 6 ++++-- JuliaLowering/test/macros.jl | 2 +- JuliaSyntax/src/porcelain/syntax_tree.jl | 10 ++++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/JuliaLowering/src/syntax_graph.jl b/JuliaLowering/src/syntax_graph.jl index 4f05384a530ac..60af1c72ad38a 100644 --- a/JuliaLowering/src/syntax_graph.jl +++ b/JuliaLowering/src/syntax_graph.jl @@ -1,4 +1,3 @@ -# TODO: This whole file should probably be moved to JuliaSyntax. import .JuliaSyntax: ParseStream, RedTreeCursor, reverse_toplevel_siblings, has_toplevel_siblings, _unsafe_wrap_substring, parse_julia_literal, is_trivia, is_prefix_op_call, @isexpr, SyntaxHead, COLON_QUOTE, is_syntactic_operator, @@ -192,7 +191,12 @@ function is_compatible_graph(x, y) syntax_graph(x).edges === syntax_graph(y).edges end -#------------------------------------------------------------------------------- +""" + struct SyntaxTree + +An ECS-style AST used in JuliaLowering. Unstable, but may eventually replace +SyntaxNode. +""" struct SyntaxTree{GraphType} _graph::GraphType _id::NodeId @@ -793,8 +797,7 @@ end # end #------------------------------------------------------------------------------- -# Conversion from the raw parsed tree -# TODO: move to JuliaSyntax. Replace SyntaxNode? +# RawGreenNode->SyntaxTree function JuliaSyntax.build_tree(::Type{SyntaxTree}, stream::ParseStream; filename=nothing, first_line=1) diff --git a/JuliaLowering/src/utils.jl b/JuliaLowering/src/utils.jl index 2803da01e30dc..914d4c2cd009e 100644 --- a/JuliaLowering/src/utils.jl +++ b/JuliaLowering/src/utils.jl @@ -3,8 +3,10 @@ TODO(msg::AbstractString) = throw(ErrorException("Lowering TODO: $msg")) TODO(ex::SyntaxTree, msg="") = throw(LoweringError(ex, "Lowering TODO: $msg")) -# Errors found during lowering will result in LoweringError being thrown to -# indicate the syntax causing the error. +""" +An error generated while lowering user code `ex` (flisp: `Expr(:error, msg)`). +For errors in lowering itself, use `@assert`. +""" struct LoweringError <: Exception ex::SyntaxTree msg::String diff --git a/JuliaLowering/test/macros.jl b/JuliaLowering/test/macros.jl index 9d368122aa715..9f88bea4dff9a 100644 --- a/JuliaLowering/test/macros.jl +++ b/JuliaLowering/test/macros.jl @@ -336,7 +336,7 @@ end @sig_mismatch(1, 2, 3, 4) └───────────────────────┘ ── Error expanding macro Caused by: - MethodError: no method matching var"@sig_mismatch"(::JuliaLowering.MacroContext, ::JuliaLowering.SyntaxTree""") + MethodError: no method matching var"@sig_mismatch"(""") end @testset "old macros producing exotic expr heads" begin diff --git a/JuliaSyntax/src/porcelain/syntax_tree.jl b/JuliaSyntax/src/porcelain/syntax_tree.jl index ad08b25e6a1df..3724d1e517566 100644 --- a/JuliaSyntax/src/porcelain/syntax_tree.jl +++ b/JuliaSyntax/src/porcelain/syntax_tree.jl @@ -82,11 +82,13 @@ function Base.:(==)(a::SyntaxData, b::SyntaxData) end """ - SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}; - keep_parens=false, position::Integer=1) + SyntaxNode(source::SourceFile, cursor::RedTreeCursor, + raw::GreenNode{SyntaxHead}; keep_parens=false) -An AST node with a similar layout to `Expr`. Typically constructed from source -text by calling one of the parser API functions such as [`parseall`](@ref) +A pointer-y AST constructed by removing all trivia from the raw parser output +(see [`RawGreenNode`](@ref)). The layout of `SyntaxNode` is different from that +of `Expr` due to the invariant that each node's children are in the order they +appeared in the source text. """ const SyntaxNode = TreeNode{SyntaxData} From 742239ce60f8657f28a17fd253bdc0d2b4d84551 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Mon, 8 Dec 2025 13:19:16 -0800 Subject: [PATCH 2/6] Remove everything specific to lowering from syntax_graph.jl - Move `SyntaxTree`->`Expr` hacks to compat.jl - Move `@SyntaxTree` to utils.jl - Move `Base.show(::SyntaxTree)` to utils.jl (odd, but currently depends on lowering-only `Kind`s) - Move AST creation utilities from ast.jl to syntax_graph.jl. There's no good way of creating a parsed SyntaxTree otherwise. --- JuliaLowering/src/ast.jl | 219 ++-------------- JuliaLowering/src/compat.jl | 42 +++ JuliaLowering/src/syntax_graph.jl | 411 +++++++++++++----------------- JuliaLowering/src/utils.jl | 204 ++++++++++++++- 4 files changed, 442 insertions(+), 434 deletions(-) diff --git a/JuliaLowering/src/ast.jl b/JuliaLowering/src/ast.jl index f27f88729f666..ae4cc2ec81aaf 100644 --- a/JuliaLowering/src/ast.jl +++ b/JuliaLowering/src/ast.jl @@ -90,80 +90,10 @@ Lexical scope ID """ const ScopeId = Int -#------------------------------------------------------------------------------- -# AST creation utilities -_node_id(graph::SyntaxGraph, ex::SyntaxTree) = (check_compatible_graph(graph, ex); ex._id) -function _node_id(graph::SyntaxGraph, ex) - # Fallback to give a comprehensible error message for use with the @ast macro - error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`") -end -function _node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree}) - # Fallback to give a comprehensible error message for use with the @ast macro - error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)") -end - -_node_ids(graph::SyntaxGraph) = () -_node_ids(graph::SyntaxGraph, ::Nothing, cs...) = _node_ids(graph, cs...) -_node_ids(graph::SyntaxGraph, c, cs...) = (_node_id(graph, c), _node_ids(graph, cs...)...) -_node_ids(graph::SyntaxGraph, cs::SyntaxList, cs1...) = (_node_ids(graph, cs...)..., _node_ids(graph, cs1...)...) -function _node_ids(graph::SyntaxGraph, cs::SyntaxList) - check_compatible_graph(graph, cs) - cs.ids -end - -_unpack_srcref(graph, srcref::SyntaxTree) = _node_id(graph, srcref) -_unpack_srcref(graph, srcref::Tuple) = _node_ids(graph, srcref...) -_unpack_srcref(graph, srcref) = srcref - -function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val) - push!(ids, _node_id(graph, val)) -end -function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val::Nothing) - nothing -end -function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals) - for v in vals - _push_nodeid!(graph, ids, v) - end -end -function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals::SyntaxList) - check_compatible_graph(graph, vals) - append!(ids, vals.ids) -end - -# TODO: "proto", if SyntaxTree, is rarely different from srcref. reorganize to: -# newnode/newleaf(ctx, srcref, k::Kind[, attrs]) -# makenode/makeleaf(ctx, old::SyntaxTree[, attrs]) - -function makeleaf(graph::SyntaxGraph, srcref, proto::Union{Kind, SyntaxTree}) - id = newnode!(graph) - ex = SyntaxTree(graph, id) - copy_attrs!(ex, proto, true) - ex.source = _unpack_srcref(graph, srcref) - return ex -end - function makeleaf(ctx::AbstractLoweringContext, srcref, proto) makeleaf(syntax_graph(ctx), srcref, proto) end -function makeleaf(ctx, srcref, proto, @nospecialize(attrs::AbstractVector)) - graph = syntax_graph(ctx) - ex = makeleaf(graph, srcref, proto) - for (k, v) in attrs - setattr!(graph, ex._id, k, v) - end - return ex -end - -function makenode(ctx, srcref, proto, children, attrs=nothing) - graph = syntax_graph(ctx) - ex = isnothing(attrs) ? makeleaf(graph, srcref, proto) : - makeleaf(graph, srcref, proto, attrs) - setchildren!(graph, ex._id, children isa SyntaxList ? children.ids : children) - return ex -end - function newleaf(ctx, srcref, k::Kind, @nospecialize(value)) leaf = makeleaf(ctx, srcref, k) if k == K"Identifier" || k == K"core" || k == K"top" || k == K"Symbol" || @@ -192,15 +122,6 @@ function newleaf(ctx, srcref, k::Kind, @nospecialize(value)) leaf end -# TODO: Replace this with makeleaf variant? -function mapleaf(ctx, src, kind) - ex = makeleaf(syntax_graph(ctx), src, kind) - # TODO: Value coercion might be broken here due to use of `name_val` vs - # `value` vs ... ? - copy_attrs!(ex, src) - ex -end - # Convenience functions to create leaf nodes referring to identifiers within # the Core and Top modules. core_ref(ctx, ex, name) = newleaf(ctx, ex, K"core", name) @@ -228,6 +149,32 @@ end #------------------------------------------------------------------------------- # @ast macro + +function _node_id(graph::SyntaxGraph, ex) + # Fallback to give a comprehensible error message for use with the @ast macro + error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`") +end +function _node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree}) + # Fallback to give a comprehensible error message for use with the @ast macro + error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)") +end + +function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val) + push!(ids, _node_id(graph, val)) +end +function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val::Nothing) + nothing +end +function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals) + for v in vals + _push_nodeid!(graph, ids, v) + end +end +function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals::SyntaxList) + check_compatible_graph(graph, vals) + append!(ids, vals.ids) +end + function _match_srcref(ex) if Meta.isexpr(ex, :macrocall) && ex.args[1] == Symbol("@HERE") QuoteNode(ex.args[2]) @@ -398,120 +345,6 @@ macro ast(ctx, srcref, tree) end end -#------------------------------------------------------------------------------- -# Mapping and copying of AST nodes -function copy_attrs!(dest, src, all=false) - # TODO: Make this faster? - for (name, attr) in pairs(src._graph.attributes) - if (all || (name !== :source && name !== :kind && name !== :syntax_flags)) && - haskey(attr, src._id) - dest_attr = getattr(dest._graph, name, nothing) - if !isnothing(dest_attr) - dest_attr[dest._id] = attr[src._id] - end - end - end -end - -function copy_attrs!(dest, head::Union{Kind,JuliaSyntax.SyntaxHead}, all=false) - if all - setattr!(dest._graph, dest._id, :kind, kind(head)) - !(head isa Kind) && setattr!(dest._graph, dest._id, :syntax_flags, flags(head)) - end -end - -function mapchildren(f::Function, ctx, ex::SyntaxTree, do_map_child::Function) - if is_leaf(ex) - return ex - end - orig_children = children(ex) - cs = nothing - for (i,e) in enumerate(orig_children) - newchild = do_map_child(i) ? f(e) : e - if isnothing(cs) - if newchild == e - continue - else - cs = SyntaxList(ctx) - append!(cs, orig_children[1:i-1]) - end - end - push!(cs::SyntaxList, newchild) - end - if isnothing(cs) - # This function should be allocation-free if no children were changed - # by the mapping and there's no extra_attrs - return ex - end - cs::SyntaxList - ex2 = makenode(ctx, ex, ex, cs) - return ex2 -end - -function mapchildren(f::Function, ctx, ex::SyntaxTree, - mapped_children::AbstractVector{<:Integer}) - j = Ref(firstindex(mapped_children)) - function do_map_child(i) - ind = j[] - if ind <= lastindex(mapped_children) && mapped_children[ind] == i - j[] += 1 - true - else - false - end - end - mapchildren(f, ctx, ex, do_map_child) -end - -function mapchildren(f::Function, ctx, ex::SyntaxTree) - mapchildren(f, ctx, ex, i->true) -end - - -""" -Recursively copy AST `ex` into `ctx`. - -Special provenance handling: If `copy_source` is true, treat the `.source` -attribute as a reference and recurse on its contents. Otherwise, treat it like -any other attribute. -""" -function copy_ast(ctx, ex::SyntaxTree; copy_source=true) - graph1 = syntax_graph(ex) - graph2 = syntax_graph(ctx) - !copy_source && check_same_graph(graph1, graph2) - id2 = _copy_ast(graph2, graph1, ex._id, Dict{NodeId, NodeId}(), copy_source) - return SyntaxTree(graph2, id2) -end - -function _copy_ast(graph2::SyntaxGraph, graph1::SyntaxGraph, - id1::NodeId, seen, copy_source) - let copied = get(seen, id1, nothing) - isnothing(copied) || return copied - end - id2 = newnode!(graph2) - seen[id1] = id2 - src1 = get(SyntaxTree(graph1, id1), :source, nothing) - src2 = if !copy_source - src1 - elseif src1 isa NodeId - _copy_ast(graph2, graph1, src1, seen, copy_source) - elseif src1 isa Tuple - map(i->_copy_ast(graph2, graph1, i, seen, copy_source), src1) - else - src1 - end - copy_attrs!(SyntaxTree(graph2, id2), SyntaxTree(graph1, id1), true) - setattr!(graph2, id2, :source, src2) - if !is_leaf(graph1, id1) - cs = NodeId[] - for cid in children(graph1, id1) - push!(cs, _copy_ast(graph2, graph1, cid, seen, copy_source)) - end - setchildren!(graph2, id2, cs) - end - return id2 -end - #------------------------------------------------------------------------------- function set_scope_layer(ctx, ex, layer_id, force) k = kind(ex) diff --git a/JuliaLowering/src/compat.jl b/JuliaLowering/src/compat.jl index fe7376f83486d..0718b9714f01e 100644 --- a/JuliaLowering/src/compat.jl +++ b/JuliaLowering/src/compat.jl @@ -584,3 +584,45 @@ function _insert_child_exprs(head::Symbol, child_exprs::Vector{Any}, end return st_child_ids, last_src end + +#------------------------------------------------------------------------------- +# SyntaxTree->Expr overrides + +function JuliaSyntax._expr_leaf_val(ex::SyntaxTree, _...) + name = get(ex, :name_val, nothing) + if !isnothing(name) + n = Symbol(name) + if kind(ex) === K"Symbol" + return QuoteNode(n) + elseif hasattr(ex, :scope_layer) + Expr(:scope_layer, n, ex.scope_layer) + else + n + end + else + val = get(ex, :value, nothing) + if kind(ex) == K"Value" && val isa Expr || val isa LineNumberNode + # Expr AST embedded in a SyntaxTree should be quoted rather than + # becoming part of the output AST. + QuoteNode(val) + else + val + end + end +end + +function JuliaSyntax.fixup_Expr_child(::Type{<:SyntaxTree}, head::SyntaxHead, + @nospecialize(arg), first::Bool) + isa(arg, Expr) || return arg + k = kind(head) + coalesce_dot = k in KSet"call dotcall curly" || + (k == K"quote" && has_flags(head, COLON_QUOTE)) + if @isexpr(arg, :., 1) && arg.args[1] isa Tuple + h, a = arg.args[1]::Tuple{SyntaxHead,Any} + arg = ((coalesce_dot && first) || is_syntactic_operator(h)) ? + Symbol(".", a) : Expr(:., a) + end + return arg +end + +Base.Expr(ex::SyntaxTree) = JuliaSyntax.to_expr(ex) diff --git a/JuliaLowering/src/syntax_graph.jl b/JuliaLowering/src/syntax_graph.jl index 60af1c72ad38a..26d3a2307503e 100644 --- a/JuliaLowering/src/syntax_graph.jl +++ b/JuliaLowering/src/syntax_graph.jl @@ -417,107 +417,6 @@ end const SourceAttrType = Union{SourceRef,LineNumberNode,NodeId,Tuple} -attrsummary(name, value) = string(name) -attrsummary(name, value::Number) = "$name=$value" - -function _value_string(ex) - k = kind(ex) - str = k == K"Identifier" || is_operator(k) ? ex.name_val : - k == K"Placeholder" ? ex.name_val : - k == K"SSAValue" ? "%" : - k == K"BindingId" ? "#" : - k == K"label" ? "label" : - k == K"core" ? "core.$(ex.name_val)" : - k == K"top" ? "top.$(ex.name_val)" : - k == K"Symbol" ? ":$(ex.name_val)" : - k == K"globalref" ? "$(ex.mod).$(ex.name_val)" : - k == K"slot" ? "slot" : - k == K"latestworld" ? "latestworld" : - k == K"static_parameter" ? "static_parameter" : - k == K"symbolic_label" ? "label:$(ex.name_val)" : - k == K"symbolic_goto" ? "goto:$(ex.name_val)" : - k == K"SourceLocation" ? - "SourceLocation:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : - k == K"Value" && ex.value isa SourceRef ? - "SourceRef:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : - repr(get(ex, :value, nothing)) - id = get(ex, :var_id, nothing) - if isnothing(id) - id = get(ex, :id, nothing) - end - if !isnothing(id) - idstr = subscript_str(id) - str = "$(str)$idstr" - end - if k == K"slot" || k == K"BindingId" - p = provenance(ex)[1] - while p isa SyntaxTree - if kind(p) == K"Identifier" - str = "$(str)/$(p.name_val)" - break - end - p = provenance(p)[1] - end - end - return str -end - -function _show_syntax_tree(io, ex, indent, show_kinds) - val = get(ex, :value, nothing) - nodestr = !is_leaf(ex) ? "[$(untokenize(head(ex)))]" : _value_string(ex) - - treestr = rpad(string(indent, nodestr), 40) - if show_kinds && is_leaf(ex) - treestr = treestr*" :: "*string(kind(ex)) - end - - std_attrs = Set([:name_val,:value,:kind,:syntax_flags,:source,:var_id]) - attrstr = join([attrsummary(n, getproperty(ex, n)) - for n in attrnames(ex) if n ∉ std_attrs], ",") - treestr = string(rpad(treestr, 60), " │ $attrstr") - - println(io, treestr) - if !is_leaf(ex) - new_indent = indent*" " - for n in children(ex) - _show_syntax_tree(io, n, new_indent, show_kinds) - end - end -end - -function Base.show(io::IO, ::MIME"text/plain", ex::SyntaxTree, show_kinds=true) - anames = join(string.(attrnames(syntax_graph(ex))), ",") - println(io, "SyntaxTree with attributes $anames") - _show_syntax_tree(io, ex, "", show_kinds) -end - -function _show_syntax_tree_sexpr(io, ex) - if is_leaf(ex) - if is_error(ex) - print(io, "(", untokenize(head(ex)), ")") - else - print(io, _value_string(ex)) - end - else - print(io, "(", untokenize(head(ex))) - first = true - for n in children(ex) - print(io, ' ') - _show_syntax_tree_sexpr(io, n) - first = false - end - print(io, ')') - end -end - -function Base.show(io::IO, ::MIME"text/x.sexpression", node::SyntaxTree) - _show_syntax_tree_sexpr(io, node) -end - -function Base.show(io::IO, node::SyntaxTree) - _show_syntax_tree_sexpr(io, node) -end - function reparent(ctx, ex::SyntaxTree) # Ensure `ex` has the same parent graph, in a somewhat loose sense. # Could relax by copying if necessary? @@ -537,145 +436,6 @@ syntax_graph(ex::SyntaxTree) = ex._graph JuliaSyntax.sourcefile(ex::SyntaxTree) = sourcefile(sourceref(ex)) JuliaSyntax.byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) -function JuliaSyntax._expr_leaf_val(ex::SyntaxTree, _...) - name = get(ex, :name_val, nothing) - if !isnothing(name) - n = Symbol(name) - if kind(ex) === K"Symbol" - return QuoteNode(n) - elseif hasattr(ex, :scope_layer) - Expr(:scope_layer, n, ex.scope_layer) - else - n - end - else - val = get(ex, :value, nothing) - if kind(ex) == K"Value" && val isa Expr || val isa LineNumberNode - # Expr AST embedded in a SyntaxTree should be quoted rather than - # becoming part of the output AST. - QuoteNode(val) - else - val - end - end -end - -function JuliaSyntax.fixup_Expr_child(::Type{<:SyntaxTree}, head::SyntaxHead, - @nospecialize(arg), first::Bool) - isa(arg, Expr) || return arg - k = kind(head) - coalesce_dot = k in KSet"call dotcall curly" || - (k == K"quote" && has_flags(head, COLON_QUOTE)) - if @isexpr(arg, :., 1) && arg.args[1] isa Tuple - h, a = arg.args[1]::Tuple{SyntaxHead,Any} - arg = ((coalesce_dot && first) || is_syntactic_operator(h)) ? - Symbol(".", a) : Expr(:., a) - end - return arg -end - -Base.Expr(ex::SyntaxTree) = JuliaSyntax.to_expr(ex) - -#-------------------------------------------------- -function _find_SyntaxTree_macro(ex, line) - @assert !is_leaf(ex) - for c in children(ex) - rng = byte_range(c) - firstline = JuliaSyntax.source_line(sourcefile(c), first(rng)) - lastline = JuliaSyntax.source_line(sourcefile(c), last(rng)) - if line < firstline || lastline < line - continue - end - # We're in the line range. Either - if firstline == line && kind(c) == K"macrocall" && begin - name = c[1] - if kind(name) == K"." - name = name[2] - end - @assert kind(name) == K"Identifier" - name.name_val == "@SyntaxTree" - end - # We find the node we're looking for. NB: Currently assuming a max - # of one @SyntaxTree invocation per line. Though we could relax - # this with more heuristic matching of the Expr-AST... - @assert numchildren(c) == 2 - return c[2] - elseif !is_leaf(c) - # Recurse - ex1 = _find_SyntaxTree_macro(c, line) - if !isnothing(ex1) - return ex1 - end - end - end - return nothing # Will get here if multiple children are on the same line. -end - -# Translate JuliaLowering hygiene to esc() for use in @SyntaxTree -function _scope_layer_1_to_esc!(ex) - if ex isa Expr - if ex.head == :scope_layer - @assert ex.args[2] === 1 - return esc(_scope_layer_1_to_esc!(ex.args[1])) - else - map!(_scope_layer_1_to_esc!, ex.args, ex.args) - return ex - end - else - return ex - end -end - -""" -Macro to construct quoted SyntaxTree literals (instead of quoted Expr literals) -in normal Julia source code. - -Example: - -```julia -tree1 = @SyntaxTree :(some_unique_identifier) -tree2 = @SyntaxTree quote - x = 1 - \$tree1 = x -end -``` -""" -macro SyntaxTree(ex_old) - # The implementation here is hilarious and arguably very janky: we - # 1. Briefly check but throw away the Expr-AST - if !(Meta.isexpr(ex_old, :quote) || ex_old isa QuoteNode) - throw(ArgumentError("@SyntaxTree expects a `quote` block or `:`-quoted expression")) - end - # 2. Re-parse the current source file as SyntaxTree instead - fname = isnothing(__source__.file) ? error("No current file") : String(__source__.file) - if occursin(r"REPL\[\d+\]", fname) - # Assume we should look at last history entry in REPL - try - # Wow digging in like this is an awful hack but `@SyntaxTree` is - # already a hack so let's go for it I guess 😆 - text = Base.active_repl.mistate.interface.modes[1].hist.history[end] - if !occursin("@SyntaxTree", text) - error("Text not found in last REPL history line") - end - catch - error("Text not found in REPL history") - end - else - text = read(fname, String) - end - full_ex = parseall(SyntaxTree, text) - # 3. Using the current file and line number, dig into the re-parsed tree and - # discover the piece of AST which should be returned. - ex = _find_SyntaxTree_macro(full_ex, __source__.line) - isnothing(ex) && error("_find_SyntaxTree_macro failed") - # 4. Do the first step of JuliaLowering's syntax lowering to get - # syntax interpolations to work - _, ex1 = expand_forms_1(__module__, ex, false, Base.tls_world_age()) - @assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast - Expr(:call, :interpolate_ast, SyntaxTree, ex1[3][1], - map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[4:end])...) -end - #------------------------------------------------------------------------------- # Lightweight vector of nodes ids with associated pointer to graph stored separately. mutable struct SyntaxList{GraphType, NodeIdVecType} <: AbstractVector{SyntaxTree} @@ -796,6 +556,177 @@ end # out # end + +#------------------------------------------------------------------------------- +# AST creation utilities + +# TODO: "proto", if SyntaxTree, is rarely different from srcref. reorganize to: +# newnode/newleaf(ctx, srcref, k::Kind[, attrs]) +# makenode/makeleaf(ctx, old::SyntaxTree[, attrs]) + +_node_id(graph::SyntaxGraph, ex::SyntaxTree) = (check_compatible_graph(graph, ex); ex._id) + +_node_ids(graph::SyntaxGraph) = () +_node_ids(graph::SyntaxGraph, ::Nothing, cs...) = _node_ids(graph, cs...) +_node_ids(graph::SyntaxGraph, c, cs...) = (_node_id(graph, c), _node_ids(graph, cs...)...) +_node_ids(graph::SyntaxGraph, cs::SyntaxList, cs1...) = (_node_ids(graph, cs...)..., _node_ids(graph, cs1...)...) +function _node_ids(graph::SyntaxGraph, cs::SyntaxList) + check_compatible_graph(graph, cs) + cs.ids +end + +_unpack_srcref(graph, srcref::SyntaxTree) = _node_id(graph, srcref) +_unpack_srcref(graph, srcref::Tuple) = _node_ids(graph, srcref...) +_unpack_srcref(graph, srcref) = srcref + +function makeleaf(graph::SyntaxGraph, srcref, proto::Union{Kind, SyntaxTree}) + id = newnode!(graph) + ex = SyntaxTree(graph, id) + copy_attrs!(ex, proto, true) + ex.source = _unpack_srcref(graph, srcref) + return ex +end + +function makeleaf(ctx, srcref, proto, @nospecialize(attrs::AbstractVector)) + graph = syntax_graph(ctx) + ex = makeleaf(graph, srcref, proto) + for (k, v) in attrs + setattr!(graph, ex._id, k, v) + end + return ex +end + +function makenode(ctx, srcref, proto, children, attrs=nothing) + graph = syntax_graph(ctx) + ex = isnothing(attrs) ? makeleaf(graph, srcref, proto) : + makeleaf(graph, srcref, proto, attrs) + setchildren!(graph, ex._id, children isa SyntaxList ? children.ids : children) + return ex +end + +# TODO: Replace this with makeleaf variant? +function mapleaf(ctx, src, kind) + ex = makeleaf(syntax_graph(ctx), src, kind) + # TODO: Value coercion might be broken here due to use of `name_val` vs + # `value` vs ... ? + copy_attrs!(ex, src) + ex +end + +#------------------------------------------------------------------------------- +# Mapping and copying of AST nodes +function copy_attrs!(dest, src, all=false) + # TODO: Make this faster? + for (name, attr) in pairs(src._graph.attributes) + if (all || (name !== :source && name !== :kind && name !== :syntax_flags)) && + haskey(attr, src._id) + dest_attr = getattr(dest._graph, name, nothing) + if !isnothing(dest_attr) + dest_attr[dest._id] = attr[src._id] + end + end + end +end + +function copy_attrs!(dest, head::Union{Kind,JuliaSyntax.SyntaxHead}, all=false) + if all + setattr!(dest._graph, dest._id, :kind, kind(head)) + !(head isa Kind) && setattr!(dest._graph, dest._id, :syntax_flags, flags(head)) + end +end + +function mapchildren(f::Function, ctx, ex::SyntaxTree, do_map_child::Function) + if is_leaf(ex) + return ex + end + orig_children = children(ex) + cs = nothing + for (i,e) in enumerate(orig_children) + newchild = do_map_child(i) ? f(e) : e + if isnothing(cs) + if newchild == e + continue + else + cs = SyntaxList(ctx) + append!(cs, orig_children[1:i-1]) + end + end + push!(cs::SyntaxList, newchild) + end + if isnothing(cs) + # This function should be allocation-free if no children were changed + # by the mapping and there's no extra_attrs + return ex + end + cs::SyntaxList + ex2 = makenode(ctx, ex, ex, cs) + return ex2 +end + +function mapchildren(f::Function, ctx, ex::SyntaxTree, + mapped_children::AbstractVector{<:Integer}) + j = Ref(firstindex(mapped_children)) + function do_map_child(i) + ind = j[] + if ind <= lastindex(mapped_children) && mapped_children[ind] == i + j[] += 1 + true + else + false + end + end + mapchildren(f, ctx, ex, do_map_child) +end + +function mapchildren(f::Function, ctx, ex::SyntaxTree) + mapchildren(f, ctx, ex, i->true) +end + + +""" +Recursively copy AST `ex` into `ctx`. + +Special provenance handling: If `copy_source` is true, treat the `.source` +attribute as a reference and recurse on its contents. Otherwise, treat it like +any other attribute. +""" +function copy_ast(ctx, ex::SyntaxTree; copy_source=true) + graph1 = syntax_graph(ex) + graph2 = syntax_graph(ctx) + !copy_source && check_same_graph(graph1, graph2) + id2 = _copy_ast(graph2, graph1, ex._id, Dict{NodeId, NodeId}(), copy_source) + return SyntaxTree(graph2, id2) +end + +function _copy_ast(graph2::SyntaxGraph, graph1::SyntaxGraph, + id1::NodeId, seen, copy_source) + let copied = get(seen, id1, nothing) + isnothing(copied) || return copied + end + id2 = newnode!(graph2) + seen[id1] = id2 + src1 = get(SyntaxTree(graph1, id1), :source, nothing) + src2 = if !copy_source + src1 + elseif src1 isa NodeId + _copy_ast(graph2, graph1, src1, seen, copy_source) + elseif src1 isa Tuple + map(i->_copy_ast(graph2, graph1, i, seen, copy_source), src1) + else + src1 + end + copy_attrs!(SyntaxTree(graph2, id2), SyntaxTree(graph1, id1), true) + setattr!(graph2, id2, :source, src2) + if !is_leaf(graph1, id1) + cs = NodeId[] + for cid in children(graph1, id1) + push!(cs, _copy_ast(graph2, graph1, cid, seen, copy_source)) + end + setchildren!(graph2, id2, cs) + end + return id2 +end + #------------------------------------------------------------------------------- # RawGreenNode->SyntaxTree diff --git a/JuliaLowering/src/utils.jl b/JuliaLowering/src/utils.jl index 914d4c2cd009e..e011811626b93 100644 --- a/JuliaLowering/src/utils.jl +++ b/JuliaLowering/src/utils.jl @@ -1,3 +1,104 @@ +attrsummary(name, value) = string(name) +attrsummary(name, value::Number) = "$name=$value" + +function _value_string(ex) + k = kind(ex) + str = k == K"Identifier" || is_operator(k) ? ex.name_val : + k == K"Placeholder" ? ex.name_val : + k == K"SSAValue" ? "%" : + k == K"BindingId" ? "#" : + k == K"label" ? "label" : + k == K"core" ? "core.$(ex.name_val)" : + k == K"top" ? "top.$(ex.name_val)" : + k == K"Symbol" ? ":$(ex.name_val)" : + k == K"globalref" ? "$(ex.mod).$(ex.name_val)" : + k == K"slot" ? "slot" : + k == K"latestworld" ? "latestworld" : + k == K"static_parameter" ? "static_parameter" : + k == K"symbolic_label" ? "label:$(ex.name_val)" : + k == K"symbolic_goto" ? "goto:$(ex.name_val)" : + k == K"SourceLocation" ? + "SourceLocation:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : + k == K"Value" && ex.value isa SourceRef ? + "SourceRef:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : + repr(get(ex, :value, nothing)) + id = get(ex, :var_id, nothing) + if isnothing(id) + id = get(ex, :id, nothing) + end + if !isnothing(id) + idstr = subscript_str(id) + str = "$(str)$idstr" + end + if k == K"slot" || k == K"BindingId" + p = provenance(ex)[1] + while p isa SyntaxTree + if kind(p) == K"Identifier" + str = "$(str)/$(p.name_val)" + break + end + p = provenance(p)[1] + end + end + return str +end + +function _show_syntax_tree(io, ex, indent, show_kinds) + val = get(ex, :value, nothing) + nodestr = !is_leaf(ex) ? "[$(untokenize(head(ex)))]" : _value_string(ex) + + treestr = rpad(string(indent, nodestr), 40) + if show_kinds && is_leaf(ex) + treestr = treestr*" :: "*string(kind(ex)) + end + + std_attrs = Set([:name_val,:value,:kind,:syntax_flags,:source,:var_id]) + attrstr = join([attrsummary(n, getproperty(ex, n)) + for n in attrnames(ex) if n ∉ std_attrs], ",") + treestr = string(rpad(treestr, 60), " │ $attrstr") + + println(io, treestr) + if !is_leaf(ex) + new_indent = indent*" " + for n in children(ex) + _show_syntax_tree(io, n, new_indent, show_kinds) + end + end +end + +function Base.show(io::IO, ::MIME"text/plain", ex::SyntaxTree, show_kinds=true) + anames = join(string.(attrnames(syntax_graph(ex))), ",") + println(io, "SyntaxTree with attributes $anames") + _show_syntax_tree(io, ex, "", show_kinds) +end +function _show_syntax_tree_sexpr(io, ex) + if is_leaf(ex) + if is_error(ex) + print(io, "(", untokenize(head(ex)), ")") + else + print(io, _value_string(ex)) + end + else + print(io, "(", untokenize(head(ex))) + first = true + for n in children(ex) + print(io, ' ') + _show_syntax_tree_sexpr(io, n) + first = false + end + print(io, ')') + end +end + +function Base.show(io::IO, ::MIME"text/x.sexpression", node::SyntaxTree) + _show_syntax_tree_sexpr(io, node) +end + +function Base.show(io::IO, node::SyntaxTree) + _show_syntax_tree_sexpr(io, node) +end + +#------------------------------------------------------------------------------- # Error handling TODO(msg::AbstractString) = throw(ErrorException("Lowering TODO: $msg")) @@ -23,7 +124,6 @@ function Base.showerror(io::IO, exc::LoweringError; show_detail=true) end end -#------------------------------------------------------------------------------- function _show_provtree(io::IO, ex::SyntaxTree, indent) print(io, ex, "\n") prov = provenance(ex) @@ -186,3 +286,105 @@ else esc(f) end end + +#------------------------------------------------------------------------------- +# @SyntaxTree(::Expr) + +function _find_SyntaxTree_macro(ex, line) + @assert !is_leaf(ex) + for c in children(ex) + rng = byte_range(c) + firstline = JuliaSyntax.source_line(sourcefile(c), first(rng)) + lastline = JuliaSyntax.source_line(sourcefile(c), last(rng)) + if line < firstline || lastline < line + continue + end + # We're in the line range. Either + if firstline == line && kind(c) == K"macrocall" && begin + name = c[1] + if kind(name) == K"." + name = name[2] + end + @assert kind(name) == K"Identifier" + name.name_val == "@SyntaxTree" + end + # We find the node we're looking for. NB: Currently assuming a max + # of one @SyntaxTree invocation per line. Though we could relax + # this with more heuristic matching of the Expr-AST... + @assert numchildren(c) == 2 + return c[2] + elseif !is_leaf(c) + # Recurse + ex1 = _find_SyntaxTree_macro(c, line) + if !isnothing(ex1) + return ex1 + end + end + end + return nothing # Will get here if multiple children are on the same line. +end + +# Translate JuliaLowering hygiene to esc() for use in @SyntaxTree +function _scope_layer_1_to_esc!(ex) + if ex isa Expr + if ex.head == :scope_layer + @assert ex.args[2] === 1 + return esc(_scope_layer_1_to_esc!(ex.args[1])) + else + map!(_scope_layer_1_to_esc!, ex.args, ex.args) + return ex + end + else + return ex + end +end + +""" +Macro to construct quoted SyntaxTree literals (instead of quoted Expr literals) +in normal Julia source code. + +Example: + +```julia +tree1 = @SyntaxTree :(some_unique_identifier) +tree2 = @SyntaxTree quote + x = 1 + \$tree1 = x +end +``` +""" +macro SyntaxTree(ex_old) + # The implementation here is hilarious and arguably very janky: we + # 1. Briefly check but throw away the Expr-AST + if !(Meta.isexpr(ex_old, :quote) || ex_old isa QuoteNode) + throw(ArgumentError("@SyntaxTree expects a `quote` block or `:`-quoted expression")) + end + # 2. Re-parse the current source file as SyntaxTree instead + fname = isnothing(__source__.file) ? error("No current file") : String(__source__.file) + if occursin(r"REPL\[\d+\]", fname) + # Assume we should look at last history entry in REPL + try + # Wow digging in like this is an awful hack but `@SyntaxTree` is + # already a hack so let's go for it I guess 😆 + text = Base.active_repl.mistate.interface.modes[1].hist.history[end] + if !occursin("@SyntaxTree", text) + error("Text not found in last REPL history line") + end + catch + error("Text not found in REPL history") + end + else + text = read(fname, String) + end + full_ex = parseall(SyntaxTree, text) + # 3. Using the current file and line number, dig into the re-parsed tree and + # discover the piece of AST which should be returned. + ex = _find_SyntaxTree_macro(full_ex, __source__.line) + isnothing(ex) && error("_find_SyntaxTree_macro failed") + # 4. Do the first step of JuliaLowering's syntax lowering to get + # syntax interpolations to work + _, ex1 = expand_forms_1(__module__, ex, false, Base.tls_world_age()) + @assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast + Expr(:call, :interpolate_ast, SyntaxTree, ex1[3][1], + map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[4:end])...) +end From 162eddeae4269d1b538b2da6f23fb94ec9c728b2 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Mon, 8 Dec 2025 14:01:33 -0800 Subject: [PATCH 3/6] Move `syntax_graph.jl` from JL to JS --- JuliaLowering/src/JuliaLowering.jl | 10 ++- JuliaLowering/src/ast.jl | 8 +- JuliaLowering/test/ast.jl | 20 +++++ JuliaLowering/test/runtests.jl | 2 +- JuliaLowering/test/utils.jl | 16 ---- JuliaSyntax/src/JuliaSyntax.jl | 1 + .../src/porcelain}/syntax_graph.jl | 78 +++++++++++-------- JuliaSyntax/test/runtests.jl | 1 + .../test/syntax_graph.jl | 69 +++++++--------- 9 files changed, 106 insertions(+), 99 deletions(-) create mode 100644 JuliaLowering/test/ast.jl rename {JuliaLowering/src => JuliaSyntax/src/porcelain}/syntax_graph.jl (92%) rename {JuliaLowering => JuliaSyntax}/test/syntax_graph.jl (58%) diff --git a/JuliaLowering/src/JuliaLowering.jl b/JuliaLowering/src/JuliaLowering.jl index d8ff05dd013e8..191cae6df4729 100644 --- a/JuliaLowering/src/JuliaLowering.jl +++ b/JuliaLowering/src/JuliaLowering.jl @@ -16,12 +16,18 @@ using .JuliaSyntax: highlight, Kind, @KSet_str, is_leaf, children, numchildren, head, kind, flags, has_flags, numeric_flags, filename, first_byte, last_byte, byte_range, sourcefile, source_location, span, sourcetext, is_literal, is_number, is_operator, is_prec_assignment, is_prefix_call, - is_infix_op_call, is_postfix_op_call, is_error + is_infix_op_call, is_postfix_op_call, is_error, + @isexpr, SyntaxHead, COLON_QUOTE, is_syntactic_operator, SyntaxGraph, + SyntaxTree, SyntaxList, NodeId, SourceRef, SourceAttrType, + ensure_attributes, ensure_attributes!, delete_attributes, newnode!, hasattr, + setattr, setattr!, deleteattr!, syntax_graph, is_compatible_graph, + check_compatible_graph, copy_node, copy_ast, provenance, sourceref, + reparent, _node_id, _node_ids, makeleaf, makenode, mapchildren, mapleaf, + flattened_provenance, setchildren!, attrnames _include("kinds.jl") _register_kinds() -_include("syntax_graph.jl") _include("ast.jl") _include("bindings.jl") _include("utils.jl") diff --git a/JuliaLowering/src/ast.jl b/JuliaLowering/src/ast.jl index ae4cc2ec81aaf..976cb40b506c0 100644 --- a/JuliaLowering/src/ast.jl +++ b/JuliaLowering/src/ast.jl @@ -56,7 +56,7 @@ collected later. """ current_lambda_bindings(ctx::AbstractLoweringContext) = nothing -function syntax_graph(ctx::AbstractLoweringContext) +function JuliaSyntax.syntax_graph(ctx::AbstractLoweringContext) ctx.graph end @@ -90,7 +90,7 @@ Lexical scope ID """ const ScopeId = Int -function makeleaf(ctx::AbstractLoweringContext, srcref, proto) +function JuliaSyntax.makeleaf(ctx::AbstractLoweringContext, srcref, proto) makeleaf(syntax_graph(ctx), srcref, proto) end @@ -150,11 +150,11 @@ end #------------------------------------------------------------------------------- # @ast macro -function _node_id(graph::SyntaxGraph, ex) +function JuliaSyntax._node_id(graph::SyntaxGraph, ex) # Fallback to give a comprehensible error message for use with the @ast macro error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`") end -function _node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree}) +function JuliaSyntax._node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree}) # Fallback to give a comprehensible error message for use with the @ast macro error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)") end diff --git a/JuliaLowering/test/ast.jl b/JuliaLowering/test/ast.jl new file mode 100644 index 0000000000000..6f21205693785 --- /dev/null +++ b/JuliaLowering/test/ast.jl @@ -0,0 +1,20 @@ +@testset "SyntaxTree" begin + # Expr conversion + @test Expr(parsestmt(SyntaxTree, "begin a + b ; c end", filename="none")) == + Meta.parse("begin a + b ; c end") + + # @SyntaxTree + tree1 = JuliaLowering.@SyntaxTree :(some_unique_identifier) + @test tree1 isa SyntaxTree + @test kind(tree1) == K"Identifier" + @test tree1.name_val == "some_unique_identifier" + + tree2 = JuliaLowering.@SyntaxTree quote + x + $tree1 + end + @test tree2 isa SyntaxTree + @test kind(tree2) == K"block" + @test kind(tree2[1]) == K"Identifier" && tree2[1].name_val == "x" + @test kind(tree2[2]) == K"Identifier" && tree2[2].name_val == "some_unique_identifier" +end diff --git a/JuliaLowering/test/runtests.jl b/JuliaLowering/test/runtests.jl index 7451ecb5c179f..95e3635d8829f 100644 --- a/JuliaLowering/test/runtests.jl +++ b/JuliaLowering/test/runtests.jl @@ -1,7 +1,7 @@ include("utils.jl") @testset "JuliaLowering.jl" begin - include("syntax_graph.jl") + include("ast.jl") include("ir_tests.jl") diff --git a/JuliaLowering/test/utils.jl b/JuliaLowering/test/utils.jl index 0a150a38c1351..6ca0207f2dff3 100644 --- a/JuliaLowering/test/utils.jl +++ b/JuliaLowering/test/utils.jl @@ -47,22 +47,6 @@ macro ast_(tree) end end -function ≈(ex1, ex2) - if kind(ex1) != kind(ex2) || is_leaf(ex1) != is_leaf(ex2) - return false - end - if is_leaf(ex1) - return get(ex1, :value, nothing) == get(ex2, :value, nothing) && - get(ex1, :name_val, nothing) == get(ex2, :name_val, nothing) - else - if numchildren(ex1) != numchildren(ex2) - return false - end - return all(c1 ≈ c2 for (c1,c2) in zip(children(ex1), children(ex2))) - end -end - - #------------------------------------------------------------------------------- function _format_as_ast_macro(io, ex, indent) k = kind(ex) diff --git a/JuliaSyntax/src/JuliaSyntax.jl b/JuliaSyntax/src/JuliaSyntax.jl index da5861c0d5b62..1d500988539a8 100644 --- a/JuliaSyntax/src/JuliaSyntax.jl +++ b/JuliaSyntax/src/JuliaSyntax.jl @@ -100,6 +100,7 @@ include("julia/literal_parsing.jl") include("porcelain/green_node.jl") include("porcelain/syntax_tree.jl") include("integration/expr.jl") +include("porcelain/syntax_graph.jl") # Hooks to integrate the parser with Base include("integration/hooks.jl") diff --git a/JuliaLowering/src/syntax_graph.jl b/JuliaSyntax/src/porcelain/syntax_graph.jl similarity index 92% rename from JuliaLowering/src/syntax_graph.jl rename to JuliaSyntax/src/porcelain/syntax_graph.jl index 26d3a2307503e..37fc35462cd0f 100644 --- a/JuliaLowering/src/syntax_graph.jl +++ b/JuliaSyntax/src/porcelain/syntax_graph.jl @@ -1,8 +1,3 @@ -import .JuliaSyntax: ParseStream, RedTreeCursor, reverse_toplevel_siblings, - has_toplevel_siblings, _unsafe_wrap_substring, parse_julia_literal, is_trivia, - is_prefix_op_call, @isexpr, SyntaxHead, COLON_QUOTE, is_syntactic_operator, - lower_identifier_name - const NodeId = Int """ @@ -112,19 +107,19 @@ function setchildren!(graph::SyntaxGraph, id::NodeId, append!(graph.edges, children) end -function JuliaSyntax.is_leaf(graph::SyntaxGraph, id) +function is_leaf(graph::SyntaxGraph, id) first(graph.edge_ranges[id]) == 0 end -function JuliaSyntax.numchildren(graph::SyntaxGraph, id) +function numchildren(graph::SyntaxGraph, id) length(graph.edge_ranges[id]) end -function JuliaSyntax.children(graph::SyntaxGraph, id) +function children(graph::SyntaxGraph, id) @view graph.edges[graph.edge_ranges[id]] end -function JuliaSyntax.children(graph::SyntaxGraph, id, r::UnitRange) +function children(graph::SyntaxGraph, id, r::UnitRange) @view graph.edges[graph.edge_ranges[id][r]] end @@ -239,6 +234,21 @@ end Base.firstindex(ex::SyntaxTree) = 1 Base.lastindex(ex::SyntaxTree) = numchildren(ex) +function Base.:≈(ex1::SyntaxTree, ex2::SyntaxTree) + if kind(ex1) != kind(ex2) || is_leaf(ex1) != is_leaf(ex2) + return false + end + if is_leaf(ex1) + return get(ex1, :value, nothing) == get(ex2, :value, nothing) && + get(ex1, :name_val, nothing) == get(ex2, :name_val, nothing) + else + if numchildren(ex1) != numchildren(ex2) + return false + end + return all(c1 ≈ c2 for (c1,c2) in zip(children(ex1), children(ex2))) + end +end + function hasattr(ex::SyntaxTree, name::Symbol) attr = getattr(ex._graph, name, nothing) return !isnothing(attr) && haskey(attr, ex._id) @@ -273,27 +283,27 @@ end # JuliaSyntax tree API -function JuliaSyntax.is_leaf(ex::SyntaxTree) +function is_leaf(ex::SyntaxTree) is_leaf(ex._graph, ex._id) end -function JuliaSyntax.numchildren(ex::SyntaxTree) +function numchildren(ex::SyntaxTree) numchildren(ex._graph, ex._id) end -function JuliaSyntax.children(ex::SyntaxTree) +function children(ex::SyntaxTree) SyntaxList(ex._graph, children(ex._graph, ex._id)) end -function JuliaSyntax.head(ex::SyntaxTree) - JuliaSyntax.SyntaxHead(kind(ex), flags(ex)) +function head(ex::SyntaxTree) + SyntaxHead(kind(ex), flags(ex)) end -function JuliaSyntax.kind(ex::SyntaxTree) - ex.kind::JuliaSyntax.Kind +function kind(ex::SyntaxTree) + ex.kind::Kind end -function JuliaSyntax.flags(ex::SyntaxTree) +function flags(ex::SyntaxTree) get(ex, :syntax_flags, 0x0000) end @@ -305,29 +315,29 @@ struct SourceRef last_byte::Int end -JuliaSyntax.sourcefile(src::SourceRef) = src.file -JuliaSyntax.byte_range(src::SourceRef) = src.first_byte:src.last_byte +sourcefile(src::SourceRef) = src.file +byte_range(src::SourceRef) = src.first_byte:src.last_byte # TODO: Adding these methods to support LineNumberNode is kind of hacky but we # can remove these after JuliaLowering becomes self-bootstrapping for macros # and we a proper SourceRef for @ast's @HERE form. -JuliaSyntax.byte_range(src::LineNumberNode) = 0:0 -JuliaSyntax.source_location(src::LineNumberNode) = (src.line, 0) -JuliaSyntax.source_location(::Type{LineNumberNode}, src::LineNumberNode) = src -JuliaSyntax.source_line(src::LineNumberNode) = src.line +byte_range(src::LineNumberNode) = 0:0 +source_location(src::LineNumberNode) = (src.line, 0) +source_location(::Type{LineNumberNode}, src::LineNumberNode) = src +source_line(src::LineNumberNode) = src.line # The follow somewhat strange cases are for where LineNumberNode is standing in # for SourceFile because we've only got Expr-based provenance info -JuliaSyntax.sourcefile(src::LineNumberNode) = src -JuliaSyntax.sourcetext(src::LineNumberNode) = SubString("") -JuliaSyntax.source_location(src::LineNumberNode, byte_index::Integer) = (src.line, 0) -JuliaSyntax.source_location(::Type{LineNumberNode}, src::LineNumberNode, byte_index::Integer) = src -JuliaSyntax.filename(src::LineNumberNode) = string(src.file) +sourcefile(src::LineNumberNode) = src +sourcetext(src::LineNumberNode) = SubString("") +source_location(src::LineNumberNode, byte_index::Integer) = (src.line, 0) +source_location(::Type{LineNumberNode}, src::LineNumberNode, byte_index::Integer) = src +filename(src::LineNumberNode) = string(src.file) -function JuliaSyntax.highlight(io::IO, src::LineNumberNode; note="") +function highlight(io::IO, src::LineNumberNode; note="") print(io, src, " - ", note) end -function JuliaSyntax.highlight(io::IO, src::SourceRef; kws...) +function highlight(io::IO, src::SourceRef; kws...) highlight(io, src.file, first_byte(src):last_byte(src); kws...) end @@ -433,8 +443,8 @@ end syntax_graph(ex::SyntaxTree) = ex._graph -JuliaSyntax.sourcefile(ex::SyntaxTree) = sourcefile(sourceref(ex)) -JuliaSyntax.byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) +sourcefile(ex::SyntaxTree) = sourcefile(sourceref(ex)) +byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) #------------------------------------------------------------------------------- # Lightweight vector of nodes ids with associated pointer to graph stored separately. @@ -628,7 +638,7 @@ function copy_attrs!(dest, src, all=false) end end -function copy_attrs!(dest, head::Union{Kind,JuliaSyntax.SyntaxHead}, all=false) +function copy_attrs!(dest, head::Union{Kind,SyntaxHead}, all=false) if all setattr!(dest._graph, dest._id, :kind, kind(head)) !(head isa Kind) && setattr!(dest._graph, dest._id, :syntax_flags, flags(head)) @@ -730,7 +740,7 @@ end #------------------------------------------------------------------------------- # RawGreenNode->SyntaxTree -function JuliaSyntax.build_tree(::Type{SyntaxTree}, stream::ParseStream; +function build_tree(::Type{SyntaxTree}, stream::ParseStream; filename=nothing, first_line=1) cursor = RedTreeCursor(stream) graph = SyntaxGraph() diff --git a/JuliaSyntax/test/runtests.jl b/JuliaSyntax/test/runtests.jl index 644f073124982..65946babb0a76 100644 --- a/JuliaSyntax/test/runtests.jl +++ b/JuliaSyntax/test/runtests.jl @@ -19,6 +19,7 @@ include("parse_stream.jl") include("parser.jl") include("green_node.jl") include("syntax_tree.jl") +include("syntax_graph.jl") include("diagnostics.jl") include("parser_api.jl") include("expr.jl") diff --git a/JuliaLowering/test/syntax_graph.jl b/JuliaSyntax/test/syntax_graph.jl similarity index 58% rename from JuliaLowering/test/syntax_graph.jl rename to JuliaSyntax/test/syntax_graph.jl index 68196e14b6a50..b5e7769cc7ccd 100644 --- a/JuliaLowering/test/syntax_graph.jl +++ b/JuliaSyntax/test/syntax_graph.jl @@ -1,8 +1,10 @@ +using JuliaSyntax: SyntaxGraph, SyntaxTree, SyntaxList, freeze_attrs, unfreeze_attrs, ensure_attributes, ensure_attributes!, delete_attributes, copy_ast, attrdefs + @testset "SyntaxGraph attrs" begin st = parsestmt(SyntaxTree, "function foo end") - g_init = JuliaLowering.unfreeze_attrs(st._graph) - gf1 = JuliaLowering.freeze_attrs(g_init) - gu1 = JuliaLowering.unfreeze_attrs(gf1) + g_init = unfreeze_attrs(st._graph) + gf1 = freeze_attrs(g_init) + gu1 = unfreeze_attrs(gf1) # Check that freeze/unfreeze do their jobs @test gf1.attributes isa NamedTuple @@ -10,62 +12,45 @@ @test Set(keys(gf1.attributes)) == Set(keys(gu1.attributes)) # ensure_attributes - gf2 = JuliaLowering.ensure_attributes(gf1, test_attr=Symbol, foo=Type) - gu2 = JuliaLowering.ensure_attributes(gu1, test_attr=Symbol, foo=Type) + gf2 = ensure_attributes(gf1, test_attr=Symbol, foo=Type) + gu2 = ensure_attributes(gu1, test_attr=Symbol, foo=Type) # returns a graph with the same attribute storage @test gf2.attributes isa NamedTuple @test gu2.attributes isa Dict # does its job - @test (:test_attr=>Symbol) in JuliaLowering.attrdefs(gf2) - @test (:foo=>Type) in JuliaLowering.attrdefs(gf2) + @test (:test_attr=>Symbol) in attrdefs(gf2) + @test (:foo=>Type) in attrdefs(gf2) @test Set(keys(gf2.attributes)) == Set(keys(gu2.attributes)) # no mutation - @test !((:test_attr=>Symbol) in JuliaLowering.attrdefs(gf1)) - @test !((:foo=>Type) in JuliaLowering.attrdefs(gf1)) + @test !((:test_attr=>Symbol) in attrdefs(gf1)) + @test !((:foo=>Type) in attrdefs(gf1)) @test Set(keys(gf1.attributes)) == Set(keys(gu1.attributes)) # delete_attributes - gf3 = JuliaLowering.delete_attributes(gf2, :test_attr, :foo) - gu3 = JuliaLowering.delete_attributes(gu2, :test_attr, :foo) + gf3 = delete_attributes(gf2, :test_attr, :foo) + gu3 = delete_attributes(gu2, :test_attr, :foo) # returns a graph with the same attribute storage @test gf3.attributes isa NamedTuple @test gu3.attributes isa Dict # does its job - @test !((:test_attr=>Symbol) in JuliaLowering.attrdefs(gf3)) - @test !((:foo=>Type) in JuliaLowering.attrdefs(gf3)) + @test !((:test_attr=>Symbol) in attrdefs(gf3)) + @test !((:foo=>Type) in attrdefs(gf3)) @test Set(keys(gf3.attributes)) == Set(keys(gu3.attributes)) # no mutation - @test (:test_attr=>Symbol) in JuliaLowering.attrdefs(gf2) - @test (:foo=>Type) in JuliaLowering.attrdefs(gf2) + @test (:test_attr=>Symbol) in attrdefs(gf2) + @test (:foo=>Type) in attrdefs(gf2) @test Set(keys(gf2.attributes)) == Set(keys(gu2.attributes)) end -@testset "SyntaxTree" begin - # Expr conversion - @test Expr(parsestmt(SyntaxTree, "begin a + b ; c end", filename="none")) == - Meta.parse("begin a + b ; c end") - - # Parsing to SyntaxTree: errors should fall through +@testset "SyntaxTree parsing" begin + # Errors should fall through @test parsestmt(SyntaxTree, "@"; ignore_errors=true) isa SyntaxTree @test parsestmt(SyntaxTree, "@@@"; ignore_errors=true) isa SyntaxTree @test parsestmt(SyntaxTree, "(a b c)"; ignore_errors=true) isa SyntaxTree @test parsestmt(SyntaxTree, "'a b c'"; ignore_errors=true) isa SyntaxTree +end - # @SyntaxTree - tree1 = JuliaLowering.@SyntaxTree :(some_unique_identifier) - @test tree1 isa SyntaxTree - @test kind(tree1) == K"Identifier" - @test tree1.name_val == "some_unique_identifier" - - tree2 = JuliaLowering.@SyntaxTree quote - x - $tree1 - end - @test tree2 isa SyntaxTree - @test kind(tree2) == K"block" - @test kind(tree2[1]) == K"Identifier" && tree2[1].name_val == "x" - @test kind(tree2[2]) == K"Identifier" && tree2[2].name_val == "some_unique_identifier" - +@testset "SyntaxTree utils" begin "For filling required attrs in graphs created by hand" function testgraph(edge_ranges, edges, more_attrs...) kinds = Dict(map(i->(i=>K"block"), eachindex(edge_ranges))) @@ -86,7 +71,7 @@ end map(i->i+3, 1:6)... map(LineNumberNode, 7:9)...]))) st = SyntaxTree(g, 1) - stcopy = JuliaLowering.copy_ast(g, st) + stcopy = copy_ast(g, st) # Each node should be copied once @test length(g.edge_ranges) === 18 @test st._id != stcopy._id @@ -95,7 +80,7 @@ end @test st.source[1] !== stcopy.source[1] @test st.source[1][1] !== stcopy.source[1][1] - stcopy2 = JuliaLowering.copy_ast(g, st; copy_source=false) + stcopy2 = copy_ast(g, st; copy_source=false) # Only nodes 1-3 should be copied @test length(g.edge_ranges) === 21 @test st._id != stcopy2._id @@ -105,13 +90,13 @@ end @test st.source[1][1] === stcopy2.source[1][1] # Copy into a new graph - new_g = ensure_attributes!(SyntaxGraph(); JuliaLowering.attrdefs(g)...) - stcopy3 = JuliaLowering.copy_ast(new_g, st) + new_g = ensure_attributes!(SyntaxGraph(); attrdefs(g)...) + stcopy3 = copy_ast(new_g, st) @test length(new_g.edge_ranges) === 9 @test st ≈ stcopy3 - new_g = ensure_attributes!(SyntaxGraph(); JuliaLowering.attrdefs(g)...) + new_g = ensure_attributes!(SyntaxGraph(); attrdefs(g)...) # Disallow for now, since we can't prevent dangling sourcerefs - @test_throws ErrorException JuliaLowering.copy_ast(new_g, st; copy_source=false) + @test_throws ErrorException copy_ast(new_g, st; copy_source=false) end end From 53154a3d354339a487d04e20d9ea5f811336a082 Mon Sep 17 00:00:00 2001 From: Em Chu Date: Mon, 8 Dec 2025 14:26:31 -0800 Subject: [PATCH 4/6] Trim some JS imports --- JuliaLowering/src/JuliaLowering.jl | 15 ++++++--------- JuliaLowering/src/ast.jl | 2 +- JuliaLowering/src/compat.jl | 6 +++--- JuliaLowering/src/desugaring.jl | 6 +++--- JuliaLowering/src/macro_expansion.jl | 2 +- JuliaLowering/src/utils.jl | 8 ++++---- JuliaLowering/test/compat.jl | 28 ++++++++++++++-------------- 7 files changed, 32 insertions(+), 35 deletions(-) diff --git a/JuliaLowering/src/JuliaLowering.jl b/JuliaLowering/src/JuliaLowering.jl index 191cae6df4729..6a1901540655b 100644 --- a/JuliaLowering/src/JuliaLowering.jl +++ b/JuliaLowering/src/JuliaLowering.jl @@ -13,17 +13,14 @@ else end using .JuliaSyntax: highlight, Kind, @KSet_str, is_leaf, children, numchildren, - head, kind, flags, has_flags, numeric_flags, filename, first_byte, - last_byte, byte_range, sourcefile, source_location, span, sourcetext, - is_literal, is_number, is_operator, is_prec_assignment, is_prefix_call, - is_infix_op_call, is_postfix_op_call, is_error, - @isexpr, SyntaxHead, COLON_QUOTE, is_syntactic_operator, SyntaxGraph, - SyntaxTree, SyntaxList, NodeId, SourceRef, SourceAttrType, + head, kind, flags, has_flags, filename, first_byte, last_byte, byte_range, + sourcefile, source_location, span, sourcetext, is_literal, is_infix_op_call, + is_postfix_op_call, @isexpr, SyntaxHead, is_syntactic_operator, + SyntaxGraph, SyntaxTree, SyntaxList, NodeId, SourceRef, SourceAttrType, ensure_attributes, ensure_attributes!, delete_attributes, newnode!, hasattr, - setattr, setattr!, deleteattr!, syntax_graph, is_compatible_graph, + setattr, setattr!, syntax_graph, is_compatible_graph, check_compatible_graph, copy_node, copy_ast, provenance, sourceref, - reparent, _node_id, _node_ids, makeleaf, makenode, mapchildren, mapleaf, - flattened_provenance, setchildren!, attrnames + reparent, makeleaf, makenode, mapchildren, mapleaf, flattened_provenance _include("kinds.jl") _register_kinds() diff --git a/JuliaLowering/src/ast.jl b/JuliaLowering/src/ast.jl index 976cb40b506c0..b8a70c468d5be 100644 --- a/JuliaLowering/src/ast.jl +++ b/JuliaLowering/src/ast.jl @@ -160,7 +160,7 @@ function JuliaSyntax._node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTre end function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val) - push!(ids, _node_id(graph, val)) + push!(ids, JuliaSyntax._node_id(graph, val)) end function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val::Nothing) nothing diff --git a/JuliaLowering/src/compat.jl b/JuliaLowering/src/compat.jl index 0718b9714f01e..55c946cd08472 100644 --- a/JuliaLowering/src/compat.jl +++ b/JuliaLowering/src/compat.jl @@ -209,7 +209,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA elseif e isa String st_id = _insert_tree_node(graph, K"string", src) id_inner = _insert_tree_node(graph, K"String", src, [:value=>e]) - setchildren!(graph, st_id, [id_inner]) + JuliaSyntax.setchildren!(graph, st_id, [id_inner]) return st_id, src elseif !(e isa Expr) # There are other kinds we could potentially back-convert (e.g. Float), @@ -560,7 +560,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA return st_id, src else st_child_ids, last_src = _insert_child_exprs(e.head, child_exprs, graph, src) - setchildren!(graph, st_id, st_child_ids) + JuliaSyntax.setchildren!(graph, st_id, st_child_ids) return st_id, last_src end end @@ -616,7 +616,7 @@ function JuliaSyntax.fixup_Expr_child(::Type{<:SyntaxTree}, head::SyntaxHead, isa(arg, Expr) || return arg k = kind(head) coalesce_dot = k in KSet"call dotcall curly" || - (k == K"quote" && has_flags(head, COLON_QUOTE)) + (k == K"quote" && has_flags(head, JuliaSyntax.COLON_QUOTE)) if @isexpr(arg, :., 1) && arg.args[1] isa Tuple h, a = arg.args[1]::Tuple{SyntaxHead,Any} arg = ((coalesce_dot && first) || is_syntactic_operator(h)) ? diff --git a/JuliaLowering/src/desugaring.jl b/JuliaLowering/src/desugaring.jl index a634a5587cec8..6fd3d34f89373 100644 --- a/JuliaLowering/src/desugaring.jl +++ b/JuliaLowering/src/desugaring.jl @@ -1113,7 +1113,7 @@ function flatten_ncat_rows!(flat_elems, nrow_spans, row_major, parent_layout_dim layout_dim = 1 @chk parent_layout_dim != 1 (ex,"Badly nested rows in `ncat`") elseif k == K"nrow" - dim = numeric_flags(ex) + dim = JuliaSyntax.numeric_flags(ex) @chk dim > 0 (ex,"Unsupported dimension $dim in ncat") @chk !row_major || dim != 2 (ex,"2D `nrow` cannot be mixed with `row` in `ncat`") layout_dim = nrow_flipdim(row_major, dim) @@ -1146,7 +1146,7 @@ end # - ragged column first or row first function expand_ncat(ctx, ex) is_typed = kind(ex) == K"typed_ncat" - outer_dim = numeric_flags(ex) + outer_dim = JuliaSyntax.numeric_flags(ex) @chk outer_dim > 0 (ex,"Unsupported dimension in ncat") eltype = is_typed ? ex[1] : nothing elements = is_typed ? ex[2:end] : ex[1:end] @@ -2208,7 +2208,7 @@ function expand_decls(ctx, ex) bindings = children(ex) stmts = SyntaxList(ctx) for binding in bindings - if is_prec_assignment(kind(binding)) + if JuliaSyntax.is_prec_assignment(kind(binding)) @chk numchildren(binding) == 2 # expand_assignment will create the type decls make_lhs_decls(ctx, stmts, declkind, declmeta, binding[1], false) diff --git a/JuliaLowering/src/macro_expansion.jl b/JuliaLowering/src/macro_expansion.jl index ebd85685db6cb..55802f5803032 100644 --- a/JuliaLowering/src/macro_expansion.jl +++ b/JuliaLowering/src/macro_expansion.jl @@ -377,7 +377,7 @@ function remove_scope_layer!(ex) remove_scope_layer!(c) end end - deleteattr!(ex, :scope_layer) + JuliaSyntax.deleteattr!(ex, :scope_layer) ex end diff --git a/JuliaLowering/src/utils.jl b/JuliaLowering/src/utils.jl index e011811626b93..0e74710d4e57e 100644 --- a/JuliaLowering/src/utils.jl +++ b/JuliaLowering/src/utils.jl @@ -3,7 +3,7 @@ attrsummary(name, value::Number) = "$name=$value" function _value_string(ex) k = kind(ex) - str = k == K"Identifier" || is_operator(k) ? ex.name_val : + str = k == K"Identifier" || JuliaSyntax.is_operator(k) ? ex.name_val : k == K"Placeholder" ? ex.name_val : k == K"SSAValue" ? "%" : k == K"BindingId" ? "#" : @@ -54,7 +54,7 @@ function _show_syntax_tree(io, ex, indent, show_kinds) std_attrs = Set([:name_val,:value,:kind,:syntax_flags,:source,:var_id]) attrstr = join([attrsummary(n, getproperty(ex, n)) - for n in attrnames(ex) if n ∉ std_attrs], ",") + for n in JuliaSyntax.attrnames(ex) if n ∉ std_attrs], ",") treestr = string(rpad(treestr, 60), " │ $attrstr") println(io, treestr) @@ -67,13 +67,13 @@ function _show_syntax_tree(io, ex, indent, show_kinds) end function Base.show(io::IO, ::MIME"text/plain", ex::SyntaxTree, show_kinds=true) - anames = join(string.(attrnames(syntax_graph(ex))), ",") + anames = join(string.(JuliaSyntax.attrnames(syntax_graph(ex))), ",") println(io, "SyntaxTree with attributes $anames") _show_syntax_tree(io, ex, "", show_kinds) end function _show_syntax_tree_sexpr(io, ex) if is_leaf(ex) - if is_error(ex) + if JuliaSyntax.is_error(ex) print(io, "(", untokenize(head(ex)), ")") else print(io, _value_string(ex)) diff --git a/JuliaLowering/test/compat.jl b/JuliaLowering/test/compat.jl index bb62769180996..c27109bd2be47 100644 --- a/JuliaLowering/test/compat.jl +++ b/JuliaLowering/test/compat.jl @@ -89,37 +89,37 @@ const JL = JuliaLowering if JS.is_infix_op_call(st) && (k === K"call" || k === K"dotcall") # Infix calls are not preserved in Expr; we need to re-order the children - pre_st_args = JL.NodeId[st[2]._id, st[1]._id] + pre_st_args = JS.NodeId[st[2]._id, st[1]._id] for c in st[3:end] push!(pre_st_args, c._id) end pre_st_flags = (JS.flags(st) & ~JS.INFIX_FLAG) | JS.PREFIX_CALL_FLAG - JL.setchildren!(st._graph, st._id, pre_st_args) - JL.setattr!(st._graph, st._id, :syntax_flags, pre_st_flags) + JS.setchildren!(st._graph, st._id, pre_st_args) + JS.setattr!(st._graph, st._id, :syntax_flags, pre_st_flags) elseif JS.is_postfix_op_call(st) && (k === K"call" || k === K"dotcall") - pre_st_args = JL.NodeId[st[end]._id] + pre_st_args = JS.NodeId[st[end]._id] for c in st[1:end-1] push!(pre_st_args, c._id) end pre_st_flags = (JS.flags(st) & ~JS.POSTFIX_OP_FLAG) | JS.PREFIX_CALL_FLAG - JL.setchildren!(st._graph, st._id, pre_st_args) - JL.setattr!(st._graph, st._id, :syntax_flags, pre_st_flags) + JS.setchildren!(st._graph, st._id, pre_st_args) + JS.setattr!(st._graph, st._id, :syntax_flags, pre_st_flags) elseif k in JS.KSet"tuple block macrocall" - JL.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.PARENS_FLAG) + JS.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.PARENS_FLAG) elseif k === K"toplevel" - JL.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.TOPLEVEL_SEMICOLONS_FLAG) + JS.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.TOPLEVEL_SEMICOLONS_FLAG) end if k in JS.KSet"tuple call dotcall macrocall vect curly braces <: >:" - JL.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.TRAILING_COMMA_FLAG) + JS.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.TRAILING_COMMA_FLAG) end - k === K"quote" && JL.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.COLON_QUOTE) - k === K"wrapper" && JL.setattr!(st._graph, st._id, :kind, K"block") + k === K"quote" && JS.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.COLON_QUOTE) + k === K"wrapper" && JS.setattr!(st._graph, st._id, :kind, K"block") # All ops are prefix ops in an expr. # Ignore trivia (shows up on some K"error"s) - JL.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & + JS.setattr!(st._graph, st._id, :syntax_flags, JS.flags(st) & ~JS.PREFIX_OP_FLAG & ~JS.INFIX_FLAG & ~JS.TRIVIA_FLAG & ~JS.NON_TERMINAL_FLAG) for c in JS.children(st) @@ -311,7 +311,7 @@ const JL = JuliaLowering for p in programs @testset "`$(repr(p))`" begin - st_good = JS.parsestmt(JL.SyntaxTree, p; ignore_errors=true) + st_good = JS.parsestmt(JS.SyntaxTree, p; ignore_errors=true) st_test = JL.expr_to_syntaxtree(Expr(st_good)) @test st_roughly_equal(;st_good, st_test) end @@ -324,7 +324,7 @@ const JL = JuliaLowering ] for p in toplevel_programs @testset "`$(repr(p))`" begin - st_good = JS.parseall(JL.SyntaxTree, p; ignore_errors=true) + st_good = JS.parseall(JS.SyntaxTree, p; ignore_errors=true) st_test = JL.expr_to_syntaxtree(Expr(st_good)) @test st_roughly_equal(;st_good, st_test) end From 8ee591a3f6ec3c1fa48a9e5b3b29be9a1b71333f Mon Sep 17 00:00:00 2001 From: Em Chu Date: Mon, 8 Dec 2025 14:03:25 -0800 Subject: [PATCH 5/6] Rename old syntax_tree.jl to syntax_node.jl --- JuliaSyntax/src/JuliaSyntax.jl | 2 +- JuliaSyntax/src/porcelain/{syntax_tree.jl => syntax_node.jl} | 0 JuliaSyntax/test/runtests.jl | 2 +- JuliaSyntax/test/{syntax_tree.jl => syntax_node.jl} | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename JuliaSyntax/src/porcelain/{syntax_tree.jl => syntax_node.jl} (100%) rename JuliaSyntax/test/{syntax_tree.jl => syntax_node.jl} (100%) diff --git a/JuliaSyntax/src/JuliaSyntax.jl b/JuliaSyntax/src/JuliaSyntax.jl index 1d500988539a8..704fd1c5924cd 100644 --- a/JuliaSyntax/src/JuliaSyntax.jl +++ b/JuliaSyntax/src/JuliaSyntax.jl @@ -98,7 +98,7 @@ include("julia/literal_parsing.jl") # Tree data structures include("porcelain/green_node.jl") -include("porcelain/syntax_tree.jl") +include("porcelain/syntax_node.jl") include("integration/expr.jl") include("porcelain/syntax_graph.jl") diff --git a/JuliaSyntax/src/porcelain/syntax_tree.jl b/JuliaSyntax/src/porcelain/syntax_node.jl similarity index 100% rename from JuliaSyntax/src/porcelain/syntax_tree.jl rename to JuliaSyntax/src/porcelain/syntax_node.jl diff --git a/JuliaSyntax/test/runtests.jl b/JuliaSyntax/test/runtests.jl index 65946babb0a76..68ed1566b4b3a 100644 --- a/JuliaSyntax/test/runtests.jl +++ b/JuliaSyntax/test/runtests.jl @@ -18,7 +18,7 @@ end include("parse_stream.jl") include("parser.jl") include("green_node.jl") -include("syntax_tree.jl") +include("syntax_node.jl") include("syntax_graph.jl") include("diagnostics.jl") include("parser_api.jl") diff --git a/JuliaSyntax/test/syntax_tree.jl b/JuliaSyntax/test/syntax_node.jl similarity index 100% rename from JuliaSyntax/test/syntax_tree.jl rename to JuliaSyntax/test/syntax_node.jl From c01b49d68e95e48c2cace56223d103b96401718a Mon Sep 17 00:00:00 2001 From: Em Chu Date: Thu, 11 Dec 2025 12:34:45 -0800 Subject: [PATCH 6/6] fix --- JuliaSyntax/src/JuliaSyntax.jl | 4 +++- JuliaSyntax/test/runtests.jl | 4 +++- JuliaSyntax/test/syntax_graph.jl | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/JuliaSyntax/src/JuliaSyntax.jl b/JuliaSyntax/src/JuliaSyntax.jl index 704fd1c5924cd..7413e4d54a72f 100644 --- a/JuliaSyntax/src/JuliaSyntax.jl +++ b/JuliaSyntax/src/JuliaSyntax.jl @@ -100,7 +100,9 @@ include("julia/literal_parsing.jl") include("porcelain/green_node.jl") include("porcelain/syntax_node.jl") include("integration/expr.jl") -include("porcelain/syntax_graph.jl") +if VERSION >= v"1.12" + include("porcelain/syntax_graph.jl") +end # Hooks to integrate the parser with Base include("integration/hooks.jl") diff --git a/JuliaSyntax/test/runtests.jl b/JuliaSyntax/test/runtests.jl index 68ed1566b4b3a..bdfaa12934bc6 100644 --- a/JuliaSyntax/test/runtests.jl +++ b/JuliaSyntax/test/runtests.jl @@ -19,7 +19,9 @@ include("parse_stream.jl") include("parser.jl") include("green_node.jl") include("syntax_node.jl") -include("syntax_graph.jl") +if VERSION >= v"1.12" + include("syntax_graph.jl") +end include("diagnostics.jl") include("parser_api.jl") include("expr.jl") diff --git a/JuliaSyntax/test/syntax_graph.jl b/JuliaSyntax/test/syntax_graph.jl index b5e7769cc7ccd..42583bab76bd7 100644 --- a/JuliaSyntax/test/syntax_graph.jl +++ b/JuliaSyntax/test/syntax_graph.jl @@ -1,4 +1,4 @@ -using JuliaSyntax: SyntaxGraph, SyntaxTree, SyntaxList, freeze_attrs, unfreeze_attrs, ensure_attributes, ensure_attributes!, delete_attributes, copy_ast, attrdefs +using .JuliaSyntax: SyntaxGraph, SyntaxTree, SyntaxList, freeze_attrs, unfreeze_attrs, ensure_attributes, ensure_attributes!, delete_attributes, copy_ast, attrdefs @testset "SyntaxGraph attrs" begin st = parsestmt(SyntaxTree, "function foo end")