Move SyntaxGraph/SyntaxTree from JuliaLowering to JuliaSyntax

JuliaLowering/src/JuliaLowering.jl

@@ -13,15 +13,18 @@ else
 end
 
 using .JuliaSyntax: highlight, Kind, @KSet_str, is_leaf, children, numchildren,
-    head, kind, flags, has_flags, numeric_flags, filename, first_byte,
-    last_byte, byte_range, sourcefile, source_location, span, sourcetext,
-    is_literal, is_number, is_operator, is_prec_assignment, is_prefix_call,
-    is_infix_op_call, is_postfix_op_call, is_error
+    head, kind, flags, has_flags, filename, first_byte, last_byte, byte_range,
+    sourcefile, source_location, span, sourcetext, is_literal, is_infix_op_call,
+    is_postfix_op_call, @isexpr, SyntaxHead, is_syntactic_operator,
+    SyntaxGraph, SyntaxTree, SyntaxList, NodeId, SourceRef, SourceAttrType,
+    ensure_attributes, ensure_attributes!, delete_attributes, newnode!, hasattr,
+    setattr, setattr!, syntax_graph, is_compatible_graph,
+    check_compatible_graph, copy_node, copy_ast, provenance, sourceref,
+    reparent, makeleaf, makenode, mapchildren, mapleaf, flattened_provenance
 
 _include("kinds.jl")
 _register_kinds()
 
-_include("syntax_graph.jl")
 _include("ast.jl")
 _include("bindings.jl")
 _include("utils.jl")

JuliaLowering/src/ast.jl

@@ -56,7 +56,7 @@ collected later.
 """
 current_lambda_bindings(ctx::AbstractLoweringContext) = nothing
 
-function syntax_graph(ctx::AbstractLoweringContext)
+function JuliaSyntax.syntax_graph(ctx::AbstractLoweringContext)
     ctx.graph
 end
 
@@ -90,80 +90,10 @@ Lexical scope ID
 """
 const ScopeId = Int
 
-#-------------------------------------------------------------------------------
-# AST creation utilities
-_node_id(graph::SyntaxGraph, ex::SyntaxTree) = (check_compatible_graph(graph, ex); ex._id)
-function _node_id(graph::SyntaxGraph, ex)
-    # Fallback to give a comprehensible error message for use with the @ast macro
-    error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`")
-end
-function _node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree})
-    # Fallback to give a comprehensible error message for use with the @ast macro
-    error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)")
-end
-
-_node_ids(graph::SyntaxGraph) = ()
-_node_ids(graph::SyntaxGraph, ::Nothing, cs...) = _node_ids(graph, cs...)
-_node_ids(graph::SyntaxGraph, c, cs...) = (_node_id(graph, c), _node_ids(graph, cs...)...)
-_node_ids(graph::SyntaxGraph, cs::SyntaxList, cs1...) = (_node_ids(graph, cs...)..., _node_ids(graph, cs1...)...)
-function _node_ids(graph::SyntaxGraph, cs::SyntaxList)
-    check_compatible_graph(graph, cs)
-    cs.ids
-end
-
-_unpack_srcref(graph, srcref::SyntaxTree) = _node_id(graph, srcref)
-_unpack_srcref(graph, srcref::Tuple)      = _node_ids(graph, srcref...)
-_unpack_srcref(graph, srcref)             = srcref
-
-function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val)
-    push!(ids, _node_id(graph, val))
-end
-function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val::Nothing)
-    nothing
-end
-function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals)
-    for v in vals
-        _push_nodeid!(graph, ids, v)
-    end
-end
-function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals::SyntaxList)
-    check_compatible_graph(graph, vals)
-    append!(ids, vals.ids)
-end
-
-# TODO: "proto", if SyntaxTree, is rarely different from srcref. reorganize to:
-# newnode/newleaf(ctx, srcref, k::Kind[, attrs])
-# makenode/makeleaf(ctx, old::SyntaxTree[, attrs])
-
-function makeleaf(graph::SyntaxGraph, srcref, proto::Union{Kind, SyntaxTree})
-    id = newnode!(graph)
-    ex = SyntaxTree(graph, id)
-    copy_attrs!(ex, proto, true)
-    ex.source = _unpack_srcref(graph, srcref)
-    return ex
-end
-
-function makeleaf(ctx::AbstractLoweringContext, srcref, proto)
+function JuliaSyntax.makeleaf(ctx::AbstractLoweringContext, srcref, proto)
     makeleaf(syntax_graph(ctx), srcref, proto)
 end
 
-function makeleaf(ctx, srcref, proto, @nospecialize(attrs::AbstractVector))
-    graph = syntax_graph(ctx)
-    ex = makeleaf(graph, srcref, proto)
-    for (k, v) in attrs
-        setattr!(graph, ex._id, k, v)
-    end
-    return ex
-end
-
-function makenode(ctx, srcref, proto, children, attrs=nothing)
-    graph = syntax_graph(ctx)
-    ex = isnothing(attrs) ? makeleaf(graph, srcref, proto) :
-        makeleaf(graph, srcref, proto, attrs)
-    setchildren!(graph, ex._id, children isa SyntaxList ? children.ids : children)
-    return ex
-end
-
 function newleaf(ctx, srcref, k::Kind, @nospecialize(value))
     leaf = makeleaf(ctx, srcref, k)
     if k == K"Identifier" || k == K"core" || k == K"top" || k == K"Symbol" ||
@@ -192,15 +122,6 @@ function newleaf(ctx, srcref, k::Kind, @nospecialize(value))
     leaf
 end
 
-# TODO: Replace this with makeleaf variant?
-function mapleaf(ctx, src, kind)
-    ex = makeleaf(syntax_graph(ctx), src, kind)
-    # TODO: Value coercion might be broken here due to use of `name_val` vs
-    # `value` vs ... ?
-    copy_attrs!(ex, src)
-    ex
-end
-
 # Convenience functions to create leaf nodes referring to identifiers within
 # the Core and Top modules.
 core_ref(ctx, ex, name) = newleaf(ctx, ex, K"core", name)
@@ -228,6 +149,32 @@ end
 
 #-------------------------------------------------------------------------------
 # @ast macro
+
+function JuliaSyntax._node_id(graph::SyntaxGraph, ex)
+    # Fallback to give a comprehensible error message for use with the @ast macro
+    error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`")
+end
+function JuliaSyntax._node_id(graph::SyntaxGraph, ex::AbstractVector{<:SyntaxTree})
+    # Fallback to give a comprehensible error message for use with the @ast macro
+    error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)")
+end
+
+function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val)
+    push!(ids, JuliaSyntax._node_id(graph, val))
+end
+function _push_nodeid!(graph::SyntaxGraph, ids::Vector{NodeId}, val::Nothing)
+    nothing
+end
+function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals)
+    for v in vals
+        _push_nodeid!(graph, ids, v)
+    end
+end
+function _append_nodeids!(graph::SyntaxGraph, ids::Vector{NodeId}, vals::SyntaxList)
+    check_compatible_graph(graph, vals)
+    append!(ids, vals.ids)
+end
+
 function _match_srcref(ex)
     if Meta.isexpr(ex, :macrocall) && ex.args[1] == Symbol("@HERE")
         QuoteNode(ex.args[2])
@@ -398,120 +345,6 @@ macro ast(ctx, srcref, tree)
     end
 end
 
-#-------------------------------------------------------------------------------
-# Mapping and copying of AST nodes
-function copy_attrs!(dest, src, all=false)
-    # TODO: Make this faster?
-    for (name, attr) in pairs(src._graph.attributes)
-        if (all || (name !== :source && name !== :kind && name !== :syntax_flags)) &&
-                haskey(attr, src._id)
-            dest_attr = getattr(dest._graph, name, nothing)
-            if !isnothing(dest_attr)
-                dest_attr[dest._id] = attr[src._id]
-            end
-        end
-    end
-end
-
-function copy_attrs!(dest, head::Union{Kind,JuliaSyntax.SyntaxHead}, all=false)
-    if all
-        setattr!(dest._graph, dest._id, :kind, kind(head))
-        !(head isa Kind) && setattr!(dest._graph, dest._id, :syntax_flags, flags(head))
-    end
-end
-
-function mapchildren(f::Function, ctx, ex::SyntaxTree, do_map_child::Function)
-    if is_leaf(ex)
-        return ex
-    end
-    orig_children = children(ex)
-    cs = nothing
-    for (i,e) in enumerate(orig_children)
-        newchild = do_map_child(i) ? f(e) : e
-        if isnothing(cs)
-            if newchild == e
-                continue
-            else
-                cs = SyntaxList(ctx)
-                append!(cs, orig_children[1:i-1])
-            end
-        end
-        push!(cs::SyntaxList, newchild)
-    end
-    if isnothing(cs)
-        # This function should be allocation-free if no children were changed
-        # by the mapping and there's no extra_attrs
-        return ex
-    end
-    cs::SyntaxList
-    ex2 = makenode(ctx, ex, ex, cs)
-    return ex2
-end
-
-function mapchildren(f::Function, ctx, ex::SyntaxTree,
-                     mapped_children::AbstractVector{<:Integer})
-    j = Ref(firstindex(mapped_children))
-    function do_map_child(i)
-        ind = j[]
-        if ind <= lastindex(mapped_children) && mapped_children[ind] == i
-            j[] += 1
-            true
-        else
-            false
-        end
-    end
-    mapchildren(f, ctx, ex, do_map_child)
-end
-
-function mapchildren(f::Function, ctx, ex::SyntaxTree)
-    mapchildren(f, ctx, ex, i->true)
-end
-
-
-"""
-Recursively copy AST `ex` into `ctx`.
-
-Special provenance handling: If `copy_source` is true, treat the `.source`
-attribute as a reference and recurse on its contents.  Otherwise, treat it like
-any other attribute.
-"""
-function copy_ast(ctx, ex::SyntaxTree; copy_source=true)
-    graph1 = syntax_graph(ex)
-    graph2 = syntax_graph(ctx)
-    !copy_source && check_same_graph(graph1, graph2)
-    id2 = _copy_ast(graph2, graph1, ex._id, Dict{NodeId, NodeId}(), copy_source)
-    return SyntaxTree(graph2, id2)
-end
-
-function _copy_ast(graph2::SyntaxGraph, graph1::SyntaxGraph,
-                   id1::NodeId, seen, copy_source)
-    let copied = get(seen, id1, nothing)
-        isnothing(copied) || return copied
-    end
-    id2 = newnode!(graph2)
-    seen[id1] = id2
-    src1 = get(SyntaxTree(graph1, id1), :source, nothing)
-    src2 = if !copy_source
-        src1
-    elseif src1 isa NodeId
-        _copy_ast(graph2, graph1, src1, seen, copy_source)
-    elseif src1 isa Tuple
-        map(i->_copy_ast(graph2, graph1, i, seen, copy_source), src1)
-    else
-        src1
-    end
-    copy_attrs!(SyntaxTree(graph2, id2), SyntaxTree(graph1, id1), true)
-    setattr!(graph2, id2, :source, src2)
-    if !is_leaf(graph1, id1)
-        cs = NodeId[]
-        for cid in children(graph1, id1)
-            push!(cs, _copy_ast(graph2, graph1, cid, seen, copy_source))
-        end
-        setchildren!(graph2, id2, cs)
-    end
-    return id2
-end
-
 #-------------------------------------------------------------------------------
 function set_scope_layer(ctx, ex, layer_id, force)
     k = kind(ex)

JuliaLowering/src/compat.jl

@@ -209,7 +209,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
     elseif e isa String
         st_id = _insert_tree_node(graph, K"string", src)
         id_inner = _insert_tree_node(graph, K"String", src, [:value=>e])
-        setchildren!(graph, st_id, [id_inner])
+        JuliaSyntax.setchildren!(graph, st_id, [id_inner])
         return st_id, src
     elseif !(e isa Expr)
         # There are other kinds we could potentially back-convert (e.g. Float),
@@ -560,7 +560,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
         return st_id, src
     else
         st_child_ids, last_src = _insert_child_exprs(e.head, child_exprs, graph, src)
-        setchildren!(graph, st_id, st_child_ids)
+        JuliaSyntax.setchildren!(graph, st_id, st_child_ids)
         return st_id, last_src
     end
 end
@@ -584,3 +584,45 @@ function _insert_child_exprs(head::Symbol, child_exprs::Vector{Any},
     end
     return st_child_ids, last_src
 end
+
+#-------------------------------------------------------------------------------
+# SyntaxTree->Expr overrides
+
+function JuliaSyntax._expr_leaf_val(ex::SyntaxTree, _...)
+    name = get(ex, :name_val, nothing)
+    if !isnothing(name)
+        n = Symbol(name)
+        if kind(ex) === K"Symbol"
+            return QuoteNode(n)
+        elseif hasattr(ex, :scope_layer)
+            Expr(:scope_layer, n, ex.scope_layer)
+        else
+            n
+        end
+    else
+        val = get(ex, :value, nothing)
+        if kind(ex) == K"Value" && val isa Expr || val isa LineNumberNode
+            # Expr AST embedded in a SyntaxTree should be quoted rather than
+            # becoming part of the output AST.
+            QuoteNode(val)
+        else
+            val
+        end
+    end
+end
+
+function JuliaSyntax.fixup_Expr_child(::Type{<:SyntaxTree}, head::SyntaxHead,
+                                      @nospecialize(arg), first::Bool)
+    isa(arg, Expr) || return arg
+    k = kind(head)
+    coalesce_dot = k in KSet"call dotcall curly" ||
+                   (k == K"quote" && has_flags(head, JuliaSyntax.COLON_QUOTE))
+    if @isexpr(arg, :., 1) && arg.args[1] isa Tuple
+        h, a = arg.args[1]::Tuple{SyntaxHead,Any}
+        arg = ((coalesce_dot && first) || is_syntactic_operator(h)) ?
+            Symbol(".", a) : Expr(:., a)
+    end
+    return arg
+end
+
+Base.Expr(ex::SyntaxTree) = JuliaSyntax.to_expr(ex)

JuliaLowering/src/desugaring.jl

@@ -1113,7 +1113,7 @@ function flatten_ncat_rows!(flat_elems, nrow_spans, row_major, parent_layout_dim
         layout_dim = 1
         @chk parent_layout_dim != 1 (ex,"Badly nested rows in `ncat`")
     elseif k == K"nrow"
-        dim = numeric_flags(ex)
+        dim = JuliaSyntax.numeric_flags(ex)
         @chk dim > 0                (ex,"Unsupported dimension $dim in ncat")
         @chk !row_major || dim != 2 (ex,"2D `nrow` cannot be mixed with `row` in `ncat`")
         layout_dim = nrow_flipdim(row_major, dim)
@@ -1146,7 +1146,7 @@ end
 # - ragged column first or row first
 function expand_ncat(ctx, ex)
     is_typed = kind(ex) == K"typed_ncat"
-    outer_dim = numeric_flags(ex)
+    outer_dim = JuliaSyntax.numeric_flags(ex)
     @chk outer_dim > 0 (ex,"Unsupported dimension in ncat")
     eltype      = is_typed ? ex[1]     : nothing
     elements    = is_typed ? ex[2:end] : ex[1:end]
@@ -2208,7 +2208,7 @@ function expand_decls(ctx, ex)
     bindings = children(ex)
     stmts = SyntaxList(ctx)
     for binding in bindings
-        if is_prec_assignment(kind(binding))
+        if JuliaSyntax.is_prec_assignment(kind(binding))
             @chk numchildren(binding) == 2
             # expand_assignment will create the type decls
             make_lhs_decls(ctx, stmts, declkind, declmeta, binding[1], false)

JuliaLowering/src/macro_expansion.jl

@@ -377,7 +377,7 @@ function remove_scope_layer!(ex)
             remove_scope_layer!(c)
         end
     end
-    deleteattr!(ex, :scope_layer)
+    JuliaSyntax.deleteattr!(ex, :scope_layer)
     ex
 end