diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl index 735939c2f1890..cd6381462b595 100644 --- a/JuliaSyntax/src/julia/parser.jl +++ b/JuliaSyntax/src/julia/parser.jl @@ -21,24 +21,27 @@ struct ParseState whitespace_newline::Bool # Enable parsing `where` with high precedence where_enabled::Bool + # Treat newline like ordinary whitespace, but specifically for macro arguments + macro_whitespace_newline::Bool end # Normal context function ParseState(stream::ParseStream) - ParseState(stream, true, false, false, false, false, true) + ParseState(stream, true, false, false, false, false, true, false) end function ParseState(ps::ParseState; range_colon_enabled=nothing, space_sensitive=nothing, for_generator=nothing, end_symbol=nothing, whitespace_newline=nothing, - where_enabled=nothing) + where_enabled=nothing, macro_whitespace_newline=false) ParseState(ps.stream, range_colon_enabled === nothing ? ps.range_colon_enabled : range_colon_enabled, space_sensitive === nothing ? ps.space_sensitive : space_sensitive, for_generator === nothing ? ps.for_generator : for_generator, end_symbol === nothing ? ps.end_symbol : end_symbol, whitespace_newline === nothing ? ps.whitespace_newline : whitespace_newline, - where_enabled === nothing ? ps.where_enabled : where_enabled) + where_enabled === nothing ? ps.where_enabled : where_enabled, + macro_whitespace_newline) end # Functions to change parse state @@ -50,13 +53,15 @@ function normal_context(ps::ParseState) where_enabled=true, for_generator=false, end_symbol=false, - whitespace_newline=false) + whitespace_newline=false, + macro_whitespace_newline=false) end function with_space_sensitive(ps::ParseState) ParseState(ps, space_sensitive=true, - whitespace_newline=false) + whitespace_newline=false, + macro_whitespace_newline=ps.macro_whitespace_newline) end # Convenient wrappers for ParseStream @@ -377,7 +382,7 @@ function parse_LtoR(ps::ParseState, down, is_op) is_op(tk) || break isdot && bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG bump(ps, remap_kind=K"Identifier") - down(ps) + down(ParseState(ps)) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) end end @@ -392,7 +397,7 @@ function parse_RtoL(ps::ParseState, down, is_op, self) isdot, tk = peek_dotted_op_token(ps) if is_op(tk) bump_dotted(ps, isdot, remap_kind=K"Identifier") - self(ps) + self(ParseState(ps)) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) end end @@ -620,7 +625,7 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where { # [a~b] ==> (vect (call-i a ~ b)) bump_dotted(ps, isdot, remap_kind=K"Identifier") bump_trivia(ps) - parse_assignment(ps, down) + parse_assignment(ParseState(ps), down) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) else # f() = 1 ==> (function-= (call f) 1) @@ -642,7 +647,7 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where { bump_trivia(ps) # Syntax Edition TODO: We'd like to call `down` here when # is_short_form_func is true, to prevent `f() = 1 = 2` from parsing. - parse_assignment(ps, down) + parse_assignment(ParseState(ps), down) emit(ps, mark, is_short_form_func ? K"function" : (isdot ? dotted(k) : k), is_short_form_func ? SHORT_FORM_FUNCTION_FLAG : flags(t)) @@ -740,7 +745,7 @@ function parse_cond(ps::ParseState) else # A[x ? y : end] ==> (ref A (? x y end)) end - parse_eq_star(ps) + parse_eq_star(ParseState(ps)) emit(ps, mark, K"?") end @@ -756,7 +761,7 @@ function parse_arrow(ps::ParseState) if kind(t) == K"-->" && !isdot && !is_suffixed(t) # x --> y ==> (--> x y) # The only syntactic arrow bump(ps, TRIVIA_FLAG) - parse_arrow(ps) + parse_arrow(ParseState(ps)) emit(ps, mark, k, flags(t)) else # x → y ==> (call-i x → y) @@ -764,7 +769,7 @@ function parse_arrow(ps::ParseState) # x .--> y ==> (dotcall-i x --> y) # x -->₁ y ==> (call-i x -->₁ y) bump_dotted(ps, isdot, remap_kind=K"Identifier") - parse_arrow(ps) + parse_arrow(ParseState(ps)) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) end end @@ -792,7 +797,7 @@ function parse_lazy_cond(ps::ParseState, down, is_op, self) k = kind(t) if is_op(k) bump_dotted(ps, isdot, TRIVIA_FLAG) - self(ps) + self(ParseState(ps)) emit(ps, mark, isdot ? dotted(k) : k, flags(t)) if isdot min_supported_version(v"1.7", ps, mark, "dotted operators `.||` and `.&&`") @@ -840,7 +845,7 @@ function parse_comparison(ps::ParseState, subtype_comparison=false) n_comparisons += 1 op_dotted = isdot op_pos = bump_dotted(ps, isdot, emit_dot_node=true, remap_kind=K"Identifier") - parse_pipe_lt(ps) + parse_pipe_lt(ParseState(ps)) end if n_comparisons == 1 if is_type_operator(initial_tok, initial_dot) @@ -1027,11 +1032,11 @@ function parse_with_chains(ps::ParseState, down, is_op, chain_ops) break end bump_dotted(ps, isdot, remap_kind=K"Identifier") - down(ps) + down(ParseState(ps)) if kind(t) in chain_ops && !is_suffixed(t) && !isdot # a + b + c ==> (call-i a + b c) # a + b .+ c ==> (dotcall-i (call-i a + b) + c) - parse_chain(ps, down, kind(t)) + parse_chain(ParseState(ps), down, kind(t)) end # a +₁ b +₁ c ==> (call-i (call-i a +₁ b) +₁ c) # a .+ b .+ c ==> (dotcall-i (dotcall-i a + b) + c) @@ -1055,7 +1060,7 @@ function parse_chain(ps::ParseState, down, op_kind) break end bump(ps, TRIVIA_FLAG) - down(ps) + down(ParseState(ps)) end end @@ -1125,7 +1130,7 @@ function parse_where_chain(ps0::ParseState, mark) # x where T ==> (where x T) # x where \n T ==> (where x T) # x where T<:S ==> (where x (<: T S)) - parse_comparison(ps) + parse_comparison(ParseState(ps)) emit(ps, mark, K"where") end end @@ -1428,7 +1433,7 @@ function parse_decl_with_initial_ex(ps::ParseState, mark) while peek(ps) == K"::" # a::b::c ==> (::-i (::-i a b) c) bump(ps, TRIVIA_FLAG) - parse_where(ps, parse_call) + parse_where(ParseState(ps), parse_call) emit(ps, mark, K"::", INFIX_FLAG) end if peek(ps) == K"->" @@ -1449,7 +1454,7 @@ function parse_decl_with_initial_ex(ps::ParseState, mark) end bump(ps, TRIVIA_FLAG) # -> is unusual: it binds tightly on the left and loosely on the right. - parse_eq_star(ps) + parse_eq_star(ParseState(ps)) emit(ps, mark, K"->") end end @@ -2846,11 +2851,21 @@ function parse_space_separated_exprs(ps::ParseState) n_sep = 0 while true k = peek(ps) - if is_closing_token(ps, k) || k == K"NewlineWs" || - (ps.for_generator && k == K"for") + if is_closing_token(ps, k) || (ps.for_generator && k == K"for") break end - parse_eq(ps) + if k == K"NewlineWs" + if ps.macro_whitespace_newline + bump(ps, TRIVIA_FLAG) + continue + else + break + end + end + # Disable macro_whitespace_newline for sub-expressions so that only the + # outermost macro in a parenthesized context eats newline-separated args. + # E.g. in (@foo @bar x y\nz), z should be an arg of @foo not @bar. + parse_eq(ParseState(ps)) n_sep += 1 end return n_sep @@ -3294,7 +3309,8 @@ function parse_brackets(after_parse::F, ps = ParseState(ps, range_colon_enabled=true, space_sensitive=false, where_enabled=true, - whitespace_newline=true) + whitespace_newline=true, + macro_whitespace_newline=closing_kind==K")") params_positions = acquire_positions(ps.stream) num_subexprs = 0 num_semis = 0 diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index a260e34e54612..c0c9ff824af4e 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -29,6 +29,7 @@ end PARSE_ERROR = r"\(error" + with_version(v::VersionNumber, (i,o)::Pair) = ((;v=v), i) => o # TODO: @@ -346,7 +347,6 @@ tests = [ # parse_call_chain "f(a).g(b)" => "(call (. (call f a) g) b)" "\$A.@x" => "(macrocall (. (\$ A) (macro_name x)))" - # non-errors in space sensitive contexts "[f (x)]" => "(hcat f (parens x))" "[f x]" => "(hcat f x)" @@ -364,6 +364,37 @@ tests = [ "A.@var\"#\" a"=> "(macrocall (. A (macro_name (var #))) a)" "@+x y" => "(macrocall (macro_name +) x y)" "A.@.x" => "(macrocall (. A (macro_name .)) x)" + + # newline macro calls: only apply when macrocall is the immediate content of () + "(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))" + # When @foo is on the RHS of a binary op, it does not eat newline args; + # y becomes an unexpected token (error) since it has no comma before it. + "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x)) (error-t y))" + # For backwards compatibility (1 + @foo x\n + 2) must parse as (+ 1 (@foo x) 2) + "(1 + @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" + "(1 +\n @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" + "(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))" + "(@foo \n (@bar \n x))" => "(parens (macrocall (macro_name foo) (parens (macrocall (macro_name bar) x))))" + # Don't change parsing rules for [] and {} cases! + "[x, @foo y\n z]" => "(vect x (macrocall (macro_name foo) y) (error-t z))" + "{@foo x\n y}" => "(bracescat (macrocall (macro_name foo) x) y)" + # Nested macros: newline args belong to outer macro, not inner + "(@foo @bar x\n y)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x) y))" + "(@foo @bar x y\n z)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x y) z))" + # Commas in call args with macro on previous line still parse correctly + "f(a, b, @bar 1\n, 2)" => "(call f a b (macrocall (macro_name bar) 1) 2)" + # In function call brackets, @foo IS the direct child so it eats newline args + "f(@foo x\n y)" => "(call f (macrocall (macro_name foo) x y))" + # When @foo is the direct content (LHS of nothing), it can eat unary across newline + "(@foo x\n + 2)" => "(parens (macrocall (macro_name foo) x (call-pre + 2)))" + # All binary/logical/comparison/arrow operators block newline eating on their RHS + "(a || @foo x\n y)" => "(parens (|| a (macrocall (macro_name foo) x)) (error-t y))" + "(a && @foo x\n y)" => "(parens (&& a (macrocall (macro_name foo) x)) (error-t y))" + "(a == @foo x\n y)" => "(parens (call-i a == (macrocall (macro_name foo) x)) (error-t y))" + "(a <: @foo x\n y)" => "(parens (<: a (macrocall (macro_name foo) x)) (error-t y))" + "(a --> @foo x\n y)" => "(parens (--> a (macrocall (macro_name foo) x)) (error-t y))" + "(a = @foo x\n y)" => "(parens (= a (macrocall (macro_name foo) x)) (error-t y))" + # Macro names "@! x" => "(macrocall (macro_name !) x)" "@.. x" => "(macrocall (macro_name ..) x)"