From 2a8a19d467bd9f7ad45e01159e0131122328a65b Mon Sep 17 00:00:00 2001 From: Mason Protter Date: Thu, 20 Nov 2025 11:16:52 +0100 Subject: [PATCH 1/5] Allow macros enclosed in parens to treat newlines as whitespace --- JuliaSyntax/src/julia/parser.jl | 26 +++++++++++++++++++------- JuliaSyntax/test/parser.jl | 8 +++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl index 4142a010bb6b9..5f819cfc0547a 100644 --- a/JuliaSyntax/src/julia/parser.jl +++ b/JuliaSyntax/src/julia/parser.jl @@ -21,24 +21,27 @@ struct ParseState whitespace_newline::Bool # Enable parsing `where` with high precedence where_enabled::Bool + # Treat newline like ordinary whitespace, but specifically for macro arguments + macro_whitespace_newline::Bool end # Normal context function ParseState(stream::ParseStream) - ParseState(stream, true, false, false, false, false, true) + ParseState(stream, true, false, false, false, false, true, false) end function ParseState(ps::ParseState; range_colon_enabled=nothing, space_sensitive=nothing, for_generator=nothing, end_symbol=nothing, whitespace_newline=nothing, - where_enabled=nothing) + where_enabled=nothing, macro_whitespace_newline=nothing) ParseState(ps.stream, range_colon_enabled === nothing ? ps.range_colon_enabled : range_colon_enabled, space_sensitive === nothing ? ps.space_sensitive : space_sensitive, for_generator === nothing ? ps.for_generator : for_generator, end_symbol === nothing ? ps.end_symbol : end_symbol, whitespace_newline === nothing ? ps.whitespace_newline : whitespace_newline, - where_enabled === nothing ? ps.where_enabled : where_enabled) + where_enabled === nothing ? ps.where_enabled : where_enabled, + macro_whitespace_newline === nothing ? ps.macro_whitespace_newline : macro_whitespace_newline) end # Functions to change parse state @@ -50,7 +53,8 @@ function normal_context(ps::ParseState) where_enabled=true, for_generator=false, end_symbol=false, - whitespace_newline=false) + whitespace_newline=false, + macro_whitespace_newline=false) end function with_space_sensitive(ps::ParseState) @@ -2727,10 +2731,17 @@ function parse_space_separated_exprs(ps::ParseState) n_sep = 0 while true k = peek(ps) - if is_closing_token(ps, k) || k == K"NewlineWs" || - (ps.for_generator && k == K"for") + if is_closing_token(ps, k) || (ps.for_generator && k == K"for") break end + if k == K"NewlineWs" + if ps.macro_whitespace_newline + bump(ps, TRIVIA_FLAG) + continue + else + break + end + end parse_eq(ps) n_sep += 1 end @@ -3176,7 +3187,8 @@ function parse_brackets(after_parse::Function, ps = ParseState(ps, range_colon_enabled=true, space_sensitive=false, where_enabled=true, - whitespace_newline=true) + whitespace_newline=true, + macro_whitespace_newline=true) params_positions = acquire_positions(ps.stream) last_eq_before_semi = 0 num_subexprs = 0 diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index 6c66ea40123e0..abca5c8f12f6c 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -29,6 +29,7 @@ end PARSE_ERROR = r"\(error-t " + with_version(v::VersionNumber, (i,o)::Pair) = ((;v=v), i) => o # TODO: @@ -346,7 +347,6 @@ tests = [ # parse_call_chain "f(a).g(b)" => "(call (. (call f a) g) b)" "\$A.@x" => "(macrocall (. (\$ A) (macro_name x)))" - # non-errors in space sensitive contexts "[f (x)]" => "(hcat f (parens x))" "[f x]" => "(hcat f x)" @@ -364,6 +364,12 @@ tests = [ "A.@var\"#\" a"=> "(macrocall (. A (macro_name (var #))) a)" "@+x y" => "(macrocall (macro_name +) x y)" "A.@.x" => "(macrocall (. A (macro_name .)) x)" + + # newline macro calls + "(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))" + "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x y)))" + "(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))" + # Macro names "@! x" => "(macrocall (macro_name !) x)" "@.. x" => "(macrocall (macro_name ..) x)" From 9966bbaf16352181cf5907e5933f4b49c0f8163b Mon Sep 17 00:00:00 2001 From: Mason Protter Date: Tue, 13 Jan 2026 10:07:24 +0100 Subject: [PATCH 2/5] Only allow macro_whitespace_newline inside round brackets --- JuliaSyntax/src/julia/parser.jl | 2 +- JuliaSyntax/test/parser.jl | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl index 5f819cfc0547a..04c7bddad2a98 100644 --- a/JuliaSyntax/src/julia/parser.jl +++ b/JuliaSyntax/src/julia/parser.jl @@ -3188,7 +3188,7 @@ function parse_brackets(after_parse::Function, space_sensitive=false, where_enabled=true, whitespace_newline=true, - macro_whitespace_newline=true) + macro_whitespace_newline=closing_kind==K")") params_positions = acquire_positions(ps.stream) last_eq_before_semi = 0 num_subexprs = 0 diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index abca5c8f12f6c..a41c1880401ac 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -366,9 +366,12 @@ tests = [ "A.@.x" => "(macrocall (. A (macro_name .)) x)" # newline macro calls - "(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))" - "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x y)))" + "(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))" + "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x y)))" "(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))" + # Don't change parsing rules for [] and {} cases! + "[x, @foo y\n z]" => "(vect x (macrocall (macro_name foo) y) (error-t z))" + "{@foo x\n y}" => "(bracescat (macrocall (macro_name foo) x) y)" # Macro names "@! x" => "(macrocall (macro_name !) x)" From 1f542f8009ab24819158ac89dbc75f35c90e7b73 Mon Sep 17 00:00:00 2001 From: Mason Protter Date: Thu, 2 Apr 2026 21:26:29 +0200 Subject: [PATCH 3/5] Only outermost macro in parenthesized expression eats newline args --- JuliaSyntax/src/julia/parser.jl | 5 ++++- JuliaSyntax/test/parser.jl | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl index 957f8b6a45520..63a8d0a8d4a59 100644 --- a/JuliaSyntax/src/julia/parser.jl +++ b/JuliaSyntax/src/julia/parser.jl @@ -2860,7 +2860,10 @@ function parse_space_separated_exprs(ps::ParseState) break end end - parse_eq(ps) + # Disable macro_whitespace_newline for sub-expressions so that only the + # outermost macro in a parenthesized context eats newline-separated args. + # E.g. in (@foo @bar x y\nz), z should be an arg of @foo not @bar. + parse_eq(ParseState(ps, macro_whitespace_newline=false)) n_sep += 1 end return n_sep diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index 96b328939a1ee..27120a2cd498e 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -369,9 +369,15 @@ tests = [ "(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))" "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x y)))" "(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))" + "(@foo \n (@bar \n x))" => "(parens (macrocall (macro_name foo) (parens (macrocall (macro_name bar) x))))" # Don't change parsing rules for [] and {} cases! "[x, @foo y\n z]" => "(vect x (macrocall (macro_name foo) y) (error-t z))" "{@foo x\n y}" => "(bracescat (macrocall (macro_name foo) x) y)" + # Nested macros: newline args belong to outer macro, not inner + "(@foo @bar x\n y)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x) y))" + "(@foo @bar x y\n z)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x y) z))" + # Commas in call args with macro on previous line still parse correctly + "f(a, b, @bar 1\n, 2)" => "(call f a b (macrocall (macro_name bar) 1) 2)" # Macro names "@! x" => "(macrocall (macro_name !) x)" From 2427eda8b404007b2a2cbb4204bd929b99d142a2 Mon Sep 17 00:00:00 2001 From: MasonProtter Date: Wed, 20 May 2026 13:18:25 +0200 Subject: [PATCH 4/5] only use the macro_whitespace_newline immediately following parens --- JuliaSyntax/src/julia/parser.jl | 39 +++++++++++++++++---------------- JuliaSyntax/test/parser.jl | 20 +++++++++++++++-- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl index b393ac7a13e40..cd6381462b595 100644 --- a/JuliaSyntax/src/julia/parser.jl +++ b/JuliaSyntax/src/julia/parser.jl @@ -33,7 +33,7 @@ end function ParseState(ps::ParseState; range_colon_enabled=nothing, space_sensitive=nothing, for_generator=nothing, end_symbol=nothing, whitespace_newline=nothing, - where_enabled=nothing, macro_whitespace_newline=nothing) + where_enabled=nothing, macro_whitespace_newline=false) ParseState(ps.stream, range_colon_enabled === nothing ? ps.range_colon_enabled : range_colon_enabled, space_sensitive === nothing ? ps.space_sensitive : space_sensitive, @@ -41,7 +41,7 @@ function ParseState(ps::ParseState; range_colon_enabled=nothing, end_symbol === nothing ? ps.end_symbol : end_symbol, whitespace_newline === nothing ? ps.whitespace_newline : whitespace_newline, where_enabled === nothing ? ps.where_enabled : where_enabled, - macro_whitespace_newline === nothing ? ps.macro_whitespace_newline : macro_whitespace_newline) + macro_whitespace_newline) end # Functions to change parse state @@ -60,7 +60,8 @@ end function with_space_sensitive(ps::ParseState) ParseState(ps, space_sensitive=true, - whitespace_newline=false) + whitespace_newline=false, + macro_whitespace_newline=ps.macro_whitespace_newline) end # Convenient wrappers for ParseStream @@ -381,7 +382,7 @@ function parse_LtoR(ps::ParseState, down, is_op) is_op(tk) || break isdot && bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG bump(ps, remap_kind=K"Identifier") - down(ps) + down(ParseState(ps)) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) end end @@ -396,7 +397,7 @@ function parse_RtoL(ps::ParseState, down, is_op, self) isdot, tk = peek_dotted_op_token(ps) if is_op(tk) bump_dotted(ps, isdot, remap_kind=K"Identifier") - self(ps) + self(ParseState(ps)) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) end end @@ -624,7 +625,7 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where { # [a~b] ==> (vect (call-i a ~ b)) bump_dotted(ps, isdot, remap_kind=K"Identifier") bump_trivia(ps) - parse_assignment(ps, down) + parse_assignment(ParseState(ps), down) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) else # f() = 1 ==> (function-= (call f) 1) @@ -646,7 +647,7 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where { bump_trivia(ps) # Syntax Edition TODO: We'd like to call `down` here when # is_short_form_func is true, to prevent `f() = 1 = 2` from parsing. - parse_assignment(ps, down) + parse_assignment(ParseState(ps), down) emit(ps, mark, is_short_form_func ? K"function" : (isdot ? dotted(k) : k), is_short_form_func ? SHORT_FORM_FUNCTION_FLAG : flags(t)) @@ -744,7 +745,7 @@ function parse_cond(ps::ParseState) else # A[x ? y : end] ==> (ref A (? x y end)) end - parse_eq_star(ps) + parse_eq_star(ParseState(ps)) emit(ps, mark, K"?") end @@ -760,7 +761,7 @@ function parse_arrow(ps::ParseState) if kind(t) == K"-->" && !isdot && !is_suffixed(t) # x --> y ==> (--> x y) # The only syntactic arrow bump(ps, TRIVIA_FLAG) - parse_arrow(ps) + parse_arrow(ParseState(ps)) emit(ps, mark, k, flags(t)) else # x → y ==> (call-i x → y) @@ -768,7 +769,7 @@ function parse_arrow(ps::ParseState) # x .--> y ==> (dotcall-i x --> y) # x -->₁ y ==> (call-i x -->₁ y) bump_dotted(ps, isdot, remap_kind=K"Identifier") - parse_arrow(ps) + parse_arrow(ParseState(ps)) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) end end @@ -796,7 +797,7 @@ function parse_lazy_cond(ps::ParseState, down, is_op, self) k = kind(t) if is_op(k) bump_dotted(ps, isdot, TRIVIA_FLAG) - self(ps) + self(ParseState(ps)) emit(ps, mark, isdot ? dotted(k) : k, flags(t)) if isdot min_supported_version(v"1.7", ps, mark, "dotted operators `.||` and `.&&`") @@ -844,7 +845,7 @@ function parse_comparison(ps::ParseState, subtype_comparison=false) n_comparisons += 1 op_dotted = isdot op_pos = bump_dotted(ps, isdot, emit_dot_node=true, remap_kind=K"Identifier") - parse_pipe_lt(ps) + parse_pipe_lt(ParseState(ps)) end if n_comparisons == 1 if is_type_operator(initial_tok, initial_dot) @@ -1031,11 +1032,11 @@ function parse_with_chains(ps::ParseState, down, is_op, chain_ops) break end bump_dotted(ps, isdot, remap_kind=K"Identifier") - down(ps) + down(ParseState(ps)) if kind(t) in chain_ops && !is_suffixed(t) && !isdot # a + b + c ==> (call-i a + b c) # a + b .+ c ==> (dotcall-i (call-i a + b) + c) - parse_chain(ps, down, kind(t)) + parse_chain(ParseState(ps), down, kind(t)) end # a +₁ b +₁ c ==> (call-i (call-i a +₁ b) +₁ c) # a .+ b .+ c ==> (dotcall-i (dotcall-i a + b) + c) @@ -1059,7 +1060,7 @@ function parse_chain(ps::ParseState, down, op_kind) break end bump(ps, TRIVIA_FLAG) - down(ps) + down(ParseState(ps)) end end @@ -1129,7 +1130,7 @@ function parse_where_chain(ps0::ParseState, mark) # x where T ==> (where x T) # x where \n T ==> (where x T) # x where T<:S ==> (where x (<: T S)) - parse_comparison(ps) + parse_comparison(ParseState(ps)) emit(ps, mark, K"where") end end @@ -1432,7 +1433,7 @@ function parse_decl_with_initial_ex(ps::ParseState, mark) while peek(ps) == K"::" # a::b::c ==> (::-i (::-i a b) c) bump(ps, TRIVIA_FLAG) - parse_where(ps, parse_call) + parse_where(ParseState(ps), parse_call) emit(ps, mark, K"::", INFIX_FLAG) end if peek(ps) == K"->" @@ -1453,7 +1454,7 @@ function parse_decl_with_initial_ex(ps::ParseState, mark) end bump(ps, TRIVIA_FLAG) # -> is unusual: it binds tightly on the left and loosely on the right. - parse_eq_star(ps) + parse_eq_star(ParseState(ps)) emit(ps, mark, K"->") end end @@ -2864,7 +2865,7 @@ function parse_space_separated_exprs(ps::ParseState) # Disable macro_whitespace_newline for sub-expressions so that only the # outermost macro in a parenthesized context eats newline-separated args. # E.g. in (@foo @bar x y\nz), z should be an arg of @foo not @bar. - parse_eq(ParseState(ps, macro_whitespace_newline=false)) + parse_eq(ParseState(ps)) n_sep += 1 end return n_sep diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index 27120a2cd498e..f4bc11aeae32c 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -365,9 +365,14 @@ tests = [ "@+x y" => "(macrocall (macro_name +) x y)" "A.@.x" => "(macrocall (. A (macro_name .)) x)" - # newline macro calls + # newline macro calls: only apply when macrocall is the immediate content of () "(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))" - "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x y)))" + # When @foo is on the RHS of a binary op, it does not eat newline args; + # y becomes an unexpected token (error) since it has no comma before it. + "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x)) (error-t y))" + # The main regression: (1 + @foo x\n + 2) must parse as (+ 1 (@foo x) 2) + "(1 + @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" + "(1 +\n @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" "(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))" "(@foo \n (@bar \n x))" => "(parens (macrocall (macro_name foo) (parens (macrocall (macro_name bar) x))))" # Don't change parsing rules for [] and {} cases! @@ -378,6 +383,17 @@ tests = [ "(@foo @bar x y\n z)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x y) z))" # Commas in call args with macro on previous line still parse correctly "f(a, b, @bar 1\n, 2)" => "(call f a b (macrocall (macro_name bar) 1) 2)" + # In function call brackets, @foo IS the direct child so it eats newline args + "f(@foo x\n y)" => "(call f (macrocall (macro_name foo) x y))" + # When @foo is the direct content (LHS of nothing), it can eat unary across newline + "(@foo x\n + 2)" => "(parens (macrocall (macro_name foo) x (call-pre + 2)))" + # All binary/logical/comparison/arrow operators block newline eating on their RHS + "(a || @foo x\n y)" => "(parens (|| a (macrocall (macro_name foo) x)) (error-t y))" + "(a && @foo x\n y)" => "(parens (&& a (macrocall (macro_name foo) x)) (error-t y))" + "(a == @foo x\n y)" => "(parens (call-i a == (macrocall (macro_name foo) x)) (error-t y))" + "(a <: @foo x\n y)" => "(parens (<: a (macrocall (macro_name foo) x)) (error-t y))" + "(a --> @foo x\n y)" => "(parens (--> a (macrocall (macro_name foo) x)) (error-t y))" + "(a = @foo x\n y)" => "(parens (= a (macrocall (macro_name foo) x)) (error-t y))" # Macro names "@! x" => "(macrocall (macro_name !) x)" From efcacc7df17b176f7215a332c867a0fcf8a566b0 Mon Sep 17 00:00:00 2001 From: MasonProtter Date: Wed, 20 May 2026 13:26:12 +0200 Subject: [PATCH 5/5] fix msg --- JuliaSyntax/test/parser.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index f4bc11aeae32c..c0c9ff824af4e 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -370,7 +370,7 @@ tests = [ # When @foo is on the RHS of a binary op, it does not eat newline args; # y becomes an unexpected token (error) since it has no comma before it. "(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x)) (error-t y))" - # The main regression: (1 + @foo x\n + 2) must parse as (+ 1 (@foo x) 2) + # For backwards compatibility (1 + @foo x\n + 2) must parse as (+ 1 (@foo x) 2) "(1 + @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" "(1 +\n @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" "(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))"