Skip to content

[JuliaSyntax] Allow macros enclosed in parens to treat newlines as whitespace #60181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MasonProtter wants to merge 8 commits into
base: master
Choose a base branch
from
+72 −25
Open

[JuliaSyntax] Allow macros enclosed in parens to treat newlines as whitespace #60181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2a8a19d
Allow macros enclosed in parens to treat newlines as whitespace
MasonProtter Nov 20, 2025
9966bba
Only allow macro_whitespace_newline inside round brackets
MasonProtter Jan 13, 2026
a5b48aa
Merge branch 'master' into mp/newline-macros
MasonProtter Mar 31, 2026
1f542f8
Only outermost macro in parenthesized expression eats newline args
MasonProtter Apr 2, 2026
18dfdd0
Merge branch 'master' into mp/newline-macros
MasonProtter Apr 3, 2026
2427eda
only use the macro_whitespace_newline immediately following parens
MasonProtter May 20, 2026
efcacc7
fix msg
MasonProtter May 20, 2026
c822d36
Merge branch 'master' into mp/newline-macros
MasonProtter May 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 40 additions & 24 deletions JuliaSyntax/src/julia/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,27 @@ struct ParseState
whitespace_newline::Bool
# Enable parsing `where` with high precedence
where_enabled::Bool
# Treat newline like ordinary whitespace, but specifically for macro arguments
macro_whitespace_newline::Bool
end

# Normal context
function ParseState(stream::ParseStream)
ParseState(stream, true, false, false, false, false, true)
ParseState(stream, true, false, false, false, false, true, false)
end

function ParseState(ps::ParseState; range_colon_enabled=nothing,
space_sensitive=nothing, for_generator=nothing,
end_symbol=nothing, whitespace_newline=nothing,
where_enabled=nothing)
where_enabled=nothing, macro_whitespace_newline=false)
ParseState(ps.stream,
range_colon_enabled === nothing ? ps.range_colon_enabled : range_colon_enabled,
space_sensitive === nothing ? ps.space_sensitive : space_sensitive,
for_generator === nothing ? ps.for_generator : for_generator,
end_symbol === nothing ? ps.end_symbol : end_symbol,
whitespace_newline === nothing ? ps.whitespace_newline : whitespace_newline,
where_enabled === nothing ? ps.where_enabled : where_enabled)
where_enabled === nothing ? ps.where_enabled : where_enabled,
macro_whitespace_newline)
end

# Functions to change parse state
Expand All @@ -50,13 +53,15 @@ function normal_context(ps::ParseState)
where_enabled=true,
for_generator=false,
end_symbol=false,
whitespace_newline=false)
whitespace_newline=false,
macro_whitespace_newline=false)
end

function with_space_sensitive(ps::ParseState)
ParseState(ps,
space_sensitive=true,
whitespace_newline=false)
whitespace_newline=false,
macro_whitespace_newline=ps.macro_whitespace_newline)
end

# Convenient wrappers for ParseStream
Expand Down Expand Up @@ -377,7 +382,7 @@ function parse_LtoR(ps::ParseState, down, is_op)
is_op(tk) || break
isdot && bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG
bump(ps, remap_kind=K"Identifier")
down(ps)
down(ParseState(ps))
emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG)
end
end
Expand All @@ -392,7 +397,7 @@ function parse_RtoL(ps::ParseState, down, is_op, self)
isdot, tk = peek_dotted_op_token(ps)
if is_op(tk)
bump_dotted(ps, isdot, remap_kind=K"Identifier")
self(ps)
self(ParseState(ps))
emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG)
end
end
Expand Down Expand Up @@ -620,7 +625,7 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where {
# [a~b] ==> (vect (call-i a ~ b))
bump_dotted(ps, isdot, remap_kind=K"Identifier")
bump_trivia(ps)
parse_assignment(ps, down)
parse_assignment(ParseState(ps), down)
emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG)
else
# f() = 1 ==> (function-= (call f) 1)
Expand All @@ -642,7 +647,7 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where {
bump_trivia(ps)
# Syntax Edition TODO: We'd like to call `down` here when
# is_short_form_func is true, to prevent `f() = 1 = 2` from parsing.
parse_assignment(ps, down)
parse_assignment(ParseState(ps), down)
emit(ps, mark,
is_short_form_func ? K"function" : (isdot ? dotted(k) : k),
is_short_form_func ? SHORT_FORM_FUNCTION_FLAG : flags(t))
Expand Down Expand Up @@ -740,7 +745,7 @@ function parse_cond(ps::ParseState)
else
# A[x ? y : end] ==> (ref A (? x y end))
end
parse_eq_star(ps)
parse_eq_star(ParseState(ps))
emit(ps, mark, K"?")
end

Expand All @@ -756,15 +761,15 @@ function parse_arrow(ps::ParseState)
if kind(t) == K"-->" && !isdot && !is_suffixed(t)
# x --> y ==> (--> x y) # The only syntactic arrow
bump(ps, TRIVIA_FLAG)
parse_arrow(ps)
parse_arrow(ParseState(ps))
emit(ps, mark, k, flags(t))
else
# x → y ==> (call-i x → y)
# x <--> y ==> (call-i x <--> y)
# x .--> y ==> (dotcall-i x --> y)
# x -->₁ y ==> (call-i x -->₁ y)
bump_dotted(ps, isdot, remap_kind=K"Identifier")
parse_arrow(ps)
parse_arrow(ParseState(ps))
emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG)
end
end
Expand Down Expand Up @@ -792,7 +797,7 @@ function parse_lazy_cond(ps::ParseState, down, is_op, self)
k = kind(t)
if is_op(k)
bump_dotted(ps, isdot, TRIVIA_FLAG)
self(ps)
self(ParseState(ps))
emit(ps, mark, isdot ? dotted(k) : k, flags(t))
if isdot
min_supported_version(v"1.7", ps, mark, "dotted operators `.||` and `.&&`")
Expand Down Expand Up @@ -840,7 +845,7 @@ function parse_comparison(ps::ParseState, subtype_comparison=false)
n_comparisons += 1
op_dotted = isdot
op_pos = bump_dotted(ps, isdot, emit_dot_node=true, remap_kind=K"Identifier")
parse_pipe_lt(ps)
parse_pipe_lt(ParseState(ps))
end
if n_comparisons == 1
if is_type_operator(initial_tok, initial_dot)
Expand Down Expand Up @@ -1027,11 +1032,11 @@ function parse_with_chains(ps::ParseState, down, is_op, chain_ops)
break
end
bump_dotted(ps, isdot, remap_kind=K"Identifier")
down(ps)
down(ParseState(ps))
if kind(t) in chain_ops && !is_suffixed(t) && !isdot
# a + b + c ==> (call-i a + b c)
# a + b .+ c ==> (dotcall-i (call-i a + b) + c)
parse_chain(ps, down, kind(t))
parse_chain(ParseState(ps), down, kind(t))
end
# a +₁ b +₁ c ==> (call-i (call-i a +₁ b) +₁ c)
# a .+ b .+ c ==> (dotcall-i (dotcall-i a + b) + c)
Expand All @@ -1055,7 +1060,7 @@ function parse_chain(ps::ParseState, down, op_kind)
break
end
bump(ps, TRIVIA_FLAG)
down(ps)
down(ParseState(ps))
end
end

Expand Down Expand Up @@ -1125,7 +1130,7 @@ function parse_where_chain(ps0::ParseState, mark)
# x where T ==> (where x T)
# x where \n T ==> (where x T)
# x where T<:S ==> (where x (<: T S))
parse_comparison(ps)
parse_comparison(ParseState(ps))
emit(ps, mark, K"where")
end
end
Expand Down Expand Up @@ -1428,7 +1433,7 @@ function parse_decl_with_initial_ex(ps::ParseState, mark)
while peek(ps) == K"::"
# a::b::c ==> (::-i (::-i a b) c)
bump(ps, TRIVIA_FLAG)
parse_where(ps, parse_call)
parse_where(ParseState(ps), parse_call)
emit(ps, mark, K"::", INFIX_FLAG)
end
if peek(ps) == K"->"
Expand All @@ -1449,7 +1454,7 @@ function parse_decl_with_initial_ex(ps::ParseState, mark)
end
bump(ps, TRIVIA_FLAG)
# -> is unusual: it binds tightly on the left and loosely on the right.
parse_eq_star(ps)
parse_eq_star(ParseState(ps))
emit(ps, mark, K"->")
end
end
Expand Down Expand Up @@ -2846,11 +2851,21 @@ function parse_space_separated_exprs(ps::ParseState)
n_sep = 0
while true
k = peek(ps)
if is_closing_token(ps, k) || k == K"NewlineWs" ||
(ps.for_generator && k == K"for")
if is_closing_token(ps, k) || (ps.for_generator && k == K"for")
break
end
parse_eq(ps)
if k == K"NewlineWs"
if ps.macro_whitespace_newline
bump(ps, TRIVIA_FLAG)
continue
else
break
end
end
# Disable macro_whitespace_newline for sub-expressions so that only the
# outermost macro in a parenthesized context eats newline-separated args.
# E.g. in (@foo @bar x y\nz), z should be an arg of @foo not @bar.
parse_eq(ParseState(ps))
n_sep += 1
end
return n_sep
Expand Down Expand Up @@ -3294,7 +3309,8 @@ function parse_brackets(after_parse::F,
ps = ParseState(ps, range_colon_enabled=true,
space_sensitive=false,
where_enabled=true,
whitespace_newline=true)
whitespace_newline=true,
macro_whitespace_newline=closing_kind==K")")
params_positions = acquire_positions(ps.stream)
num_subexprs = 0
num_semis = 0
Expand Down
33 changes: 32 additions & 1 deletion JuliaSyntax/test/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ end

PARSE_ERROR = r"\(error"


with_version(v::VersionNumber, (i,o)::Pair) = ((;v=v), i) => o

# TODO:
Expand Down Expand Up @@ -346,7 +347,6 @@ tests = [
# parse_call_chain
"f(a).g(b)" => "(call (. (call f a) g) b)"
"\$A.@x" => "(macrocall (. (\$ A) (macro_name x)))"

# non-errors in space sensitive contexts
"[f (x)]" => "(hcat f (parens x))"
"[f x]" => "(hcat f x)"
Expand All @@ -364,6 +364,37 @@ tests = [
"A.@var\"#\" a"=> "(macrocall (. A (macro_name (var #))) a)"
"@+x y" => "(macrocall (macro_name +) x y)"
"A.@.x" => "(macrocall (. A (macro_name .)) x)"

# newline macro calls: only apply when macrocall is the immediate content of ()
"(@foo x\n y)" => "(parens (macrocall (macro_name foo) x y))"
# When @foo is on the RHS of a binary op, it does not eat newline args;
# y becomes an unexpected token (error) since it has no comma before it.
"(1 + @foo x\n y)" => "(parens (call-i 1 + (macrocall (macro_name foo) x)) (error-t y))"
# For backwards compatibility (1 + @foo x\n + 2) must parse as (+ 1 (@foo x) 2)
"(1 + @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))"
"(1 +\n @foo x\n + 2)" => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))"
"(@foo function bar()\n @baz \n x \n end)" => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))"
"(@foo \n (@bar \n x))" => "(parens (macrocall (macro_name foo) (parens (macrocall (macro_name bar) x))))"
# Don't change parsing rules for [] and {} cases!
"[x, @foo y\n z]" => "(vect x (macrocall (macro_name foo) y) (error-t z))"
"{@foo x\n y}" => "(bracescat (macrocall (macro_name foo) x) y)"
# Nested macros: newline args belong to outer macro, not inner
"(@foo @bar x\n y)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x) y))"
"(@foo @bar x y\n z)" => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x y) z))"
# Commas in call args with macro on previous line still parse correctly
"f(a, b, @bar 1\n, 2)" => "(call f a b (macrocall (macro_name bar) 1) 2)"
# In function call brackets, @foo IS the direct child so it eats newline args
"f(@foo x\n y)" => "(call f (macrocall (macro_name foo) x y))"
# When @foo is the direct content (LHS of nothing), it can eat unary across newline
"(@foo x\n + 2)" => "(parens (macrocall (macro_name foo) x (call-pre + 2)))"
# All binary/logical/comparison/arrow operators block newline eating on their RHS
"(a || @foo x\n y)" => "(parens (|| a (macrocall (macro_name foo) x)) (error-t y))"
"(a && @foo x\n y)" => "(parens (&& a (macrocall (macro_name foo) x)) (error-t y))"
"(a == @foo x\n y)" => "(parens (call-i a == (macrocall (macro_name foo) x)) (error-t y))"
"(a <: @foo x\n y)" => "(parens (<: a (macrocall (macro_name foo) x)) (error-t y))"
"(a --> @foo x\n y)" => "(parens (--> a (macrocall (macro_name foo) x)) (error-t y))"
"(a = @foo x\n y)" => "(parens (= a (macrocall (macro_name foo) x)) (error-t y))"

# Macro names
"@! x" => "(macrocall (macro_name !) x)"
"@.. x" => "(macrocall (macro_name ..) x)"
Expand Down