[JuliaSyntax] Remove separate syntax heads for each operator

JuliaLowering/src/compat.jl

@@ -6,10 +6,18 @@ function find_kind(s::String)
 end
 
 # flisp: dot-operators
+#
+# We work from the operator's name here (rather than its `Kind`) because by this
+# point operators are represented uniformly as identifier-like names: this code
+# also runs on trees converted from `Expr`, where an operator such as `.^` is
+# simply the `Symbol` `:.^` with no token or `Kind` to inspect. `Base.isoperator`
+# is the same operator-name test already used for `op=` in `est_to_dst` below;
+# note we can't look up a `Kind` by name (eg via `find_kind`), since most
+# operators no longer have their own kind - they share `K"Operator"`.
 function is_dotted_operator(s::AbstractString)
     return length(s) >= 2 &&
         s[1] === '.' && s[2] !== '.' &&
-        JS.is_operator(something(find_kind(s[2:end]), K"None"))
+        Base.isoperator(s[2:end])
 end
 
 function is_eventually_call(e)

JuliaSyntax/docs/src/api.md

@@ -101,7 +101,6 @@ JuliaSyntax.is_infix_op_call
 JuliaSyntax.is_prefix_op_call
 JuliaSyntax.is_postfix_op_call
 JuliaSyntax.is_dotted
-JuliaSyntax.is_suffixed
 JuliaSyntax.is_decorated
 JuliaSyntax.numeric_flags
 ```

JuliaSyntax/docs/src/reference.md

@@ -80,7 +80,7 @@ class of tokenization errors and lets the parser deal with them.
 * We use flags rather than child nodes to represent the difference between `struct` and `mutable struct`, `module` and `baremodule` (#220)
 * Iterations are represented with the `iteration` and `in` heads rather than `=` within the header of a `for`. Thus `for i=is ; body end` parses to `(for (iteration (in i is)) (block body))`. Cartesian iteration as in `for a=as, b=bs body end` are represented with a nested `(iteration (in a as) (in b bs))` rather than a `block` containing `=` because these lists of iterators are neither semantically nor syntactically a sequence of statements, unlike other uses of `block`. Generators also use the `iteration` head - see information on that below.
 * Short form functions like `f(x) = x + 1` are represented with the `function` head rather than the `=` head. In this case the `SHORT_FORM_FUNCTION_FLAG` flag is set to allow the surface syntactic form to be easily distinguished from long form functions.
-* All kinds of updating assignment operators like `+=` are represented with a single `K"op="` head, with the operator itself in infix position. For example, `x += 1` is `(op= x + 1)`, where the plus token is of kind `K"Identifier"`. This greatly reduces the number of distinct forms here from a rather big list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=` `^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as kind `K"Identifier"`, as it should. It also makes it possible to add further unicode updating operators while keeping the AST stable.
+* All kinds of updating assignment operators like `+=` are represented with a single `K"op="` head, with the operator itself in infix position. For example, `x += 1` is `(op= x + 1)`, where the plus token is of kind `K"Identifier"`. This greatly reduces the number of distinct forms here from a rather big list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=` `^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as kind `K"Identifier"`, as it should. It also makes it possible to add further unicode updating operators while keeping the AST stable. The broadcasting form `x .+= 1` uses the `K".op="` head, ie `(.op= x + 1)`. When an updating operator appears without operands - for example when quoted as `:(+=)` - it is represented by the one-argument form `(op= +)` (and likewise `(.op= +)` for `:(.+=)`), which converts to the `Symbol` `:+=` (resp. `:.+=`).
 * The lexer never emits the multi-dot sequences `..` and `...` as single tokens; it always produces a sequence of `K"."` tokens which the parser groups as required (#573). When `..`/`...` are used as ordinary identifiers (for example the `..` operator in `a .. b`, or `...` quoted as in `:(...)`) the dots are gathered into a single leaf of kind `K"DotsIdentifier"` rather than `K"Identifier"`, with the number of dots stored in the numeric flags. Like `K"var"` and `K"MacroName"`, `K"DotsIdentifier"` is one of the kinds which may appear where an identifier is expected, so consumers matching identifiers should account for it rather than testing for `K"Identifier"` alone. `K"..."` is now used exclusively for splatting/slurping (as in `f(x...)`).
 
 ## More detail on tree differences

JuliaSyntax/src/JuliaSyntax.jl

@@ -39,6 +39,12 @@ export SourceFile
 # Expression predicates, kinds and flags
 export @K_str, kind
 @_public Kind
+@_public PrecedenceLevel, PREC_NONE, PREC_ASSIGNMENT,
+    PREC_PAIRARROW, PREC_CONDITIONAL, PREC_ARROW, PREC_LAZYOR, PREC_LAZYAND,
+    PREC_COMPARISON, PREC_PIPE_LT, PREC_PIPE_GT, PREC_COLON, PREC_PLUS,
+    PREC_BITSHIFT, PREC_TIMES, PREC_RATIONAL, PREC_POWER, PREC_DECL,
+    PREC_WHERE, PREC_DOT, PREC_QUOTE, PREC_UNICODE_OPS, PREC_COMPOUND_ASSIGN,
+    generic_operators_by_level
 
 @_public flags,
     SyntaxHead,
@@ -49,7 +55,6 @@ export @K_str, kind
     is_prefix_op_call,
     is_postfix_op_call,
     is_dotted,
-    is_suffixed,
     is_decorated,
     numeric_flags,
     has_flags,

JuliaSyntax/src/core/parse_stream.jl

@@ -45,7 +45,7 @@ kind(head::SyntaxHead) = head.kind
 
 Return the flag bits of a syntactic construct. Prefer to query these with the
 predicates `is_trivia`, `is_prefix_call`, `is_infix_op_call`,
-`is_prefix_op_call`, `is_postfix_op_call`, `is_dotted`, `is_suffixed`,
+`is_prefix_op_call`, `is_postfix_op_call`, `is_dotted`,
 `is_decorated`.
 
 Or extract numeric portion of the flags with `numeric_flags`.
@@ -376,7 +376,10 @@ function _buffer_lookahead_tokens(lexer, lookahead)
         was_whitespace = is_whitespace(k)
         had_whitespace |= was_whitespace
         f = EMPTY_FLAGS
-        raw.suffix     && (f |= SUFFIXED_FLAG)
+        if k == K"Operator" && raw.op_precedence != Tokenize.PREC_NONE
+            # Store operator precedence in numeric flags
+            f |= set_numeric_flags(Int(raw.op_precedence))
+        end
         push!(lookahead, SyntaxToken(SyntaxHead(k, f), k,
                                      had_whitespace, raw.endbyte + 2))
         token_count += 1

JuliaSyntax/src/integration/expr.jl

@@ -346,20 +346,28 @@ end
     elseif k == K"DotsIdentifier"
         n = numeric_flags(flags(nodehead))
         return n == 2 ? :(..) : :(...)
-    elseif k == K"op=" && length(args) == 3
-        lhs = args[1]
-        op = args[2]
-        rhs = args[3]
-        headstr = string(args[2], '=')
-        retexpr.head = Symbol(headstr)
-        retexpr.args = Any[lhs, rhs]
-    elseif k == K".op=" && length(args) == 3
-        lhs = args[1]
-        op = args[2]
-        rhs = args[3]
-        headstr = '.' * string(args[2], '=')
-        retexpr.head = Symbol(headstr)
-        retexpr.args = Any[lhs, rhs]
+    elseif k == K"op="
+        if length(args) == 3
+            lhs = args[1]
+            op = args[2]
+            rhs = args[3]
+            headstr = string(args[2], '=')
+            retexpr.head = Symbol(headstr)
+            retexpr.args = Any[lhs, rhs]
+        elseif length(args) == 1
+            return Symbol(string(args[1], '='))
+        end
+    elseif k == K".op="
+        if length(args) == 3
+            lhs = args[1]
+            op = args[2]
+            rhs = args[3]
+            headstr = '.' * string(args[2], '=')
+            retexpr.head = Symbol(headstr)
+            retexpr.args = Any[lhs, rhs]
+        else
+            return Symbol(string('.', args[1], '='))
+        end
     elseif k == K"macrocall"
         if length(args) >= 2
             a2 = args[2]

JuliaSyntax/src/julia/julia_parse_stream.jl

@@ -1,7 +1,3 @@
-# Token flags - may be set for operator kinded tokens
-# Operator has a suffix
-const SUFFIXED_FLAG = RawFlags(1<<2)
-
 # Set for K"call", K"dotcall" or any syntactic operator heads
 # Distinguish various syntaxes which are mapped to K"call"
 const PREFIX_CALL_FLAG = RawFlags(0<<3)
@@ -110,15 +106,6 @@ Return true for postfix operator calls such as the `'ᵀ` call node parsed from
 """
 is_postfix_op_call(x) = call_type_flags(x) == POSTFIX_OP_FLAG
 
-
-"""
-    is_suffixed(x)
-
-Return true for operators which have suffixes, such as `+₁`
-"""
-is_suffixed(x) = has_flags(x, SUFFIXED_FLAG)
-
-
 """
     numeric_flags(x)
 
@@ -164,7 +151,6 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
                 str *= "-,"
             end
         end
-        is_suffixed(head) && (str = str*"-suf")
     end
     str
 end
@@ -261,45 +247,6 @@ function validate_tokens(stream::ParseStream)
     sort!(stream.diagnostics, by=first_byte)
 end
 
-"""
-    bump_split(stream, token_spec1, [token_spec2 ...])
-
-Bump the next token, splitting it into several pieces
-
-Tokens are defined by a number of `token_spec` of shape `(nbyte, kind, flags)`.
-If all `nbyte` are positive, the sum must equal the token length. If one
-`nbyte` is negative, that token is given `tok_len + nbyte` bytes and the sum of
-all `nbyte` must equal zero.
-
-This is a hack which helps resolves the occasional lexing ambiguity. For
-example
-* Whether .+ should be a single token or the composite (. +) which is used for
-  standalone operators.
-* Whether ... is splatting (most of the time) or three . tokens in import paths
-
-TODO: Are these the only cases?  Can we replace this general utility with a
-simpler one which only splits preceding dots?
-"""
-function bump_split(stream::ParseStream, split_spec::Vararg{Any, N}) where {N}
-    tok = stream.lookahead[stream.lookahead_index]
-    stream.lookahead_index += 1
-    start_b = _next_byte(stream)
-    toklen = tok.next_byte - start_b
-    prev_b = start_b
-    for (nbyte, k, f) in split_spec
-        h = SyntaxHead(k, f)
-        actual_nbyte = nbyte < 0 ? (toklen + nbyte) : nbyte
-        orig_k = k == K"." ? K"." : kind(tok)
-        node = RawGreenNode(h, actual_nbyte, orig_k)
-        push!(stream.output, node)
-        prev_b += actual_nbyte
-        stream.next_byte += actual_nbyte
-    end
-    @assert tok.next_byte == prev_b
-    stream.peek_count = 0
-    return position(stream)
-end
-
 function peek_dotted_op_token(ps)
     # Peek the next token, but if it is a dot, peek the next one as well
     t = peek_token(ps)
@@ -317,7 +264,14 @@ function peek_dotted_op_token(ps)
             t = t2
         end
     end
-    return (isdotted, t)
+    # A compound assignment such as `x += y` is lexed as an operator carrying
+    # the special `PREC_COMPOUND_ASSIGN` precedence (immediately followed by an
+    # `=` token). Operators which don't form a compound assignment - eg `⋅`,
+    # `√`, or suffixed operators like `+₁` - are not marked this way, so `x ⋅= y`
+    # is left to be parsed as the identifier `⋅` followed by `=`, matching the
+    # reference parser.
+    isassign = is_prec_compound_assign(t)
+    return (isdotted, isassign, t)
 end
 
 function bump_dotted(ps, isdot, t, flags=EMPTY_FLAGS; emit_dot_node=false, remap_kind=K"None")