From aadd88fb906684a96a9e378683bb554aabaeb102 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 10 Jul 2025 20:28:16 +0000 Subject: [PATCH] Remove separate syntax heads for each operator This replaces all the specialized operator heads by a single K"Operator" head that encodes the precedence level in its flags (except for operators that are also used for non-operator purposes). The operators are already K"Identifier" in the final parse tree. There is very little reason to spend all of the extra effort separating them into separate heads only to undo this later. Moreover, I think it's actively misleading, because it makes people think that they can query things about an operator by looking at the head, which doesn't work for suffixed operators. Additionally, this removes the `op=` token, replacing it by two tokens, one K"Operator" with a special precedence level and one `=`. An operator only forms a compound assignment when followed by the single token `=`; when followed by `==`, `===` or `=>` it is emitted as a plain operator token. This then removes the last use of `bump_split` (since this PR is on top of JuliaLang/JuliaSyntax.jl#573). As a free bonus this prepares us for having compound assignment syntax for suffixed operators, which was infeasible in the flips parser. That syntax change is not part of this PR but would be trivial (this PR makes it an explicit error). Because most operators no longer have a dedicated kind, the in-tree consumers of the syntax tree are updated to match: * `_green_to_est` now lowers the one-argument `K"op="`/`K".op="` forms (the quoted compound-assignment operators such as `:(+=)`) to the corresponding identifier name. * JuliaLowering's `is_dotted_operator` no longer looks up an operator by `Kind` name (which no longer exists for most operators); it uses the operator-name test `Base.isoperator`, as already done for `op=` in `est_to_dst`. * JuliaSyntaxHighlighting is updated similarly (detect `===` by text, drop the removed `is_syntactic_assignment`, and recognize the now-kindless operators) and its stdlib version pin is bumped accordingly. Fixes JuliaLang/JuliaSyntax.jl#334 (cherry picked from commit 9c41dfa55e460ff50906644f21050d1352f01f20) Co-Authored-By: Claude Fable 5 --- JuliaLowering/src/compat.jl | 10 +- JuliaSyntax/docs/src/api.md | 1 - JuliaSyntax/docs/src/reference.md | 2 +- JuliaSyntax/src/JuliaSyntax.jl | 7 +- JuliaSyntax/src/core/parse_stream.jl | 7 +- JuliaSyntax/src/integration/expr.jl | 36 +- JuliaSyntax/src/julia/julia_parse_stream.jl | 62 +- JuliaSyntax/src/julia/kinds.jl | 912 ++++-------------- JuliaSyntax/src/julia/parser.jl | 134 +-- JuliaSyntax/src/julia/tokenize.jl | 296 +++--- JuliaSyntax/src/porcelain/syntax_graph.jl | 6 + JuliaSyntax/test/parser.jl | 64 +- JuliaSyntax/test/parser_api.jl | 4 +- JuliaSyntax/test/tokenize.jl | 147 ++- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/JuliaSyntaxHighlighting.version | 8 +- 19 files changed, 605 insertions(+), 1095 deletions(-) create mode 100644 deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/sha512 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/sha512 diff --git a/JuliaLowering/src/compat.jl b/JuliaLowering/src/compat.jl index bfd2482793111..5d6e0a3e8e0b4 100644 --- a/JuliaLowering/src/compat.jl +++ b/JuliaLowering/src/compat.jl @@ -6,10 +6,18 @@ function find_kind(s::String) end # flisp: dot-operators +# +# We work from the operator's name here (rather than its `Kind`) because by this +# point operators are represented uniformly as identifier-like names: this code +# also runs on trees converted from `Expr`, where an operator such as `.^` is +# simply the `Symbol` `:.^` with no token or `Kind` to inspect. `Base.isoperator` +# is the same operator-name test already used for `op=` in `est_to_dst` below; +# note we can't look up a `Kind` by name (eg via `find_kind`), since most +# operators no longer have their own kind - they share `K"Operator"`. function is_dotted_operator(s::AbstractString) return length(s) >= 2 && s[1] === '.' && s[2] !== '.' && - JS.is_operator(something(find_kind(s[2:end]), K"None")) + Base.isoperator(s[2:end]) end function is_eventually_call(e) diff --git a/JuliaSyntax/docs/src/api.md b/JuliaSyntax/docs/src/api.md index 5dfbec6e4fcc3..b2440f014ea60 100644 --- a/JuliaSyntax/docs/src/api.md +++ b/JuliaSyntax/docs/src/api.md @@ -101,7 +101,6 @@ JuliaSyntax.is_infix_op_call JuliaSyntax.is_prefix_op_call JuliaSyntax.is_postfix_op_call JuliaSyntax.is_dotted -JuliaSyntax.is_suffixed JuliaSyntax.is_decorated JuliaSyntax.numeric_flags ``` diff --git a/JuliaSyntax/docs/src/reference.md b/JuliaSyntax/docs/src/reference.md index 373cbf4c51144..050b5f08d7d88 100644 --- a/JuliaSyntax/docs/src/reference.md +++ b/JuliaSyntax/docs/src/reference.md @@ -80,7 +80,7 @@ class of tokenization errors and lets the parser deal with them. * We use flags rather than child nodes to represent the difference between `struct` and `mutable struct`, `module` and `baremodule` (#220) * Iterations are represented with the `iteration` and `in` heads rather than `=` within the header of a `for`. Thus `for i=is ; body end` parses to `(for (iteration (in i is)) (block body))`. Cartesian iteration as in `for a=as, b=bs body end` are represented with a nested `(iteration (in a as) (in b bs))` rather than a `block` containing `=` because these lists of iterators are neither semantically nor syntactically a sequence of statements, unlike other uses of `block`. Generators also use the `iteration` head - see information on that below. * Short form functions like `f(x) = x + 1` are represented with the `function` head rather than the `=` head. In this case the `SHORT_FORM_FUNCTION_FLAG` flag is set to allow the surface syntactic form to be easily distinguished from long form functions. -* All kinds of updating assignment operators like `+=` are represented with a single `K"op="` head, with the operator itself in infix position. For example, `x += 1` is `(op= x + 1)`, where the plus token is of kind `K"Identifier"`. This greatly reduces the number of distinct forms here from a rather big list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=` `^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as kind `K"Identifier"`, as it should. It also makes it possible to add further unicode updating operators while keeping the AST stable. +* All kinds of updating assignment operators like `+=` are represented with a single `K"op="` head, with the operator itself in infix position. For example, `x += 1` is `(op= x + 1)`, where the plus token is of kind `K"Identifier"`. This greatly reduces the number of distinct forms here from a rather big list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=` `^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as kind `K"Identifier"`, as it should. It also makes it possible to add further unicode updating operators while keeping the AST stable. The broadcasting form `x .+= 1` uses the `K".op="` head, ie `(.op= x + 1)`. When an updating operator appears without operands - for example when quoted as `:(+=)` - it is represented by the one-argument form `(op= +)` (and likewise `(.op= +)` for `:(.+=)`), which converts to the `Symbol` `:+=` (resp. `:.+=`). * The lexer never emits the multi-dot sequences `..` and `...` as single tokens; it always produces a sequence of `K"."` tokens which the parser groups as required (#573). When `..`/`...` are used as ordinary identifiers (for example the `..` operator in `a .. b`, or `...` quoted as in `:(...)`) the dots are gathered into a single leaf of kind `K"DotsIdentifier"` rather than `K"Identifier"`, with the number of dots stored in the numeric flags. Like `K"var"` and `K"MacroName"`, `K"DotsIdentifier"` is one of the kinds which may appear where an identifier is expected, so consumers matching identifiers should account for it rather than testing for `K"Identifier"` alone. `K"..."` is now used exclusively for splatting/slurping (as in `f(x...)`). ## More detail on tree differences diff --git a/JuliaSyntax/src/JuliaSyntax.jl b/JuliaSyntax/src/JuliaSyntax.jl index a814d673784cf..b265c2b034c39 100644 --- a/JuliaSyntax/src/JuliaSyntax.jl +++ b/JuliaSyntax/src/JuliaSyntax.jl @@ -39,6 +39,12 @@ export SourceFile # Expression predicates, kinds and flags export @K_str, kind @_public Kind +@_public PrecedenceLevel, PREC_NONE, PREC_ASSIGNMENT, + PREC_PAIRARROW, PREC_CONDITIONAL, PREC_ARROW, PREC_LAZYOR, PREC_LAZYAND, + PREC_COMPARISON, PREC_PIPE_LT, PREC_PIPE_GT, PREC_COLON, PREC_PLUS, + PREC_BITSHIFT, PREC_TIMES, PREC_RATIONAL, PREC_POWER, PREC_DECL, + PREC_WHERE, PREC_DOT, PREC_QUOTE, PREC_UNICODE_OPS, PREC_COMPOUND_ASSIGN, + generic_operators_by_level @_public flags, SyntaxHead, @@ -49,7 +55,6 @@ export @K_str, kind is_prefix_op_call, is_postfix_op_call, is_dotted, - is_suffixed, is_decorated, numeric_flags, has_flags, diff --git a/JuliaSyntax/src/core/parse_stream.jl b/JuliaSyntax/src/core/parse_stream.jl index 64a5a019e6921..8bb15fcbe4a4a 100644 --- a/JuliaSyntax/src/core/parse_stream.jl +++ b/JuliaSyntax/src/core/parse_stream.jl @@ -45,7 +45,7 @@ kind(head::SyntaxHead) = head.kind Return the flag bits of a syntactic construct. Prefer to query these with the predicates `is_trivia`, `is_prefix_call`, `is_infix_op_call`, -`is_prefix_op_call`, `is_postfix_op_call`, `is_dotted`, `is_suffixed`, +`is_prefix_op_call`, `is_postfix_op_call`, `is_dotted`, `is_decorated`. Or extract numeric portion of the flags with `numeric_flags`. @@ -376,7 +376,10 @@ function _buffer_lookahead_tokens(lexer, lookahead) was_whitespace = is_whitespace(k) had_whitespace |= was_whitespace f = EMPTY_FLAGS - raw.suffix && (f |= SUFFIXED_FLAG) + if k == K"Operator" && raw.op_precedence != Tokenize.PREC_NONE + # Store operator precedence in numeric flags + f |= set_numeric_flags(Int(raw.op_precedence)) + end push!(lookahead, SyntaxToken(SyntaxHead(k, f), k, had_whitespace, raw.endbyte + 2)) token_count += 1 diff --git a/JuliaSyntax/src/integration/expr.jl b/JuliaSyntax/src/integration/expr.jl index 32519516bf83c..58ef255a85ae1 100644 --- a/JuliaSyntax/src/integration/expr.jl +++ b/JuliaSyntax/src/integration/expr.jl @@ -346,20 +346,28 @@ end elseif k == K"DotsIdentifier" n = numeric_flags(flags(nodehead)) return n == 2 ? :(..) : :(...) - elseif k == K"op=" && length(args) == 3 - lhs = args[1] - op = args[2] - rhs = args[3] - headstr = string(args[2], '=') - retexpr.head = Symbol(headstr) - retexpr.args = Any[lhs, rhs] - elseif k == K".op=" && length(args) == 3 - lhs = args[1] - op = args[2] - rhs = args[3] - headstr = '.' * string(args[2], '=') - retexpr.head = Symbol(headstr) - retexpr.args = Any[lhs, rhs] + elseif k == K"op=" + if length(args) == 3 + lhs = args[1] + op = args[2] + rhs = args[3] + headstr = string(args[2], '=') + retexpr.head = Symbol(headstr) + retexpr.args = Any[lhs, rhs] + elseif length(args) == 1 + return Symbol(string(args[1], '=')) + end + elseif k == K".op=" + if length(args) == 3 + lhs = args[1] + op = args[2] + rhs = args[3] + headstr = '.' * string(args[2], '=') + retexpr.head = Symbol(headstr) + retexpr.args = Any[lhs, rhs] + else + return Symbol(string('.', args[1], '=')) + end elseif k == K"macrocall" if length(args) >= 2 a2 = args[2] diff --git a/JuliaSyntax/src/julia/julia_parse_stream.jl b/JuliaSyntax/src/julia/julia_parse_stream.jl index 296659b8b9e25..5430be6ff9bee 100644 --- a/JuliaSyntax/src/julia/julia_parse_stream.jl +++ b/JuliaSyntax/src/julia/julia_parse_stream.jl @@ -1,7 +1,3 @@ -# Token flags - may be set for operator kinded tokens -# Operator has a suffix -const SUFFIXED_FLAG = RawFlags(1<<2) - # Set for K"call", K"dotcall" or any syntactic operator heads # Distinguish various syntaxes which are mapped to K"call" const PREFIX_CALL_FLAG = RawFlags(0<<3) @@ -110,15 +106,6 @@ Return true for postfix operator calls such as the `'ᵀ` call node parsed from """ is_postfix_op_call(x) = call_type_flags(x) == POSTFIX_OP_FLAG - -""" - is_suffixed(x) - -Return true for operators which have suffixes, such as `+₁` -""" -is_suffixed(x) = has_flags(x, SUFFIXED_FLAG) - - """ numeric_flags(x) @@ -164,7 +151,6 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true) str *= "-," end end - is_suffixed(head) && (str = str*"-suf") end str end @@ -261,45 +247,6 @@ function validate_tokens(stream::ParseStream) sort!(stream.diagnostics, by=first_byte) end -""" - bump_split(stream, token_spec1, [token_spec2 ...]) - -Bump the next token, splitting it into several pieces - -Tokens are defined by a number of `token_spec` of shape `(nbyte, kind, flags)`. -If all `nbyte` are positive, the sum must equal the token length. If one -`nbyte` is negative, that token is given `tok_len + nbyte` bytes and the sum of -all `nbyte` must equal zero. - -This is a hack which helps resolves the occasional lexing ambiguity. For -example -* Whether .+ should be a single token or the composite (. +) which is used for - standalone operators. -* Whether ... is splatting (most of the time) or three . tokens in import paths - -TODO: Are these the only cases? Can we replace this general utility with a -simpler one which only splits preceding dots? -""" -function bump_split(stream::ParseStream, split_spec::Vararg{Any, N}) where {N} - tok = stream.lookahead[stream.lookahead_index] - stream.lookahead_index += 1 - start_b = _next_byte(stream) - toklen = tok.next_byte - start_b - prev_b = start_b - for (nbyte, k, f) in split_spec - h = SyntaxHead(k, f) - actual_nbyte = nbyte < 0 ? (toklen + nbyte) : nbyte - orig_k = k == K"." ? K"." : kind(tok) - node = RawGreenNode(h, actual_nbyte, orig_k) - push!(stream.output, node) - prev_b += actual_nbyte - stream.next_byte += actual_nbyte - end - @assert tok.next_byte == prev_b - stream.peek_count = 0 - return position(stream) -end - function peek_dotted_op_token(ps) # Peek the next token, but if it is a dot, peek the next one as well t = peek_token(ps) @@ -317,7 +264,14 @@ function peek_dotted_op_token(ps) t = t2 end end - return (isdotted, t) + # A compound assignment such as `x += y` is lexed as an operator carrying + # the special `PREC_COMPOUND_ASSIGN` precedence (immediately followed by an + # `=` token). Operators which don't form a compound assignment - eg `⋅`, + # `√`, or suffixed operators like `+₁` - are not marked this way, so `x ⋅= y` + # is left to be parsed as the identifier `⋅` followed by `=`, matching the + # reference parser. + isassign = is_prec_compound_assign(t) + return (isdotted, isassign, t) end function bump_dotted(ps, isdot, t, flags=EMPTY_FLAGS; emit_dot_node=false, remap_kind=K"None") diff --git a/JuliaSyntax/src/julia/kinds.jl b/JuliaSyntax/src/julia/kinds.jl index 678a59fa238ec..69b5ba4db3389 100644 --- a/JuliaSyntax/src/julia/kinds.jl +++ b/JuliaSyntax/src/julia/kinds.jl @@ -187,7 +187,6 @@ Return the `Kind` of `x`. """ kind(k::Kind) = k - #------------------------------------------------------------------------------- # Kinds used by JuliaSyntax register_kinds!(JuliaSyntax, 0, [ @@ -199,6 +198,7 @@ register_kinds!(JuliaSyntax, 0, [ # Identifiers "BEGIN_IDENTIFIERS" "Identifier" + "Operator" "Placeholder" # Used for empty catch variables, and all-underscore identifiers in lowering # String and command macro names are modeled as a special kind of # identifier as they need to be mangled before lookup. @@ -291,725 +291,56 @@ register_kinds!(JuliaSyntax, 0, [ "ErrorInvalidOperator" "Error**" - # Level 1 + # Various operators that have special parsing rules and thus get explicit heads. + # All other operators (including suffixed versions of these) are K"Operator". "BEGIN_ASSIGNMENTS" - "BEGIN_SYNTACTIC_ASSIGNMENTS" "=" ".=" - "op=" # Updating assignment operator ( $= %= &= *= += -= //= /= <<= >>= >>>= \= ^= |= ÷= ⊻= ) - ".op=" ":=" - "END_SYNTACTIC_ASSIGNMENTS" "~" "≔" "⩴" "≕" + # Compound assignments + "op=" + ".op=" "END_ASSIGNMENTS" - - "BEGIN_PAIRARROW" - "=>" - "END_PAIRARROW" - - # Level 2 - "BEGIN_CONDITIONAL" - "?" - "END_CONDITIONAL" - - # Level 3 - "BEGIN_ARROW" - "-->" - "<--" - "<-->" - "←" - "→" - "↔" - "↚" - "↛" - "↞" - "↠" - "↢" - "↣" - "↤" - "↦" - "↮" - "⇎" - "⇍" - "⇏" - "⇐" - "⇒" - "⇔" - "⇴" - "⇶" - "⇷" - "⇸" - "⇹" - "⇺" - "⇻" - "⇼" - "⇽" - "⇾" - "⇿" - "⟵" - "⟶" - "⟷" - "⟹" - "⟺" - "⟻" - "⟼" - "⟽" - "⟾" - "⟿" - "⤀" - "⤁" - "⤂" - "⤃" - "⤄" - "⤅" - "⤆" - "⤇" - "⤌" - "⤍" - "⤎" - "⤏" - "⤐" - "⤑" - "⤔" - "⤕" - "⤖" - "⤗" - "⤘" - "⤝" - "⤞" - "⤟" - "⤠" - "⥄" - "⥅" - "⥆" - "⥇" - "⥈" - "⥊" - "⥋" - "⥎" - "⥐" - "⥒" - "⥓" - "⥖" - "⥗" - "⥚" - "⥛" - "⥞" - "⥟" - "⥢" - "⥤" - "⥦" - "⥧" - "⥨" - "⥩" - "⥪" - "⥫" - "⥬" - "⥭" - "⥰" - "⧴" - "⬱" - "⬰" - "⬲" - "⬳" - "⬴" - "⬵" - "⬶" - "⬷" - "⬸" - "⬹" - "⬺" - "⬻" - "⬼" - "⬽" - "⬾" - "⬿" - "⭀" - "⭁" - "⭂" - "⭃" - "⥷" - "⭄" - "⥺" - "⭇" - "⭈" - "⭉" - "⭊" - "⭋" - "⭌" - "￩" - "￫" - "⇜" - "⇝" - "↜" - "↝" - "↩" - "↪" - "↫" - "↬" - "↼" - "↽" - "⇀" - "⇁" - "⇄" - "⇆" - "⇇" - "⇉" - "⇋" - "⇌" - "⇚" - "⇛" - "⇠" - "⇢" - "↷" - "↶" - "↺" - "↻" - "🢲" - "END_ARROW" - - # Level 4 - "BEGIN_LAZYOR" - "||" - ".||" - "END_LAZYOR" - - # Level 5 - "BEGIN_LAZYAND" - "&&" - ".&&" - "END_LAZYAND" - - # Level 6 - "BEGIN_COMPARISON" - "<:" - ">:" - ">" - "<" - ">=" - "≥" - "<=" - "≤" - "==" - "===" - "≡" - "!=" - "≠" - "!==" - "≢" - "∈" - "in" - "isa" - "∉" - "∋" - "∌" - "⊆" - "⊈" - "⊂" - "⊄" - "⊊" - "∝" - "∊" - "∍" - "∥" - "∦" - "∷" - "∺" - "∻" - "∽" - "∾" - "≁" - "≃" - "≂" - "≄" - "≅" - "≆" - "≇" - "≈" - "≉" - "≊" - "≋" - "≌" - "≍" - "≎" - "≐" - "≑" - "≒" - "≓" - "≖" - "≗" - "≘" - "≙" - "≚" - "≛" - "≜" - "≝" - "≞" - "≟" - "≣" - "≦" - "≧" - "≨" - "≩" - "≪" - "≫" - "≬" - "≭" - "≮" - "≯" - "≰" - "≱" - "≲" - "≳" - "≴" - "≵" - "≶" - "≷" - "≸" - "≹" - "≺" - "≻" - "≼" - "≽" - "≾" - "≿" - "⊀" - "⊁" - "⊃" - "⊅" - "⊇" - "⊉" - "⊋" - "⊏" - "⊐" - "⊑" - "⊒" - "⊜" - "⊩" - "⊬" - "⊮" - "⊰" - "⊱" - "⊲" - "⊳" - "⊴" - "⊵" - "⊶" - "⊷" - "⋍" - "⋐" - "⋑" - "⋕" - "⋖" - "⋗" - "⋘" - "⋙" - "⋚" - "⋛" - "⋜" - "⋝" - "⋞" - "⋟" - "⋠" - "⋡" - "⋢" - "⋣" - "⋤" - "⋥" - "⋦" - "⋧" - "⋨" - "⋩" - "⋪" - "⋫" - "⋬" - "⋭" - "⋲" - "⋳" - "⋴" - "⋵" - "⋶" - "⋷" - "⋸" - "⋹" - "⋺" - "⋻" - "⋼" - "⋽" - "⋾" - "⋿" - "⟈" - "⟉" - "⟒" - "⦷" - "⧀" - "⧁" - "⧡" - "⧣" - "⧤" - "⧥" - "⩦" - "⩧" - "⩪" - "⩫" - "⩬" - "⩭" - "⩮" - "⩯" - "⩰" - "⩱" - "⩲" - "⩳" - "⩵" - "⩶" - "⩷" - "⩸" - "⩹" - "⩺" - "⩻" - "⩼" - "⩽" - "⩾" - "⩿" - "⪀" - "⪁" - "⪂" - "⪃" - "⪄" - "⪅" - "⪆" - "⪇" - "⪈" - "⪉" - "⪊" - "⪋" - "⪌" - "⪍" - "⪎" - "⪏" - "⪐" - "⪑" - "⪒" - "⪓" - "⪔" - "⪕" - "⪖" - "⪗" - "⪘" - "⪙" - "⪚" - "⪛" - "⪜" - "⪝" - "⪞" - "⪟" - "⪠" - "⪡" - "⪢" - "⪣" - "⪤" - "⪥" - "⪦" - "⪧" - "⪨" - "⪩" - "⪪" - "⪫" - "⪬" - "⪭" - "⪮" - "⪯" - "⪰" - "⪱" - "⪲" - "⪳" - "⪴" - "⪵" - "⪶" - "⪷" - "⪸" - "⪹" - "⪺" - "⪻" - "⪼" - "⪽" - "⪾" - "⪿" - "⫀" - "⫁" - "⫂" - "⫃" - "⫄" - "⫅" - "⫆" - "⫇" - "⫈" - "⫉" - "⫊" - "⫋" - "⫌" - "⫍" - "⫎" - "⫏" - "⫐" - "⫑" - "⫒" - "⫓" - "⫔" - "⫕" - "⫖" - "⫗" - "⫘" - "⫙" - "⫷" - "⫸" - "⫹" - "⫺" - "⊢" - "⊣" - "⟂" - # ⫪,⫫ see https://github.com/JuliaLang/julia/issues/39350 - "⫪" - "⫫" - "END_COMPARISON" - - # Level 7 - "BEGIN_PIPE" - "<|" - "|>" - "END_PIPE" - - # Level 8 - "BEGIN_COLON" - ":" - "…" - "⁝" - "⋮" - "⋱" - "⋰" - "⋯" - "END_COLON" - - # Level 9 - "BEGIN_PLUS" - "\$" - "+" - "-" # also used for "−" - "++" - "⊕" - "⊖" - "⊞" - "⊟" - "|" - "∪" - "∨" - "⊔" - "±" - "∓" - "∔" - "∸" - "≏" - "⊎" - "⊻" - "⊽" - "⋎" - "⋓" - "⟇" - "⧺" - "⧻" - "⨈" - "⨢" - "⨣" - "⨤" - "⨥" - "⨦" - "⨧" - "⨨" - "⨩" - "⨪" - "⨫" - "⨬" - "⨭" - "⨮" - "⨹" - "⨺" - "⩁" - "⩂" - "⩅" - "⩊" - "⩌" - "⩏" - "⩐" - "⩒" - "⩔" - "⩖" - "⩗" - "⩛" - "⩝" - "⩡" - "⩢" - "⩣" - "¦" - "END_PLUS" - - # Level 10 - "BEGIN_TIMES" - "*" - "/" - "÷" - "%" - "⋅" # also used for lookalikes "·" and "·" - "∘" - "×" - "\\" - "&" - "∩" - "∧" - "⊗" - "⊘" - "⊙" - "⊚" - "⊛" - "⊠" - "⊡" - "⊓" - "∗" - "∙" - "∤" - "⅋" - "≀" - "⊼" - "⋄" - "⋆" - "⋇" - "⋉" - "⋊" - "⋋" - "⋌" - "⋏" - "⋒" - "⟑" - "⦸" - "⦼" - "⦾" - "⦿" - "⧶" - "⧷" - "⨇" - "⨰" - "⨱" - "⨲" - "⨳" - "⨴" - "⨵" - "⨶" - "⨷" - "⨸" - "⨻" - "⨼" - "⨽" - "⩀" - "⩃" - "⩄" - "⩋" - "⩍" - "⩎" - "⩑" - "⩓" - "⩕" - "⩘" - "⩚" - "⩜" - "⩞" - "⩟" - "⩠" - "⫛" - "⊍" - "▷" - "⨝" - "⟕" - "⟖" - "⟗" - "⌿" - "⨟" - "END_TIMES" - - # Level 11 - "BEGIN_RATIONAL" - "//" - "END_RATIONAL" - - # Level 12 - "BEGIN_BITSHIFTS" - "<<" - ">>" - ">>>" - "END_BITSHIFTS" - - # Level 13 - "BEGIN_POWER" - "^" - "↑" - "↓" - "⇵" - "⟰" - "⟱" - "⤈" - "⤉" - "⤊" - "⤋" - "⤒" - "⤓" - "⥉" - "⥌" - "⥍" - "⥏" - "⥑" - "⥔" - "⥕" - "⥘" - "⥙" - "⥜" - "⥝" - "⥠" - "⥡" - "⥣" - "⥥" - "⥮" - "⥯" - "￪" - "￬" - "END_POWER" - - # Level 14 - "BEGIN_DECL" - "::" - "END_DECL" - - # Level 15 - "BEGIN_WHERE" - "where" - "END_WHERE" - - # Level 16 - "BEGIN_DOT" - "." - "END_DOT" - - "!" - "'" - ".'" - "->" - - "BEGIN_UNICODE_OPS" - "¬" - "√" - "∛" - "∜" - "END_UNICODE_OPS" + "?" # ternary operator + "||" # not an operator call + ".||" # dotted of above (not emitted by lexer) + "&&" # not an operator call + ".&&" # dotted of above (not emitted by lexer) + "<:" # subtype syntax + ">:" # supertype syntax + "::" # field type syntax + "." # various dot syntax + ".." # .. operator (not emitted by lexer) + "in" # iteration syntax + "isa" + "where" + "!" # syntactic unary + "'" # special postfix parsing + ".'" # special postfix parsing + "->" # syntactic arrow + "-->" # syntactic arrow + ":" # used for quoting + "+" # used in numeric constants + "++" # special chaining syntax + "*" # special chaining syntax + "<" # recovery path for :< + ">" # recovery path for :> + "\$" # interpolation + "-" # negated constants + "&" # syntactic unary + "∈" # iteration syntax + # all syntactic unary + "⋆" + "±" + "∓" + "¬" + "√" + "∛" + "∜" "END_OPS" # 2. Nonterminals which are exposed in the AST, but where the surface @@ -1101,6 +432,110 @@ register_kinds!(JuliaSyntax, 0, [ "END_ERRORS" ]) +@enum PrecedenceLevel begin + PREC_NONE + PREC_ASSIGNMENT + PREC_PAIRARROW + PREC_CONDITIONAL + PREC_ARROW + PREC_LAZYOR + PREC_LAZYAND + PREC_COMPARISON + PREC_PIPE_LT + PREC_PIPE_GT + PREC_COLON + PREC_PLUS + PREC_BITSHIFT + PREC_TIMES + PREC_RATIONAL + PREC_POWER + PREC_DECL + PREC_WHERE + PREC_DOT + PREC_QUOTE + PREC_UNICODE_OPS + # Special precedence to only allow compound assignment for designated operators, for + # compatibility with flisp + PREC_COMPOUND_ASSIGN +end + +const generic_operators_by_level = Dict{PrecedenceLevel, Vector{Char}}( + # Operators which have their own kinds are commented out in these lists + PREC_ASSIGNMENT => Char[#= = .= := ~ ≔ ⩴ ≕ =#], + PREC_PAIRARROW => Char[#= => =#], + PREC_CONDITIONAL => Char[#= ? =#], + PREC_ARROW => + [#= -> --> <-- <--> =# + '←', '→', '↔', '↚', '↛', '↞', '↠', '↢', + '↣', '↤', '↦', '↮', '⇎', '⇍', '⇏', '⇐', '⇒', '⇔', '⇴', + '⇶', '⇷', '⇸', '⇹', '⇺', '⇻', '⇼', '⇽', '⇾', '⇿', '⟵', + '⟶', '⟷', '⟹', '⟺', '⟻', '⟼', '⟽', '⟾', '⟿', '⤀', '⤁', + '⤂', '⤃', '⤄', '⤅', '⤆', '⤇', '⤌', '⤍', '⤎', '⤏', '⤐', '⤑', + '⤔', '⤕', '⤖', '⤗', '⤘', '⤝', '⤞', '⤟', '⤠', '⥄', '⥅', '⥆', + '⥇', '⥈', '⥊', '⥋', '⥎', '⥐', '⥒', '⥓', '⥖', '⥗', '⥚', '⥛', + '⥞', '⥟', '⥢', '⥤', '⥦', '⥧', '⥨', '⥩', '⥪', '⥫', '⥬', '⥭', + '⥰', '⧴', '⬱', '⬰', '⬲', '⬳', '⬴', '⬵', '⬶', '⬷', '⬸', '⬹', + '⬺', '⬻', '⬼', '⬽', '⬾', '⬿', '⭀', '⭁', '⭂', '⭃', '⥷', '⭄', + '⥺', '⭇', '⭈', '⭉', '⭊', '⭋', '⭌', '￩', '￫', '⇜', '⇝', '↜', '↝', + '↩', '↪', '↫', '↬', '↼', '↽', '⇀', '⇁', '⇄', '⇆', '⇇', '⇉', '⇋', + '⇌', '⇚', '⇛', '⇠', '⇢', '↷', '↶', '↺', '↻', '🢲'], + PREC_LAZYOR => Char[#= || =#], + PREC_LAZYAND => Char[#= && =#], + PREC_COMPARISON => + [#= <: >: in isa < > ∈ == != !== =# + '≥', '≤', '≡', '≠', '≢', '∉', '∋', + '∌', '⊆', '⊈', '⊂', '⊄', '⊊', '∝', '∊', '∍', '∥', '∦', + '∷', '∺', '∻', '∽', '∾', '≁', '≃', '≂', '≄', '≅', '≆', + '≇', '≈', '≉', '≊', '≋', '≌', '≍', '≎', '≐', '≑', '≒', + '≓', '≖', '≗', '≘', '≙', '≚', '≛', '≜', '≝', '≞', '≟', + '≣', '≦', '≧', '≨', '≩', '≪', '≫', '≬', '≭', '≮', '≯', + '≰', '≱', '≲', '≳', '≴', '≵', '≶', '≷', '≸', '≹', '≺', + '≻', '≼', '≽', '≾', '≿', '⊀', '⊁', '⊃', '⊅', '⊇', '⊉', + '⊋', '⊏', '⊐', '⊑', '⊒', '⊜', '⊩', '⊬', '⊮', '⊰', '⊱', + '⊲', '⊳', '⊴', '⊵', '⊶', '⊷', '⋍', '⋐', '⋑', '⋕', '⋖', + '⋗', '⋘', '⋙', '⋚', '⋛', '⋜', '⋝', '⋞', '⋟', '⋠', '⋡', + '⋢', '⋣', '⋤', '⋥', '⋦', '⋧', '⋨', '⋩', '⋪', '⋫', '⋬', + '⋭', '⋲', '⋳', '⋴', '⋵', '⋶', '⋷', '⋸', '⋹', '⋺', '⋻', + '⋼', '⋽', '⋾', '⋿', '⟈', '⟉', '⟒', '⦷', '⧀', '⧁', '⧡', + '⧣', '⧤', '⧥', '⩦', '⩧', '⩪', '⩫', '⩬', '⩭', '⩮', '⩯', + '⩰', '⩱', '⩲', '⩳', '⩵', '⩶', '⩷', '⩸', '⩹', '⩺', '⩻', + '⩼', '⩽', '⩾', '⩿', '⪀', '⪁', '⪂', '⪃', '⪄', '⪅', '⪆', '⪇', + '⪈', '⪉', '⪊', '⪋', '⪌', '⪍', '⪎', '⪏', '⪐', '⪑', '⪒', '⪓', + '⪔', '⪕', '⪖', '⪗', '⪘', '⪙', '⪚', '⪛', '⪜', '⪝', '⪞', '⪟', + '⪠', '⪡', '⪢', '⪣', '⪤', '⪥', '⪦', '⪧', '⪨', '⪩', '⪪', + '⪫', '⪬', '⪭', '⪮', '⪯', '⪰', '⪱', '⪲', '⪳', '⪴', '⪵', + '⪶', '⪷', '⪸', '⪹', '⪺', '⪻', '⪼', '⪽', '⪾', '⪿', '⫀', + '⫁', '⫂', '⫃', '⫄', '⫅', '⫆', '⫇', '⫈', '⫉', '⫊', '⫋', + '⫌', '⫍', '⫎', '⫏', '⫐', '⫑', '⫒', '⫓', '⫔', '⫕', '⫖', + '⫗', '⫘', '⫙', '⫷', '⫸', '⫹', '⫺', '⊢', '⊣', '⟂', '⫪', '⫫'], + PREC_PIPE_LT => Char[#= <| =#], + PREC_PIPE_GT => Char[#= |> =#], + PREC_COLON => [ #= : .. =# '…', '⁝', '⋮', '⋱', '⋰', '⋯'], + PREC_PLUS => + [ #= + - ± ∓ ++ =# + '⊕', '⊖', '⊞', '⊟', '|', '∪', '∨', + '⊔', '±', '∓', '∔', '∸', '≏', '⊎', '⊻', '⊽', '⋎', '⋓', '⟇', '⧺', + '⧻', '⨈', '⨢', '⨣', '⨤', '⨥', '⨦', '⨧', '⨨', '⨩', '⨪', '⨫', '⨬', '⨭', + '⨮', '⨹', '⨺', '⩁', '⩂', '⩅', '⩊', '⩌', '⩏', '⩐', '⩒', '⩔', '⩖', '⩗', + '⩛', '⩝', '⩡', '⩢', '⩣', '¦'], + PREC_TIMES => + [ #= * ⋆ & =# + '/', '÷', '%', '⋅', '·', '·', '∘', '×', '\\', '∩', '∧', '⊗', + '⊘', '⊙', '⊚', '⊛', '⊠', '⊡', '⊓', '∗', '∙', '∤', '⅋', '≀', '⊼', '⋄', '⋆', + '⋇', '⋉', '⋊', '⋋', '⋌', '⋏', '⋒', '⟑', '⦸', '⦼', '⦾', '⦿', '⧶', '⧷', + '⨇', '⨰', '⨱', '⨲', '⨳', '⨴', '⨵', '⨶', '⨷', '⨸', '⨻', '⨼', '⨽', '⩀', + '⩃', '⩄', '⩋', '⩍', '⩎', '⩑', '⩓', '⩕', '⩘', '⩚', '⩜', '⩞', '⩟', '⩠', + '⫛', '⊍', '▷', '⨝', '⟕', '⟖', '⟗', '⌿', '⨟', + '\u00b7', # '·' Middle Dot + '\u0387' # '·' Greek Ano Teleia + ], + PREC_RATIONAL => Char[#= // =#], + PREC_BITSHIFT => Char[#= << >> >>> =#], + PREC_POWER => ['^', '↑', '↓', '⇵', '⟰', '⟱', '⤈', '⤉', '⤊', '⤋', '⤒', '⤓', '⥉', + '⥌', '⥍', '⥏', '⥑', '⥔', '⥕', '⥘', '⥙', '⥜', '⥝', '⥠', '⥡', '⥣', '⥥', + '⥮', '⥯', '￪', '￬'], +) + #------------------------------------------------------------------------------- const _nonunique_kind_names = Set([ K"Comment" @@ -1187,7 +622,7 @@ is_keyword(k::Kind) = K"BEGIN_KEYWORDS" <= k <= K"END_KEYWORDS" is_block_continuation_keyword(k::Kind) = K"BEGIN_BLOCK_CONTINUATION_KEYWORDS" <= k <= K"END_BLOCK_CONTINUATION_KEYWORDS" is_literal(k::Kind) = K"BEGIN_LITERAL" <= k <= K"END_LITERAL" is_number(k::Kind) = K"BEGIN_NUMBERS" <= k <= K"END_NUMBERS" -is_operator(k::Kind) = K"BEGIN_OPS" <= k <= K"END_OPS" +is_operator(k::Kind) = k == K"Operator" || K"BEGIN_OPS" <= k <= K"END_OPS" is_word_operator(k::Kind) = (k == K"in" || k == K"isa" || k == K"where") is_identifier(x) = is_identifier(kind(x)) @@ -1202,28 +637,36 @@ is_word_operator(x) = is_word_operator(kind(x)) # Predicates for operator precedence # FIXME: Review how precedence depends on dottedness, eg # https://github.com/JuliaLang/julia/pull/36725 + + +# Most operators no longer have a dedicated kind - they're represented by +# K"Operator" with the precedence level stored in the numeric flags. A few +# operators are still kept as distinct kinds because they're treated specially +# during parsing, so the precedence predicates below additionally check for them. +_is_op_prec(x, prec) = kind(x) == K"Operator" && numeric_flags(head(x)) == Int(prec) + is_prec_assignment(x) = K"BEGIN_ASSIGNMENTS" <= kind(x) <= K"END_ASSIGNMENTS" -is_prec_pair(x) = K"BEGIN_PAIRARROW" <= kind(x) <= K"END_PAIRARROW" -is_prec_conditional(x) = K"BEGIN_CONDITIONAL" <= kind(x) <= K"END_CONDITIONAL" -is_prec_arrow(x) = K"BEGIN_ARROW" <= kind(x) <= K"END_ARROW" -is_prec_lazy_or(x) = K"BEGIN_LAZYOR" <= kind(x) <= K"END_LAZYOR" -is_prec_lazy_and(x) = K"BEGIN_LAZYAND" <= kind(x) <= K"END_LAZYAND" -is_prec_comparison(x) = K"BEGIN_COMPARISON" <= kind(x) <= K"END_COMPARISON" -is_prec_pipe(x) = K"BEGIN_PIPE" <= kind(x) <= K"END_PIPE" -is_prec_colon(x) = K"BEGIN_COLON" <= kind(x) <= K"END_COLON" -is_prec_plus(x) = K"BEGIN_PLUS" <= kind(x) <= K"END_PLUS" -is_prec_bitshift(x) = K"BEGIN_BITSHIFTS" <= kind(x) <= K"END_BITSHIFTS" -is_prec_times(x) = K"BEGIN_TIMES" <= kind(x) <= K"END_TIMES" -is_prec_rational(x) = K"BEGIN_RATIONAL" <= kind(x) <= K"END_RATIONAL" -is_prec_power(x) = K"BEGIN_POWER" <= kind(x) <= K"END_POWER" -is_prec_decl(x) = K"BEGIN_DECL" <= kind(x) <= K"END_DECL" -is_prec_where(x) = K"BEGIN_WHERE" <= kind(x) <= K"END_WHERE" -is_prec_dot(x) = K"BEGIN_DOT" <= kind(x) <= K"END_DOT" -is_prec_unicode_ops(x) = K"BEGIN_UNICODE_OPS" <= kind(x) <= K"END_UNICODE_OPS" -is_prec_pipe_lt(x) = kind(x) == K"<|" -is_prec_pipe_gt(x) = kind(x) == K"|>" +is_prec_pair(x) = _is_op_prec(x, PREC_PAIRARROW) +is_prec_conditional(x) = kind(x) == K"?" +is_prec_arrow(x) = _is_op_prec(x, PREC_ARROW) || kind(x) == K"-->" +is_prec_lazy_or(x) = _is_op_prec(x, PREC_LAZYOR) || kind(x) in KSet"||" +is_prec_lazy_and(x) = _is_op_prec(x, PREC_LAZYAND) || kind(x) in KSet"&&" +is_prec_comparison(x) = _is_op_prec(x, PREC_COMPARISON) || kind(x) in KSet"<: >: in isa < > ∈" +is_prec_pipe_lt(x) = _is_op_prec(x, PREC_PIPE_LT) +is_prec_pipe_gt(x) = _is_op_prec(x, PREC_PIPE_GT) +is_prec_pipe(x) = is_prec_pipe_lt(x) || is_prec_pipe_gt(x) +is_prec_colon(x) = _is_op_prec(x, PREC_COLON) || kind(x) == K".." +is_prec_plus(x) = _is_op_prec(x, PREC_PLUS) || kind(x) in KSet"+ - ± ∓ $ ++" +is_prec_bitshift(x) = _is_op_prec(x, PREC_BITSHIFT) +is_prec_times(x) = _is_op_prec(x, PREC_TIMES) || kind(x) in KSet"* ⋆ &" +is_prec_rational(x) = _is_op_prec(x, PREC_RATIONAL) +is_prec_power(x) = _is_op_prec(x, PREC_POWER) +is_prec_decl(x) = _is_op_prec(x, PREC_DECL) || kind(x) == K"::" +is_prec_where(x) = _is_op_prec(x, PREC_WHERE) || kind(x) == K"where" +is_prec_dot(x) = _is_op_prec(x, PREC_DOT) || kind(x) == K"." +is_prec_quote(x) = _is_op_prec(x, PREC_QUOTE) || kind(x) == K"'" is_syntax_kind(x) = K"BEGIN_SYNTAX_KINDS"<= kind(x) <= K"END_SYNTAX_KINDS" -is_syntactic_assignment(x) = K"BEGIN_SYNTACTIC_ASSIGNMENTS" <= kind(x) <= K"END_SYNTACTIC_ASSIGNMENTS" +is_prec_compound_assign(x) = _is_op_prec(x, PREC_COMPOUND_ASSIGN) function is_string_delim(x) kind(x) in (K"\"", K"\"\"\"") @@ -1247,5 +690,8 @@ function is_syntactic_operator(x) # in the parser? The lexer itself usually disallows such tokens, so it's # not clear whether we need to handle them. (Though note `.->` is a # token...) - return k in KSet"&& || . ... ->" || is_syntactic_assignment(k) + # Note the assignment-like kinds `= .= op= .op= :=` are all syntactic, just + # as they were when each had its own kind (before they were collapsed into + # `K"Operator"`). + return k in KSet"&& || . ... -> = := .= op= .op=" end diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl index bd9ae676e97a2..7e3d1406d9640 100644 --- a/JuliaSyntax/src/julia/parser.jl +++ b/JuliaSyntax/src/julia/parser.jl @@ -101,10 +101,6 @@ function bump_glue(ps::ParseState, args...; kws...) bump_glue(ps.stream, args...; kws...) end -function bump_split(ps::ParseState, args...; kws...) - bump_split(ps.stream, args...; kws...) -end - function reset_node!(ps::ParseState, args...; kws...) reset_node!(ps.stream, args...; kws...) end @@ -221,10 +217,6 @@ end # # All these take either a raw kind or a token. -function is_plain_equals(t) - kind(t) == K"=" && !is_suffixed(t) -end - function is_closing_token(ps::ParseState, k) k = kind(k) return k in KSet"else elseif catch finally , ) ] } ; EndMarker" || @@ -304,20 +296,14 @@ end function is_unary_op(t, isdot) k = kind(t) - !is_suffixed(t) && ( - (k in KSet"<: >:" && !isdot) || - k in KSet"+ - ! ~ ¬ √ ∛ ∜ ⋆ ± ∓" # dotop allowed - ) + (k in KSet"<: >:" && !isdot) || + k in KSet"+ - ! ~ ¬ √ ∛ ∜ ⋆ ± ∓" # dotop allowed end # Operators that are both unary and binary function is_both_unary_and_binary(t, isdot) k = kind(t) - # Preventing is_suffixed here makes this consistent with the flisp parser. - # But is this by design or happenstance? - !is_suffixed(t) && ( - k in KSet"+ - ⋆ ± ∓" || (k in KSet"$ & ~" && !isdot) - ) + k in KSet"+ - ⋆ ± ∓" || (k in KSet"$ & ~" && !isdot) end function is_string_macro_suffix(k) @@ -373,8 +359,8 @@ function parse_LtoR(ps::ParseState, down, is_op) mark = position(ps) down(ps) while true - isdot, tk = peek_dotted_op_token(ps) - is_op(tk) || break + isdot, isassign, tk = peek_dotted_op_token(ps) + (is_op(tk) && !isassign) || break isdot && bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG bump(ps, remap_kind=K"Identifier") down(ps) @@ -389,8 +375,8 @@ end function parse_RtoL(ps::ParseState, down, is_op, self) mark = position(ps) down(ps) - isdot, tk = peek_dotted_op_token(ps) - if is_op(tk) + isdot, isassign, tk = peek_dotted_op_token(ps) + if is_op(tk) && !isassign bump_dotted(ps, isdot, tk, remap_kind=K"Identifier") self(ps) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) @@ -601,11 +587,13 @@ function parse_assignment(ps::ParseState, down) end function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where {T} # where => specialize on `down` - isdot, t = peek_dotted_op_token(ps) + isdot, is_compound_assignment, t = peek_dotted_op_token(ps) k = kind(t) - if !is_prec_assignment(k) + + if !is_prec_assignment(t) && !is_compound_assignment return end + if k == K"~" if ps.space_sensitive && preceding_whitespace(t) && !preceding_whitespace(peek_token(ps, 2)) # Unary ~ in space sensitive context is not assignment precedence @@ -625,17 +613,15 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where { else # f() = 1 ==> (function-= (call f) 1) # f() .= 1 ==> (.= (call f) 1) - # a += b ==> (+= a b) # a .= b ==> (.= a b) is_short_form_func = k == K"=" && !isdot && was_eventually_call(ps) - if k == K"op=" + if is_compound_assignment # x += y ==> (op= x + y) # x .+= y ==> (.op= x + y) bump_trivia(ps) - isdot && bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG - bump_split(ps, - (-1, K"Identifier", EMPTY_FLAGS), # op - (1, K"=", TRIVIA_FLAG)) + bump_dotted(ps, isdot, t, remap_kind=K"Identifier") + bump(ps, TRIVIA_FLAG) # bump the = + k = K"op=" # Set k for the emit below else bump_dotted(ps, isdot, t, TRIVIA_FLAG) end @@ -665,7 +651,7 @@ function parse_comma(ps::ParseState, do_emit=true) end bump(ps, TRIVIA_FLAG) n_commas += 1 - if is_plain_equals(peek_token(ps)) + if kind(peek_token(ps)) == K"=" # Allow trailing comma before `=` # x, = xs ==> (tuple x) continue @@ -750,10 +736,10 @@ end function parse_arrow(ps::ParseState) mark = position(ps) parse_or(ps) - isdot, t = peek_dotted_op_token(ps) + isdot, isassign, t = peek_dotted_op_token(ps) k = kind(t) - if is_prec_arrow(k) - if kind(t) == K"-->" && !isdot && !is_suffixed(t) + if is_prec_arrow(t) + if kind(t) == K"-->" && !isdot # x --> y ==> (--> x y) # The only syntactic arrow bump(ps, TRIVIA_FLAG) parse_arrow(ps) @@ -788,9 +774,9 @@ end function parse_lazy_cond(ps::ParseState, down, is_op, self) mark = position(ps) down(ps) - (isdot, t) = peek_dotted_op_token(ps) + (isdot, isassign, t) = peek_dotted_op_token(ps) k = kind(t) - if is_op(k) + if is_op(t) bump_dotted(ps, isdot, t, TRIVIA_FLAG) self(ps) emit(ps, mark, isdot ? dotted(k) : k, flags(t)) @@ -835,8 +821,8 @@ function parse_comparison(ps::ParseState, subtype_comparison=false) n_comparisons = 0 op_pos = NO_POSITION op_dotted = false - (initial_dot, initial_tok) = peek_dotted_op_token(ps) - while ((isdot, t) = peek_dotted_op_token(ps); is_prec_comparison(t)) + (initial_dot, initial_isassign, initial_tok) = peek_dotted_op_token(ps) + while ((isdot, isassign, t) = peek_dotted_op_token(ps); is_prec_comparison(t)) n_comparisons += 1 op_dotted = isdot op_pos = bump_dotted(ps, isdot, t, emit_dot_node=true, remap_kind=K"Identifier") @@ -894,9 +880,11 @@ function parse_range(ps::ParseState) mark = position(ps) parse_invalid_ops(ps) - (initial_dot, initial_tok) = peek_dotted_op_token(ps) + # The compound-assignment flag (`a ..= b` etc.) is intentionally ignored + # here; such forms are rejected in `parse_assignment_with_initial_ex`. + (initial_dot, _, initial_tok) = peek_dotted_op_token(ps) initial_kind = kind(initial_tok) - if initial_kind != K":" && (is_prec_colon(initial_kind) || (initial_dot && initial_kind == K".")) + if initial_kind != K":" && (is_prec_colon(initial_tok) || (initial_dot && initial_kind == K".")) # a..b ==> (call-i a (DotsIdentifier-2) b) # a … b ==> (call-i a … b) # a .… b ==> (dotcall-i a … b) @@ -986,7 +974,7 @@ end function parse_invalid_ops(ps::ParseState) mark = position(ps) parse_expr(ps) - while ((isdot, t) = peek_dotted_op_token(ps); kind(t) in KSet"ErrorInvalidOperator Error**") + while ((isdot, isassign, t) = peek_dotted_op_token(ps); kind(t) in KSet"ErrorInvalidOperator Error**") bump_trivia(ps) bump_dotted(ps, isdot, t) parse_expr(ps) @@ -1016,7 +1004,7 @@ end function parse_with_chains(ps::ParseState, down, is_op, chain_ops) mark = position(ps) down(ps) - while ((isdot, t) = peek_dotted_op_token(ps); is_op(kind(t))) + while ((isdot, isassign, t) = peek_dotted_op_token(ps); is_op(t) && !isassign) if ps.space_sensitive && preceding_whitespace(t) && is_both_unary_and_binary(t, isdot) && !preceding_whitespace(peek_token(ps, 2)) @@ -1031,7 +1019,7 @@ function parse_with_chains(ps::ParseState, down, is_op, chain_ops) end bump_dotted(ps, isdot, t, remap_kind=K"Identifier") down(ps) - if kind(t) in chain_ops && !is_suffixed(t) && !isdot + if kind(t) in chain_ops && !isdot # a + b + c ==> (call-i a + b c) # a + b .+ c ==> (dotcall-i (call-i a + b) + c) parse_chain(ps, down, kind(t)) @@ -1047,8 +1035,8 @@ end # flisp: parse-chain function parse_chain(ps::ParseState, down, op_kind) while true - isdot, t = peek_dotted_op_token(ps) - if kind(t) != op_kind || is_suffixed(t) || isdot + isdot, isassign, t = peek_dotted_op_token(ps) + if kind(t) != op_kind || isdot break end if ps.space_sensitive && preceding_whitespace(t) && @@ -1165,7 +1153,8 @@ function parse_juxtapose(ps::ParseState) is_syntactic_unary_op(prev_k) || is_initial_reserved_word(ps, prev_k) ))) && (!is_operator(k) || is_radical_op(k)) && - !is_closing_token(ps, k) + !is_closing_token(ps, k) && + k != K"ErrorInvalidOperator" && k != K"Error**" if prev_k == K"string" || is_string_delim(t) bump_invisible(ps, K"error", TRIVIA_FLAG, error="cannot juxtapose string literal") @@ -1214,7 +1203,7 @@ end function parse_unary(ps::ParseState) mark = position(ps) bump_trivia(ps) - (op_dotted, op_t) = peek_dotted_op_token(ps) + (op_dotted, op_isassign, op_t) = peek_dotted_op_token(ps) op_k = kind(op_t) if ( !is_operator(op_k) || @@ -1232,12 +1221,12 @@ function parse_unary(ps::ParseState) end t2 = peek_token(ps, 2+op_dotted) k2 = kind(t2) - if op_k in KSet"- +" && !is_suffixed(op_t) && !op_dotted + if op_k in KSet"- +" && !op_dotted if !preceding_whitespace(t2) && (k2 in KSet"Integer Float Float32" || (op_k == K"+" && k2 in KSet"BinInt HexInt OctInt")) - k3 = peek(ps, 3) - if is_prec_power(k3) || k3 in KSet"[ {" + t3 = peek_token(ps, 3) + if is_prec_power(t3) || kind(t3) in KSet"[ {" # `[`, `{` (issue #18851) and `^` have higher precedence than # unary negation # -2^x ==> (call-pre - (call-i 2 ^ x)) @@ -1410,7 +1399,7 @@ end # flisp: parse-factor-with-initial-ex function parse_factor_with_initial_ex(ps::ParseState, mark) parse_decl_with_initial_ex(ps, mark) - if ((isdot, t) = peek_dotted_op_token(ps); is_prec_power(kind(t))) + if ((isdot, isassign, t) = peek_dotted_op_token(ps); is_prec_power(t) && !isassign) bump_dotted(ps, isdot, t, remap_kind=K"Identifier") parse_factor_after(ps) emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) @@ -1482,7 +1471,7 @@ end # flisp: parse-unary-prefix function parse_unary_prefix(ps::ParseState, has_unary_prefix=false) mark = position(ps) - (isdot, t) = peek_dotted_op_token(ps) + (isdot, isassign, t) = peek_dotted_op_token(ps) k = kind(t) if is_syntactic_unary_op(k) && !isdot k2 = peek(ps, 2) @@ -1796,7 +1785,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false) maybe_strmac_1 = true emit(ps, mark, K".") end - elseif k == K"'" && !preceding_whitespace(t) + elseif is_prec_quote(t) && !preceding_whitespace(t) # f' ==> (call-post f ') # f'ᵀ ==> (call-post f 'ᵀ) bump(ps, remap_kind=K"Identifier") @@ -2259,15 +2248,18 @@ end function parse_global_local_const_vars(ps) mark = position(ps) n_commas = parse_comma(ps, false) - (isdot, t) = peek_dotted_op_token(ps) - if is_prec_assignment(t) + # `isassign` is true for a (possibly dotted) operator followed by `=`, ie a + # compound assignment like `x += 1` or `x .+= 1`. + (isdot, isassign, t) = peek_dotted_op_token(ps) + + if is_prec_assignment(t) || isassign if n_commas >= 1 # const x,y = 1,2 ==> (const (= (tuple x y) (tuple 1 2))) emit(ps, mark, K"tuple") end # const x = 1 ==> (const (= x 1)) # global x ~ 1 ==> (global (call-i x ~ 1)) - # global x += 1 ==> (global (+= x 1)) + # global x += 1 ==> (global (op= x + 1)) parse_assignment_with_initial_ex(ps, mark, parse_comma) else # global x,y ==> (global x y) @@ -2362,7 +2354,7 @@ function parse_function_signature(ps::ParseState, is_function::Bool) # function (:)() end ==> (function (call (parens :)) (block)) # function (x::T)() end ==> (function (call (parens (::-i x T))) (block)) # function (::T)() end ==> (function (call (parens (::-pre T))) (block)) - # function (:*=(f))() end ==> (function (call (parens (call (quote-: *=) f))) (block)) + # function (:*=(f))() end ==> (function (call (parens (call (quote-: (op= *)) f))) (block)) emit(ps, mark, K"parens", PARENS_FLAG) end end @@ -3213,13 +3205,13 @@ function parse_paren(ps::ParseState, check_identifiers=true, has_unary_prefix=fa mark = position(ps) @check peek(ps) == K"(" bump(ps, TRIVIA_FLAG) # K"(" - (_isdot, tok) = peek_dotted_op_token(ps) + (_isdot, isassign, tok) = peek_dotted_op_token(ps) k = kind(tok) if k == K")" # () ==> (tuple-p) bump(ps, TRIVIA_FLAG) emit(ps, mark, K"tuple", PARENS_FLAG) - elseif is_syntactic_operator(k) + elseif is_syntactic_operator(k) || isassign # allow :(=) etc in unchecked contexts, eg quotes # :(=) ==> (quote-: (parens =)) parse_atom(ps, check_identifiers) @@ -3613,7 +3605,7 @@ end function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=false) bump_trivia(ps) mark = position(ps) - (leading_dot, leading_tok) = peek_dotted_op_token(ps) + (leading_dot, leading_isassign, leading_tok) = peek_dotted_op_token(ps) leading_kind = kind(leading_tok) # todo: Reorder to put most likely tokens first? if leading_dot @@ -3706,7 +3698,7 @@ function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=fal # a[:(end)] ==> (ref a (quote-: (error-t end))) parse_atom(ParseState(ps, end_symbol=false), false) emit(ps, mark, K"quote", COLON_QUOTE) - elseif check_identifiers && leading_kind == K"=" && is_plain_equals(peek_token(ps)) && !leading_dot + elseif check_identifiers && leading_kind == K"=" && kind(peek_token(ps)) == K"=" && !leading_dot # = ==> (error =) bump(ps, error="unexpected `=`") elseif leading_kind == K"Identifier" @@ -3720,16 +3712,26 @@ function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=fal @label is_operator # + ==> + # .+ ==> (. +) - bump_dotted(ps, leading_dot, leading_tok, emit_dot_node=true, remap_kind= + is_compound_assignment = !is_prec_assignment(leading_tok) && leading_isassign + bump_dotted(ps, leading_dot, leading_tok, emit_dot_node=!is_compound_assignment, remap_kind= is_syntactic_operator(leading_kind) ? leading_kind : K"Identifier") + + if is_compound_assignment + bump(ps, TRIVIA_FLAG) # consume the = but mark as trivia + emit(ps, mark, leading_dot ? K".op=" : K"op=") + if check_identifiers + # += ==> (error (op= +)) + # .+= ==> (error (.op= +)) + emit(ps, mark, K"error", error="invalid identifier") + end + # Quoted syntactic operators are allowed + # :+= ==> (quote-: (op= +)) + return + end + if check_identifiers && !(is_valid_identifier(leading_kind) || (leading_dot && leading_kind == K".")) - # += ==> (error (op= +)) # ? ==> (error ?) - # .+= ==> (error (. (op= +))) emit(ps, mark, K"error", error="invalid identifier") - else - # Quoted syntactic operators allowed - # :+= ==> (quote-: (op= +)) end elseif is_keyword(leading_kind) if leading_kind == K"var" && (t = peek_token(ps,2); diff --git a/JuliaSyntax/src/julia/tokenize.jl b/JuliaSyntax/src/julia/tokenize.jl index b89da199f22de..477fedb1d87d1 100644 --- a/JuliaSyntax/src/julia/tokenize.jl +++ b/JuliaSyntax/src/julia/tokenize.jl @@ -2,9 +2,15 @@ module Tokenize export tokenize, untokenize -using ..JuliaSyntax: @KSet_str, @K_str, @callsite_inline, Kind +using ..JuliaSyntax: JuliaSyntax, Kind, @K_str, @KSet_str, @callsite_inline, + generic_operators_by_level, PrecedenceLevel, PREC_NONE, PREC_ASSIGNMENT, + PREC_PAIRARROW, PREC_CONDITIONAL, PREC_ARROW, PREC_LAZYOR, PREC_LAZYAND, + PREC_COMPARISON, PREC_PIPE_LT, PREC_PIPE_GT, PREC_COLON, PREC_PLUS, + PREC_BITSHIFT, PREC_TIMES, PREC_RATIONAL, PREC_POWER, PREC_DECL, + PREC_WHERE, PREC_DOT, PREC_QUOTE, PREC_UNICODE_OPS, PREC_COMPOUND_ASSIGN -import ..JuliaSyntax: is_contextual_keyword, is_literal, is_word_operator, kind +import ..JuliaSyntax: kind, + is_literal, is_contextual_keyword, is_word_operator, is_operator #------------------------------------------------------------------------------- # Character-based predicates for tokenization @@ -71,32 +77,6 @@ end readchar(io::IO) = eof(io) ? EOF_CHAR : read(io, Char) -# Some unicode operators are normalized by the tokenizer into their equivalent -# kinds. See also normalize_identifier() -const _ops_with_unicode_aliases = [ - # \minus '−' is normalized into K"-", - '−' => K"-" - # Lookalikes which are normalized into K"⋅", - # https://github.com/JuliaLang/julia/pull/25157, - '\u00b7' => K"⋅" # '·' Middle Dot,, - '\u0387' => K"⋅" # '·' Greek Ano Teleia,, -] - -function _nondot_symbolic_operator_kinds() - op_range = reinterpret(UInt16, K"BEGIN_OPS"):reinterpret(UInt16, K"END_OPS") - setdiff(reinterpret.(Kind, op_range), [ - K"ErrorInvalidOperator" - K"Error**" - K"..." - K"." - K"where" - K"isa" - K"in" - K".'" - K"op=" - ]) -end - function _char_in_set_expr(varname, firstchars) codes = sort!(UInt32.(unique(firstchars))) terms = [] @@ -120,15 +100,18 @@ end if c == EOF_CHAR || !isvalid(c) return false end - u = UInt32(c) - return $(_char_in_set_expr(:u, - append!(first.(string.(_nondot_symbolic_operator_kinds())), - first.(_ops_with_unicode_aliases)))) + # Check if character is a known operator char or in our unicode ops + # dictionary. The second tuple lists the unicode operators which don't + # appear in `_unicode_ops` because they have their own kinds (and explicit + # branches in `_next_token`). + return c in ('!', '#', '$', '%', '&', '*', '+', '-', '−', '/', ':', '<', '=', '>', '?', '@', '\\', '^', '|', '~', '÷', '⊻', '\'') || + c in ('∈', '≔', '⩴', '≕', '¬', '√', '∛', '∜') || + haskey(_unicode_ops, c) end # Checks whether a Char is an operator which can be prefixed with a dot `.` function is_dottable_operator_start_char(c) - return c != '?' && c != '$' && c != ':' && c != '\'' && is_operator_start_char(c) + return c != '?' && c != '$' && c != ':' && c != '\'' && c != '#' && c != '@' && is_operator_start_char(c) end @eval function isopsuffix(c::Char) @@ -150,7 +133,8 @@ end end function optakessuffix(k) - (K"BEGIN_OPS" <= k <= K"END_OPS") && + # Most operators can take suffix except for specific ones + is_operator(k) && !( K"BEGIN_ASSIGNMENTS" <= k <= K"END_ASSIGNMENTS" || k == K"?" || @@ -160,8 +144,6 @@ function optakessuffix(k) k == K"||" || k == K"in" || k == K"isa" || - k == K"≔" || - k == K"⩴" || k == K":" || k == K"$" || k == K"::" || @@ -170,19 +152,21 @@ function optakessuffix(k) k == K"!" || k == K".'" || k == K"->" || - K"BEGIN_UNICODE_OPS" <= k <= K"END_UNICODE_OPS" + K"¬" <= k <= K"∜" ) end const _unicode_ops = let - ks = _nondot_symbolic_operator_kinds() - ss = string.(ks) + # Map single-character unicode operators to their precedence levels + ops = Dict{Char, PrecedenceLevel}() - ops = Dict{Char, Kind}([first(s)=>k for (k,s) in zip(ks,ss) - if length(s) == 1 && !isascii(s[1])]) - for ck in _ops_with_unicode_aliases - push!(ops, ck) + # Add operators from generic_operators_by_level + for (prec, chars) in generic_operators_by_level + for c in chars + ops[c] = prec + end end + ops end @@ -194,12 +178,12 @@ struct RawToken # Offsets into a string or buffer startbyte::Int # The byte where the token start in the buffer endbyte::Int # The byte where the token ended in the buffer - suffix::Bool + op_precedence::PrecedenceLevel # If K"Operator", the operator's precedence level end function RawToken(kind::Kind, startbyte::Int, endbyte::Int) - RawToken(kind, startbyte, endbyte, false) + RawToken(kind, startbyte, endbyte, PREC_NONE) end -RawToken() = RawToken(K"error", 0, 0, false) +RawToken() = RawToken(K"error", 0, 0, PREC_NONE) const EMPTY_TOKEN = RawToken() @@ -424,17 +408,47 @@ end Returns a `RawToken` of kind `kind` with contents `str` and starts a new `RawToken`. """ -function emit(l::Lexer, kind::Kind, maybe_op=true) - suffix = false - if optakessuffix(kind) && maybe_op +function emit(l::Lexer, kind::Kind) + tok = RawToken(kind, startpos(l), position(l) - 1, PREC_NONE) + + l.last_token = kind + return tok +end + +function emit_operator(l::Lexer, kind::Kind, precedence::PrecedenceLevel, take_suffix=optakessuffix(kind)) + if take_suffix while isopsuffix(peekchar(l)) readchar(l) - suffix = true + kind = K"Operator" end end + tok = RawToken(kind, startpos(l), position(l) - 1, precedence) + + l.last_token = kind + return tok +end - tok = RawToken(kind, startpos(l), position(l) - 1, suffix) +# Check whether the operator just lexed forms a compound assignment like `+=`, +# ie whether the input continues with the single token `=`. Peeking one +# character is not enough: the other tokens starting with `=` (`==`, `===` and +# `=>`) do not form compound assignments, eg `a +== b` is `a + (== b)`. +function compound_assign_follows(l::Lexer) + pc, ppc = dpeekchar(l) + return pc == '=' && ppc != '=' && ppc != '>' +end +# Emit `kind` with precedence `prec`, unless the operator is immediately followed +# by `=`, in which case it forms a compound assignment like `+=` and is emitted as +# a `K"Operator"` with `PREC_COMPOUND_ASSIGN`. +function emit_operator_or_compound_assign(l::Lexer, kind::Kind, prec::PrecedenceLevel) + if compound_assign_follows(l) + return emit_operator(l, K"Operator", PREC_COMPOUND_ASSIGN) + end + return emit_operator(l, kind, prec) +end + +function emit_trivia(l::Lexer, kind::Kind) + tok = RawToken(kind, startpos(l), position(l) - 1, PREC_NONE) l.last_token = kind return tok end @@ -447,9 +461,9 @@ Returns the next `RawToken`. function next_token(l::Lexer, start = true) start && start_token!(l) if !isempty(l.string_states) - lex_string_chunk(l) + return lex_string_chunk(l) else - _next_token(l, readchar(l)) + return _next_token(l, readchar(l)) end end @@ -522,18 +536,40 @@ function _next_token(l::Lexer, c) return lex_plus(l); elseif c == '-' return lex_minus(l); - elseif c == '−' # \minus '−' treated as hyphen '-' - return emit(l, accept(l, '=') ? K"op=" : K"-") elseif c == '`' return lex_backtick(l); + elseif c == '−' # \minus '−' treated as hyphen '-' + return emit_operator_or_compound_assign(l, K"-", PREC_PLUS) + elseif c == '∈' + return emit_operator(l, K"∈", PREC_COMPARISON) + elseif c == '⋆' + return emit_operator(l, K"⋆", PREC_TIMES) + elseif c == '±' + return emit_operator(l, K"±", PREC_PLUS) + elseif c == '∓' + return emit_operator(l, K"∓", PREC_PLUS) + elseif c == '¬' + return emit(l, K"¬") + elseif c == '√' + return emit(l, K"√") + elseif c == '∛' + return emit(l, K"∛") + elseif c == '∜' + return emit(l, K"∜") + elseif c == '≔' + return emit_operator(l, K"≔", PREC_ASSIGNMENT) + elseif c == '⩴' + return emit_operator(l, K"⩴", PREC_ASSIGNMENT) + elseif c == '≕' + return emit_operator(l, K"≕", PREC_ASSIGNMENT) + elseif haskey(_unicode_ops, c) + return emit_operator(l, K"Operator", _unicode_ops[c]) elseif is_identifier_start_char(c) return lex_identifier(l, c) elseif isdigit(c) return lex_digit(l, K"Integer") - elseif (k = get(_unicode_ops, c, K"None")) != K"None" - return emit(l, k) else - emit(l, + return emit(l, !isvalid(c) ? K"ErrorInvalidUTF8" : is_invisible_char(c) ? K"ErrorInvisibleChar" : is_identifier_char(c) ? K"ErrorIdentifierStart" : @@ -639,7 +675,7 @@ function lex_string_chunk(l) K"\"\"\"" : K"```") else return emit(l, state.delim == '"' ? K"\"" : - state.delim == '`' ? K"`" : K"'", false) + state.delim == '`' ? K"`" : K"'") end end # Read a chunk of string characters @@ -738,7 +774,7 @@ function lex_whitespace(l::Lexer, c) end c = readchar(l) end - return emit(l, k) + return emit_trivia(l, k) end function lex_comment(l::Lexer) @@ -747,7 +783,8 @@ function lex_comment(l::Lexer) while true pc, ppc = dpeekchar(l) if pc == '\n' || (pc == '\r' && ppc == '\n') || pc == EOF_CHAR - return emit(l, valid ? K"Comment" : K"ErrorInvalidUTF8") + return valid ? emit_trivia(l, K"Comment") : + emit(l, K"ErrorInvalidUTF8") end valid &= isvalid(pc) readchar(l) @@ -791,55 +828,44 @@ end # Lex a greater char, a '>' has been consumed function lex_greater(l::Lexer) if accept(l, '>') - if accept(l, '>') - if accept(l, '=') - return emit(l, K"op=") - else # >>>?, ? not a = - return emit(l, K">>>") - end - elseif accept(l, '=') - return emit(l, K"op=") - else - return emit(l, K">>") - end + # >> >>> >>= >>>= + accept(l, '>') + return emit_operator_or_compound_assign(l, K"Operator", PREC_BITSHIFT) elseif accept(l, '=') - return emit(l, K">=") + return emit_operator(l, K"Operator", PREC_COMPARISON) # >= elseif accept(l, ':') return emit(l, K">:") else - return emit(l, K">") + return emit_operator(l, K">", PREC_COMPARISON) end end # Lex a less char, a '<' has been consumed function lex_less(l::Lexer) if accept(l, '<') - if accept(l, '=') - return emit(l, K"op=") - else # '<') - return emit(l, K"<-->") + return emit_operator(l, K"Operator", PREC_ARROW) # <--> elseif accept(l, '-') return emit(l, K"ErrorInvalidOperator") else - return emit(l, K"<--") + return emit_operator(l, K"Operator", PREC_ARROW) # <-- end end else - return emit(l, K"<") + return emit_operator(l, K"<", PREC_COMPARISON) end end @@ -847,15 +873,12 @@ end # An '=' char has been consumed function lex_equal(l::Lexer) if accept(l, '=') - if accept(l, '=') - emit(l, K"===") - else - emit(l, K"==") - end + accept(l, '=') + return emit_operator(l, K"Operator", PREC_COMPARISON) # ==, === elseif accept(l, '>') - emit(l, K"=>") + return emit_operator(l, K"Operator", PREC_PAIRARROW) else - emit(l, K"=") + return emit(l, K"=") end end @@ -866,16 +889,16 @@ function lex_colon(l::Lexer) elseif accept(l, '=') return emit(l, K":=") else - return emit(l, K":") + return emit_operator(l, K":", PREC_COLON) end end function lex_exclaim(l::Lexer) if accept(l, '=') if accept(l, '=') - return emit(l, K"!==") + return emit_operator(l, K"Operator", PREC_COMPARISON) # !== else - return emit(l, K"!=") + return emit_operator(l, K"Operator", PREC_COMPARISON) # != end else return emit(l, K"!") @@ -883,84 +906,60 @@ function lex_exclaim(l::Lexer) end function lex_percent(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - else - return emit(l, K"%") - end + return emit_operator_or_compound_assign(l, K"Operator", PREC_TIMES) end function lex_bar(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - elseif accept(l, '>') - return emit(l, K"|>") + if accept(l, '>') + return emit_operator(l, K"Operator", PREC_PIPE_GT) # |> elseif accept(l, '|') return emit(l, K"||") else - emit(l, K"|") + return emit_operator_or_compound_assign(l, K"Operator", PREC_PLUS) end end function lex_plus(l::Lexer) if accept(l, '+') - return emit(l, K"++") - elseif accept(l, '=') - return emit(l, K"op=") + return emit_operator(l, K"++", PREC_PLUS) end - return emit(l, K"+") + return emit_operator_or_compound_assign(l, K"+", PREC_PLUS) end function lex_minus(l::Lexer) if accept(l, '-') if accept(l, '>') - return emit(l, K"-->") + return emit_operator(l, K"-->", PREC_ARROW) else return emit(l, K"ErrorInvalidOperator") # "--" is an invalid operator end elseif l.last_token != K"." && accept(l, '>') - return emit(l, K"->") - elseif accept(l, '=') - return emit(l, K"op=") + return emit_operator(l, K"->", PREC_ARROW) end - return emit(l, K"-") + return emit_operator_or_compound_assign(l, K"-", PREC_PLUS) end function lex_star(l::Lexer) if accept(l, '*') return emit(l, K"Error**") # "**" is an invalid operator use ^ - elseif accept(l, '=') - return emit(l, K"op=") end - return emit(l, K"*") + return emit_operator_or_compound_assign(l, K"*", PREC_TIMES) end function lex_circumflex(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - end - return emit(l, K"^") + return emit_operator_or_compound_assign(l, K"Operator", PREC_POWER) # ^ end function lex_division(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - end - return emit(l, K"÷") + return emit_operator_or_compound_assign(l, K"Operator", PREC_TIMES) # / end function lex_dollar(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - end - return emit(l, K"$") + return emit_operator_or_compound_assign(l, K"$", PREC_PLUS) end function lex_xor(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - end - return emit(l, K"⊻") + return emit_operator_or_compound_assign(l, K"Operator", PREC_PLUS) end function accept_number(l::Lexer, f::F) where {F} @@ -1104,20 +1103,18 @@ function lex_prime(l) is_literal(l.last_token) # FIXME ^ This doesn't cover all cases - probably needs involvement # from the parser state. - return emit(l, K"'") + return emit_operator(l, K"'", PREC_QUOTE) else push!(l.string_states, StringState(false, true, '\'', 0)) - return emit(l, K"'", false) + return emit(l, K"'") end end function lex_amper(l::Lexer) if accept(l, '&') return emit(l, K"&&") - elseif accept(l, '=') - return emit(l, K"op=") else - return emit(l, K"&") + return emit_operator_or_compound_assign(l, K"&", PREC_TIMES) end end @@ -1151,24 +1148,12 @@ end # Parse a token starting with a forward slash. # A '/' has been consumed function lex_forwardslash(l::Lexer) - if accept(l, '/') - if accept(l, '=') - return emit(l, K"op=") - else - return emit(l, K"//") - end - elseif accept(l, '=') - return emit(l, K"op=") - else - return emit(l, K"/") - end + prec = accept(l, '/') ? PREC_RATIONAL : PREC_TIMES + return emit_operator_or_compound_assign(l, K"Operator", prec) # // or / end function lex_backslash(l::Lexer) - if accept(l, '=') - return emit(l, K"op=") - end - return emit(l, K"\\") + return emit_operator_or_compound_assign(l, K"Operator", PREC_TIMES) end function lex_dot(l::Lexer) @@ -1239,11 +1224,12 @@ function lex_identifier(l::Lexer, c) end if n > MAX_KW_LENGTH - emit(l, K"Identifier") + return emit(l, K"Identifier") elseif h == _true_hash || h == _false_hash - emit(l, K"Bool") + return emit(l, K"Bool") else - emit(l, get(_kw_hash, h, K"Identifier")) + k = get(_kw_hash, h, K"Identifier") + return emit(l, k) end end diff --git a/JuliaSyntax/src/porcelain/syntax_graph.jl b/JuliaSyntax/src/porcelain/syntax_graph.jl index bba78d86c4a2f..9ec32b45901e4 100644 --- a/JuliaSyntax/src/porcelain/syntax_graph.jl +++ b/JuliaSyntax/src/porcelain/syntax_graph.jl @@ -1375,6 +1375,12 @@ function _green_to_est(parent::SyntaxTree, parent_i::Int, rhs = _green_to_est(st, 0, cs[3]) out = newnode(graph, st, K"unknown_head", tree_ids(lhs, rhs)) return setattr!(out, :name_val, op_s) + elseif k === K"op=" && n_cs === 1 + # (op= +) => += (the operator name itself, eg when quoted as `:(+=)`) + return symleaf(string(cs[1]) * '=') + elseif k === K".op=" && n_cs === 1 + # (.op= +) => .+= + return symleaf('.' * string(cs[1]) * '=') elseif k === K"macrocall" && n_cs > 0 # LineNumberNodes are not usually added to the tree as they are in Expr, # but this specifically inserts the macrocall child for compatibility diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl index 63746b06afc14..cdd9a1210c8ad 100644 --- a/JuliaSyntax/test/parser.jl +++ b/JuliaSyntax/test/parser.jl @@ -76,6 +76,19 @@ tests = [ "f(x) where S where U = 1" => "(function-= (where (where (call f x) S) U) 1)" "(f(x)::T) where S = 1" => "(function-= (where (parens (::-i (call f x) T)) S) 1)" "f(x) = 1 = 2" => "(function-= (call f x) (= 1 2))" # Should be a warning! + # Suffixed operators don't form compound assignments (matching the + # reference parser): `+₁` is parsed as the operator, leaving a stray `=` + "a +₁= b" => "(call-i a +₁ (error =))" + # ... and likewise operators which simply have no compound-assignment + # form are parsed as an identifier being assigned to + "⋅ = 5" => "(= ⋅ 5)" + "⋅=5" => "(= ⋅ 5)" + # Operators followed by `==`, `===` or `=>` (rather than the single + # token `=`) are not compound assignments + "a +== b" => "(call-i a + (call-pre (error ==) b))" + "a -=> b" => "(call-i a - (call-pre (error =>) b))" + "a >>>== b" => "(call-i a >>> (call-pre (error ==) b))" + "a .+== b" => "(dotcall-i a + (call-pre (error ==) b))" ], JuliaSyntax.parse_pair => [ "a => b" => "(call-i a => b)" @@ -119,6 +132,8 @@ tests = [ "x < y" => "(call-i x < y)" "x .< y" => "(dotcall-i x < y)" "x .<: y" => "(dotcall-i x <: y)" + # A dotted operator directly following a float literal + "1.1.∈a" => "(dotcall-i 1.1 ∈ a)" ":. == :." => "(call-i (quote-: .) == (quote-: .))" # Comparison chains "x < y < z" => "(comparison x < y < z)" @@ -206,6 +221,9 @@ tests = [ "x 'y" => "x" "x@y" => "x" "(begin end)x" => "(parens (block))" + # Invalid operators (`**`, `--`) are not juxtaposed + "2**2" => "2" + "2--2" => "2" ], JuliaSyntax.parse_unary => [ ":T" => "(quote-: T)" @@ -642,7 +660,7 @@ tests = [ "function (\n ::T\n )(x, y) end" => "(function (call (parens (::-pre T)) x y) (block))" "function (\n f::T{g(i)}\n )() end" => "(function (call (parens (::-i f (curly T (call g i))))) (block))" "function (\n x, y\n ) x + y end" => "(function (tuple-p x y) (block (call-i x + y)))" - "function (:*=(f))() end" => "(function (call (parens (call (quote-: *=) f))) (block))" + "function (:*=(f))() end" => "(function (call (parens (call (quote-: (op= *)) f))) (block))" "function begin() end" => "(function (call (error begin)) (block))" "function f() end" => "(function (call f) (block))" "function type() end" => "(function (call type) (block))" @@ -842,24 +860,24 @@ tests = [ "||" => "(error ||)" "." => "(error .)" "..." => "(error (DotsIdentifier-3))" - "+=" => "(error +=)" - "-=" => "(error -=)" - "*=" => "(error *=)" - "/=" => "(error /=)" - "//=" => "(error //=)" - "|=" => "(error |=)" - "^=" => "(error ^=)" - "÷=" => "(error ÷=)" - "%=" => "(error %=)" - "<<=" => "(error <<=)" - ">>=" => "(error >>=)" - ">>>="=> "(error >>>=)" - "\\=" => "(error \\=)" - "&=" => "(error &=)" - ":=" => "(error :=)" - "\$=" => "(error \$=)" - "⊻=" => "(error ⊻=)" - ".+=" => "(error (. +=))" + "+=" => "(error (op= +))" + "-=" => "(error (op= -))" + "*=" => "(error (op= *))" + "/=" => "(error (op= /))" + "//=" => "(error (op= //))" + "|=" => "(error (op= |))" + "^=" => "(error (op= ^))" + "÷=" => "(error (op= ÷))" + "%=" => "(error (op= %))" + "<<=" => "(error (op= <<))" + ">>=" => "(error (op= >>))" + ">>>="=> "(error (op= >>>))" + "\\=" => "(error (op= \\))" + "&=" => "(error (op= &))" + ":=" => "(error :=)" # Assignment operator, not `:`-update + "\$=" => "(error (op= \$))" + "⊻=" => "(error (op= ⊻))" + ".+=" => "(error (.op= +))" # Normal operators "+" => "+" # Assignment-precedence operators which can be used as identifiers @@ -868,8 +886,8 @@ tests = [ "⩴" => "⩴" "≕" => "≕" # Quoted syntactic operators allowed - ":+=" => "(quote-: +=)" - ":.+=" => "(quote-: (. +=))" + ":+=" => "(quote-: (op= +))" + ":.+=" => "(quote-: (.op= +))" ":.=" => "(quote-: (. =))" ":.&&" => "(quote-: (. &&))" # Special symbols quoted @@ -1259,8 +1277,8 @@ parsestmt_with_kind_tests = [ "a += b" => "(op= a::Identifier +::Identifier b::Identifier)" "a .+= b" => "(.op= a::Identifier +::Identifier b::Identifier)" "a >>= b" => "(op= a::Identifier >>::Identifier b::Identifier)" - ":+=" => "(quote-: +=::op=)" - ":.+=" => "(quote-: (. +=::op=))" + ":+=" => "(quote-: (op= +::Identifier))" + ":.+=" => "(quote-: (.op= +::Identifier))" # str/cmd macro name kinds "x\"str\"" => """(macrocall x::StrMacroName (string-r "str"::String))""" "x`str`" => """(macrocall x::CmdMacroName (cmdstring-r "str"::CmdString))""" diff --git a/JuliaSyntax/test/parser_api.jl b/JuliaSyntax/test/parser_api.jl index 10a09d3ace585..85f80839fe977 100644 --- a/JuliaSyntax/test/parser_api.jl +++ b/JuliaSyntax/test/parser_api.jl @@ -214,8 +214,8 @@ tokensplit(str; kws...) = [kind(tok) => untokenize(tok, str) for tok in tokenize K"Integer" => "1", ] - # A predicate based on flags() - @test JuliaSyntax.is_suffixed(tokenize("+₁")[1]) + # +₁ is tokenized as a single identifier token (subscripts are valid in identifiers) + @test tokensplit("+₁") == [K"Identifier"=>"+₁"] # Buffer interface @test tokenize(Vector{UInt8}("a + b")) == tokenize("a + b") diff --git a/JuliaSyntax/test/tokenize.jl b/JuliaSyntax/test/tokenize.jl index 301a29ed25d1e..db5de2c2b7094 100644 --- a/JuliaSyntax/test/tokenize.jl +++ b/JuliaSyntax/test/tokenize.jl @@ -129,11 +129,11 @@ end # testset K"NewlineWs",K"Comment", - K"NewlineWs",K"Integer",K"%",K"Integer", + K"NewlineWs",K"Integer",K"Operator",K"Integer", - K"NewlineWs",K"Identifier",K"'",K"/",K"Identifier",K"'", + K"NewlineWs",K"Identifier",K"'",K"Operator",K"Identifier",K"'", - K"NewlineWs",K"Identifier",K".",K"'",K"\\",K"Identifier",K".",K"'", + K"NewlineWs",K"Identifier",K".",K"'",K"Operator",K"Identifier",K".",K"'", K"NewlineWs",K"`",K"CmdString",K"`", @@ -175,19 +175,33 @@ end end @testset "test added operators" begin - @test tok("1+=2", 2).kind == K"op=" - @test tok("1-=2", 2).kind == K"op=" - @test tok("1*=2", 2).kind == K"op=" - @test tok("1^=2", 2).kind == K"op=" - @test tok("1÷=2", 2).kind == K"op=" - @test tok("1\\=2", 2).kind == K"op=" - @test tok("1\$=2", 2).kind == K"op=" - @test tok("1⊻=2", 2).kind == K"op=" - @test tok("1:=2", 2).kind == K":=" - @test tok("1-->2", 2).kind == K"-->" - @test tok("1<--2", 2).kind == K"<--" - @test tok("1<-->2", 2).kind == K"<-->" - @test tok("1>:2", 2).kind == K">:" + # Compound assignments now emit separate operator and = tokens. The operator + # itself is emitted as K"Operator" when immediately followed by `=`. + @test toks("1+=2")[2:3] == ["+"=>K"Operator", "="=>K"="] + @test toks("1-=2")[2:3] == ["-"=>K"Operator", "="=>K"="] + @test toks("1*=2")[2:3] == ["*"=>K"Operator", "="=>K"="] + @test toks("1^=2")[2:3] == ["^"=>K"Operator", "="=>K"="] + @test toks("1÷=2")[2:3] == ["÷"=>K"Operator", "="=>K"="] + @test toks("1\\=2")[2:3] == ["\\"=>K"Operator", "="=>K"="] + @test toks("1\$=2")[2:3] == ["\$"=>K"Operator", "="=>K"="] + @test toks("1⊻=2")[2:3] == ["⊻"=>K"Operator", "="=>K"="] + @test toks("1:=2")[2] == (":="=>K":=") + @test toks("1-->2")[2] == ("-->"=>K"-->") + @test toks("1<--2")[2] == ("<--"=>K"Operator") + @test toks("1<-->2")[2] == ("<-->"=>K"Operator") + @test toks("1>:2")[2] == (">:"=>K">:") + + # Operators followed by `==`, `===` or `=>` (rather than the single token + # `=`) do not form compound assignments + @test toks("1+==2")[2:3] == ["+"=>K"+", "=="=>K"Operator"] + @test toks("1-=>2")[2:3] == ["-"=>K"-", "=>"=>K"Operator"] + @test toks("1*===2")[2:3] == ["*"=>K"*", "==="=>K"Operator"] + @test toks("1&==2")[2:3] == ["&"=>K"&", "=="=>K"Operator"] + @test toks("1−==2")[2:3] == ["−"=>K"-", "=="=>K"Operator"] + @test toks("1^==2")[2:3] == ["^"=>K"Operator", "=="=>K"Operator"] + @test toks("1<<==2")[2:3] == ["<<"=>K"Operator", "=="=>K"Operator"] + @test toks("1>>=>2")[2:3] == [">>"=>K"Operator", "=>"=>K"Operator"] + @test toks("1>>>==2")[2:3] == [">>>"=>K"Operator", "=="=>K"Operator"] end @testset "infix" begin @@ -567,6 +581,22 @@ end "⫪"=>K"String" "\""=>K"\"" ] + # Operators which have their own kind rather than being in the + # generic operator table are also allowed + @test toks("\"\$x∈y\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "∈y"=>K"String" + "\""=>K"\"" + ] + @test toks("\"\$x√y\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "√y"=>K"String" + "\""=>K"\"" + ] # Some chars disallowed (eg, U+0DF4) @test toks("\"\$x෴\"") == [ "\""=>K"\"" @@ -584,9 +614,9 @@ end end @testset "modifying function names (!) followed by operator" begin - @test toks("a!=b") == ["a"=>K"Identifier", "!="=>K"!=", "b"=>K"Identifier"] - @test toks("a!!=b") == ["a!"=>K"Identifier", "!="=>K"!=", "b"=>K"Identifier"] - @test toks("!=b") == ["!="=>K"!=", "b"=>K"Identifier"] + @test toks("a!=b") == ["a"=>K"Identifier", "!="=>K"Operator", "b"=>K"Identifier"] + @test toks("a!!=b") == ["a!"=>K"Identifier", "!="=>K"Operator", "b"=>K"Identifier"] + @test toks("!=b") == ["!="=>K"Operator", "b"=>K"Identifier"] end @testset "integer literals" begin @@ -729,13 +759,18 @@ end "f"=>K"Identifier", "("=>K"(", "a"=>K"Identifier", ")"=>K")"] @test toks("1234.0 .f(a)") == ["1234.0"=>K"Float", " "=>K"Whitespace", "."=>K".", "f"=>K"Identifier", "("=>K"(", "a"=>K"Identifier", ")"=>K")"] - @test toks("1f0./1") == ["1f0"=>K"Float32", "."=>K".", "/"=>K"/", "1"=>K"Integer"] + @test toks("1f0./1") == ["1f0"=>K"Float32", "."=>K".", "/"=>K"Operator", "1"=>K"Integer"] # Dotted operators after numeric constants are ok - @test toks("1e1.⫪") == ["1e1"=>K"Float", "."=>K".", "⫪"=>K"⫪"] - @test toks("1.1.⫪") == ["1.1"=>K"Float", "."=>K".", "⫪"=>K"⫪"] + @test toks("1e1.⫪") == ["1e1"=>K"Float", "."=>K".", "⫪"=>K"Operator"] + @test toks("1.1.⫪") == ["1.1"=>K"Float", "."=>K".", "⫪"=>K"Operator"] @test toks("1e1.−") == ["1e1"=>K"Float", "."=>K".", "−"=>K"-"] @test toks("1.1.−") == ["1.1"=>K"Float", "."=>K".", "−"=>K"-"] + # ... including operators which have their own kind rather than being in + # the generic operator table + @test toks("1e1.∈") == ["1e1"=>K"Float", "."=>K".", "∈"=>K"∈"] + @test toks("1.1.∈") == ["1.1"=>K"Float", "."=>K".", "∈"=>K"∈"] + @test toks("1.1.√") == ["1.1"=>K"Float", "."=>K".", "√"=>K"√"] # Non-dottable operators are not ok @test toks("1e1.\$") == ["1e1."=>K"ErrorInvalidNumericConstant", "\$"=>K"$"] @test toks("1.1.\$") == ["1.1."=>K"ErrorInvalidNumericConstant", "\$"=>K"$"] @@ -743,8 +778,11 @@ end # Ambiguous dotted operators @test toks("1.+") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+"] @test toks("1.+ ") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+", " "=>K"Whitespace"] - @test toks("1.⤋") == ["1."=>K"ErrorAmbiguousNumericConstant", "⤋"=>K"⤋"] - @test toks("1.⫪") == ["1."=>K"ErrorAmbiguousNumericConstant", "⫪"=>K"⫪"] + @test toks("1.⤋") == ["1."=>K"ErrorAmbiguousNumericConstant", "⤋"=>K"Operator"] + @test toks("1.⫪") == ["1."=>K"ErrorAmbiguousNumericConstant", "⫪"=>K"Operator"] + @test toks("1.∈") == ["1."=>K"ErrorAmbiguousNumericConstant", "∈"=>K"∈"] + @test toks("1.√") == ["1."=>K"ErrorAmbiguousNumericConstant", "√"=>K"√"] + @test toks("1.≔") == ["1."=>K"ErrorAmbiguousNumericConstant", "≔"=>K"≔"] # non-dottable ops are the exception @test toks("1.:") == ["1."=>K"Float", ":"=>K":"] @test toks("1.\$") == ["1."=>K"Float", "\$"=>K"$"] @@ -797,10 +835,45 @@ end @test length(collect(tokenize(io))) == 4 end +# Multi-character operators which share `K"Operator"` rather than having their +# own kind. Their precedence is assigned directly in the `lex_*` functions +# rather than via `generic_operators_by_level`, so they're listed here. +const _LEXER_MULTICHAR_OPERATORS = [ + "==", "===", "!=", "!==", "<=", ">=", + "<<", ">>", ">>>", "//", "|>", "<|", "=>", "<--", "<-->", +] + +# Operators which aren't symbolic infix/prefix operators (or aren't operators at +# all) and so can't be exercised by the dotted/suffixed forms below. +const _NON_SYMBOLIC_OPERATORS = Set([ + "ErrorInvalidOperator", "Error**", "Operator", + ".", "..", "where", "isa", "in", ".'", "op=", +]) + +# The non-dotted symbolic operators in the language. Most operators no longer +# have their own `Kind` (they share `K"Operator"`, distinguished by precedence +# flags), so rather than hard-coding the full list we derive it: single-character +# operators come from the precedence table `generic_operators_by_level`, the +# operators which still have a dedicated kind from the `BEGIN_OPS:END_OPS` range, +# and the multi-character `K"Operator"`s from `_LEXER_MULTICHAR_OPERATORS`. +function _all_symbolic_operators() + ops = String[] + # Normalize the chars from the precedence table the same way the lexer does, + # so eg the `·` variants collapse to `⋅` (matching how they're tokenized). + for chars in values(JuliaSyntax.generic_operators_by_level), c in chars + push!(ops, JuliaSyntax.normalize_identifier(string(c))) + end + op_range = reinterpret(UInt16, K"BEGIN_OPS"):reinterpret(UInt16, K"END_OPS") + for k in reinterpret.(Kind, op_range) + push!(ops, string(k)) + end + append!(ops, _LEXER_MULTICHAR_OPERATORS) + return unique!(filter(s -> !(s in _NON_SYMBOLIC_OPERATORS), ops)) +end + @testset "dotted and suffixed operators" begin -for opkind in Tokenize._nondot_symbolic_operator_kinds() - op = string(opkind) +for op in _all_symbolic_operators() strs = [ 1 => [ # unary "$(op)b", @@ -857,19 +930,21 @@ end @testset "Normalization of Unicode symbols" begin # https://github.com/JuliaLang/julia/pull/25157 - @test tok("\u00b7").kind == K"⋅" - @test tok("\u0387").kind == K"⋅" - @test toks(".\u00b7") == ["."=>K".", "\u00b7"=>K"⋅"] - @test toks(".\u0387") == ["."=>K".", "\u0387"=>K"⋅"] + @test tok("\u00b7").kind == K"Operator" + @test tok("\u0387").kind == K"Operator" + @test toks(".\u00b7") == ["."=>K".", "\u00b7"=>K"Operator"] + @test toks(".\u0387") == ["."=>K".", "\u0387"=>K"Operator"] # https://github.com/JuliaLang/julia/pull/40948 @test tok("−").kind == K"-" - @test tok("−=").kind == K"op=" + # −= now emits separate tokens + @test tok("−=").kind == K"Operator" # − before = + @test tok("−=", 2).kind == K"=" @test toks(".−") == ["."=>K".", "−"=>K"-"] end @testset "perp" begin - @test tok("1 ⟂ 2", 3).kind==K"⟂" + @test tok("1 ⟂ 2", 3).kind==K"Operator" end @testset "outer" begin @@ -898,7 +973,7 @@ end @testset "circ arrow right op" begin s = "↻" - @test collect(tokenize(s))[1].kind == K"↻" + @test collect(tokenize(s))[1].kind == K"Operator" end @testset "invalid float" begin @@ -922,8 +997,8 @@ end raw"<|" raw"|>" raw": .. … ⁝ ⋮ ⋱ ⋰ ⋯" - raw"$ + - ¦ | ⊕ ⊖ ⊞ ⊟ ++ ∪ ∨ ⊔ ± ∓ ∔ ∸ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣" - raw"* / ⌿ ÷ % & ⋅ ∘ × \ ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗" + raw"$ + - | ⊕ ⊖ ⊞ ⊟ ++ ∪ ∨ ⊔ ± ∓ ∔ ∸ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣" + raw"* / ÷ % & ⋅ ∘ × \ ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗" raw"//" raw"<< >> >>>" raw"^ ↑ ↓ ⇵ ⟰ ⟱ ⤈ ⤉ ⤊ ⤋ ⤒ ⤓ ⥉ ⥌ ⥍ ⥏ ⥑ ⥔ ⥕ ⥘ ⥙ ⥜ ⥝ ⥠ ⥡ ⥣ ⥥ ⥮ ⥯ ￪ ￬" @@ -931,7 +1006,7 @@ end raw"." ] if VERSION >= v"1.6.0" - push!(ops, raw"<-- <-->") + push!(ops, raw"<-- <--> ¦ ⌿") end if VERSION >= v"1.7.0" append!(ops, [ diff --git a/deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/md5 new file mode 100644 index 0000000000000..7928110675e53 --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/md5 @@ -0,0 +1 @@ +ca4bf605f5e8caaaff390bb7501c00d4 diff --git a/deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/sha512 new file mode 100644 index 0000000000000..9517b9b9e36e8 --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-4409eba6a0fee8d7188a8d8c5f31a9d0113fd237.tar.gz/sha512 @@ -0,0 +1 @@ +a12dca7625f32c63fb5036791226369559b58dd40ceffbf1efb780ca4203fb8d951861d3cb8adea86ac5daeb5a6d9cc07543f9310184ea973f7dccfbe58a07a8 diff --git a/deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/md5 deleted file mode 100644 index 0906f0b00dd4c..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -af997270e65acae4e5c853c5748053d8 diff --git a/deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/sha512 deleted file mode 100644 index 71b22b2dadae5..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-8cf666da6de43e437d8f09fe1dfc052acac98d31.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -0aad9e7288cd1653d9f73e47df3bf29016b56e053445ebefdeda74bcd3554ab4ce276e983ee0e91db9fccdc34aa360bfb16c69a75e6ecd45be2cfec5098f4793 diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index a11a85f04774e..b626994da6a06 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ -JULIASYNTAXHIGHLIGHTING_BRANCH = main -JULIASYNTAXHIGHLIGHTING_SHA1 = 8cf666da6de43e437d8f09fe1dfc052acac98d31 -JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/julialang/JuliaSyntaxHighlighting.jl.git -JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/julialang/JuliaSyntaxHighlighting.jl/tarball/$1 +JULIASYNTAXHIGHLIGHTING_BRANCH = kf/op-kind-removal +JULIASYNTAXHIGHLIGHTING_SHA1 = 4409eba6a0fee8d7188a8d8c5f31a9d0113fd237 +JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/KenoAIStaging/JuliaSyntaxHighlighting.jl.git +JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/KenoAIStaging/JuliaSyntaxHighlighting.jl/tarball/$1