stricter token recognition in macro/syntax handler

fn/graphics.fn

@@ -320,57 +320,7 @@ fn key_nine() { 57 }
 export macro key_held_action: Expr keyHeldActionHead;
 export macro key_pressed_action: Expr keyPressedActionHead;
 
-syntax keyHeldActionKeyExpr ::= "space" quote { key_space() }
-| "escape" quote { key_escape() }
-| "enter" quote { key_enter() }
-| "tab" quote { key_tab() }
-| "backspace" quote { key_backspace() }
-| "right" quote { arrow_right() }
-| "left" quote { arrow_left() }
-| "down" quote { arrow_down() }
-| "up" quote { arrow_up() }
-| "shift" quote { key_left_shift() }
-| "ctrl" quote { key_left_ctrl() }
-| "alt" quote { key_left_alt() }
-| "a" quote { key_a() }
-| "b" quote { key_b() }
-| "c" quote { key_c() }
-| "d" quote { key_d() }
-| "e" quote { key_e() }
-| "f" quote { key_f() }
-| "g" quote { key_g() }
-| "h" quote { key_h() }
-| "i" quote { key_i() }
-| "j" quote { key_j() }
-| "k" quote { key_k() }
-| "l" quote { key_l() }
-| "m" quote { key_m() }
-| "n" quote { key_n() }
-| "o" quote { key_o() }
-| "p" quote { key_p() }
-| "q" quote { key_q() }
-| "r" quote { key_r() }
-| "s" quote { key_s() }
-| "t" quote { key_t() }
-| "u" quote { key_u() }
-| "v" quote { key_v() }
-| "w" quote { key_w() }
-| "x" quote { key_x() }
-| "y" quote { key_y() }
-| "z" quote { key_z() }
-| "zero" quote { key_zero() }
-| "one" quote { key_one() }
-| "two" quote { key_two() }
-| "three" quote { key_three() }
-| "four" quote { key_four() }
-| "five" quote { key_five() }
-| "six" quote { key_six() }
-| "seven" quote { key_seven() }
-| "eight" quote { key_eight() }
-| "nine" quote { key_nine() }
-;
-
-syntax keyPressedActionKeyExpr ::= "space" quote { key_space() }
+syntax keyActionKeyExpr ::= "space" quote { key_space() }
 | "escape" quote { key_escape() }
 | "enter" quote { key_enter() }
 | "tab" quote { key_tab() }
@@ -424,7 +374,7 @@ syntax keyHeldActionHead ::= "[" body: Syntax(keyHeldActionClauses) "]" {
     body
 };
 
-syntax keyHeldActionClauses ::= key: Syntax(keyHeldActionKeyExpr) "=>" body: Nest ";" rest: Syntax(keyHeldActionClauses) quote {
+syntax keyHeldActionClauses ::= key: Syntax(keyActionKeyExpr) "=>" body: Nest ";" rest: Syntax(keyHeldActionClauses) quote {
     if (key_held(unquote(key))) {
         unquote(body)
     } else {
@@ -439,7 +389,7 @@ syntax keyPressedActionHead ::= "[" body: Syntax(keyPressedActionClauses) "]" {
     body
 };
 
-syntax keyPressedActionClauses ::= key: Syntax(keyPressedActionKeyExpr) "=>" body: Nest ";" rest: Syntax(keyPressedActionClauses) quote {
+syntax keyPressedActionClauses ::= key: Syntax(keyActionKeyExpr) "=>" body: Nest ";" rest: Syntax(keyPressedActionClauses) quote {
     if (key_pressed(unquote(key))) {
         unquote(body)
     } else {

src/pratt_parser.c

@@ -23,6 +23,7 @@
 #include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <wchar.h>
 #include <wctype.h>
 
@@ -227,6 +228,10 @@ static bool stageImportedSyntaxSpec(PrattParser *parser, PrattExportedOps *ops,
                                     PrattMacroSpec *source, int nsRef,
                                     HashSymbol *nsSymbol,
                                     PrattMacroTable *stagedSpecs);
+static PrattMacroSpec *findReusableImportedSyntaxSpec(PrattParser *parser,
+                                                      PrattMacroSpec *source,
+                                                      int nsRef,
+                                                      HashSymbol *nsSymbol);
 static PrattRecord *ensureTargetRecord(PrattParser *parser, HashSymbol *op);
 static AstSyntaxEntryKind
 convertPrattSyntaxEntryKind(PrattSyntaxEntryKind entryKind);
@@ -247,6 +252,8 @@ static void resolvePendingMacroFixupsForHelper(PrattParser *parser,
                                                HashSymbol *helperName,
                                                PrattMacroSpec *helperSpec);
 static void finalizePendingMacroFixups(PrattParser *parser);
+static void registerSyntaxSpecTerminals(PrattParser *parser,
+                                        PrattMacroSpec *spec);
 static void registerExprSyntaxHead(PrattParser *parser, HashSymbol *head);
 static void mergeFixityImport(PrattParser *parser, PrattRecord *target,
                               PrattRecord *source, int nsRef,
@@ -273,6 +280,73 @@ static bool symbolArrayContains(SymbolArray *symbols, HashSymbol *symbol) {
 
     return false;
 }
+
+static bool syntaxTerminalNeedsTrieRegistration(HashSymbol *symbol) {
+    if (symbol == NULL || symbol->name == NULL) {
+        return false;
+    }
+
+    size_t byteLen = strlen(symbol->name);
+    if (byteLen <= 1) {
+        return false;
+    }
+
+    wchar_t wide[byteLen + 1];
+    size_t wideLen = mbstowcs(wide, symbol->name, byteLen + 1);
+    if (wideLen == (size_t)-1 || wideLen <= 1) {
+        return false;
+    }
+
+    for (size_t i = 0; i < wideLen; ++i) {
+        if (unicode_isalnum(wide[i]) || unicode_isspace(wide[i])) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+static void registerSyntaxPatternTerminals(PrattParser *parser,
+                                           PrattMacroPatternItems *items) {
+    if (parser == NULL || items == NULL) {
+        return;
+    }
+
+    for (Index i = 0; i < countPrattMacroPatternItems(items); ++i) {
+        PrattMacroPatternItem *item = getPrattMacroPatternItems(items, i);
+        if (item == NULL ||
+            item->type != PRATTMACROPATTERNITEM_TYPE_QUOTEDTERMINAL) {
+            continue;
+        }
+
+        HashSymbol *terminal = getPrattMacroPatternItem_QuotedTerminal(item);
+        if (syntaxTerminalNeedsTrieRegistration(terminal)) {
+            parser->trie = insertPrattTrie(parser->trie, terminal);
+        }
+    }
+}
+
+static void registerSyntaxSpecTerminals(PrattParser *parser,
+                                        PrattMacroSpec *spec) {
+    if (parser == NULL || spec == NULL) {
+        return;
+    }
+
+    if (spec->alternatives != NULL) {
+        for (Index i = 0; i < sizePrattMacroAlternatives(spec->alternatives);
+             ++i) {
+            PrattMacroAlternative *alternative =
+                getPrattMacroAlternatives(spec->alternatives, i);
+            if (alternative != NULL) {
+                registerSyntaxPatternTerminals(parser,
+                                               alternative->patternItems);
+            }
+        }
+        return;
+    }
+
+    registerSyntaxPatternTerminals(parser, spec->patternItems);
+}
 // if you're wondering where the arithmetic primitives are, they're
 // defined in the preamble.
 
@@ -1373,6 +1447,13 @@ static bool stageImportedSyntaxSpec(PrattParser *parser, PrattExportedOps *ops,
         return true;
     }
 
+    PrattMacroSpec *reused =
+        findReusableImportedSyntaxSpec(parser, source, nsRef, nsSymbol);
+    if (reused != NULL) {
+        setPrattMacroTable(stagedSpecs, reused->headSymbol, reused);
+        return true;
+    }
+
     int declarationId = prattNextDeclarationId();
     PrattMacroSpec *clone =
         cloneImportedSyntaxSpec(source, declarationId, nsRef, nsSymbol);
@@ -1483,6 +1564,33 @@ static bool stageImportedSyntaxSpec(PrattParser *parser, PrattExportedOps *ops,
     return true;
 }
 
+static PrattMacroSpec *findReusableImportedSyntaxSpec(PrattParser *parser,
+                                                      PrattMacroSpec *source,
+                                                      int nsRef,
+                                                      HashSymbol *nsSymbol) {
+    if (parser == NULL || source == NULL) {
+        return NULL;
+    }
+
+    PrattMacroSpec *existing = NULL;
+    getPrattMacroTable(parser->macros, source->headSymbol, &existing);
+    if (existing == NULL) {
+        return NULL;
+    }
+
+    if (existing->importNsRef != nsRef ||
+        existing->importNsSymbol != nsSymbol) {
+        return NULL;
+    }
+
+    if (existing->entryKind != source->entryKind ||
+        existing->resultKind != source->resultKind) {
+        return NULL;
+    }
+
+    return existing;
+}
+
 static PrattRecord *ensureTargetRecord(PrattParser *parser, HashSymbol *op) {
     PrattRecord *target = NULL;
     if (!getPrattRecordTable(parser->rules, op, &target) || target == NULL) {
@@ -3224,6 +3332,7 @@ static AstDefinition *syntaxDefinition(PrattParser *parser) {
     int declarationId = prattNextDeclarationId();
     spec->declarationId = declarationId;
     setPrattMacroTable(parser->macros, ruleName, spec);
+    registerSyntaxSpecTerminals(parser, spec);
 
     AstSyntaxAlternatives *astAlts = newAstSyntaxAlternatives();
     int save2 = PROTECT(astAlts);
@@ -3596,7 +3705,9 @@ static AstDefinition *importOp(PrattParser *parser) {
                 PrattMacroSpec *staged = NULL;
                 while ((name = iteratePrattMacroTable(stagedSpecs, &i,
                                                       &staged)) != NULL) {
-                    if (getPrattMacroTable(parser->macros, name, NULL)) {
+                    PrattMacroSpec *installed = NULL;
+                    if (getPrattMacroTable(parser->macros, name, &installed) &&
+                        installed != NULL && installed != staged) {
                         parserErrorAt(TOKPI(tok), parser,
                                       "import macro conflicts with existing "
                                       "syntax %s",
@@ -3609,8 +3720,16 @@ static AstDefinition *importOp(PrattParser *parser) {
                     i = 0;
                     while (iteratePrattMacroTable(stagedSpecs, &i, &staged) !=
                            NULL) {
+                        PrattMacroSpec *installed = NULL;
+                        if (getPrattMacroTable(parser->macros,
+                                               staged->headSymbol,
+                                               &installed) &&
+                            installed == staged) {
+                            continue;
+                        }
                         setPrattMacroTable(parser->macros, staged->headSymbol,
                                            staged);
+                        registerSyntaxSpecTerminals(parser, staged);
                         if (staged->entryKind ==
                             PRATTSYNTAXENTRYKIND_TYPE_EXPR) {
                             registerExprSyntaxHead(parser, staged->headSymbol);

src/syntax_parse.c

@@ -23,9 +23,6 @@
 #include "pratt_scanner.h"
 #include "syntax_template.h"
 
-#include <ctype.h>
-#include <string.h>
-
 static int sNextDeclarationId = 1;
 
 int prattNextDeclarationId(void) { return sNextDeclarationId++; }
@@ -393,82 +390,14 @@ static void restoreSyntaxLexerCheckpoint(PrattParser *parser,
     parser->panicMode = checkpoint->panicMode;
 }
 
-static bool isAsciiPunctuationSymbolName(const char *name) {
-    if (name == NULL || name[0] == '\0') {
-        return false;
-    }
-
-    for (const unsigned char *cursor = (const unsigned char *)name;
-         *cursor != '\0'; ++cursor) {
-        if (*cursor > 0x7f || isalnum((int)*cursor) || isspace((int)*cursor)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-static bool stringStartsWith(const char *text, const char *prefix) {
-    while (*prefix != '\0') {
-        if (*text == '\0' || *text != *prefix) {
-            return false;
-        }
-        ++text;
-        ++prefix;
-    }
-
-    return true;
-}
-
-static bool consumeQuotedTerminalPunctuationFallback(PrattParser *parser,
-                                                     HashSymbol *expected) {
-    if (expected == NULL || expected->name == NULL ||
-        expected->name[1] == '\0' ||
-        !isAsciiPunctuationSymbolName(expected->name)) {
-        return false;
-    }
-
-    SyntaxLexerCheckpoint checkpoint = captureSyntaxLexerCheckpoint(parser);
-    int save = STARTPROTECT();
-    if (checkpoint.bufList != NULL) {
-        PROTECT(checkpoint.bufList);
-    }
-    if (checkpoint.queuedTokens != NULL) {
-        PROTECT(checkpoint.queuedTokens);
-    }
-    if (checkpoint.snapshots != NULL) {
-        PROTECT(checkpoint.snapshots);
-    }
-
-    const char *remaining = expected->name;
-    while (*remaining != '\0') {
-        PrattToken *token = peek(parser);
-        HashSymbol *actual = prattTokenTypeOrAtom(token);
-
-        if (actual == NULL || actual->name == NULL ||
-            !isAsciiPunctuationSymbolName(actual->name) ||
-            !stringStartsWith(remaining, actual->name)) {
-            restoreSyntaxLexerCheckpoint(parser, &checkpoint);
-            UNPROTECT(save);
-            return false;
-        }
-
-        next(parser);
-        remaining += strlen(actual->name);
-    }
-
-    UNPROTECT(save);
-    return true;
-}
-
 static bool matchQuotedTerminal(PrattParser *parser, HashSymbol *expected) {
     PrattToken *token = peek(parser);
-    if (prattIsTokenTypeOrAtom(token, expected)) {
-        next(parser);
-        return true;
+    if (!prattIsTokenTypeOrAtom(token, expected)) {
+        return false;
     }
 
-    return consumeQuotedTerminalPunctuationFallback(parser, expected);
+    next(parser);
+    return true;
 }
 
 static AstExpression *lookupSyntaxBindingCopy(SyntaxExprBindings *bindings,

tests/fn/fail_gfx_input_action_split_arrow.fn

@@ -0,0 +1,10 @@
+let
+    link "graphics.fn" as gfx;
+    import gfx macro key_held_action;
+
+    broken = key_held_action[
+        w = > { 1 };
+        default => { 2 }
+    ];
+in
+    broken

tests/fn/fail_syntax_composite_punctuation_split.fn

@@ -0,0 +1,20 @@
+let
+    macro choose_first: Expr chooseFirstHead;
+
+    syntax chooseFirstHead ::= "[" body: Syntax(chooseFirstClauses) "]" {
+        body
+    };
+
+    syntax chooseFirstClauses ::= "on" "=>" value: Expr ";" rest: Syntax(chooseFirstClauses) {
+        value
+    }
+    | "default" "=>" value: Expr {
+        value
+    };
+
+    broken = choose_first[
+        on = > 11;
+        default => 22
+    ];
+in
+    broken