diff --git a/docs/PRATT_REGEX_SCANNER_DESIGN.md b/docs/PRATT_REGEX_SCANNER_DESIGN.md new file mode 100644 index 00000000..db5080e8 --- /dev/null +++ b/docs/PRATT_REGEX_SCANNER_DESIGN.md @@ -0,0 +1,374 @@ +# Pratt Scanner Regex Integration + +This note investigates using the existing regex engine inside the core Pratt +scanner to replace the hand-written state-driven number and string recognizers +in `src/pratt_scanner.c`. + +The short version is: + +* number scanning is a good fit for regex-based recognition. +* string and char scanning are only a partial fit, because the current scanner + also performs escape decoding and detailed error recovery. +* a third `RegexSource` leg is not required for a first experiment, but a + zero-copy source path is required for any sensible final implementation. + +## Current Scanner Surface + +The relevant scanner entry points are: + +* `parseNumeric(...)` for numeric lexemes. +* `parseString(...)` for string and char literals. +* `parseRegex(...)` for `#/.../` literals. +* `next(...)` for top-level scanner dispatch. + +The current Pratt buffer model is already close to what the regex engine wants: + +* `PrattBuffer` stores a contiguous `WCharVec`. +* `buffer->start` is a moving `Character *` into that contiguous data. +* the remaining scanner input is null-terminated because the backing + `WCharVec` is null-terminated. + +That means scanner input is already available as a contiguous wide-character +slice. + +## Current Regex Interface Fit + +The regex engine already has three relevant entry surfaces: + +* `regexMatchp(const Regex *pattern, const Character *text, ...)` +* `regexMatchCharArrayp(const Regex *pattern, CharacterArray *text, ...)` +* `regexMatchSourcep(const Regex *pattern, RegexSource *source, ...)` + +For a quick prototype, the scanner could call `regexMatchp(...)` directly on +`buffer->start`. + +That does work functionally, but it is not zero-copy. `regexMatchp(...)` +currently copies the entire remaining null-terminated text into a +`CharacterArray` before matching. + +So the current situation is: + +* the regex engine can already be applied to Pratt buffers immediately. +* the current direct path copies the remaining scanner tail. +* the existing no-copy source path cannot be used directly because it accepts + `CharacterArray`, while Pratt buffers are backed by `WCharVec`. + +## Prefix Matching Versus Search + +The scanner needs left-prefix recognition. + +The current matcher entry point `regexMatchSourcep(...)` is a search routine: +it tries the pattern at offset `0`, then `1`, then `2`, and so on until it +finds a match or reaches end-of-input. + +For scanner use, that means: + +* anchoring a pattern with `^` is functionally correct. +* even with `^`, the current search loop still retries later offsets after a + failed prefix attempt. + +That is unnecessary work for tokenization. If regex becomes part of the Pratt +scanner, a dedicated prefix-only matcher would be a better fit than routing +scanner code through the generic search API. + +A minimal helper would look like: + +```c +bool regexMatchPrefixSource(const Regex *pattern, RegexSource *source, + Index *matchLength); +``` + +or, if the first step stays on contiguous text rather than `RegexSource`: + +```c +bool regexMatchPrefixText(const Regex *pattern, const Character *text, + Index *matchLength); +``` + +## Numeric Literals + +### Feasibility + +Numeric scanning is the best target for the first integration step. + +The current numeric scanner is mostly doing two jobs: + +* recognizing the token boundary. +* converting the matched slice into `MaybeBigInt` or irrational form. + +That split already exists in the code. `parseNumeric(...)` identifies the +lexeme, and the conversion logic lives in `makeMaybeBigInt(...)` and +`makeIrrational(...)`. + +Regex can replace the boundary-recognition part while leaving the existing +numeric conversion code in place. + +### Important Compatibility Detail + +The current state machine is more permissive than a cleaned-up numeric regex +would naturally be. + +Notable current behavior includes: + +* `0` receives special treatment that other Unicode decimal digits do not. +* hexadecimal literals are ASCII-only after `0x` or `0X`. +* floats such as `1.` and `0.` are accepted. +* underscores are accepted in places that may not all be intentional. +* `i` is handled as an optional suffix for imaginary literals. + +Because of that, the first design decision is not technical but semantic: + +* either preserve current scanner behavior exactly. +* or use the regex migration to tighten the intended numeric grammar. + +The safer path is to decide this explicitly before changing the scanner. + +### Recommended Scanner Regex Split + +Because only ASCII `0` gets the special zero/hex prefix behavior, the easiest +way to preserve the current shape is to choose the regex based on the first +character rather than forcing everything through one pattern. + +Recommended split: + +* if the first character is ASCII `0`, use a zero-prefixed pattern. +* otherwise, use a decimal pattern. + +Candidate patterns, written in the current regex surface rather than C string +escaping, are: + +```text +zeroOrHex := ^0((x|X)[0-9A-Fa-f_]*i?|([.]\d*)?i?)? +decimal := ^\d[\d_]*([.]\d*)?i? +``` + +This is still only a proposal. The exact pattern should be chosen after +deciding whether to preserve the current permissive edge cases or intentionally +trim them. + +### Recommended Numeric Integration + +The cleanest first step is: + +1. Compile the scanner numeric regexes once. +2. Match against `buffer->start`. +3. Use only the matched length from the regex. +4. Keep `makeMaybeBigInt(...)` and `makeIrrational(...)` unchanged. + +That minimizes behavioral change by keeping token interpretation in one place. + +## Strings and Chars + +### Why Strings Are Harder + +`parseString(...)` is not only identifying token boundaries. It is also: + +* decoding escapes into a `WCharArray`. +* handling `\u...;` and `\U...;` escapes. +* reporting specific scanner errors. +* tracking line numbers on malformed multi-line input. +* handling char-specific conditions such as empty chars and missing + terminators. + +So string handling is not just a recognizer. It is part recognizer, part +decoder, and part error-recovery routine. + +### Where Regex Still Helps + +Regex can still help as a valid-lexeme fast path. + +A practical split would be: + +* regex determines whether the next token is a lexically valid complete string + or char literal, and returns its length. +* existing or slightly refactored C code performs escape decoding over the + matched slice. +* malformed literals continue to use the current hand-written path so that the + scanner keeps its existing diagnostics. + +That avoids forcing detailed error reporting into the regex layer. + +### Candidate Fast-Path Patterns + +Valid double-quoted string, using current string semantics: + +```text +^"([^"\\\n]|\\([^uUntr\n]|[ntr]|[uU][0-9A-Fa-f]+;))*" +``` + +Valid single-quoted char literal: + +```text +^'([^'\\\n]|\\([^uUntr\n]|[ntr]|[uU][0-9A-Fa-f]+;))' +``` + +These are intended only for the successful fast path. + +They do not replace the need for the current state machine on malformed input, +because today malformed unicode escapes and unterminated literals still +produce specific scanner errors and partial recovery behavior. + +## Regex Literals + +`parseRegex(...)` is already small and specialized. + +It preserves regex-literal transport semantics rather than string-literal +semantics, especially around `\/` handling and preservation of regex escape +content. + +There is little benefit in rewriting that path via regex. It is reasonable to +leave regex-literal scanning hand-written even if numbers and strings move +toward regex-assisted recognition. + +## Source-Layer Options + +### Option A: No New Source Type Yet + +For an experiment, call the regex engine directly on `buffer->start` via +`regexMatchp(...)`. + +Advantages: + +* smallest change. +* proves whether scanner regexes improve clarity and maintainability. +* does not require touching `regex.yaml` or the source abstraction. + +Disadvantage: + +* copies the entire remaining Pratt buffer tail on every match attempt. + +This is acceptable only as a proof-of-concept technique. It is not an +acceptable steady-state scanner architecture, because a final implementation +cannot afford to copy the remaining Pratt buffer tail on each match attempt. + +### Option B: Add a Third Regex Source Leg + +If the scanner is going to rely on regex matching long-term, a third source +variant or equivalent zero-copy path is required. + +The important recommendation is to keep it generic rather than Pratt-specific. +The source should represent an existing contiguous character buffer, not a +Pratt-only concept. + +In shape, that wants something closer to: + +```yaml +structs: + RegexTextSource: + data: + data: WCharVec + start: index + exhausted: bool=true + +unions: + RegexSource: + data: + string: RegexStringSource + file: RegexFileSource + text: RegexTextSource + +external: + - !include cekfs.yaml + - !include utils.yaml +``` + +The implementation would then be thin: + +* `regexSourceGet(...)` returns `data->entries[start + position]` until it + reaches `L'\0'`. +* `regexSourceSetPosition(...)` is a no-op for this source kind. +* scanner code constructs the source from the existing `PrattBuffer` without + copying characters. + +### Why This Should Not Store Only a Raw Pointer + +A raw `Character *` is enough to read characters, but it is not an ideal +stored representation for a GC-managed source object. + +Holding the owning `WCharVec` plus a start offset is safer because: + +* the owner object remains visible to the GC. +* the source remains generic over any existing contiguous wide-character + buffer. +* the source does not have to know about Pratt-specific structs. + +## Compiled Regex Lifetime + +If the scanner compiles regexes once and reuses them, those compiled regexes +must be rooted. + +The builtin regex cache already does this via `markRegexCache()` during GC. +Scanner-owned compiled regexes would need a similar arrangement. + +That means a production implementation should include either: + +* a dedicated scanner regex cache with a mark hook. +* or compile-on-demand plus immediate use for the first iteration, then add + caching once the scanner patterns have settled. + +## Recommended Phasing + +### Phase 1 + +Replace numeric recognition only. + +* keep `parseNumeric(...)` as the entry point. +* replace its hand-written boundary-recognition loop with regex matching. +* continue to use the current conversion helpers. +* use the existing raw-text regex path first, even though it copies. + +This is the highest-value, lowest-risk step, but it should be treated +explicitly as a temporary proof of concept rather than the intended end state. + +### Phase 2 + +Add a prefix-only regex matcher entry point. + +That avoids routing scanner tokenization through the generic search API. + +### Phase 3 + +Add a generic contiguous-text `RegexSource` variant. + +That removes the Pratt-buffer copy while keeping the source abstraction clean. +This phase is not an optional optimization pass. It is the point where the +scanner integration becomes architecturally viable as a final implementation. + +### Phase 4 + +Optionally add a string fast path. + +At this stage the recommended approach is still: + +* regex for valid-lexeme recognition. +* hand-written decode and error handling for semantics and diagnostics. + +## Risks + +The main risks are semantic, not mechanical. + +* numeric-literal compatibility may change if the regex surface is cleaner + than the current state machine. +* string diagnostics may regress if regex tries to replace error-oriented + scanning logic instead of only supplementing it. +* any scanner path that keeps the copy inside `regexMatchp(...)` is suitable + only for experiments, not for the final implementation. +* a scanner-side compiled-regex cache needs GC rooting. + +## Recommendation + +The proposal is sound, but it should be staged. + +Recommended final position: + +* use regex to replace numeric lexeme recognition first. +* do not try to replace string decoding and error handling wholesale. +* keep `parseRegex(...)` hand-written. +* do not require a third `RegexSource` leg for the first experiment. +* require a generic contiguous-text source or equivalent zero-copy path before + treating the scanner integration as complete. +* pair that zero-copy path with a prefix-only matcher so the scanner can use + the regex engine without copying or search-loop overhead. + +That path gets most of the maintainability benefit while keeping scanner +behavior understandable and testable. diff --git a/docs/REGEX.md b/docs/REGEX.md index 8f027695..a2eb1597 100644 --- a/docs/REGEX.md +++ b/docs/REGEX.md @@ -2,6 +2,12 @@ Regex support is no longer just a standalone engine. +For the current implementation status, including the `RegexSource` +string/file source model and the existing parser-combinator library, see +[REGEX_STATUS.md](./REGEX_STATUS.md). This document still contains useful +design background, but parts of its status and future-work discussion are now +historical. + The current system already includes: * a unicode-centric regex compiler and matcher over `wchar_t`-style diff --git a/docs/REGEX_STATUS.md b/docs/REGEX_STATUS.md new file mode 100644 index 00000000..e40324d5 --- /dev/null +++ b/docs/REGEX_STATUS.md @@ -0,0 +1,274 @@ +# Regex Status + +This note summarizes the current regex implementation as it exists in the +codebase now. It is intended as a status update to complement +`docs/REGEX.md`, which still contains older design discussion and now-outdated +future work. + +## Summary + +Regex support is already integrated at three levels: + +* the engine compiles to a regex AST and matches against an abstract + `RegexSource`. +* the runtime exposes both string-backed and file-backed regex matching. +* the F♮ library already contains reusable parser combinator modules for both + deterministic and `amb`-based parsing, including regex-driven parsers. + +For parser combinators, the important point is that this is no longer just a +prototype direction. There is already a small working library in `fn/` and it +is covered by focused tests. + +## Engine and Source Model + +The current matcher no longer depends on fully materialized null-terminated +input buffers as its only operating mode. + +The key implementation split is: + +* `src/regex_helper.c` and `src/regex_helper.h` implement regex compile and + match logic. +* `src/regex_source.c` and `src/regex_source.h` implement the input-source + abstraction used by the matcher. +* `src/regex.yaml` defines the generated data structures that back the source + layer. + +The source layer is cursor-based and offset-based: + +* matching proceeds through `regexMatchSourcep(...)`. +* source access is routed through `regexSourceGet(source, position)`. +* file-backed cursor updates are routed through + `regexSourceSetPosition(source, position)`. +* string-backed splitting and file-backed prefix extraction are routed through + `regexSourceSplitAt(...)`. + +The generated source types are: + +* `RegexStringSource`: a grow-only cache backed by a CEKF string list tail. +* `RegexFileSource`: a grow-only cache backed by a file handle plus cached + `fpos_t` positions. +* `RegexSource`: a tagged union over those two concrete source kinds. + +Operationally, the model is: + +* string input is consumed lazily from the list representation into a cached + `CharacterArray`. +* file input is read incrementally from the `FILE *`, decoded into wide + characters, and each cached character records the corresponding file + position after that read. +* the matcher works in logical offsets rather than raw input pointers. +* the file source can restore the underlying file handle to a cached logical + position. + +That means the earlier design idea in `docs/REGEX.md` about adding a source +abstraction has already happened. Both string and file sources are now part of +the implementation surface. + +## Language and Builtin Surface + +The language-level `regex` type and `#/.../` literal syntax remain in place. +The current runtime entry points are registered in `src/builtin_regex.c`: + +```fn +regex_match: regex -> list(char) -> maybe(#(list(char), list(char))) +regex_match_file: regex -> opaque:file -> maybe(list(char)) +``` + +Their current behavior is: + +* `regex_match` matches against a string source and returns the matched prefix + plus the remaining suffix. +* `regex_match_file` matches against a file source and returns the matched + prefix while advancing the file cursor on success. +* regex compilation is cached by pattern string in the builtin layer unless + caching is explicitly disabled. + +The matcher-facing entry point inside the engine is +`regexMatchSourcep(const Regex *pattern, RegexSource *source, Index +*matchLength)`. + +## Current Grammar Support + +The implemented grammar is still a deliberately small parser-oriented one. +In rough form, the supported surface is: + +```text +pattern := leading_flags? expression +leading_flags := "(?" flag+ ")" +flag := "i" + +expression := alternation +alternation := sequence ("|" sequence)* +sequence := quantified* +quantified := primary ["*" | "+" | "?"] +primary := literal + | "." + | "^" + | "$" + | "(" expression ")" + | char_class + | named_category + | escaped_atom + +char_class := "[" "^"? class_item+ "]" +class_item := literal + | literal "-" literal + | named_category + | escaped_class_item + +named_category := "[[" category_name "]]" +``` + +In practical terms, the current grammar includes: + +* alternation, concatenation, grouping, and `*`, `+`, `?` quantifiers. +* `.`, `^`, and `$`. +* bracket character classes, including negated classes such as + `[^[[Lu]]]`. +* named Unicode categories as both standalone atoms and class items. +* escapes including `\n`, `\r`, `\t`, `\d`, `\D`, `\s`, `\S`, `\w`, + `\W`, `\u...;`, and `\U...;`. +* a leading whole-pattern `(?i)` case-insensitive flag. + +Some current semantics are worth stating explicitly: + +* `.` matches any code point except newline and carriage return. +* `^` and `$` are anchors for the whole input slice being matched. +* character-class ranges are literal-to-literal ranges such as `[a-z]`. +* escapes inside `#/.../` literals are regex escapes, not ordinary string + escapes. +* only a leading whole-pattern `(?i)` flag is supported; unsupported flags + such as `(?s)` are rejected with `REGEX_STATUS_INVALID_INLINE_FLAG`. + +The current implementation still does not include counted repetition, +non-greedy quantifiers, capture groups, backreferences, lookaround, or scoped +inline modifiers. + +## Unicode General Category Classes + +Unicode general category support is a real part of the current grammar, not a +placeholder. + +Named categories use the `[[...]]` form and are parsed into `RegexCategory` +nodes. The matcher then checks each candidate character with +`unicode_category(...)` from the Unicode support layer. + +Two category shapes are supported: + +* major categories such as `[[L]]`, `[[N]]`, `[[P]]`, and `[[Z]]`. +* exact subcategories such as `[[Lu]]`, `[[Ll]]`, `[[Nd]]`, `[[Pc]]`, and + the other standard Unicode general-category codes listed in + `docs/REGEX.md`. + +The distinction matters in matching: + +* major categories match by category family. For example, `[[L]]` matches any + letter category, not just uppercase or lowercase letters. +* exact categories match only that specific category. For example, `[[Lu]]` + matches uppercase letters and `[[Nd]]` matches decimal digits. + +Current behavior confirmed by the implementation and tests includes: + +* category atoms work directly in sequences, for example `^[[Lu]][[Ll]]+$`. +* category items work inside bracket classes. +* negated classes can contain category items. +* invalid category names such as `[[Qx]]` are rejected with + `REGEX_STATUS_UNKNOWN_CATEGORY`. +* `(?i)` affects literal and range comparisons, but it does not reinterpret + category membership. `(?i)^[[Lu]]+$` still means uppercase-letter category + membership, so it matches `ABC` but not `abc`. + +Compile-time status reporting is represented by `RegexStatus` in +`src/regex.yaml` and exported through the `REGEX_STATUS_*` macros in +`src/regex_helper.h`. + +## Parser Combinator Status + +Regex support for parser combinators is already present and usable. + +The current reusable parser surfaces are: + +* `fn/parsercore.fn`: shared higher-order helpers such as `map_with`, + `pair_with`, `apply_with`, and `sequence_with`. +* `fn/parserdet.fn`: deterministic parser combinators built around + `success(...)` and `failure(...)`. +* `fn/parseramb.fn`: nondeterministic combinators built around `amb`, `back`, + and `cut`. +* `fn/parserdo.fn`: a small `pdo[...]` macro surface for parser do-notation. +* `fn/regexutils.fn`: regex/file helpers for `amb`-style rollback. + +Regex integration is already part of those parser libraries rather than living +only in playground examples: + +* `parserdet.match_regex` wraps `regex_match`. +* `parserdet.match_regex_file` wraps `regex_match_file`. +* `parseramb.match_regex` wraps `regex_match`. +* `parseramb.match_regex_file` saves and restores file position around + backtracking. +* `regexutils.regex_match_file_amb` exposes the same file rollback behavior as + a standalone helper. + +So the status here is stronger than “there are experiments”. The old +`parser-playground.fn` and `parser-playground-amb.fn` files still exist, but +they now sit alongside a factored parser library that is already used in +tests. + +## File Parsing and Backtracking + +File-backed parsing is a first-class part of the current surface. + +For deterministic parsing: + +* `parserdet.match_regex_file` returns `success(#(matched, file))` or + `failure(...)`. +* `parserdet.parse_complete_file` composes a parser with `file_eof`. + +For `amb`-based parsing: + +* `parseramb.match_regex_file` saves the current file position with `fgetpos`. +* on success it yields the match and arranges to restore the file position if + backtracking later revisits that branch. +* on failure it restores the original file position before `back`. + +That means the parser-facing file story is not just hypothetical. There is +already a concrete rollback-aware adapter layer for regex over files. + +## Focused Validation in the Tree + +The current codebase already has focused tests covering the main surfaces: + +* `tests/src/test_regex.c` covers the regex engine directly. +* `tests/fn/test_regex_literal.fn` covers literal transport. +* `tests/fn/test_regex_match.fn` covers string-backed language-level matching. +* `tests/fn/test_regex_match_file.fn` covers file-backed matching. +* `tests/fn/test_regex_match_file_amb.fn` covers rollback-aware file matching. +* `tests/fn/test_parserdet.fn` covers deterministic parser combinators. +* `tests/fn/test_parserdet_file.fn` covers deterministic file parsing with + regex parsers. +* `tests/fn/test_parseramb.fn` covers `amb`-based parser combinators. +* `tests/fn/test_parseramb_file.fn` covers `amb`-based file parsing with + regex parsers. + +Taken together, those tests support the current practical status: + +* regexes are a language feature. +* the matcher works over both string and file sources. +* parser-combinator support is already implemented for both deterministic and + nondeterministic styles. +* file-backed regex parsing already has a working cursor/rollback story. + +## Practical Status + +The current situation is best described like this: + +* the regex engine has already crossed the boundary from string-only matching + to a shared source abstraction. +* file-backed regex matching is already implemented, not planned. +* parser combinator support is already present as library code in `fn/`, not + just as design sketches. +* the parser library is still small and likely not the final long-term API, + but it is already real, reusable, and tested. + +In short: regex support for parser combinators is complete enough to use now, +and the codebase already contains the beginnings of a proper parser-combinator +library built on top of it. diff --git a/docs/TODO.md b/docs/TODO.md index 5ff171d0..ecbab23c 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -5,7 +5,7 @@ More of a wish-list than a hard and fast plan. * More folding opportunities. * fold boolean expressions `true and false => false`. * tricky because `and`, `or` etc. are not primitive, they are lazy operators defined in terms of `if` in the preamble. - * fold boolean comparisons `a == a => true`, `a >= a => true` etc. DONE + * fold comparisons `a == a => true`, `a >= a => true` etc. DONE * fold constant conditions `(if true a b) => a`. DONE * This solves the boolean expression folding problem, after β/η-reduction: * `true and false => (if true false false) => false` @@ -13,6 +13,8 @@ More of a wish-list than a hard and fast plan. * Continuations. * Reinstate `cut` (prunes current back continuation). DONE * Implement delimited continuations. +* Regular Expressions. + * Enlist the new regex engine to support the core Pratt scanner. * Types. * Consider type classes as a general solution to `EQ `, `map` etc. * Records should create accessor functions for each tag. diff --git a/docs/generated/regex.md b/docs/generated/regex.md index 6652b6d7..f4a856d7 100644 --- a/docs/generated/regex.md +++ b/docs/generated/regex.md @@ -25,6 +25,8 @@ RegexFileSource --startPos--> filepos RegexFileSource --positions--> RegexFilePosArray RegexFileSource --cache--> CharacterArray RegexFileSource --exhausted--> bool +RegexTextSource --data--> WCharVec +RegexTextSource --start--> wstring RegexClassItem --literal--> character RegexClassItem --range--> RegexRange RegexClassItem --meta--> RegexMetaType @@ -43,6 +45,7 @@ RegexNode --alternation--> RegexNodeArray RegexNode --repeat--> RegexRepeat RegexSource --string--> RegexStringSource RegexSource --file--> RegexFileSource +RegexSource --text--> RegexTextSource RegexStatus["enum RegexStatus"] RegexMetaType["enum RegexMetaType"] RegexClassItemArray["RegexClassItemArray[]"] --entries--> RegexClassItem diff --git a/src/memory.c b/src/memory.c index 714e599b..86ba839b 100644 --- a/src/memory.c +++ b/src/memory.c @@ -33,6 +33,7 @@ #include "memory.h" #include "minlam_runtime.h" #include "opaque.h" +#include "pratt_scanner.h" #include "step.h" #include "symbol.h" #include "wrapper_synthesis.h" @@ -551,6 +552,7 @@ static void mark() { markProtected(); markNameSpaces(); markMemBufs(); + markPrattScannerRegexCache(); markRegexCache(); minlam_runtime_mark_reg(); #ifdef DEBUG_LOG_GC diff --git a/src/pratt_scanner.c b/src/pratt_scanner.c index bbc62091..5f4ec169 100644 --- a/src/pratt_scanner.c +++ b/src/pratt_scanner.c @@ -26,8 +26,11 @@ #include #include "bigint.h" +#include "memory.h" #include "pratt_debug.h" #include "pratt_scanner.h" +#include "regex_helper.h" +#include "regex_source.h" #include "symbol.h" #include "unicode.h" @@ -114,6 +117,11 @@ TOKFN(WILDCARD, "_") #undef TOKFN +static Regex *leadingZeroNumericRegex = NULL; +static Regex *decimalNumericRegex = NULL; +static Regex *stringLiteralRegex = NULL; +static Regex *charLiteralRegex = NULL; + /** * @brief Checks if a symbol is an internal symbol. */ @@ -121,6 +129,21 @@ static inline bool isInternal(HashSymbol *symbol) { return symbol->name[0] == ' '; } +void markPrattScannerRegexCache(void) { + if (leadingZeroNumericRegex != NULL) { + markObject((Header *)leadingZeroNumericRegex); + } + if (decimalNumericRegex != NULL) { + markObject((Header *)decimalNumericRegex); + } + if (stringLiteralRegex != NULL) { + markObject((Header *)stringLiteralRegex); + } + if (charLiteralRegex != NULL) { + markObject((Header *)charLiteralRegex); + } +} + /** * @brief Constructs a ParserInfo value from the argument PrattLexer. */ @@ -535,136 +558,208 @@ static MaybeBigInt *makeIrrational(Character *str, int length) { return irrationalBigInt(f / div, imag); } -/** - * @brief Parses a numeric token from the current position in the buffer. - */ -static PrattToken *parseNumeric(PrattLexer *lexer) { - PrattBuffer *buffer = lexer->bufList->buffer; - HashSymbol *type = TOK_NUMBER(); - PrattNumberState state = PRATTNUMBERSTATE_TYPE_START; - bool floating = false; - while (state != PRATTNUMBERSTATE_TYPE_END) { - if (buffer->start[buffer->offset] == L'_') { - ++buffer->offset; +static Regex *getCachedRegex(const Character *pattern, Regex **cacheSlot) { + if (*cacheSlot == NULL) { + RegexStatus status = REGEX_STATUS_OK; + Index errorOffset = 0; + + *cacheSlot = regexCompile(pattern, &status, &errorOffset); + if (*cacheSlot == NULL || status != REGEX_STATUS_OK) { + cant_happen("invalid scanner numeric regex at offset %d", + (int)errorOffset); + } + } + + return *cacheSlot; +} + +static int matchCachedPrefixRegex(PrattBuffer *buffer, const Character *pattern, + Regex **cacheSlot, Index *matchLength) { + int save = STARTPROTECT(); + Regex *compiled = getCachedRegex(pattern, cacheSlot); + RegexSource *source = + regexSourceFromWCharVecSlice(buffer->data, buffer->start); + + PROTECT(source); + int matchStart = regexMatchPrefixSourcep(compiled, source, matchLength); + UNPROTECT(save); + return matchStart; +} + +static Character hexDigitValue(Character c) { + if (c >= L'0' && c <= L'9') { + return c - L'0'; + } + if (c >= L'a' && c <= L'f') { + return 10 + c - L'a'; + } + if (c >= L'A' && c <= L'F') { + return 10 + c - L'A'; + } + cant_happen("invalid hex digit %lc in validated string literal", c); +} + +static bool tryHexDigitValue(Character c, Character *value) { + if (c >= L'0' && c <= L'9') { + *value = c - L'0'; + return true; + } + if (c >= L'a' && c <= L'f') { + *value = 10 + c - L'a'; + return true; + } + if (c >= L'A' && c <= L'F') { + *value = 10 + c - L'A'; + return true; + } + return false; +} + +static Character extendUnicodeEscape(Character value, Character hexDigit) { + return (value << 4) | hexDigit; +} + +static bool tryDecodeSimpleEscape(Character escaped, Character *decoded) { + switch (escaped) { + case L'n': + *decoded = L'\n'; + return true; + case L't': + *decoded = L'\t'; + return true; + case L'r': + *decoded = L'\r'; + return true; + default: + return false; + } +} + +static WCharArray *decodeValidatedStringLiteral(const Character *start, + Index length) { + WCharArray *string = newWCharArray(); + int save = PROTECT(string); + + ASSERT(length >= 2); + for (Index i = 1; i + 1 < length; ++i) { + if (start[i] != L'\\') { + pushWCharArray(string, start[i]); continue; } - switch (state) { - case PRATTNUMBERSTATE_TYPE_START: - switch (buffer->start[buffer->offset]) { - case L'0': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_ZERO; - break; - default: - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_DEC; - break; - } - break; - case PRATTNUMBERSTATE_TYPE_ZERO: - if (unicode_isdigit(buffer->start[buffer->offset])) { - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_DEC; - break; - } else { - switch (buffer->start[buffer->offset]) { - case L'x': - case L'X': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_HEX; - break; - case L'.': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_FLOAT; - floating = true; - break; - case L'i': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_END; - break; - default: - state = PRATTNUMBERSTATE_TYPE_END; - break; - } - } - break; - case PRATTNUMBERSTATE_TYPE_HEX: - switch (buffer->start[buffer->offset]) { - case L'0': - case L'1': - case L'2': - case L'3': - case L'4': - case L'5': - case L'6': - case L'7': - case L'8': - case L'9': - case L'a': - case L'b': - case L'c': - case L'd': - case L'e': - case L'f': - case L'A': - case L'B': - case L'C': - case L'D': - case L'E': - case L'F': - case L'_': - ++buffer->offset; - break; - case L'i': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_END; - break; - default: - state = PRATTNUMBERSTATE_TYPE_END; - break; - } - break; - case PRATTNUMBERSTATE_TYPE_DEC: - if (unicode_isdigit(buffer->start[buffer->offset])) { - ++buffer->offset; - break; - } else { - switch (buffer->start[buffer->offset]) { - case L'.': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_FLOAT; - floating = true; - break; - case L'i': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_END; - break; - default: - state = PRATTNUMBERSTATE_TYPE_END; - break; - } + + i++; + Character decoded = 0; + + if (tryDecodeSimpleEscape(start[i], &decoded)) { + pushWCharArray(string, decoded); + continue; + } + + switch (start[i]) { + case L'u': + case L'U': { + Character uni = 0; + + for (i++; start[i] != L';'; ++i) { + uni = extendUnicodeEscape(uni, hexDigitValue(start[i])); } + pushWCharArray(string, uni); break; - case PRATTNUMBERSTATE_TYPE_FLOAT: - if (unicode_isdigit(buffer->start[buffer->offset])) { - ++buffer->offset; - break; - } else { - switch (buffer->start[buffer->offset]) { - case L'i': - ++buffer->offset; - state = PRATTNUMBERSTATE_TYPE_END; - break; - default: - state = PRATTNUMBERSTATE_TYPE_END; - break; - } - } + } + default: + pushWCharArray(string, start[i]); break; - case PRATTNUMBERSTATE_TYPE_END: - cant_happen("end state in loop"); } } + + pushWCharArray(string, L'\0'); + UNPROTECT(save); + return string; +} + +static PrattToken *finishStringToken(PrattLexer *lexer, WCharArray *string, + HashSymbol *tokenType, Index length) { + PrattBuffer *buffer = lexer->bufList->buffer; + int save = PROTECT(string); + + buffer->offset = (int)length; + PrattToken *token = tokenFromString(lexer->bufList, string, tokenType); + advance(buffer); + UNPROTECT(save); + return token; +} + +static Index stringLiteralMatchLength(PrattBuffer *buffer, + bool parsingSingleChar) { + static const Character *stringLiteralPattern = + L"^\"([^\"\\\\\n]|\\\\([ntr]|[uU][0-9A-Fa-f]+;|[^uUntr\n]))*\""; + static const Character *charLiteralPattern = + L"^'([^'\\\\\n]|\\\\([ntr]|[uU][0-9A-Fa-f]+;|[^uUntr\n]))'"; + const Character *pattern = + parsingSingleChar ? charLiteralPattern : stringLiteralPattern; + Regex **cacheSlot = + parsingSingleChar ? &charLiteralRegex : &stringLiteralRegex; + Index matchLength = 0; + int matchStart = + matchCachedPrefixRegex(buffer, pattern, cacheSlot, &matchLength); + + if (matchStart == 0) { + return matchLength; + } + + return 0; +} + +static PrattStringState nextStringContentState(bool parsingSingleChar) { + return parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR + : PRATTSTRINGSTATE_TYPE_STR; +} + +static PrattStringState appendDecodedStringCharacter(WCharArray *string, + Character decoded, + bool parsingSingleChar) { + pushWCharArray(string, decoded); + return nextStringContentState(parsingSingleChar); +} + +static Index numericMatchLength(PrattBuffer *buffer) { + static const Character *leadingZeroPattern = + L"^0_*([xX][0-9A-Fa-f_]*i?|\\d[\\d_]*(\\.[\\d_]*)?i?|\\.[\\d_]*i?|i?)?"; + static const Character *decimalPattern = L"^\\d[\\d_]*(\\.[\\d_]*)?i?"; + Index matchLength = 0; + int matchStart = + buffer->start[0] == L'0' + ? matchCachedPrefixRegex(buffer, leadingZeroPattern, + &leadingZeroNumericRegex, &matchLength) + : matchCachedPrefixRegex(buffer, decimalPattern, + &decimalNumericRegex, &matchLength); + + if (matchStart != 0 || matchLength == 0) { + cant_happen("regex numeric scanner mismatch at %lc", buffer->start[0]); + } + + return matchLength; +} + +static bool numericTokenIsFloating(PrattBuffer *buffer) { + for (int i = 0; i < buffer->offset; i++) { + if (buffer->start[i] == L'.') { + return true; + } + } + + return false; +} + +/** + * @brief Parses a numeric token from the current position in the buffer. + */ +static PrattToken *parseNumeric(PrattLexer *lexer) { + PrattBuffer *buffer = lexer->bufList->buffer; + HashSymbol *type = TOK_NUMBER(); + buffer->offset = (int)numericMatchLength(buffer); + bool floating = numericTokenIsFloating(buffer); + MaybeBigInt *bi = NULL; if (floating) { bi = makeIrrational(buffer->start, buffer->offset); @@ -735,14 +830,14 @@ static PrattToken *tokenERROR(PrattLexer *lexer) { } /** - * @brief Parses a string or character from the current position in the buffer. + * @brief Slow-path string and char parser used for malformed literals. * - * This function handles both single quoted characters and double-quoted - * strings, including escape sequences. - * It returns a PrattToken containing the parsed character or string. + * This preserves the original state-machine behavior, including detailed error + * reporting and local recovery, when the regex fast path does not recognize a + * complete valid literal. */ -static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, - Character sep) { +static PrattToken *parseStringSlow(PrattParser *parser, bool parsingSingleChar, + Character sep) { PrattLexer *lexer = parser->lexer; PrattBuffer *buffer = lexer->bufList->buffer; WCharArray *string = newWCharArray(); @@ -791,13 +886,15 @@ static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, parserError(parser, "unexpected EOF"); state = PRATTSTRINGSTATE_TYPE_END; break; - default: - pushWCharArray(string, buffer->start[buffer->offset]); + default: { + Character decoded = buffer->start[buffer->offset]; + ++buffer->offset; - state = parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR - : PRATTSTRINGSTATE_TYPE_STR; + state = appendDecodedStringCharacter(string, decoded, + parsingSingleChar); break; } + } } } else { // PRATTSTRINGSTATE_TYPE_ESCS switch (buffer->start[buffer->offset]) { @@ -810,13 +907,15 @@ static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, parserError(parser, "unexpected EOF"); state = PRATTSTRINGSTATE_TYPE_END; break; - default: - pushWCharArray(string, buffer->start[buffer->offset]); + default: { + Character decoded = buffer->start[buffer->offset]; + ++buffer->offset; - state = parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR - : PRATTSTRINGSTATE_TYPE_STR; + state = appendDecodedStringCharacter(string, decoded, + parsingSingleChar); break; } + } } break; @@ -824,6 +923,16 @@ static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, DEBUG("parseString %s %d (sep %lc) ESC: %lc", lexer->bufList->fileName->name, lexer->bufList->lineNo, sep, buffer->start[buffer->offset]); + Character decoded = 0; + + if (tryDecodeSimpleEscape(buffer->start[buffer->offset], + &decoded)) { + ++buffer->offset; + state = appendDecodedStringCharacter(string, decoded, + parsingSingleChar); + break; + } + switch (buffer->start[buffer->offset]) { case L'u': case L'U': @@ -831,24 +940,6 @@ static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, uni = 0; // reset state = PRATTSTRINGSTATE_TYPE_UNI; break; - case L'n': - pushWCharArray(string, L'\n'); - ++buffer->offset; - state = parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR - : PRATTSTRINGSTATE_TYPE_STR; - break; - case L't': - pushWCharArray(string, L'\t'); - ++buffer->offset; - state = parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR - : PRATTSTRINGSTATE_TYPE_STR; - break; - case L'r': - pushWCharArray(string, L'\r'); - ++buffer->offset; - state = parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR - : PRATTSTRINGSTATE_TYPE_STR; - break; case L'\n': parserError(parser, "unexpected EOL"); ++buffer->offset; @@ -865,54 +956,26 @@ static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, DEBUG("parseString %s %d (sep %lc) UNI: %lc", lexer->bufList->fileName->name, lexer->bufList->lineNo, sep, buffer->start[buffer->offset]); - switch (buffer->start[buffer->offset]) { - case L'0': - case L'1': - case L'2': - case L'3': - case L'4': - case L'5': - case L'6': - case L'7': - case L'8': - case L'9': { - Character c = buffer->start[buffer->offset] - L'0'; - uni <<= 4; - uni |= c; - buffer->offset++; - } break; - case L'a': - case L'b': - case L'c': - case L'd': - case L'e': - case L'f': { - Character c = 10 + buffer->start[buffer->offset] - L'a'; - uni <<= 4; - uni |= c; - buffer->offset++; - } break; - case L'A': - case L'B': - case L'C': - case L'D': - case L'E': - case L'F': { - Character c = 10 + buffer->start[buffer->offset] - L'A'; - uni <<= 4; - uni |= c; + Character hexDigit = 0; + + if (tryHexDigitValue(buffer->start[buffer->offset], &hexDigit)) { + uni = extendUnicodeEscape(uni, hexDigit); buffer->offset++; - } break; + break; + } + + switch (buffer->start[buffer->offset]) { case L';': ++buffer->offset; if (uni == 0) { parserError(parser, "Empty Unicode escape while parsing string"); } else { - pushWCharArray(string, uni); + state = appendDecodedStringCharacter(string, uni, + parsingSingleChar); + break; } - state = parsingSingleChar ? PRATTSTRINGSTATE_TYPE_CHR - : PRATTSTRINGSTATE_TYPE_STR; + state = nextStringContentState(parsingSingleChar); break; case L'\0': parserError(parser, "EOF while parsing unicode escape"); @@ -944,11 +1007,34 @@ static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, } } pushWCharArray(string, '\0'); - PrattToken *token = tokenFromString( - lexer->bufList, string, parsingSingleChar ? TOK_CHAR() : TOK_STRING()); - advance(buffer); UNPROTECT(save); - return token; + return finishStringToken(lexer, string, + parsingSingleChar ? TOK_CHAR() : TOK_STRING(), + buffer->offset); +} + +/** + * @brief Parses a string or character from the current position in the buffer. + * + * This function handles both single quoted characters and double-quoted + * strings, including escape sequences. + * It returns a PrattToken containing the parsed character or string. + */ +static PrattToken *parseString(PrattParser *parser, bool parsingSingleChar, + Character sep) { + PrattLexer *lexer = parser->lexer; + PrattBuffer *buffer = lexer->bufList->buffer; + Index matchLength = stringLiteralMatchLength(buffer, parsingSingleChar); + + if (matchLength > 0) { + WCharArray *string = + decodeValidatedStringLiteral(buffer->start, matchLength); + return finishStringToken(lexer, string, + parsingSingleChar ? TOK_CHAR() : TOK_STRING(), + matchLength); + } + + return parseStringSlow(parser, parsingSingleChar, sep); } static PrattToken *parseRegex(PrattParser *parser) { diff --git a/src/pratt_scanner.h b/src/pratt_scanner.h index d2aed28f..a724a52f 100644 --- a/src/pratt_scanner.h +++ b/src/pratt_scanner.h @@ -23,6 +23,7 @@ PrattLexer *makePrattLexerFromMbString(char *input, char *origin); PrattTrie *insertPrattTrie(PrattTrie *current, HashSymbol *symbol); +void markPrattScannerRegexCache(void); void enqueueToken(PrattLexer *lexer, PrattToken *token); diff --git a/src/regex.yaml b/src/regex.yaml index e9e04e07..624f2149 100644 --- a/src/regex.yaml +++ b/src/regex.yaml @@ -2,10 +2,13 @@ config: name: regex description: Regex AST and helper data structures parserInfo: false + includes: + - utils.h limited_includes: - cekfs.h - cekfs_debug.h - regex_filepos.h + - utils_debug.h enums: RegexStatus: @@ -90,6 +93,13 @@ structs: cache: CharacterArray exhausted: bool=false + RegexTextSource: + meta: + brief: Regex input backed by an existing contiguous wide-character buffer. + data: + data: WCharVec + start: wstring=NULL + unions: RegexClassItem: meta: @@ -128,6 +138,7 @@ unions: data: string: RegexStringSource file: RegexFileSource + text: RegexTextSource arrays: RegexClassItemArray: @@ -157,3 +168,4 @@ primitives: !include primitives.yaml external: - !include cekfs.yaml +- !include utils.yaml diff --git a/src/regex_helper.c b/src/regex_helper.c index f1be09ab..55a31f61 100644 --- a/src/regex_helper.c +++ b/src/regex_helper.c @@ -1242,6 +1242,44 @@ static bool matchNode(const RegexNode *node, RegexSource *source, } } +int regexMatchPrefixSourcep(const Regex *pattern, RegexSource *source, + Index *matchLength) { + RegexPositionArray *matches; + int save; + + if (matchLength != NULL) { + *matchLength = 0; + } + + if (pattern == NULL || source == NULL) { + return -1; + } + + matches = newRegexPositionArray(); + save = PROTECT(matches); + + if (!matchNode(pattern->root, source, 0, 0, pattern->flags, matches)) { + UNPROTECT(save); + regexSourceSetPosition(source, 0); + return -1; + } + + if (matches->size > 0) { + Index matchedLength = positionAt(matches, 0); + + if (matchLength != NULL) { + *matchLength = matchedLength; + } + regexSourceSetPosition(source, matchedLength); + UNPROTECT(save); + return 0; + } + + UNPROTECT(save); + regexSourceSetPosition(source, 0); + return -1; +} + int regexMatchSourcep(const Regex *pattern, RegexSource *source, Index *matchLength) { RegexPosition index = 0; @@ -1283,6 +1321,31 @@ int regexMatchSourcep(const Regex *pattern, RegexSource *source, } } +int regexMatchPrefixCharArrayp(const Regex *pattern, CharacterArray *text, + Index *matchLength) { + RegexSource *source; + int save; + + if (matchLength != NULL) { + *matchLength = 0; + } + + if (pattern == NULL || text == NULL) { + return -1; + } + + ensureRegexMemoryReady(); + save = STARTPROTECT(); + PROTECT((Regex *)pattern); + PROTECT(text); + source = regexSourceFromCharArray(text); + PROTECT(source); + + int result = regexMatchPrefixSourcep(pattern, source, matchLength); + UNPROTECT(save); + return result; +} + int regexMatchCharArrayp(const Regex *pattern, CharacterArray *text, Index *matchLength) { RegexSource *source; @@ -1331,6 +1394,29 @@ int regexMatchp(const Regex *pattern, const Character *text, return result; } +int regexMatchPrefixp(const Regex *pattern, const Character *text, + Index *matchLength) { + CharacterArray *chars; + int save; + + if (matchLength != NULL) { + *matchLength = 0; + } + + if (pattern == NULL || text == NULL) { + return -1; + } + + ensureRegexMemoryReady(); + save = STARTPROTECT(); + PROTECT((Regex *)pattern); + chars = copyNullTerminatedText(text); + PROTECT(chars); + int result = regexMatchPrefixCharArrayp(pattern, chars, matchLength); + UNPROTECT(save); + return result; +} + int regexMatch(const Character *pattern, const Character *text, Index *matchLength, RegexStatus *status, Index *errorOffset) { Regex *compiled = regexCompile(pattern, status, errorOffset); @@ -1347,3 +1433,21 @@ int regexMatch(const Character *pattern, const Character *text, regexFree(compiled); return result; } + +int regexMatchPrefix(const Character *pattern, const Character *text, + Index *matchLength, RegexStatus *status, + Index *errorOffset) { + Regex *compiled = regexCompile(pattern, status, errorOffset); + int result; + + if (compiled == NULL) { + if (matchLength != NULL) { + *matchLength = 0; + } + return -1; + } + + result = regexMatchPrefixp(compiled, text, matchLength); + regexFree(compiled); + return result; +} diff --git a/src/regex_helper.h b/src/regex_helper.h index 4663fc02..0017b25f 100644 --- a/src/regex_helper.h +++ b/src/regex_helper.h @@ -28,12 +28,21 @@ Regex *regexCompile(const Character *pattern, RegexStatus *status, Index *errorOffset); void regexFree(Regex *regex); +int regexMatchPrefixSourcep(const Regex *pattern, RegexSource *source, + Index *matchLength); int regexMatchSourcep(const Regex *pattern, RegexSource *source, Index *matchLength); +int regexMatchPrefixCharArrayp(const Regex *pattern, CharacterArray *text, + Index *matchLength); int regexMatchCharArrayp(const Regex *pattern, CharacterArray *text, Index *matchLength); +int regexMatchPrefixp(const Regex *pattern, const Character *text, + Index *matchLength); int regexMatchp(const Regex *pattern, const Character *text, Index *matchLength); +int regexMatchPrefix(const Character *pattern, const Character *text, + Index *matchLength, RegexStatus *status, + Index *errorOffset); int regexMatch(const Character *pattern, const Character *text, Index *matchLength, RegexStatus *status, Index *errorOffset); diff --git a/src/regex_source.c b/src/regex_source.c index 65df92b3..84d66ffe 100644 --- a/src/regex_source.c +++ b/src/regex_source.c @@ -5,10 +5,13 @@ #include #include +#include static void ensureRegexSourceMemoryReady(void); static Value charArraySliceToList(const CharacterArray *source, Index start, Index end, Value tail); +static Value textSliceToList(const Character *source, Index start, Index end, + Value tail); static bool regexFileSourceReadNext(RegexFileSource *fileSource); static void ensureRegexSourceMemoryReady(void) { @@ -78,6 +81,11 @@ Character regexSourceGet(RegexSource *source, Index position) { } return L'\0'; } + case REGEXSOURCE_TYPE_TEXT: { + RegexTextSource *textSource = getRegexSource_Text(source); + + return textSource->start[position]; + } default: cant_happen("unrecognised regex source type %d", source->type); } @@ -133,6 +141,7 @@ void regexSourceSetPosition(RegexSource *source, Index position) { switch (source->type) { case REGEXSOURCE_TYPE_STRING: + case REGEXSOURCE_TYPE_TEXT: return; case REGEXSOURCE_TYPE_FILE: { RegexFileSource *fileSource = getRegexSource_File(source); @@ -175,6 +184,20 @@ static Value charArraySliceToList(const CharacterArray *source, Index start, return list; } +static Value textSliceToList(const Character *source, Index start, Index end, + Value tail) { + Value list = tail; + int save = protectValue(list); + + for (Index i = end; i > start; i--) { + list = makePair(value_Character(source[i - 1]), list); + protectValue(list); + } + + UNPROTECT(save); + return list; +} + void regexSourceSplitAt(RegexSource *source, Index offset, Value *prefix, Value *rest) { int save; @@ -233,6 +256,22 @@ void regexSourceSplitAt(RegexSource *source, Index offset, Value *prefix, } break; } + case REGEXSOURCE_TYPE_TEXT: { + RegexTextSource *textSource = getRegexSource_Text(source); + Value empty = makeNull(); + Index length = (Index)wcslen(textSource->start); + + protectValue(empty); + if (prefix != NULL) { + *prefix = textSliceToList(textSource->start, 0, offset, empty); + protectValue(*prefix); + } + if (rest != NULL) { + *rest = textSliceToList(textSource->start, offset, length, empty); + protectValue(*rest); + } + break; + } default: cant_happen("unrecognised regex source type %d", source->type); } @@ -285,4 +324,19 @@ RegexSource *regexSourceFromCharArray(CharacterArray *text) { getRegexSource_String(source)->exhausted = true; UNPROTECT(save); return source; +} + +RegexSource *regexSourceFromWCharVecSlice(WCharVec *text, Character *start) { + RegexTextSource *textSource; + RegexSource *source; + int save; + + ensureRegexSourceMemoryReady(); + save = PROTECT(text); + textSource = newRegexTextSource(text); + PROTECT(textSource); + textSource->start = start; + source = newRegexSource_Text(textSource); + UNPROTECT(save); + return source; } \ No newline at end of file diff --git a/src/regex_source.h b/src/regex_source.h index ba072327..17edc578 100644 --- a/src/regex_source.h +++ b/src/regex_source.h @@ -11,5 +11,6 @@ void regexSourceSplitAt(RegexSource *source, Index offset, Value *prefix, RegexSource *regexSourceFromFileHandle(FILE *handle); RegexSource *regexSourceFromStringList(Vec *tail); RegexSource *regexSourceFromCharArray(CharacterArray *text); +RegexSource *regexSourceFromWCharVecSlice(WCharVec *text, Character *start); #endif \ No newline at end of file diff --git a/tests/src/test_pratt_scanner.c b/tests/src/test_pratt_scanner.c new file mode 100644 index 00000000..3b626f46 --- /dev/null +++ b/tests/src/test_pratt_scanner.c @@ -0,0 +1,217 @@ +#include "test.h" + +#include "bigint.h" +#include "common.h" +#include "init.h" +#include "pratt_scanner.h" + +#include +#include +#include + +#ifdef SAFETY_CHECKS +extern int forceGcFlag; +#endif + +static PrattParser *makeScannerParser(char *input) { + PrattLexer *lexer = makePrattLexerFromMbString(input, "test_pratt_scanner"); + int save = PROTECT(lexer); + PrattParser *parser = newPrattParser(NULL); + PROTECT(parser); + parser->lexer = lexer; + UNPROTECT(save); + return parser; +} + +static PrattToken *scanSingleTokenOfType(char *input, HashSymbol *type, + bool expectErrors) { + int save = STARTPROTECT(); + PrattParser *parser = makeScannerParser(input); + PROTECT(parser); + + clearErrors(); + PrattToken *token = next(parser); + PROTECT(token); + PrattToken *eof = next(parser); + PROTECT(eof); + + assert(token->type == type); + assert(token->value != NULL); + assert(eof->type == TOK_EOF()); + assert(hadErrors() == expectErrors); + + UNPROTECT(save); + return token; +} + +static PrattToken *scanSingleToken(char *input) { + PrattToken *token = scanSingleTokenOfType(input, TOK_NUMBER(), false); + assert(token->value->type == PRATTVALUE_TYPE_NUMBER); + return token; +} + +static void assertSmallNumberToken(char *input, int expected, bool imag) { + int save = STARTPROTECT(); + PrattToken *token = scanSingleToken(input); + PROTECT(token); + + MaybeBigInt *number = getPrattValue_Number(token->value); + assert(number->type == BI_SMALL); + assert(number->small == expected); + assert(number->imag == imag); + + UNPROTECT(save); +} + +static void assertIrrationalToken(char *input, Double expected, bool imag) { + int save = STARTPROTECT(); + PrattToken *token = scanSingleToken(input); + PROTECT(token); + + MaybeBigInt *number = getPrattValue_Number(token->value); + assert(number->type == BI_IRRATIONAL); + assert(fabs(number->irrational - expected) < 1e-12); + assert(number->imag == imag); + + UNPROTECT(save); +} + +static void assertPrintedNumberToken(char *input, const char *expected) { + int save = STARTPROTECT(); + PrattToken *token = scanSingleToken(input); + PROTECT(token); + + MaybeBigInt *number = getPrattValue_Number(token->value); + char actual[256]; + + sprintMaybeBigInt(actual, number); + assert(strcmp(actual, expected) == 0); + + UNPROTECT(save); +} + +static void assertStringToken(char *input, const wchar_t *expected) { + int save = STARTPROTECT(); + PrattToken *token = scanSingleTokenOfType(input, TOK_STRING(), false); + PROTECT(token); + + WCharArray *string = getPrattValue_String(token->value); + assert(token->value->type == PRATTVALUE_TYPE_STRING); + assert(wcscmp(string->entries, expected) == 0); + + UNPROTECT(save); +} + +static void assertCharToken(char *input, const wchar_t *expected) { + int save = STARTPROTECT(); + PrattToken *token = scanSingleTokenOfType(input, TOK_CHAR(), false); + PROTECT(token); + + WCharArray *string = getPrattValue_String(token->value); + assert(token->value->type == PRATTVALUE_TYPE_STRING); + assert(wcscmp(string->entries, expected) == 0); + + UNPROTECT(save); +} + +static void testDecimalNumber(void) { + assertSmallNumberToken("123", 123, false); +} + +static void testFloatWithTrailingDot(void) { + assertIrrationalToken("1.", 1.0, false); +} + +static void testHexadecimalNumber(void) { + assertSmallNumberToken("0xff", 255, false); + assertSmallNumberToken("0XFF", 255, false); +} + +static void testImaginaryNumbers(void) { + assertSmallNumberToken("7i", 7, true); + assertPrintedNumberToken("0x_afa_e20d_cab2_6000i", "791193233420083200i"); +} + +static void testUnicodeDecimalDigits(void) { + assertSmallNumberToken("\u0661\u0662\u0663", 123, false); +} + +static void testPermissiveEdgeCases(void) { + assertSmallNumberToken("0x", 0, false); + assertSmallNumberToken("1__2", 12, false); +} + +static void testCachedNumericRegexesSurviveForcedGc(void) { + assertSmallNumberToken("123", 123, false); + assertSmallNumberToken("0xff", 255, false); + +#ifdef SAFETY_CHECKS + // Keep forced GC scoped to the cached scans under test; enabling it for the + // whole binary makes this test suite much slower. + int previousForceGcFlag = forceGcFlag; + forceGcFlag = 1; +#endif + + assertSmallNumberToken("456", 456, false); + assertSmallNumberToken("0x10", 16, false); + +#ifdef SAFETY_CHECKS + forceGcFlag = previousForceGcFlag; +#endif +} + +static void testStringLiterals(void) { + assertStringToken("\"hello\"", L"hello"); + assertStringToken("\"a\\n\\t\\r\"", L"a\n\t\r"); + assertStringToken("\"\\u03bb;\"", L"\u03bb"); + assertStringToken("\"\\q\"", L"q"); +} + +static void testCharLiterals(void) { + assertCharToken("'x'", L"x"); + assertCharToken("'\\n'", L"\n"); + assertCharToken("'\\u03bb;'", L"\u03bb"); +} + +static void testMalformedStringsStillUseSlowPath(void) { + scanSingleTokenOfType("''", TOK_CHAR(), true); + scanSingleTokenOfType("\"\\u;\"", TOK_STRING(), true); +} + +static void testCachedStringRegexesSurviveForcedGc(void) { + assertStringToken("\"warm\"", L"warm"); + assertCharToken("'w'", L"w"); + +#ifdef SAFETY_CHECKS + // Keep forced GC scoped to the cached scans under test; enabling it for the + // whole binary makes this test suite much slower. + int previousForceGcFlag = forceGcFlag; + forceGcFlag = 1; +#endif + + assertStringToken("\"after\"", L"after"); + assertCharToken("'\\t'", L"\t"); + +#ifdef SAFETY_CHECKS + forceGcFlag = previousForceGcFlag; +#endif +} + +int main(int argc __attribute__((unused)), + char *argv[] __attribute__((unused))) { + initAll(); + + testDecimalNumber(); + testFloatWithTrailingDot(); + testHexadecimalNumber(); + testImaginaryNumbers(); + testUnicodeDecimalDigits(); + testPermissiveEdgeCases(); + testCachedNumericRegexesSurviveForcedGc(); + testStringLiterals(); + testCharLiterals(); + testMalformedStringsStillUseSlowPath(); + testCachedStringRegexesSurviveForcedGc(); + + return 0; +} \ No newline at end of file diff --git a/tests/src/test_regex.c b/tests/src/test_regex.c index 38a2b4ed..bf0fc373 100644 --- a/tests/src/test_regex.c +++ b/tests/src/test_regex.c @@ -161,6 +161,16 @@ static void testLeadingCaseInsensitiveKeepsCategoryMeaning(void) { -1); } +static void testPrefixMatcherDoesNotSearchLaterOffsets(void) { + Index matchLength; + + assert(regexMatchPrefix(L"ab", L"abcd", &matchLength, NULL, NULL) == 0); + assert(matchLength == 2); + assert(regexMatchPrefix(L"ab", L"zabcd", &matchLength, NULL, NULL) == -1); + assert(regexMatch(L"ab", L"zabcd", &matchLength, NULL, NULL) == 1); + assert(matchLength == 2); +} + static void testInvalidCategoryReportsOffset(void) { RegexStatus status; Index errorOffset; @@ -225,6 +235,7 @@ int main(int argc __attribute__((unused)), testLeadingCaseInsensitiveFlag(); testLeadingCaseInsensitiveUnicodeLiterals(); testLeadingCaseInsensitiveKeepsCategoryMeaning(); + testPrefixMatcherDoesNotSearchLaterOffsets(); testInvalidCategoryReportsOffset(); testUnterminatedGroupReportsError(); testTrailingEscapeReportsError();