From 9687fb2730fa11be8d2a91f8412d352313a09a78 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Wed, 14 Oct 2026 20:19:11 -0400 Subject: [PATCH] makeliteral(): replace rule elements with one token of given text (#450) (4.2.28) @POST makeliteral(1, 2, "70"); @RULES _xNIL <- 7 O @@ turns OCR's "7O" into one num token "70": later passes match it as _xNUM, its $text is "70", and it keeps the offsets of the "7O" it replaced. The type is what the tokenizer would give the text (all digits num, all whitespace white, one ASCII punctuation character punct, otherwise alpha), or a 4th argument forces it: makeliteral(1, 1, "USA", "alpha"). Pat::redLiteral reduces the range as singler() does, then Pat::makeLiteral drops the node's children and gives it the text, type and a new LITERAL flag; Pn::pnStr returns a literal node's own name rather than the input buffer. The compiled Arun::makeliteral reduces with the compiled singler() and calls the same Pat::makeLiteral. Like singlezap, it replaces the matched nodes, so it goes last in @POST. Named makeliteral, not literal: parse-en-us has its own literal(node), and a post action is resolved before user functions. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-linux.yml | 31 +++++ .github/workflows/build-windows.yml | 30 +++++ .../workflows/tests/literal/input/text.txt | 1 + .../workflows/tests/literal/spec/analyzer.seq | 5 + .github/workflows/tests/literal/spec/fix.nlp | 12 ++ .../workflows/tests/literal/spec/force.nlp | 16 +++ .github/workflows/tests/literal/spec/show.nlp | 14 ++ .github/workflows/tests/literal/spec/use.nlp | 12 ++ include/Api/lite/Arun.h | 2 + include/Api/lite/pn.h | 2 + lite/Arun.cpp | 20 +++ lite/fn.cpp | 1 + lite/func_defs.h | 1 + lite/funcs.h | 1 + lite/pat.h | 3 + lite/pn.cpp | 16 +++ lite/post.cpp | 121 ++++++++++++++++++ nlp/main.cpp | 2 +- 18 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests/literal/input/text.txt create mode 100644 .github/workflows/tests/literal/spec/analyzer.seq create mode 100644 .github/workflows/tests/literal/spec/fix.nlp create mode 100644 .github/workflows/tests/literal/spec/force.nlp create mode 100644 .github/workflows/tests/literal/spec/show.nlp create mode 100644 .github/workflows/tests/literal/spec/use.nlp diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index cfc7a00a..0e041d03 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1048,6 +1048,37 @@ jobs: fi echo "[smart-wildcard] wildcards stop before later elements" + - name: makeliteral() post action (#450) (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # makeliteral(start, end, text [, type]) replaces rule elements with one + # token whose text is the given string, typed as the tokenizer would + # (#450): OCR "7O" becomes a num token "70" that later passes match + # as _xNUM, keeping the original offsets. + FIX=.github/workflows/tests/literal + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(literal): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'He alpha text=He at 0-1' \ + 'paid alpha text=paid at 3-6' \ + '70 num text=70 at 8-9' \ + 'dollars alpha text=dollars at 11-17' \ + ', punct text=, at 19-19' \ + 'USA alpha text=USA at 21-23' \ + '_xNUM matched: 70 dollars') + if [ "$got" != "$want" ]; then + echo "FAIL(literal): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[literal] makeliteral() makes a token later passes match" + - name: factorial() up to 20! (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 2a23fe5a..899e16e7 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -945,6 +945,36 @@ jobs: fi echo "[smart-wildcard] wildcards stop before later elements" + - name: makeliteral() post action (#450) + shell: bash + run: | + set -eo pipefail + # makeliteral(start, end, text [, type]) replaces rule elements with one + # token whose text is the given string, typed as the tokenizer would + # (#450): OCR "7O" becomes a num token "70" that later passes match + # as _xNUM, keeping the original offsets. + FIX=.github/workflows/tests/literal + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(literal): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'He alpha text=He at 0-1' \ + 'paid alpha text=paid at 3-6' \ + '70 num text=70 at 8-9' \ + 'dollars alpha text=dollars at 11-17' \ + ', punct text=, at 19-19' \ + 'USA alpha text=USA at 21-23' \ + '_xNUM matched: 70 dollars') + if [ "$got" != "$want" ]; then + echo "FAIL(literal): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[literal] makeliteral() makes a token later passes match" + - name: factorial() up to 20! shell: bash run: | diff --git a/.github/workflows/tests/literal/input/text.txt b/.github/workflows/tests/literal/input/text.txt new file mode 100644 index 00000000..4cfa0f95 --- /dev/null +++ b/.github/workflows/tests/literal/input/text.txt @@ -0,0 +1 @@ +He paid 7O dollars ; usa diff --git a/.github/workflows/tests/literal/spec/analyzer.seq b/.github/workflows/tests/literal/spec/analyzer.seq new file mode 100644 index 00000000..79715c6e --- /dev/null +++ b/.github/workflows/tests/literal/spec/analyzer.seq @@ -0,0 +1,5 @@ +tokenize nil # +nlp fix # makeliteral(): OCR "7O" -> num token "70" +nlp force # forced type and inferred punctuation +nlp show # what the new tokens report +nlp use # a later pass matches the literal as _xNUM diff --git a/.github/workflows/tests/literal/spec/fix.nlp b/.github/workflows/tests/literal/spec/fix.nlp new file mode 100644 index 00000000..8d283306 --- /dev/null +++ b/.github/workflows/tests/literal/spec/fix.nlp @@ -0,0 +1,12 @@ +############################################### +# FILE: fix.nlp +# SUBJ: makeliteral() (#450): OCR read "70" as "7O"; make it a number token. +# NOTE: Part of the literal regression; expected output is in the CI step. +############################################### + +@NODES _ROOT + +@POST + makeliteral(1, 2, "70"); +@RULES +_xNIL <- 7 O @@ diff --git a/.github/workflows/tests/literal/spec/force.nlp b/.github/workflows/tests/literal/spec/force.nlp new file mode 100644 index 00000000..2977ed1e --- /dev/null +++ b/.github/workflows/tests/literal/spec/force.nlp @@ -0,0 +1,16 @@ +############################################### +# FILE: force.nlp +# SUBJ: makeliteral() with a forced type, and with an inferred punctuation type. +############################################### + +@NODES _ROOT + +@POST + makeliteral(1, 1, "USA", "alpha"); +@RULES +_xNIL <- usa @@ + +@POST + makeliteral(1, 1, ","); +@RULES +_xNIL <- \; @@ diff --git a/.github/workflows/tests/literal/spec/show.nlp b/.github/workflows/tests/literal/spec/show.nlp new file mode 100644 index 00000000..5737e2cb --- /dev/null +++ b/.github/workflows/tests/literal/spec/show.nlp @@ -0,0 +1,14 @@ +############################################### +# FILE: show.nlp +# SUBJ: Name, token type, $text and original offsets of each top node. +############################################### + +@CODE +L("n") = pndown(pnroot()); +while (L("n")) { + if (pntype(L("n")) != "white") + "output.txt" << pnname(L("n")) << " " << pntype(L("n")) << " text=" << pnvar(L("n"), "$text") + << " at " << pnvar(L("n"), "$ostart") << "-" << pnvar(L("n"), "$oend") << "\n"; + L("n") = pnnext(L("n")); +} +@@CODE diff --git a/.github/workflows/tests/literal/spec/use.nlp b/.github/workflows/tests/literal/spec/use.nlp new file mode 100644 index 00000000..83d4d891 --- /dev/null +++ b/.github/workflows/tests/literal/spec/use.nlp @@ -0,0 +1,12 @@ +############################################### +# FILE: use.nlp +# SUBJ: A later pass matches the literal as an ordinary number token. +############################################### + +@NODES _ROOT + +@POST + "output.txt" << "_xNUM matched: " << N("$text", 1) << " " << N("$text", 3) << "\n"; + single(); +@RULES +_money <- _xNUM _xWHITE [s] dollars @@ diff --git a/include/Api/lite/Arun.h b/include/Api/lite/Arun.h index 1f045c9f..d435e8d2 100644 --- a/include/Api/lite/Arun.h +++ b/include/Api/lite/Arun.h @@ -425,6 +425,8 @@ class LITE_API Arun // POST ACTIONS static bool single(Nlppp *); static bool singler(Nlppp *,long long,long long); // 06/02/00 AM. + static bool makeliteral(Nlppp *,long long,long long,_TCHAR *); // #450. + static bool makeliteral(Nlppp *,long long,long long,_TCHAR *,_TCHAR *); // #450. static bool singlex(Nlppp *,long long,long long); // 06/02/00 AM. static bool singlezap(Nlppp *); // 01/04/01 AM. static bool merge(Nlppp *); // 06/02/00 AM. diff --git a/include/Api/lite/pn.h b/include/Api/lite/pn.h index d806b011..3d0547b9 100644 --- a/include/Api/lite/pn.h +++ b/include/Api/lite/pn.h @@ -100,6 +100,7 @@ class LITE_API Pn bool getUnsealed() const; // 10/09/99 AM. bool getFired() const; // 05/21/01 Dd. bool getBuilt() const; // 05/25/01 AM. + bool getLiteral() const; // Text is its own name, not the input's (#450). void getTextStartEnd(_TCHAR* &, long &, long &); // 12/29/99 AM. void getTextUstartUend(_TCHAR* &, long &, long &); // [UNICODE] // 06/15/22 AM. @@ -128,6 +129,7 @@ class LITE_API Pn void setUnsealed(bool); // 10/09/99 AM. void setFired(bool); // 05/21/01 Dd. void setBuilt(bool); // 05/25/01 AM. + void setLiteral(bool); // #450. // GET THE TEXT FOR THE CURRENT NODE. _TCHAR *pnStr(); // 11/06/98 AM. diff --git a/lite/Arun.cpp b/lite/Arun.cpp index 4d163a46..da2e9e25 100644 --- a/lite/Arun.cpp +++ b/lite/Arun.cpp @@ -5239,6 +5239,26 @@ return true; } +/******************************************** +* FN: MAKELITERAL +* SUBJ: makeliteral(start, end, text [, type]), compiled runtime (#450). +* NOTE: Reduces like singler(), then Pat::makeLiteral, as the interpreter. +********************************************/ + +bool Arun::makeliteral(Nlppp *nlppp, long long num1, long long num2, _TCHAR *text) +{ +return makeliteral(nlppp, num1, num2, text, 0); +} + +bool Arun::makeliteral(Nlppp *nlppp, long long num1, long long num2, + _TCHAR *text, _TCHAR *type) +{ +if (!singler(nlppp, num1, num2) || !nlppp->red_) + return false; +return Pat::makeLiteral(nlppp->red_, text, type, nlppp); +} + + /******************************************** * FN: SINGLER * CR: 06/02/00 AM. diff --git a/lite/fn.cpp b/lite/fn.cpp index abfcebae..fe2ee47a 100644 --- a/lite/fn.cpp +++ b/lite/fn.cpp @@ -709,6 +709,7 @@ switch (fnid) // 12/21/01 AM. return fnSetlookahead(args,nlppp,/*UP*/sem); // 12/14/03 AM. case FNsetunsealed: case FNsingle: + case FNmakeliteral: // Post action, run by Pat::postAction (#450). case FNsingler: case FNsinglex: case FNsinglezap: diff --git a/lite/func_defs.h b/lite/func_defs.h index ef1922a1..bdf12a80 100644 --- a/lite/func_defs.h +++ b/lite/func_defs.h @@ -125,6 +125,7 @@ enum funcDef FNlistadd, FNlistnode, FNLJ, + FNmakeliteral, // #450. FNlj, FNloaddict, // 06/11/26 DD. FNloadkbb, // 06/11/26 DD. diff --git a/lite/funcs.h b/lite/funcs.h index cec98ddc..cab8c92b 100644 --- a/lite/funcs.h +++ b/lite/funcs.h @@ -122,6 +122,7 @@ _T("lextagger"), // 12/11/03 AM. _T("listadd"), _T("listnode"), _T("LJ"), +_T("makeliteral"), // Post action (#450). _T("lj"), _T("loaddict"), // 06/11/26 DD. _T("loadkbb"), // 06/11/26 DD. diff --git a/lite/pat.h b/lite/pat.h index ec216f17..e4911e7e 100644 --- a/lite/pat.h +++ b/lite/pat.h @@ -290,6 +290,9 @@ class Pat : public Algo static Ielt *preWildStop(Ielt *ielt, Delt *next); // #449. static bool laterMatches(Delt *d, Node *node, bool right); // #427. + static bool redLiteral(Delt *args, Nlppp *nlppp); // #450. + static bool makeLiteral(Node *sub, _TCHAR *text, _TCHAR *type, + Nlppp *nlppp); // #450. static void markFiredRule(Nlppp *nlppp, long ruleline); // #73. static bool wildFillLeft( diff --git a/lite/pn.cpp b/lite/pn.cpp index dc190f22..f7261236 100644 --- a/lite/pn.cpp +++ b/lite/pn.cpp @@ -64,6 +64,7 @@ static const int bitBASE = 01; // Node's BASE attribute. static const int bitUNSEALED = 02; // UNSEALED attr. // 10/09/99 AM. static const int bitFIRED = 04; // FIRED flag. // 05/21/01 DD. static const int bitBUILT= 010; // BUILT flag. // 05/25/01 AM. +static const int bitLITERAL = 020; // Made by makeliteral(): text is the name (#450). // 02, 04, 010, 020, 040, 0100... @@ -440,6 +441,17 @@ if (x) else flags_ &= (~bitFIRED); } +bool Pn::getLiteral() const +{ +return (flags_ & bitLITERAL) != 0; +} +void Pn::setLiteral(bool x) +{ +if (x) + flags_ |= bitLITERAL; +else + flags_ &= (~bitLITERAL); +} void Pn::setBuilt(bool x) // 05/25/01 AM. { if (x) @@ -641,6 +653,8 @@ for (; nodes; nodes = nodes->Right()) _TCHAR *Pn::pnStr() { +if (getLiteral() && name_) // A literal's text isn't the input's (#450). + return make_str(name_, (long) _tcslen(name_)); return make_str(Text, End - Start + 1); } @@ -649,6 +663,8 @@ _TCHAR *Pn::pnStr( long &len /*UP*/ // 07/23/00 AM. ) { +if (getLiteral() && name_) // A literal's text isn't the input's (#450). + return make_str(name_, len = (long) _tcslen(name_)); return make_str(Text, len = (End - Start + 1)); } diff --git a/lite/post.cpp b/lite/post.cpp index 42faa051..acf24f01 100644 --- a/lite/post.cpp +++ b/lite/post.cpp @@ -351,6 +351,8 @@ if (!strcmp_i(func, _T("single"))) // Single-tier (simple) reduction. return redSingle(args,nlppp); // 05/26/01 AM. else if (!strcmp_i(func, _T("singler"))) // Single-tier, range. return redSingler(args,nlppp); // 01/17/99 AM. +else if (!strcmp_i(func, _T("makeliteral"))) // Range to one token (#450). + return redLiteral(args,nlppp); else if (!strcmp_i(func, _T("singlex"))) // Single-tier,range,excise. return redSinglex(args,nlppp); // 01/17/99 AM. if (!strcmp_i(func, _T("singlezap"))) @@ -900,6 +902,125 @@ return ok; // 11/19/99 AM. } +/******************************************** +* FN: MAKELITERAL +* SUBJ: Turn a node just built by singler() into a token (#450). +* NOTE: The node becomes a leaf named TEXT, of TYPE ("num", "alpha", +* "punct", "white", "ctrl", "emoji" or "node"), or of the type the +* tokenizer would give TEXT: all digits num, all whitespace white, +* one ASCII punctuation character punct, otherwise alpha. It keeps +* the offsets of the nodes it replaced, and its $text is TEXT. +* Shared by makeliteral() in the interpreter and the compiled runtime. +********************************************/ + +bool Pat::makeLiteral(Node *sub, _TCHAR *text, _TCHAR *type, Nlppp *nlppp) +{ +Parse *parse = nlppp->parse_; +if (!sub || !text || !*text) + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Needs a node and a non-empty text.]") << std::ends; + parse->errOut(&gerrStr,false); + return false; + } +enum Pntype typ = PNALPHA; +if (type && *type) + { + if (!strcmp_i(type, _T("num"))) typ = PNNUM; + else if (!strcmp_i(type, _T("alpha"))) typ = PNALPHA; + else if (!strcmp_i(type, _T("punct"))) typ = PNPUNCT; + else if (!strcmp_i(type, _T("white"))) typ = PNWHITE; + else if (!strcmp_i(type, _T("ctrl"))) typ = PNCTRL; + else if (!strcmp_i(type, _T("emoji"))) typ = PNEMOJI; + else if (!strcmp_i(type, _T("node"))) typ = PNNODE; + else + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Unknown type=") << type + << _T(". Use num, alpha, punct, white, ctrl, emoji or node.]") << std::ends; + parse->errOut(&gerrStr,false); + return false; + } + } +else + { + bool digits = true, white = true; + for (_TCHAR *p = text; *p; ++p) + { + if (*p < '0' || *p > '9') + digits = false; + if (*p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') + white = false; + } + if (digits) + typ = PNNUM; + else if (white) + typ = PNWHITE; + else if (!text[1] && (_TUCHAR) text[0] < 0x80 && !alphabetic(text[0])) + typ = PNPUNCT; + } + +// The replaced nodes go; the token has no children. +Tree *tree = (Tree *) parse->getTree(); +if (sub->Down()) + { + tree->deleteSubsNC(sub->Down()); + sub->setDown(0); + } + +Sym *sym = parse->getSym(text); +Pn *pn = sub->getData(); +pn->setName(sym->getStr()); +pn->setSym(sym); +pn->setType(typ); +pn->setLiteral(true); +pn->setBuilt(true); +nlppp->excised_ = true; // The matched nodes are gone, as with singlezap. +return true; +} + + +/******************************************** +* FN: REDLITERAL +* SUBJ: makeliteral(start, end, text [, type]): replace rule elements START..END +* with one token whose text is TEXT (#450). +* EX: @POST makeliteral(1, 2, "70"); @RULES _xNIL <- 7 O @@ +* Later passes see "70" as a num token: _xNUM matches it. +* NOTE: Like singlezap, it replaces the matched nodes, so put it last. +********************************************/ + +bool Pat::redLiteral(Delt *args, Nlppp *nlppp) +{ +Parse *parse = nlppp->parse_; +Delt *d1 = args, *d2 = args ? args->Right() : 0; +Delt *d3 = d2 ? d2->Right() : 0, *d4 = d3 ? d3->Right() : 0; +if (!d3 || (d4 && d4->Right())) + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Use makeliteral(start, end, text) or makeliteral(start, end, text, type).]") + << std::ends; + parse->errOut(&gerrStr,false); + return false; + } +Iarg *a3 = d3->getData(), *a4 = d4 ? d4->getData() : 0; +if (a3->getType() != IASTR || (a4 && a4->getType() != IASTR)) + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Text and type must be strings.]") << std::ends; + parse->errOut(&gerrStr,false); + return false; + } + +// Reduce the range as singler(start, end) does, then make it a token. +d2->setRight(0); +bool ok = redSingler(d1, nlppp); +d2->setRight(d3); +if (!ok || !nlppp->red_) + return false; +return makeLiteral(nlppp->red_, a3->getStr(), a4 ? a4->getStr() : 0, nlppp); +} + + /******************************************** * FN: REDSINGLEX * CR: 01/17/99 AM. diff --git a/nlp/main.cpp b/nlp/main.cpp index dfbb128c..d883604f 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.27" +#define NLP_ENGINE_VERSION "4.2.28" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &); void cmdHelpargs(_TCHAR *);