diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index cfc7a00a..0e041d03 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1048,6 +1048,37 @@ jobs: fi echo "[smart-wildcard] wildcards stop before later elements" + - name: makeliteral() post action (#450) (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # makeliteral(start, end, text [, type]) replaces rule elements with one + # token whose text is the given string, typed as the tokenizer would + # (#450): OCR "7O" becomes a num token "70" that later passes match + # as _xNUM, keeping the original offsets. + FIX=.github/workflows/tests/literal + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(literal): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'He alpha text=He at 0-1' \ + 'paid alpha text=paid at 3-6' \ + '70 num text=70 at 8-9' \ + 'dollars alpha text=dollars at 11-17' \ + ', punct text=, at 19-19' \ + 'USA alpha text=USA at 21-23' \ + '_xNUM matched: 70 dollars') + if [ "$got" != "$want" ]; then + echo "FAIL(literal): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[literal] makeliteral() makes a token later passes match" + - name: factorial() up to 20! (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 2a23fe5a..899e16e7 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -945,6 +945,36 @@ jobs: fi echo "[smart-wildcard] wildcards stop before later elements" + - name: makeliteral() post action (#450) + shell: bash + run: | + set -eo pipefail + # makeliteral(start, end, text [, type]) replaces rule elements with one + # token whose text is the given string, typed as the tokenizer would + # (#450): OCR "7O" becomes a num token "70" that later passes match + # as _xNUM, keeping the original offsets. + FIX=.github/workflows/tests/literal + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(literal): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'He alpha text=He at 0-1' \ + 'paid alpha text=paid at 3-6' \ + '70 num text=70 at 8-9' \ + 'dollars alpha text=dollars at 11-17' \ + ', punct text=, at 19-19' \ + 'USA alpha text=USA at 21-23' \ + '_xNUM matched: 70 dollars') + if [ "$got" != "$want" ]; then + echo "FAIL(literal): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[literal] makeliteral() makes a token later passes match" + - name: factorial() up to 20! shell: bash run: | diff --git a/.github/workflows/tests/literal/input/text.txt b/.github/workflows/tests/literal/input/text.txt new file mode 100644 index 00000000..4cfa0f95 --- /dev/null +++ b/.github/workflows/tests/literal/input/text.txt @@ -0,0 +1 @@ +He paid 7O dollars ; usa diff --git a/.github/workflows/tests/literal/spec/analyzer.seq b/.github/workflows/tests/literal/spec/analyzer.seq new file mode 100644 index 00000000..79715c6e --- /dev/null +++ b/.github/workflows/tests/literal/spec/analyzer.seq @@ -0,0 +1,5 @@ +tokenize nil # +nlp fix # makeliteral(): OCR "7O" -> num token "70" +nlp force # forced type and inferred punctuation +nlp show # what the new tokens report +nlp use # a later pass matches the literal as _xNUM diff --git a/.github/workflows/tests/literal/spec/fix.nlp b/.github/workflows/tests/literal/spec/fix.nlp new file mode 100644 index 00000000..8d283306 --- /dev/null +++ b/.github/workflows/tests/literal/spec/fix.nlp @@ -0,0 +1,12 @@ +############################################### +# FILE: fix.nlp +# SUBJ: makeliteral() (#450): OCR read "70" as "7O"; make it a number token. +# NOTE: Part of the literal regression; expected output is in the CI step. +############################################### + +@NODES _ROOT + +@POST + makeliteral(1, 2, "70"); +@RULES +_xNIL <- 7 O @@ diff --git a/.github/workflows/tests/literal/spec/force.nlp b/.github/workflows/tests/literal/spec/force.nlp new file mode 100644 index 00000000..2977ed1e --- /dev/null +++ b/.github/workflows/tests/literal/spec/force.nlp @@ -0,0 +1,16 @@ +############################################### +# FILE: force.nlp +# SUBJ: makeliteral() with a forced type, and with an inferred punctuation type. +############################################### + +@NODES _ROOT + +@POST + makeliteral(1, 1, "USA", "alpha"); +@RULES +_xNIL <- usa @@ + +@POST + makeliteral(1, 1, ","); +@RULES +_xNIL <- \; @@ diff --git a/.github/workflows/tests/literal/spec/show.nlp b/.github/workflows/tests/literal/spec/show.nlp new file mode 100644 index 00000000..5737e2cb --- /dev/null +++ b/.github/workflows/tests/literal/spec/show.nlp @@ -0,0 +1,14 @@ +############################################### +# FILE: show.nlp +# SUBJ: Name, token type, $text and original offsets of each top node. +############################################### + +@CODE +L("n") = pndown(pnroot()); +while (L("n")) { + if (pntype(L("n")) != "white") + "output.txt" << pnname(L("n")) << " " << pntype(L("n")) << " text=" << pnvar(L("n"), "$text") + << " at " << pnvar(L("n"), "$ostart") << "-" << pnvar(L("n"), "$oend") << "\n"; + L("n") = pnnext(L("n")); +} +@@CODE diff --git a/.github/workflows/tests/literal/spec/use.nlp b/.github/workflows/tests/literal/spec/use.nlp new file mode 100644 index 00000000..83d4d891 --- /dev/null +++ b/.github/workflows/tests/literal/spec/use.nlp @@ -0,0 +1,12 @@ +############################################### +# FILE: use.nlp +# SUBJ: A later pass matches the literal as an ordinary number token. +############################################### + +@NODES _ROOT + +@POST + "output.txt" << "_xNUM matched: " << N("$text", 1) << " " << N("$text", 3) << "\n"; + single(); +@RULES +_money <- _xNUM _xWHITE [s] dollars @@ diff --git a/include/Api/lite/Arun.h b/include/Api/lite/Arun.h index 1f045c9f..d435e8d2 100644 --- a/include/Api/lite/Arun.h +++ b/include/Api/lite/Arun.h @@ -425,6 +425,8 @@ class LITE_API Arun // POST ACTIONS static bool single(Nlppp *); static bool singler(Nlppp *,long long,long long); // 06/02/00 AM. + static bool makeliteral(Nlppp *,long long,long long,_TCHAR *); // #450. + static bool makeliteral(Nlppp *,long long,long long,_TCHAR *,_TCHAR *); // #450. static bool singlex(Nlppp *,long long,long long); // 06/02/00 AM. static bool singlezap(Nlppp *); // 01/04/01 AM. static bool merge(Nlppp *); // 06/02/00 AM. diff --git a/include/Api/lite/pn.h b/include/Api/lite/pn.h index d806b011..3d0547b9 100644 --- a/include/Api/lite/pn.h +++ b/include/Api/lite/pn.h @@ -100,6 +100,7 @@ class LITE_API Pn bool getUnsealed() const; // 10/09/99 AM. bool getFired() const; // 05/21/01 Dd. bool getBuilt() const; // 05/25/01 AM. + bool getLiteral() const; // Text is its own name, not the input's (#450). void getTextStartEnd(_TCHAR* &, long &, long &); // 12/29/99 AM. void getTextUstartUend(_TCHAR* &, long &, long &); // [UNICODE] // 06/15/22 AM. @@ -128,6 +129,7 @@ class LITE_API Pn void setUnsealed(bool); // 10/09/99 AM. void setFired(bool); // 05/21/01 Dd. void setBuilt(bool); // 05/25/01 AM. + void setLiteral(bool); // #450. // GET THE TEXT FOR THE CURRENT NODE. _TCHAR *pnStr(); // 11/06/98 AM. diff --git a/lite/Arun.cpp b/lite/Arun.cpp index 4d163a46..da2e9e25 100644 --- a/lite/Arun.cpp +++ b/lite/Arun.cpp @@ -5239,6 +5239,26 @@ return true; } +/******************************************** +* FN: MAKELITERAL +* SUBJ: makeliteral(start, end, text [, type]), compiled runtime (#450). +* NOTE: Reduces like singler(), then Pat::makeLiteral, as the interpreter. +********************************************/ + +bool Arun::makeliteral(Nlppp *nlppp, long long num1, long long num2, _TCHAR *text) +{ +return makeliteral(nlppp, num1, num2, text, 0); +} + +bool Arun::makeliteral(Nlppp *nlppp, long long num1, long long num2, + _TCHAR *text, _TCHAR *type) +{ +if (!singler(nlppp, num1, num2) || !nlppp->red_) + return false; +return Pat::makeLiteral(nlppp->red_, text, type, nlppp); +} + + /******************************************** * FN: SINGLER * CR: 06/02/00 AM. diff --git a/lite/fn.cpp b/lite/fn.cpp index abfcebae..fe2ee47a 100644 --- a/lite/fn.cpp +++ b/lite/fn.cpp @@ -709,6 +709,7 @@ switch (fnid) // 12/21/01 AM. return fnSetlookahead(args,nlppp,/*UP*/sem); // 12/14/03 AM. case FNsetunsealed: case FNsingle: + case FNmakeliteral: // Post action, run by Pat::postAction (#450). case FNsingler: case FNsinglex: case FNsinglezap: diff --git a/lite/func_defs.h b/lite/func_defs.h index ef1922a1..bdf12a80 100644 --- a/lite/func_defs.h +++ b/lite/func_defs.h @@ -125,6 +125,7 @@ enum funcDef FNlistadd, FNlistnode, FNLJ, + FNmakeliteral, // #450. FNlj, FNloaddict, // 06/11/26 DD. FNloadkbb, // 06/11/26 DD. diff --git a/lite/funcs.h b/lite/funcs.h index cec98ddc..cab8c92b 100644 --- a/lite/funcs.h +++ b/lite/funcs.h @@ -122,6 +122,7 @@ _T("lextagger"), // 12/11/03 AM. _T("listadd"), _T("listnode"), _T("LJ"), +_T("makeliteral"), // Post action (#450). _T("lj"), _T("loaddict"), // 06/11/26 DD. _T("loadkbb"), // 06/11/26 DD. diff --git a/lite/pat.h b/lite/pat.h index ec216f17..e4911e7e 100644 --- a/lite/pat.h +++ b/lite/pat.h @@ -290,6 +290,9 @@ class Pat : public Algo static Ielt *preWildStop(Ielt *ielt, Delt *next); // #449. static bool laterMatches(Delt *d, Node *node, bool right); // #427. + static bool redLiteral(Delt *args, Nlppp *nlppp); // #450. + static bool makeLiteral(Node *sub, _TCHAR *text, _TCHAR *type, + Nlppp *nlppp); // #450. static void markFiredRule(Nlppp *nlppp, long ruleline); // #73. static bool wildFillLeft( diff --git a/lite/pn.cpp b/lite/pn.cpp index dc190f22..f7261236 100644 --- a/lite/pn.cpp +++ b/lite/pn.cpp @@ -64,6 +64,7 @@ static const int bitBASE = 01; // Node's BASE attribute. static const int bitUNSEALED = 02; // UNSEALED attr. // 10/09/99 AM. static const int bitFIRED = 04; // FIRED flag. // 05/21/01 DD. static const int bitBUILT= 010; // BUILT flag. // 05/25/01 AM. +static const int bitLITERAL = 020; // Made by makeliteral(): text is the name (#450). // 02, 04, 010, 020, 040, 0100... @@ -440,6 +441,17 @@ if (x) else flags_ &= (~bitFIRED); } +bool Pn::getLiteral() const +{ +return (flags_ & bitLITERAL) != 0; +} +void Pn::setLiteral(bool x) +{ +if (x) + flags_ |= bitLITERAL; +else + flags_ &= (~bitLITERAL); +} void Pn::setBuilt(bool x) // 05/25/01 AM. { if (x) @@ -641,6 +653,8 @@ for (; nodes; nodes = nodes->Right()) _TCHAR *Pn::pnStr() { +if (getLiteral() && name_) // A literal's text isn't the input's (#450). + return make_str(name_, (long) _tcslen(name_)); return make_str(Text, End - Start + 1); } @@ -649,6 +663,8 @@ _TCHAR *Pn::pnStr( long &len /*UP*/ // 07/23/00 AM. ) { +if (getLiteral() && name_) // A literal's text isn't the input's (#450). + return make_str(name_, len = (long) _tcslen(name_)); return make_str(Text, len = (End - Start + 1)); } diff --git a/lite/post.cpp b/lite/post.cpp index 42faa051..acf24f01 100644 --- a/lite/post.cpp +++ b/lite/post.cpp @@ -351,6 +351,8 @@ if (!strcmp_i(func, _T("single"))) // Single-tier (simple) reduction. return redSingle(args,nlppp); // 05/26/01 AM. else if (!strcmp_i(func, _T("singler"))) // Single-tier, range. return redSingler(args,nlppp); // 01/17/99 AM. +else if (!strcmp_i(func, _T("makeliteral"))) // Range to one token (#450). + return redLiteral(args,nlppp); else if (!strcmp_i(func, _T("singlex"))) // Single-tier,range,excise. return redSinglex(args,nlppp); // 01/17/99 AM. if (!strcmp_i(func, _T("singlezap"))) @@ -900,6 +902,125 @@ return ok; // 11/19/99 AM. } +/******************************************** +* FN: MAKELITERAL +* SUBJ: Turn a node just built by singler() into a token (#450). +* NOTE: The node becomes a leaf named TEXT, of TYPE ("num", "alpha", +* "punct", "white", "ctrl", "emoji" or "node"), or of the type the +* tokenizer would give TEXT: all digits num, all whitespace white, +* one ASCII punctuation character punct, otherwise alpha. It keeps +* the offsets of the nodes it replaced, and its $text is TEXT. +* Shared by makeliteral() in the interpreter and the compiled runtime. +********************************************/ + +bool Pat::makeLiteral(Node *sub, _TCHAR *text, _TCHAR *type, Nlppp *nlppp) +{ +Parse *parse = nlppp->parse_; +if (!sub || !text || !*text) + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Needs a node and a non-empty text.]") << std::ends; + parse->errOut(&gerrStr,false); + return false; + } +enum Pntype typ = PNALPHA; +if (type && *type) + { + if (!strcmp_i(type, _T("num"))) typ = PNNUM; + else if (!strcmp_i(type, _T("alpha"))) typ = PNALPHA; + else if (!strcmp_i(type, _T("punct"))) typ = PNPUNCT; + else if (!strcmp_i(type, _T("white"))) typ = PNWHITE; + else if (!strcmp_i(type, _T("ctrl"))) typ = PNCTRL; + else if (!strcmp_i(type, _T("emoji"))) typ = PNEMOJI; + else if (!strcmp_i(type, _T("node"))) typ = PNNODE; + else + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Unknown type=") << type + << _T(". Use num, alpha, punct, white, ctrl, emoji or node.]") << std::ends; + parse->errOut(&gerrStr,false); + return false; + } + } +else + { + bool digits = true, white = true; + for (_TCHAR *p = text; *p; ++p) + { + if (*p < '0' || *p > '9') + digits = false; + if (*p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') + white = false; + } + if (digits) + typ = PNNUM; + else if (white) + typ = PNWHITE; + else if (!text[1] && (_TUCHAR) text[0] < 0x80 && !alphabetic(text[0])) + typ = PNPUNCT; + } + +// The replaced nodes go; the token has no children. +Tree *tree = (Tree *) parse->getTree(); +if (sub->Down()) + { + tree->deleteSubsNC(sub->Down()); + sub->setDown(0); + } + +Sym *sym = parse->getSym(text); +Pn *pn = sub->getData(); +pn->setName(sym->getStr()); +pn->setSym(sym); +pn->setType(typ); +pn->setLiteral(true); +pn->setBuilt(true); +nlppp->excised_ = true; // The matched nodes are gone, as with singlezap. +return true; +} + + +/******************************************** +* FN: REDLITERAL +* SUBJ: makeliteral(start, end, text [, type]): replace rule elements START..END +* with one token whose text is TEXT (#450). +* EX: @POST makeliteral(1, 2, "70"); @RULES _xNIL <- 7 O @@ +* Later passes see "70" as a num token: _xNUM matches it. +* NOTE: Like singlezap, it replaces the matched nodes, so put it last. +********************************************/ + +bool Pat::redLiteral(Delt *args, Nlppp *nlppp) +{ +Parse *parse = nlppp->parse_; +Delt *d1 = args, *d2 = args ? args->Right() : 0; +Delt *d3 = d2 ? d2->Right() : 0, *d4 = d3 ? d3->Right() : 0; +if (!d3 || (d4 && d4->Right())) + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Use makeliteral(start, end, text) or makeliteral(start, end, text, type).]") + << std::ends; + parse->errOut(&gerrStr,false); + return false; + } +Iarg *a3 = d3->getData(), *a4 = d4 ? d4->getData() : 0; +if (a3->getType() != IASTR || (a4 && a4->getType() != IASTR)) + { + std::_t_strstream gerrStr; + gerrStr << _T("[makeliteral: Text and type must be strings.]") << std::ends; + parse->errOut(&gerrStr,false); + return false; + } + +// Reduce the range as singler(start, end) does, then make it a token. +d2->setRight(0); +bool ok = redSingler(d1, nlppp); +d2->setRight(d3); +if (!ok || !nlppp->red_) + return false; +return makeLiteral(nlppp->red_, a3->getStr(), a4 ? a4->getStr() : 0, nlppp); +} + + /******************************************** * FN: REDSINGLEX * CR: 01/17/99 AM. diff --git a/nlp/main.cpp b/nlp/main.cpp index dfbb128c..d883604f 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.27" +#define NLP_ENGINE_VERSION "4.2.28" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &); void cmdHelpargs(_TCHAR *);