makeliteral(): replace rule elements with one token (#450) (4.2.28)

.github/workflows/build-linux.yml

@@ -1048,6 +1048,37 @@ jobs:
           fi
           echo "[smart-wildcard] wildcards stop before later elements"
 
+      - name: makeliteral() post action (#450) (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # makeliteral(start, end, text [, type]) replaces rule elements with one
+          # token whose text is the given string, typed as the tokenizer would
+          # (#450): OCR "7O" becomes a num token "70" that later passes match
+          # as _xNUM, keeping the original offsets.
+          FIX=.github/workflows/tests/literal
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          OUT="$FIX/input/text.txt_log/output.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(literal): no output.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' \
+            'He alpha text=He at 0-1' \
+            'paid alpha text=paid at 3-6' \
+            '70 num text=70 at 8-9' \
+            'dollars alpha text=dollars at 11-17' \
+            ', punct text=, at 19-19' \
+            'USA alpha text=USA at 21-23' \
+            '_xNUM matched: 70 dollars')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(literal): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[literal] makeliteral() makes a token later passes match"
+
       - name: factorial() up to 20! (Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-windows.yml

@@ -945,6 +945,36 @@ jobs:
           fi
           echo "[smart-wildcard] wildcards stop before later elements"
 
+      - name: makeliteral() post action (#450)
+        shell: bash
+        run: |
+          set -eo pipefail
+          # makeliteral(start, end, text [, type]) replaces rule elements with one
+          # token whose text is the given string, typed as the tokenizer would
+          # (#450): OCR "7O" becomes a num token "70" that later passes match
+          # as _xNUM, keeping the original offsets.
+          FIX=.github/workflows/tests/literal
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          OUT="$FIX/input/text.txt_log/output.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(literal): no output.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' \
+            'He alpha text=He at 0-1' \
+            'paid alpha text=paid at 3-6' \
+            '70 num text=70 at 8-9' \
+            'dollars alpha text=dollars at 11-17' \
+            ', punct text=, at 19-19' \
+            'USA alpha text=USA at 21-23' \
+            '_xNUM matched: 70 dollars')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(literal): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[literal] makeliteral() makes a token later passes match"
+
       - name: factorial() up to 20!
         shell: bash
         run: |

.github/workflows/tests/literal/input/text.txt

@@ -0,0 +1 @@
+He paid 7O dollars ; usa

.github/workflows/tests/literal/spec/analyzer.seq

@@ -0,0 +1,5 @@
+tokenize	nil	# 
+nlp	fix	# makeliteral(): OCR "7O" -> num token "70"
+nlp	force	# forced type and inferred punctuation
+nlp	show	# what the new tokens report
+nlp	use	# a later pass matches the literal as _xNUM

.github/workflows/tests/literal/spec/fix.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     fix.nlp
+# SUBJ:     makeliteral() (#450): OCR read "70" as "7O"; make it a number token.
+# NOTE:     Part of the literal regression; expected output is in the CI step.
+###############################################
+
+@NODES _ROOT
+
+@POST
+  makeliteral(1, 2, "70");
+@RULES
+_xNIL <- 7 O @@

.github/workflows/tests/literal/spec/force.nlp

@@ -0,0 +1,16 @@
+###############################################
+# FILE:     force.nlp
+# SUBJ:     makeliteral() with a forced type, and with an inferred punctuation type.
+###############################################
+
+@NODES _ROOT
+
+@POST
+  makeliteral(1, 1, "USA", "alpha");
+@RULES
+_xNIL <- usa @@
+
+@POST
+  makeliteral(1, 1, ",");
+@RULES
+_xNIL <- \; @@

.github/workflows/tests/literal/spec/show.nlp

@@ -0,0 +1,14 @@
+###############################################
+# FILE:     show.nlp
+# SUBJ:     Name, token type, $text and original offsets of each top node.
+###############################################
+
+@CODE
+L("n") = pndown(pnroot());
+while (L("n")) {
+    if (pntype(L("n")) != "white")
+        "output.txt" << pnname(L("n")) << " " << pntype(L("n")) << " text=" << pnvar(L("n"), "$text")
+                     << " at " << pnvar(L("n"), "$ostart") << "-" << pnvar(L("n"), "$oend") << "\n";
+    L("n") = pnnext(L("n"));
+}
+@@CODE

.github/workflows/tests/literal/spec/use.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     use.nlp
+# SUBJ:     A later pass matches the literal as an ordinary number token.
+###############################################
+
+@NODES _ROOT
+
+@POST
+  "output.txt" << "_xNUM matched: " << N("$text", 1) << " " << N("$text", 3) << "\n";
+  single();
+@RULES
+_money <- _xNUM _xWHITE [s] dollars @@

include/Api/lite/Arun.h

@@ -425,6 +425,8 @@ class LITE_API Arun
 	// POST ACTIONS
 	static bool single(Nlppp *);
 	static bool singler(Nlppp *,long long,long long);							// 06/02/00 AM.
+	static bool makeliteral(Nlppp *,long long,long long,_TCHAR *);			// #450.
+	static bool makeliteral(Nlppp *,long long,long long,_TCHAR *,_TCHAR *);	// #450.
 	static bool singlex(Nlppp *,long long,long long);							// 06/02/00 AM.
 	static bool	singlezap(Nlppp *);										// 01/04/01 AM.
 	static bool merge(Nlppp *);											// 06/02/00 AM.

include/Api/lite/pn.h

@@ -100,6 +100,7 @@ class LITE_API Pn
 	bool getUnsealed() const;		// 10/09/99 AM.
 	bool getFired() const;			// 05/21/01 Dd.
 	bool getBuilt() const;			// 05/25/01 AM.
+	bool getLiteral() const;		// Text is its own name, not the input's (#450).
 
 	void getTextStartEnd(_TCHAR* &, long &, long &);			// 12/29/99 AM.
 	void getTextUstartUend(_TCHAR* &, long &, long &);	// [UNICODE]	// 06/15/22 AM.
@@ -128,6 +129,7 @@ class LITE_API Pn
 	void setUnsealed(bool);			// 10/09/99 AM.
 	void setFired(bool);				// 05/21/01 Dd.
 	void setBuilt(bool);				// 05/25/01 AM.
+	void setLiteral(bool);				// #450.
 
 	// GET THE TEXT FOR THE CURRENT NODE.
 	_TCHAR *pnStr();																// 11/06/98 AM.

lite/Arun.cpp

@@ -5239,6 +5239,26 @@ return true;
 }
 
 
+/********************************************
+* FN:		MAKELITERAL
+* SUBJ:	makeliteral(start, end, text [, type]), compiled runtime (#450).
+* NOTE:	Reduces like singler(), then Pat::makeLiteral, as the interpreter.
+********************************************/
+
+bool Arun::makeliteral(Nlppp *nlppp, long long num1, long long num2, _TCHAR *text)
+{
+return makeliteral(nlppp, num1, num2, text, 0);
+}
+
+bool Arun::makeliteral(Nlppp *nlppp, long long num1, long long num2,
+	_TCHAR *text, _TCHAR *type)
+{
+if (!singler(nlppp, num1, num2) || !nlppp->red_)
+	return false;
+return Pat::makeLiteral(nlppp->red_, text, type, nlppp);
+}
+
+
 /********************************************
 * FN:		SINGLER
 * CR:		06/02/00 AM.

lite/fn.cpp

@@ -709,6 +709,7 @@ switch (fnid)																	// 12/21/01 AM.
 		return fnSetlookahead(args,nlppp,/*UP*/sem);					// 12/14/03 AM.
 	case FNsetunsealed:
 	case FNsingle:
+	case FNmakeliteral:		// Post action, run by Pat::postAction (#450).
 	case FNsingler:
 	case FNsinglex:
 	case FNsinglezap:

lite/func_defs.h

@@ -125,6 +125,7 @@ enum funcDef
 	FNlistadd,
 	FNlistnode,
 	FNLJ,
+	FNmakeliteral,	// #450.
 	FNlj,
 	FNloaddict,	// 06/11/26 DD.
 	FNloadkbb,	// 06/11/26 DD.

lite/funcs.h

@@ -122,6 +122,7 @@ _T("lextagger"),	// 12/11/03 AM.
 _T("listadd"),
 _T("listnode"),
 _T("LJ"),
+_T("makeliteral"),	// Post action (#450).
 _T("lj"),
 _T("loaddict"),	// 06/11/26 DD.
 _T("loadkbb"),		// 06/11/26 DD.

lite/pat.h

@@ -290,6 +290,9 @@ class Pat : public Algo
 
 	static Ielt *preWildStop(Ielt *ielt, Delt<Ielt> *next);	// #449.
 	static bool laterMatches(Delt<Ielt> *d, Node<Pn> *node, bool right);	// #427.
+	static bool redLiteral(Delt<Iarg> *args, Nlppp *nlppp);	// #450.
+	static bool makeLiteral(Node<Pn> *sub, _TCHAR *text, _TCHAR *type,
+		Nlppp *nlppp);													// #450.
 	static void markFiredRule(Nlppp *nlppp, long ruleline);	// #73.
 
 	static bool wildFillLeft(

lite/pn.cpp

@@ -64,6 +64,7 @@ static const int bitBASE = 01;		// Node's BASE attribute.
 static const int bitUNSEALED = 02;	// UNSEALED attr.				// 10/09/99 AM.
 static const int bitFIRED = 04;		// FIRED flag.					// 05/21/01 DD.
 static const int bitBUILT= 010;		// BUILT flag.					// 05/25/01 AM.
+static const int bitLITERAL = 020;	// Made by makeliteral(): text is the name (#450).
 // 02, 04, 010, 020, 040, 0100...
 
 
@@ -440,6 +441,17 @@ if (x)
 else
 	flags_ &= (~bitFIRED);
 }
+bool Pn::getLiteral() const
+{
+return (flags_ & bitLITERAL) != 0;
+}
+void Pn::setLiteral(bool x)
+{
+if (x)
+	flags_ |= bitLITERAL;
+else
+	flags_ &= (~bitLITERAL);
+}
 void Pn::setBuilt(bool x)													// 05/25/01 AM.
 {
 if (x)
@@ -641,6 +653,8 @@ for (; nodes; nodes = nodes->Right())
 
 _TCHAR *Pn::pnStr()
 {
+if (getLiteral() && name_)		// A literal's text isn't the input's (#450).
+	return make_str(name_, (long) _tcslen(name_));
 return make_str(Text, End - Start + 1);
 }
 
@@ -649,6 +663,8 @@ _TCHAR *Pn::pnStr(
 	long &len		/*UP*/													// 07/23/00 AM.
 	)
 {
+if (getLiteral() && name_)		// A literal's text isn't the input's (#450).
+	return make_str(name_, len = (long) _tcslen(name_));
 return make_str(Text, len = (End - Start + 1));
 }