Wildcards stop before later elements, not only the next one (#427) (4.2.27)

.github/workflows/build-linux.yml

@@ -1017,6 +1017,37 @@ jobs:
           fi
           echo "[element-pre] element-key conditions match like @PRE"
 
+      - name: Wildcards stop before later elements (#427) (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # With a wildcard pending, an optional element is absent if a later
+          # element (up to the next required one) matches first (#427), so
+          # the wildcard no longer runs past it. Both directions; ties keep
+          # the optional element.
+          FIX=.github/workflows/tests/smart-wildcard
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          OUT="$FIX/input/text.txt_log/output.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(smart-wildcard): no output.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' \
+            'right: [the big bears] wild=big three=0 bears=bears' \
+            'right: [the big three bears] wild=big three=three bears=bears' \
+            'present: [the big bears] three=0' \
+            'present: [the big three bears] three=three' \
+            'tie: [the big bears] opt=big' \
+            'left: [bears met]' \
+            'left: [bears met]')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(smart-wildcard): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[smart-wildcard] wildcards stop before later elements"
+
       - name: factorial() up to 20! (Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-windows.yml

@@ -915,6 +915,36 @@ jobs:
           fi
           echo "[element-pre] element-key conditions match like @PRE"
 
+      - name: Wildcards stop before later elements (#427)
+        shell: bash
+        run: |
+          set -eo pipefail
+          # With a wildcard pending, an optional element is absent if a later
+          # element (up to the next required one) matches first (#427), so
+          # the wildcard no longer runs past it. Both directions; ties keep
+          # the optional element.
+          FIX=.github/workflows/tests/smart-wildcard
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          OUT="$FIX/input/text.txt_log/output.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(smart-wildcard): no output.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' \
+            'right: [the big bears] wild=big three=0 bears=bears' \
+            'right: [the big three bears] wild=big three=three bears=bears' \
+            'present: [the big bears] three=0' \
+            'present: [the big three bears] three=three' \
+            'tie: [the big bears] opt=big' \
+            'left: [bears met]' \
+            'left: [bears met]')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(smart-wildcard): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[smart-wildcard] wildcards stop before later elements"
+
       - name: factorial() up to 20!
         shell: bash
         run: |

.github/workflows/tests/smart-wildcard/input/text.txt

@@ -0,0 +1,3 @@
+the big bears met three bears
+the big three bears
+bears big three bears met

.github/workflows/tests/smart-wildcard/spec/analyzer.seq

@@ -0,0 +1,7 @@
+tokenize	nil	# 
+nlp	lines	# one _line per input line
+nlp	nowhite	# drop whitespace so the rules read like #427
+nlp	right	# optional element absent when a later one matches first
+nlp	present	# optional element present when it comes first
+nlp	tie	# same node: the optional element wins
+nlp	left	# the same, matching leftward from a trigger

.github/workflows/tests/smart-wildcard/spec/left.nlp

@@ -0,0 +1,14 @@
+###############################################
+# FILE:     left.nlp
+# SUBJ:     The same rule matching leftward from a trigger. On "bears big
+#           three bears met", leftward from "met", the optional "three" is
+#           absent because the required "bears" matches first.
+###############################################
+
+@NODES _line
+
+@POST
+  "output.txt" << "left: [" << phrasetext() << "]\n";
+  noop();
+@RULES
+_xNIL <- bears _xWILD [opt match=(three)] _xWILD met [trig] @@

.github/workflows/tests/smart-wildcard/spec/lines.nlp

@@ -0,0 +1,9 @@
+###############################################
+# FILE:     lines.nlp
+# SUBJ:     One _line node per input line.
+###############################################
+
+@NODES _ROOT
+
+@RULES
+_line <- _xWILD [plus fail=(\n)] @@

.github/workflows/tests/smart-wildcard/spec/nowhite.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     nowhite.nlp
+# SUBJ:     Drop the spaces inside each line, so the rules can be written
+#           word by word, as in #427.
+###############################################
+
+@NODES _line
+
+@POST
+  excise(1,1);
+@RULES
+_xNIL <- _xWHITE @@

.github/workflows/tests/smart-wildcard/spec/present.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     present.nlp
+# SUBJ:     The optional element still matches when it comes first.
+###############################################
+
+@NODES _line
+
+@POST
+  "output.txt" << "present: [" << phrasetext() << "] three=" << N("$text", 3) << "\n";
+  noop();
+@RULES
+_xNIL <- the _xWILD _xWILD [opt match=(three)] bears @@

.github/workflows/tests/smart-wildcard/spec/right.nlp

@@ -0,0 +1,17 @@
+###############################################
+# FILE:     right.nlp
+# SUBJ:     #427: with a wildcard pending, an optional element is absent if a
+#           later element matches first. On "the big bears met three bears"
+#           the wildcard used to run past the first "bears" to reach "three".
+# NOTE:     Part of the smart-wildcard regression; expected output is in the
+#           CI step.
+###############################################
+
+@NODES _line
+
+@POST
+  "output.txt" << "right: [" << phrasetext() << "] wild=" << N("$text", 2)
+               << " three=" << N("$text", 3) << " bears=" << N("$text", 4) << "\n";
+  noop();
+@RULES
+_xNIL <- the _xWILD _xWILD [opt match=(three)] _xWILD [one match=(bears)] @@

.github/workflows/tests/smart-wildcard/spec/tie.nlp

@@ -0,0 +1,13 @@
+###############################################
+# FILE:     tie.nlp
+# SUBJ:     When the optional element and a later one both match the same
+#           node, the optional element wins, as before.
+###############################################
+
+@NODES _line
+
+@POST
+  "output.txt" << "tie: [" << phrasetext() << "] opt=" << N("$text", 3) << "\n";
+  noop();
+@RULES
+_xNIL <- the _xWILD _xALPHA [opt] bears @@

include/Api/lite/Arun.h

@@ -258,6 +258,7 @@ class LITE_API Arun
 	static bool match_simple(NODE *,const ELT *,Nlppp *);			// 05/23/00 AM.
 	static int  stop_ord(Nlppp *, int dir);							// #449.
 	static bool stop_match(NODE *, int ord, Nlppp *);				// #449.
+	static bool later_matches(Nlppp *, int ord, int dir);			// #427.
 	static bool match_singlet(NODE *,const ELT *,Nlppp *);
 	static bool match_tree(NODE *,const ELT *,Nlppp *);
 	static bool match_list(NODE *,const ELT *,Nlppp *);

lite/Arun.cpp

@@ -583,6 +583,9 @@ if (nlppp->wildord_)				// If pending wildcards.
 		{
 		if ((matched = match_real_left(match,nlppp)))
 			break;
+		// An optional element is absent if a later element matches first (#427).
+		if (!elt->min && later_matches(nlppp, ord, -1))
+			break;
 		++count;
 		if (limit && count > limit)
 			break;
@@ -687,6 +690,9 @@ if (nlppp->wildord_)				// If pending wildcards.
 		{
 		if ((matched = match_real_right(match,nlppp)))					// 05/23/00 AM.
 			break;
+		// An optional element is absent if a later element matches first (#427).
+		if (!elt->min && later_matches(nlppp, ord, 1))
+			break;
 		++count;
 		if (limit && count > limit)
 			break;
@@ -1035,6 +1041,34 @@ return next;
 }
 
 
+/********************************************
+* FN:		LATER_MATCHES
+* SUBJ:	Whether the current node matches one of the elements after ORD in
+*			direction DIR, up to and including the first required one (#427).
+* NOTE:	Compiled twin of Pat::laterMatches: with a wildcard pending, an
+*			optional element is absent if a later element matches first. A bare
+*			wildcard among them ends the look.
+********************************************/
+
+bool Arun::later_matches(Nlppp *nlppp, int ord, int dir)
+{
+NODE *node = (NODE *) nlppp->node_;
+if (!node)
+	return false;
+for (int k = ord + dir; k >= 1 && k <= nlppp->len_; k += dir)
+	{
+	const ELT *e = &(nlppp->elts_[k]);
+	if (e->wild)
+		return false;
+	if (stop_match(node, k, nlppp))
+		return true;
+	if (e->min)
+		return false;		// Elements past a required one don't count.
+	}
+return false;
+}
+
+
 /********************************************
 * FN:		STOP_MATCH
 * SUBJ:	Whether NODE matches compiled rule element ORD, using the match
@@ -3211,6 +3245,9 @@ if (nlppp->wildord_)				// If pending wildcards.
 		{
 		if ((matched = special_real_left(match,micro,nlppp)))
 			break;
+		// An optional element is absent if a later element matches first (#427).
+		if (!elt->min && later_matches(nlppp, ord, -1))
+			break;
 		++count;
 		if (limit && count > limit)
 			break;
@@ -3311,6 +3348,9 @@ if (nlppp->wildord_)				// If pending wildcards.
 		{
 		if ((matched = special_real_right(match,micro,nlppp)))
 			break;
+		// An optional element is absent if a later element matches first (#427).
+		if (!elt->min && later_matches(nlppp, ord, 1))
+			break;
 		++count;
 		if (limit && count > limit)
 			break;

lite/pat.cpp

@@ -1448,7 +1448,8 @@ Node<Pn> *real = 0;	// First real node matched, moving left.
 //if (!matchRealLeft(curr,start,wildone,wildmin,wildmax, /*DU*/ real,node))
 if (!matchRealLeft(ielt,nlppp->start_,wildone,wildmin,wildmax,
 				/*DU*/ real,nlppp->node_,
-				preWildStop(ielt, curr->Left())))						// #449.
+				preWildStop(ielt, curr->Left()),						// #449.
+				curr->Left()))											// #427.
 	{
 	if (optionalElt(ielt))		// If current rule elt is optional, succeed.
 		{
@@ -1600,7 +1601,8 @@ long tcount = 0;														// 02/04/05 AM.
 //if (!matchRealRight(curr,end,wildone,wildmin,wildmax, /*DU*/ real,node))
 if (!matchRealRight(ielt,nlppp->end_,wildone,wildmin,wildmax,nlppp,
 			/*DU*/ real,nlppp->node_,tcount,
-			preWildStop(ielt, curr->Right())))						// #449.
+			preWildStop(ielt, curr->Right()),						// #449.
+			curr->Right()))											// #427.
 	{
 //	nlppp->span_ += tcount;										// 02/04/05 AM.
 	if (optionalElt(ielt))		// If current rule elt is optional, succeed.
@@ -1866,7 +1868,8 @@ bool Pat::matchRealLeft(
 	Node<Pn>* &real,		// First real node matched (moving left).
 								// (Allows for repetition.)
 	Node<Pn>* &node,		// DN: Current node.  UP: Last real node matched.
-	Ielt *stop				// Stop before a match of this elt (#449).
+	Ielt *stop,				// Stop before a match of this elt (#449).
+	Delt<Ielt> *next		// Elements after an optional one (#427).
 	)
 {
 if (!node)		// This can happen now, to handle wildcard at start of rule.
@@ -1895,6 +1898,11 @@ for (;;)		// Look for real match till done.
 		return true;
 		}
 
+	// While a wildcard is pending, an optional element is absent if a later
+	// element (up to the next required one) matches before it does (#427).
+	if (wildone && next && optionalElt(ielt) && laterMatches(next, test, false))
+		return false;
+
 	if (!wildone)
 		return false;		// No wildcards pending.  Fail.
 	if (wildmax && count > wildmax)
@@ -1932,7 +1940,8 @@ bool Pat::matchRealRight(
 								// (Allows for repetition.)
 	Node<Pn>* &node,		// DN: Current node.  UP: Last real node matched.
 	long &tot,															// 02/04/05 AM.
-	Ielt *stop				// Stop before a match of this elt (#449).
+	Ielt *stop,				// Stop before a match of this elt (#449).
+	Delt<Ielt> *next		// Elements after an optional one (#427).
 	)
 {
 if (!node)		// This can happen now, to handle wildcard at end of rule.
@@ -1970,6 +1979,11 @@ for (;;)		// Look for real match till done.
 	else																		// 02/04/05 AM.
 		tot += tcount;														// 02/04/05 AM.
 
+	// While a wildcard is pending, an optional element is absent if a later
+	// element (up to the next required one) matches before it does (#427).
+	if (wildone && next && optionalElt(ielt) && laterMatches(next, test, true))
+		return false;
+
 	if (!wildone)
 		return false;		// No wildcards pending.  Fail.
 	if (wildmax && count > wildmax)
@@ -2311,6 +2325,35 @@ return true;																	// 07/31/02 AM.
 }
 
 
+/********************************************
+* FN:		LATERMATCHES
+* SUBJ:	Whether NODE matches one of the elements from D onward, in the
+*			matching direction, up to and including the first required one (#427).
+* NOTE:	With a wildcard pending, an optional element used to be searched for
+*			past the elements after it: in  _xWILD _xWILD [opt match=(three)]
+*			_xWILD [one match=(bears)]  the wildcard ran past "bears" to reach a
+*			later "three". A bare wildcard among them ends the look: what it
+*			covers isn't known yet.
+********************************************/
+
+bool Pat::laterMatches(Delt<Ielt> *d, Node<Pn> *node, bool right)
+{
+if (!node)
+	return false;
+for (; d; d = (right ? d->Right() : d->Left()))
+	{
+	Ielt *e = d->getData();
+	if (!e || wildElt(e))
+		return false;
+	if (Match(e, node))
+		return true;
+	if (!optionalElt(e))
+		return false;		// Elements past a required one don't count.
+	}
+return false;
+}
+
+
 /********************************************
 * FN:		WILDFILLLEFT
 * CR:		10/30/98 AM.

lite/pat.h

@@ -275,7 +275,8 @@ class Pat : public Algo
 		Node<Pn>* &real,		// First real node matched (moving left).
 									// (Allows for repetition.)
 		Node<Pn>* &node,		// DN: Current node.  UP: Last real node matched.
-		Ielt *stop = 0			// Stop before a match of this elt (#449).
+		Ielt *stop = 0,			// Stop before a match of this elt (#449).
+		Delt<Ielt> *next = 0	// Elements after an optional one (#427).
 		);
 
 	static bool trueMatchLeft(
@@ -288,6 +289,7 @@ class Pat : public Algo
 		);
 
 	static Ielt *preWildStop(Ielt *ielt, Delt<Ielt> *next);	// #449.
+	static bool laterMatches(Delt<Ielt> *d, Node<Pn> *node, bool right);	// #427.
 	static void markFiredRule(Nlppp *nlppp, long ruleline);	// #73.
 
 	static bool wildFillLeft(
@@ -310,7 +312,8 @@ class Pat : public Algo
 									// (Allows for repetition.)
 		Node<Pn>* &node,		// DN: Current node.  UP: Last real node matched.
 		long & tot,				// 02/04/05 AM.
-		Ielt *stop = 0			// Stop before a match of this elt (#449).
+		Ielt *stop = 0,			// Stop before a match of this elt (#449).
+		Delt<Ielt> *next = 0	// Elements after an optional one (#427).
 		);
 
 	static bool trueMatchRight(

nlp/main.cpp

@@ -15,7 +15,7 @@ All rights reserved.
 #include "lite/nlp_engine.h"
 #include "version.h"
 
-#define NLP_ENGINE_VERSION "4.2.26"
+#define NLP_ENGINE_VERSION "4.2.27"
 
 bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &);
 void cmdHelpargs(_TCHAR *);