A rec pass that never settles is stopped with a message (4.2.47)

.github/workflows/build-linux.yml

@@ -378,6 +378,35 @@ jobs:
           done
           echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs"
 
+      - name: A rec pass that never settles is stopped (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # A rec pass repeats while any rule matches, so a rule that kept
+          # matching without changing what it matched (parse-en-us pos50 on
+          # "much I have been to blame.") hung forever with nothing said. A
+          # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT
+          # scales that and 0 lifts it. The fixture's rec pass takes 3 matches
+          # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and
+          # the pass after it still runs.
+          FIX=.github/workflows/tests/rec-limit
+          rm -rf "$FIX/logs" "$FIX"/input/*_log
+          out=$(./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r')
+          if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then
+            echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1
+          fi
+          rm -rf "$FIX/logs" "$FIX"/input/*_log
+          out=$(NLP_REC_LIMIT=1 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r')
+          if [ "$out" != "matches=2" ]; then
+            echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1
+          fi
+          if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then
+            echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1
+          fi
+          rm -f rec.err
+          echo "[rec-limit] a rec pass region past its match limit is stopped, with a message"
+
       - name: Lazy kbb lookups from any pass, and << to a path (Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-macos.yml

@@ -225,6 +225,34 @@ jobs:
           done
           echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs"
 
+      - name: A rec pass that never settles is stopped
+        shell: bash
+        run: |
+          set -eo pipefail
+          # A rec pass repeats while any rule matches, so a rule that kept
+          # matching without changing what it matched (parse-en-us pos50 on
+          # "much I have been to blame.") hung forever with nothing said. A
+          # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT
+          # scales that and 0 lifts it. The fixture's rec pass takes 3 matches
+          # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and
+          # the pass after it still runs.
+          FIX=.github/workflows/tests/rec-limit
+          rm -rf "$FIX/logs" "$FIX"/input/*_log
+          out=$(./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r')
+          if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then
+            echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1
+          fi
+          rm -rf "$FIX/logs" "$FIX"/input/*_log
+          out=$(NLP_REC_LIMIT=1 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r')
+          if [ "$out" != "matches=2" ]; then
+            echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1
+          fi
+          if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then
+            echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1
+          fi
+          rm -f rec.err
+          echo "[rec-limit] a rec pass region past its match limit is stopped, with a message"
+
       - name: Lazy kbb lookups from any pass, and << to a path
         shell: bash
         run: |

.github/workflows/build-windows.yml

@@ -313,6 +313,34 @@ jobs:
           done
           echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs"
 
+      - name: A rec pass that never settles is stopped
+        shell: bash
+        run: |
+          set -eo pipefail
+          # A rec pass repeats while any rule matches, so a rule that kept
+          # matching without changing what it matched (parse-en-us pos50 on
+          # "much I have been to blame.") hung forever with nothing said. A
+          # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT
+          # scales that and 0 lifts it. The fixture's rec pass takes 3 matches
+          # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and
+          # the pass after it still runs.
+          FIX=.github/workflows/tests/rec-limit
+          rm -rf "$FIX/logs" "$FIX"/input/*_log
+          out=$(./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r')
+          if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then
+            echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1
+          fi
+          rm -rf "$FIX/logs" "$FIX"/input/*_log
+          out=$(NLP_REC_LIMIT=1 ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r')
+          if [ "$out" != "matches=2" ]; then
+            echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1
+          fi
+          if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then
+            echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1
+          fi
+          rm -f rec.err
+          echo "[rec-limit] a rec pass region past its match limit is stopped, with a message"
+
       - name: Lazy kbb lookups from any pass, and << to a path
         shell: bash
         run: |

.github/workflows/tests/rec-limit/input/text.txt

@@ -0,0 +1 @@
+one two three

.github/workflows/tests/rec-limit/spec/analyzer.seq

@@ -0,0 +1,4 @@
+tokenize	nil	# 
+nlp	number	# number the words 0, 1, 2
+rec	cycle	# matches each word once, in turn: 3 matches in one region
+nlp	report	# runs after the rec pass, stopped or not

.github/workflows/tests/rec-limit/spec/cycle.nlp

@@ -0,0 +1,14 @@
+# A rec pass that takes one match per word. With NLP_REC_LIMIT=1 the
+# region may take only one, so it is stopped after its second.
+@NODES _ROOT
+
+@CHECK
+  if (N("i",1) != G("t") % 3)
+    fail();
+@POST
+  N("hits",1) = N("hits",1) + 1;
+  G("t")++;
+@RULES
+_xNIL <-
+	_xALPHA
+	@@

.github/workflows/tests/rec-limit/spec/number.nlp

@@ -0,0 +1,9 @@
+@NODES _ROOT
+
+@POST
+  N("i",1) = G("n");
+  G("n")++;
+@RULES
+_xNIL <-
+	_xALPHA
+	@@

.github/workflows/tests/rec-limit/spec/report.nlp

@@ -0,0 +1,3 @@
+@CODE
+cout() << "matches=" << G("t") << "\n";
+@@CODE

include/Api/lite/parse.h

@@ -138,6 +138,7 @@ class LITE_API Parse
 	bool getCodeexit();				// @CODE exited the current pass (#392).
 	_TCHAR *getAlgo();					// 05/31/00 AM.
 	bool	getIntern();				// 05/26/01 AM.
+	bool	getRulesFile() {return rulesFile_;}
 	long	getCurrpass();	// REN.	// 08/22/02 AM.
 	long	getRulepass();				// 02/03/05 AM.
 	long	getInputpass();			// 08/23/02 AM.
@@ -193,6 +194,7 @@ class LITE_API Parse
 	void setAlgo(_TCHAR *);			// 05/31/00 AM.
 	void setFbatchstart(bool);		// 10/19/00 AM.
 	void setIntern(bool);			// 05/26/01 AM.
+	void setRulesFile(bool x) {rulesFile_ = x;}
 	void setCurrpass(long);// REN.	// 08/22/02 AM.
 	void setRulepass(long);			// 02/03/05 AM.
 	void setInputpass(long);		// 08/23/02 AM.
@@ -238,6 +240,7 @@ class LITE_API Parse
 	bool errOut(std::_t_strstream*,bool,bool=false);                 // 02/25/05 AM.
 	// Echo an NLP++ error to stderr: region ("@CODE"), what it skipped.
 	void nlpppError(const _TCHAR *region, const _TCHAR *effect);
+	void recLimitError(long ruleline, long matches, long nodes);
 
 	inline void prettyPassnum(int,  /*UP*/ _TCHAR *);
 	inline void prettyParsedone();
@@ -603,6 +606,7 @@ class LITE_API Parse
 	// If this parse interns an analyzers (eg, RFB or user's analyzer),
 	// then record if any syntax errors were encountered.			// 05/26/01 AM.
 	bool intern_;	// If interned analyzer successfully.			// 05/26/01 AM.
+	bool rulesFile_;	// If parsing a rules file with the RFB grammar.
 
 	Dlist<Iarg> *blobs_;	// Global list for blobs.					// 02/27/03 AM.
 
@@ -665,6 +669,7 @@ class LITE_API Parse
 	long zoneNum_;			// Zone being run, from 1; 0 when not in zone mode.
 	bool zoneFirst_;		// $isfirstzone.
 	bool zoneLast_;		// $islastzone.
+	std::set<std::pair<long,long>> recLimitErrs_;	// (pass, rule line) reported.
 
 	// Pass info to analyzer, to G("$datum") variable.				// 03/13/03 AM.
 	_TCHAR *datum_;																// 03/13/03 AM.

lite/parse.cpp

@@ -89,6 +89,7 @@ verbose = false;		// Default.  // 12/26/98 AM.
 intern_ = true;																// 05/26/01 AM.
 zoneNum_ = 0;
 zoneFirst_ = zoneLast_ = false;
+rulesFile_ = false;
 currpass_ = 0;			// 07/24/01 AM.
 rulepass_ = 0;			// 02/03/05 AM.
 inputpass_ = 0;		// 08/23/02 AM.
@@ -2010,6 +2011,46 @@ if (input && *input)
 std::_t_cerr << _T(". Details in err.log.]") << std::endl;
 }
 
+/********************************************
+* FN:		RECLIMITERROR
+* SUBJ:	Report a rec pass region stopped at the match limit.
+* NOTE:	A rule that keeps matching without changing what it matched
+*			(eg, its loop guard is never set) repeated a rec pass forever,
+*			with nothing on the console. Once per input for each pass and
+*			rule.
+********************************************/
+
+void Parse::recLimitError(long ruleline, long matches, long nodes)
+{
+if (!recLimitErrs_.insert(std::make_pair(currpass_, ruleline)).second)
+	return;
+
+_TCHAR *file = 0;
+Dlist<Seqn> *seqs = ana ? ana->getSeqlist() : 0;
+for (Delt<Seqn> *d = seqs ? seqs->getFirst() : 0; d; d = d->Right())
+	if (d->getData()->getPassnum() == currpass_)
+		{
+		file = d->getData()->getRulesfilename();
+		break;
+		}
+
+std::basic_ostringstream<_TCHAR> msg;
+msg << _T("[Error in rec pass ") << currpass_;
+if (file && *file)
+	msg << _T(" (") << file << _T(")");
+msg << _T(": rules matched ") << matches << _T(" times in a region of ")
+	<< nodes << _T(" nodes without settling, so the region was stopped there.")
+	<< _T(" Last rule to match: line ") << ruleline;
+if (input && *input)
+	msg << _T(". Input: ") << input;
+msg << _T(". NLP_REC_LIMIT sets the limit; 0 lifts it.]");
+std::_t_cerr << msg.str() << std::endl;
+
+std::_t_strstream gerrStr;
+gerrStr << msg.str() << std::ends;
+::errOut(&gerrStr,false,currpass_,ruleline);
+}
+
 /********************************************
 * FN:		NEWBLOB
 * CR:		02/27/03 AM.

lite/rec.cpp

@@ -455,6 +455,8 @@ if (!start || !rules || !parse)
 Delt<Irule> *prule;			// Current rules; also, the rule that matched.
 Node<Pn> *node;
 bool matched = false;		// If any rules matched.
+long matches = 0;			// Matches in this region, for recLimit().
+long size = regionSize(start, end);	// Nodes in this region at the start.
 
 // 11/20/99 AM. Using this object to collect the entire parse state.
 Nlppp nlppp(															// 11/20/99 AM.
@@ -525,7 +527,12 @@ for (;;)							// WHILE RECURSING....
 				}
 
 			matched = true;	// A RULE MATCHED IN THIS CYCLE.
-
+			if (overRecLimit(&nlppp, matches, size))	// Region never settles.
+				{
+				delete (Tree<Pn> *) nlppp.collect_;
+				return true;
+				}
+
 			// Optimization.  Go back according to SPAN.				// 02/04/05 AM.
 //			start = recMoveback(&nlppp);									// 02/04/05 AM.
 
@@ -589,6 +596,8 @@ Slist<Irule> *rules = 0;	// Current rules list.
 Selt<Irule> *prule = 0;		// Current rules; also, the rule that matched.
 Node<Pn> *node;
 bool matched = false;
+long matches = 0;			// Matches in this region, for recLimit().
+long size = regionSize(start, end);	// Nodes in this region at the start.
 
 // 11/20/99 AM. Using this object to collect the entire parse state.
 Nlppp nlppp(															// 11/20/99 AM.
@@ -725,6 +734,13 @@ for (;;)							// WHILE RECURSING.....
 			nlppp.sem_ = 0;		// Zero out.					// 11/20/99 AM.
 			nlppp.dsem_ = 0;		// Zero out.					// 11/20/99 AM.
 			matched = true;			// A RULE MATCHED IN THIS CYCLE.
+			if (overRecLimit(&nlppp, matches, size))	// Region never settles.
+				{
+				if (rules)
+					Slist<Irule>::DeleteSlist(rules);
+				delete (Tree<Pn> *) nlppp.collect_;
+				return true;
+				}
 			node = matchRestart(&nlppp);	// RECOPT1	// 04/27/06 AM.
 			break;						// RESTART AT BEGINNING OF PHRASE.
 			}
@@ -742,6 +758,62 @@ delete (Tree<Pn> *) nlppp.collect_;
 return true;
 }
 
+/********************************************
+* FN:		RECLIMIT
+* SUBJ:	Scale of the most rule matches one region of a rec pass may take.
+* NOTE:	RecinfLoop only counts a match that rebuilds a node over the same
+*			span, and not one that regroups or excises, so a rule whose
+*			loop guard is never set (parse-en-us pos50, "much I have been
+*			to blame.") repeated a pass forever. overRecLimit counts every
+*			match. NLP_REC_LIMIT sets the scale (default 1000); 0 means no
+*			limit.
+********************************************/
+
+long Rec::recLimit()
+{
+static long limit = -1;
+if (limit < 0)
+	{
+	const char *s = getenv("NLP_REC_LIMIT");
+	limit = (s && *s) ? atol(s) : 1000;
+	if (limit < 0)
+		limit = 0;
+	}
+return limit;
+}
+
+// Nodes in a region, start through end (or the end of the list).
+long Rec::regionSize(Node<Pn> *start, Node<Pn> *end)
+{
+long n = 0;
+for (Node<Pn> *x = start; x; x = x->Right())
+	{
+	++n;
+	if (x == end)
+		break;
+	}
+return n;
+}
+
+// A region may take recLimit() matches plus recLimit()/50 per node it
+// started with: 1,000 plus 20 a node by default. Measured over every
+// analyzer in the analyzers repo and a 738 KB novel, no region took more
+// than 2.3 matches a node. The engine's own rules-file grammar is not
+// limited: it reads a whole pass file as one region.
+bool Rec::overRecLimit(Nlppp *nlppp, long &matches, long size)
+{
+long scale = recLimit();
+if (!scale || nlppp->parse_->getRulesFile())
+	return false;
+long long limit = (long long) scale + (long long) scale * size / 50;
+if (++matches <= limit)
+	return false;
+nlppp->parse_->recLimitError(nlppp->rule_ ? nlppp->rule_->getLine() : 0,
+	matches - 1, size);
+return true;
+}
+
+
 /********************************************
 * FN:		RECINFLOOP
 * CR:		06/09/05 AM.
@@ -1172,6 +1244,8 @@ if (!start || !pass->rules || !parse)
 //Delt<Irule> *prule;			// Current rules; also, the rule that matched.
 Node<Pn> *node;
 bool matched = false;		// If any rules matched.
+long matches = 0;			// Matches in this region, for recLimit().
+long size = regionSize(start, end);	// Nodes in this region at the start.
 
 // Using this object to collect the entire parse state.
 // Reset the nlppp object.
@@ -1293,6 +1367,8 @@ for (;;)							// WHILE RECURSING....
 				}
 
 			matched = true;	// A RULE MATCHED IN THIS CYCLE.
+			if (overRecLimit(nlppp, matches, size))	// Region never settles.
+				return true;
 			// RESET START AND END OF PHRASE.
 			start = nlppp->start_ = nlppp->select_->Down();
 			end = nlppp->end_ = 0;
@@ -1596,6 +1672,8 @@ if (!start || !pass->rules || !parse)
 //Delt<Irule> *prule;			// Current rules; also, the rule that matched.
 Node<Pn> *node;
 bool matched = false;		// If any rules matched.
+long matches = 0;			// Matches in this region, for recLimit().
+long size = regionSize(start, end);	// Nodes in this region at the start.
 
 // Using this object to collect the entire parse state.
 // Reset the nlppp object.
@@ -1710,6 +1788,8 @@ for (;;)							// WHILE RECURSING....
 				}
 
 			matched = true;	// A RULE MATCHED IN THIS CYCLE.
+			if (overRecLimit(nlppp, matches, size))	// Region never settles.
+				return true;
 			// RESET START AND END OF PHRASE.
 			start = nlppp->start_ = nlppp->select_->Down();
 			end = nlppp->end_ = 0;

lite/rec.h

@@ -119,6 +119,13 @@ class Rec : public Algo
 		Node<Pn> *select
 		);
 
+	// Most rule matches one region of a rec pass may take before the
+	// region is stopped. NLP_REC_LIMIT overrides it; 0 means no limit.
+	static long recLimit();
+	// Count a match in the current region; true once over the limit.
+	static bool overRecLimit(Nlppp *nlppp, long &matches, long size);
+	static long regionSize(Node<Pn> *start, Node<Pn> *end);
+
 	static	// RECINFLOOP	// 05/10/13 AM.
 	inline bool RecinfLoop(		// 06/09/05 AM.
 		Nlppp *nlppp,