From f765a543c25170dd1118a2c544c7ee76aed8e8b1 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Mon, 26 Oct 2026 12:42:30 -0400 Subject: [PATCH] A rec pass that never settles is stopped with a message (4.2.47) A rec pass repeats a region while any rule matches. A rule that keeps matching without changing what it matched loops forever: parse-en-us hung on "much I have been to blame." (fixed in parse-en-us #10), with nothing on the console. RecinfLoop (2005) only counts a match that rebuilds a node over the same span, and skips one that regroups or excises, so it never saw this one; the basic RecmatchNodes has no check at all. All four rec match loops now count matches per region. Past the limit the region is stopped, the rest of the pass goes on, and one line goes to stderr and err.log, once per input for each pass and rule: [Error in rec pass 75 (pos50): rules matched 1120 times in a region of 6 nodes without settling, so the region was stopped there. Last rule to match: line 15004. Input: .../w1.txt. NLP_REC_LIMIT sets the limit; 0 lifts it.] The limit is 1,000 matches plus 20 per node in the region. Measured over all 18 analyzers in the analyzers repo and a 738 KB novel, no region took more than 2.3 matches a node. NLP_REC_LIMIT scales it (the default is 1000) and 0 lifts it. The engine's own rules-file grammar is not limited: it reads a whole pass file as one region (pos50.nlp is ~25,000 matches), so Parse::rulesFile_ marks those parses. CI: tests/rec-limit has a rec pass that takes 3 matches in a 6-node region. At the default it runs silently; at NLP_REC_LIMIT=1 it is stopped at 2 with the message and the next pass still runs. On master that check fails (matches=3, no message). Checked: all Windows fixture steps pass; parse-en-us output on its three inputs is byte-identical to master's (454 files); no limit errors on any of the 18 analyzers or the novel; the unfixed parse-en-us now stops in about a second where it hung. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-linux.yml | 29 +++++++ .github/workflows/build-macos.yml | 28 +++++++ .github/workflows/build-windows.yml | 28 +++++++ .../workflows/tests/rec-limit/input/text.txt | 1 + .../tests/rec-limit/spec/analyzer.seq | 4 + .../workflows/tests/rec-limit/spec/cycle.nlp | 14 ++++ .../workflows/tests/rec-limit/spec/number.nlp | 9 ++ .../workflows/tests/rec-limit/spec/report.nlp | 3 + include/Api/lite/parse.h | 5 ++ lite/parse.cpp | 41 ++++++++++ lite/rec.cpp | 82 ++++++++++++++++++- lite/rec.h | 7 ++ lite/rfa.cpp | 1 + nlp/main.cpp | 2 +- 14 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/tests/rec-limit/input/text.txt create mode 100644 .github/workflows/tests/rec-limit/spec/analyzer.seq create mode 100644 .github/workflows/tests/rec-limit/spec/cycle.nlp create mode 100644 .github/workflows/tests/rec-limit/spec/number.nlp create mode 100644 .github/workflows/tests/rec-limit/spec/report.nlp diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 30acc5c1..00dcfd38 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -378,6 +378,35 @@ jobs: done echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs" + - name: A rec pass that never settles is stopped (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # A rec pass repeats while any rule matches, so a rule that kept + # matching without changing what it matched (parse-en-us pos50 on + # "much I have been to blame.") hung forever with nothing said. A + # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT + # scales that and 0 lifts it. The fixture's rec pass takes 3 matches + # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and + # the pass after it still runs. + FIX=.github/workflows/tests/rec-limit + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then + echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1 + fi + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(NLP_REC_LIMIT=1 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=2" ]; then + echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1 + fi + if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then + echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1 + fi + rm -f rec.err + echo "[rec-limit] a rec pass region past its match limit is stopped, with a message" + - name: Lazy kbb lookups from any pass, and << to a path (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 35c4fd53..33ea8a79 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -225,6 +225,34 @@ jobs: done echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs" + - name: A rec pass that never settles is stopped + shell: bash + run: | + set -eo pipefail + # A rec pass repeats while any rule matches, so a rule that kept + # matching without changing what it matched (parse-en-us pos50 on + # "much I have been to blame.") hung forever with nothing said. A + # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT + # scales that and 0 lifts it. The fixture's rec pass takes 3 matches + # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and + # the pass after it still runs. + FIX=.github/workflows/tests/rec-limit + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then + echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1 + fi + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(NLP_REC_LIMIT=1 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=2" ]; then + echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1 + fi + if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then + echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1 + fi + rm -f rec.err + echo "[rec-limit] a rec pass region past its match limit is stopped, with a message" + - name: Lazy kbb lookups from any pass, and << to a path shell: bash run: | diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 25c5fe77..8c3706a2 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -313,6 +313,34 @@ jobs: done echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs" + - name: A rec pass that never settles is stopped + shell: bash + run: | + set -eo pipefail + # A rec pass repeats while any rule matches, so a rule that kept + # matching without changing what it matched (parse-en-us pos50 on + # "much I have been to blame.") hung forever with nothing said. A + # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT + # scales that and 0 lifts it. The fixture's rec pass takes 3 matches + # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and + # the pass after it still runs. + FIX=.github/workflows/tests/rec-limit + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then + echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1 + fi + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(NLP_REC_LIMIT=1 ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=2" ]; then + echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1 + fi + if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then + echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1 + fi + rm -f rec.err + echo "[rec-limit] a rec pass region past its match limit is stopped, with a message" + - name: Lazy kbb lookups from any pass, and << to a path shell: bash run: | diff --git a/.github/workflows/tests/rec-limit/input/text.txt b/.github/workflows/tests/rec-limit/input/text.txt new file mode 100644 index 00000000..3750c6e1 --- /dev/null +++ b/.github/workflows/tests/rec-limit/input/text.txt @@ -0,0 +1 @@ +one two three diff --git a/.github/workflows/tests/rec-limit/spec/analyzer.seq b/.github/workflows/tests/rec-limit/spec/analyzer.seq new file mode 100644 index 00000000..ab15c450 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/analyzer.seq @@ -0,0 +1,4 @@ +tokenize nil # +nlp number # number the words 0, 1, 2 +rec cycle # matches each word once, in turn: 3 matches in one region +nlp report # runs after the rec pass, stopped or not diff --git a/.github/workflows/tests/rec-limit/spec/cycle.nlp b/.github/workflows/tests/rec-limit/spec/cycle.nlp new file mode 100644 index 00000000..5173ba97 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/cycle.nlp @@ -0,0 +1,14 @@ +# A rec pass that takes one match per word. With NLP_REC_LIMIT=1 the +# region may take only one, so it is stopped after its second. +@NODES _ROOT + +@CHECK + if (N("i",1) != G("t") % 3) + fail(); +@POST + N("hits",1) = N("hits",1) + 1; + G("t")++; +@RULES +_xNIL <- + _xALPHA + @@ diff --git a/.github/workflows/tests/rec-limit/spec/number.nlp b/.github/workflows/tests/rec-limit/spec/number.nlp new file mode 100644 index 00000000..085b3c64 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/number.nlp @@ -0,0 +1,9 @@ +@NODES _ROOT + +@POST + N("i",1) = G("n"); + G("n")++; +@RULES +_xNIL <- + _xALPHA + @@ diff --git a/.github/workflows/tests/rec-limit/spec/report.nlp b/.github/workflows/tests/rec-limit/spec/report.nlp new file mode 100644 index 00000000..d7e1d2b7 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/report.nlp @@ -0,0 +1,3 @@ +@CODE +cout() << "matches=" << G("t") << "\n"; +@@CODE diff --git a/include/Api/lite/parse.h b/include/Api/lite/parse.h index 1ea10eda..06460c85 100644 --- a/include/Api/lite/parse.h +++ b/include/Api/lite/parse.h @@ -138,6 +138,7 @@ class LITE_API Parse bool getCodeexit(); // @CODE exited the current pass (#392). _TCHAR *getAlgo(); // 05/31/00 AM. bool getIntern(); // 05/26/01 AM. + bool getRulesFile() {return rulesFile_;} long getCurrpass(); // REN. // 08/22/02 AM. long getRulepass(); // 02/03/05 AM. long getInputpass(); // 08/23/02 AM. @@ -193,6 +194,7 @@ class LITE_API Parse void setAlgo(_TCHAR *); // 05/31/00 AM. void setFbatchstart(bool); // 10/19/00 AM. void setIntern(bool); // 05/26/01 AM. + void setRulesFile(bool x) {rulesFile_ = x;} void setCurrpass(long);// REN. // 08/22/02 AM. void setRulepass(long); // 02/03/05 AM. void setInputpass(long); // 08/23/02 AM. @@ -238,6 +240,7 @@ class LITE_API Parse bool errOut(std::_t_strstream*,bool,bool=false); // 02/25/05 AM. // Echo an NLP++ error to stderr: region ("@CODE"), what it skipped. void nlpppError(const _TCHAR *region, const _TCHAR *effect); + void recLimitError(long ruleline, long matches, long nodes); inline void prettyPassnum(int, /*UP*/ _TCHAR *); inline void prettyParsedone(); @@ -603,6 +606,7 @@ class LITE_API Parse // If this parse interns an analyzers (eg, RFB or user's analyzer), // then record if any syntax errors were encountered. // 05/26/01 AM. bool intern_; // If interned analyzer successfully. // 05/26/01 AM. + bool rulesFile_; // If parsing a rules file with the RFB grammar. Dlist *blobs_; // Global list for blobs. // 02/27/03 AM. @@ -665,6 +669,7 @@ class LITE_API Parse long zoneNum_; // Zone being run, from 1; 0 when not in zone mode. bool zoneFirst_; // $isfirstzone. bool zoneLast_; // $islastzone. + std::set> recLimitErrs_; // (pass, rule line) reported. // Pass info to analyzer, to G("$datum") variable. // 03/13/03 AM. _TCHAR *datum_; // 03/13/03 AM. diff --git a/lite/parse.cpp b/lite/parse.cpp index 82b7c0af..4ae5eac6 100644 --- a/lite/parse.cpp +++ b/lite/parse.cpp @@ -89,6 +89,7 @@ verbose = false; // Default. // 12/26/98 AM. intern_ = true; // 05/26/01 AM. zoneNum_ = 0; zoneFirst_ = zoneLast_ = false; +rulesFile_ = false; currpass_ = 0; // 07/24/01 AM. rulepass_ = 0; // 02/03/05 AM. inputpass_ = 0; // 08/23/02 AM. @@ -2010,6 +2011,46 @@ if (input && *input) std::_t_cerr << _T(". Details in err.log.]") << std::endl; } +/******************************************** +* FN: RECLIMITERROR +* SUBJ: Report a rec pass region stopped at the match limit. +* NOTE: A rule that keeps matching without changing what it matched +* (eg, its loop guard is never set) repeated a rec pass forever, +* with nothing on the console. Once per input for each pass and +* rule. +********************************************/ + +void Parse::recLimitError(long ruleline, long matches, long nodes) +{ +if (!recLimitErrs_.insert(std::make_pair(currpass_, ruleline)).second) + return; + +_TCHAR *file = 0; +Dlist *seqs = ana ? ana->getSeqlist() : 0; +for (Delt *d = seqs ? seqs->getFirst() : 0; d; d = d->Right()) + if (d->getData()->getPassnum() == currpass_) + { + file = d->getData()->getRulesfilename(); + break; + } + +std::basic_ostringstream<_TCHAR> msg; +msg << _T("[Error in rec pass ") << currpass_; +if (file && *file) + msg << _T(" (") << file << _T(")"); +msg << _T(": rules matched ") << matches << _T(" times in a region of ") + << nodes << _T(" nodes without settling, so the region was stopped there.") + << _T(" Last rule to match: line ") << ruleline; +if (input && *input) + msg << _T(". Input: ") << input; +msg << _T(". NLP_REC_LIMIT sets the limit; 0 lifts it.]"); +std::_t_cerr << msg.str() << std::endl; + +std::_t_strstream gerrStr; +gerrStr << msg.str() << std::ends; +::errOut(&gerrStr,false,currpass_,ruleline); +} + /******************************************** * FN: NEWBLOB * CR: 02/27/03 AM. diff --git a/lite/rec.cpp b/lite/rec.cpp index f72536e8..8fda0e1f 100644 --- a/lite/rec.cpp +++ b/lite/rec.cpp @@ -455,6 +455,8 @@ if (!start || !rules || !parse) Delt *prule; // Current rules; also, the rule that matched. Node *node; bool matched = false; // If any rules matched. +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // 11/20/99 AM. Using this object to collect the entire parse state. Nlppp nlppp( // 11/20/99 AM. @@ -525,7 +527,12 @@ for (;;) // WHILE RECURSING.... } matched = true; // A RULE MATCHED IN THIS CYCLE. - + if (overRecLimit(&nlppp, matches, size)) // Region never settles. + { + delete (Tree *) nlppp.collect_; + return true; + } + // Optimization. Go back according to SPAN. // 02/04/05 AM. // start = recMoveback(&nlppp); // 02/04/05 AM. @@ -589,6 +596,8 @@ Slist *rules = 0; // Current rules list. Selt *prule = 0; // Current rules; also, the rule that matched. Node *node; bool matched = false; +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // 11/20/99 AM. Using this object to collect the entire parse state. Nlppp nlppp( // 11/20/99 AM. @@ -725,6 +734,13 @@ for (;;) // WHILE RECURSING..... nlppp.sem_ = 0; // Zero out. // 11/20/99 AM. nlppp.dsem_ = 0; // Zero out. // 11/20/99 AM. matched = true; // A RULE MATCHED IN THIS CYCLE. + if (overRecLimit(&nlppp, matches, size)) // Region never settles. + { + if (rules) + Slist::DeleteSlist(rules); + delete (Tree *) nlppp.collect_; + return true; + } node = matchRestart(&nlppp); // RECOPT1 // 04/27/06 AM. break; // RESTART AT BEGINNING OF PHRASE. } @@ -742,6 +758,62 @@ delete (Tree *) nlppp.collect_; return true; } +/******************************************** +* FN: RECLIMIT +* SUBJ: Scale of the most rule matches one region of a rec pass may take. +* NOTE: RecinfLoop only counts a match that rebuilds a node over the same +* span, and not one that regroups or excises, so a rule whose +* loop guard is never set (parse-en-us pos50, "much I have been +* to blame.") repeated a pass forever. overRecLimit counts every +* match. NLP_REC_LIMIT sets the scale (default 1000); 0 means no +* limit. +********************************************/ + +long Rec::recLimit() +{ +static long limit = -1; +if (limit < 0) + { + const char *s = getenv("NLP_REC_LIMIT"); + limit = (s && *s) ? atol(s) : 1000; + if (limit < 0) + limit = 0; + } +return limit; +} + +// Nodes in a region, start through end (or the end of the list). +long Rec::regionSize(Node *start, Node *end) +{ +long n = 0; +for (Node *x = start; x; x = x->Right()) + { + ++n; + if (x == end) + break; + } +return n; +} + +// A region may take recLimit() matches plus recLimit()/50 per node it +// started with: 1,000 plus 20 a node by default. Measured over every +// analyzer in the analyzers repo and a 738 KB novel, no region took more +// than 2.3 matches a node. The engine's own rules-file grammar is not +// limited: it reads a whole pass file as one region. +bool Rec::overRecLimit(Nlppp *nlppp, long &matches, long size) +{ +long scale = recLimit(); +if (!scale || nlppp->parse_->getRulesFile()) + return false; +long long limit = (long long) scale + (long long) scale * size / 50; +if (++matches <= limit) + return false; +nlppp->parse_->recLimitError(nlppp->rule_ ? nlppp->rule_->getLine() : 0, + matches - 1, size); +return true; +} + + /******************************************** * FN: RECINFLOOP * CR: 06/09/05 AM. @@ -1172,6 +1244,8 @@ if (!start || !pass->rules || !parse) //Delt *prule; // Current rules; also, the rule that matched. Node *node; bool matched = false; // If any rules matched. +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // Using this object to collect the entire parse state. // Reset the nlppp object. @@ -1293,6 +1367,8 @@ for (;;) // WHILE RECURSING.... } matched = true; // A RULE MATCHED IN THIS CYCLE. + if (overRecLimit(nlppp, matches, size)) // Region never settles. + return true; // RESET START AND END OF PHRASE. start = nlppp->start_ = nlppp->select_->Down(); end = nlppp->end_ = 0; @@ -1596,6 +1672,8 @@ if (!start || !pass->rules || !parse) //Delt *prule; // Current rules; also, the rule that matched. Node *node; bool matched = false; // If any rules matched. +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // Using this object to collect the entire parse state. // Reset the nlppp object. @@ -1710,6 +1788,8 @@ for (;;) // WHILE RECURSING.... } matched = true; // A RULE MATCHED IN THIS CYCLE. + if (overRecLimit(nlppp, matches, size)) // Region never settles. + return true; // RESET START AND END OF PHRASE. start = nlppp->start_ = nlppp->select_->Down(); end = nlppp->end_ = 0; diff --git a/lite/rec.h b/lite/rec.h index c4dabfef..9b1f88c0 100644 --- a/lite/rec.h +++ b/lite/rec.h @@ -119,6 +119,13 @@ class Rec : public Algo Node *select ); + // Most rule matches one region of a rec pass may take before the + // region is stopped. NLP_REC_LIMIT overrides it; 0 means no limit. + static long recLimit(); + // Count a match in the current region; true once over the limit. + static bool overRecLimit(Nlppp *nlppp, long &matches, long size); + static long regionSize(Node *start, Node *end); + static // RECINFLOOP // 05/10/13 AM. inline bool RecinfLoop( // 06/09/05 AM. Nlppp *nlppp, diff --git a/lite/rfa.cpp b/lite/rfa.cpp index d5e3786c..d26d1ef9 100644 --- a/lite/rfa.cpp +++ b/lite/rfa.cpp @@ -305,6 +305,7 @@ if (debug) // 10/13/99 AM. *gout << _T("Rules file name=") << file << std::endl; parse.setInput(file); // Input rules file for the parse. +parse.setRulesFile(true); // The engine's own grammar: no rec match limit. //delete [] file; // 11/19/98 AM. Chars::destroy(file); diff --git a/nlp/main.cpp b/nlp/main.cpp index ffea0e85..f3040476 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.46" +#define NLP_ENGINE_VERSION "4.2.47" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&, double &); void cmdHelpargs(_TCHAR *);