diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 30acc5c1..00dcfd38 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -378,6 +378,35 @@ jobs: done echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs" + - name: A rec pass that never settles is stopped (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # A rec pass repeats while any rule matches, so a rule that kept + # matching without changing what it matched (parse-en-us pos50 on + # "much I have been to blame.") hung forever with nothing said. A + # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT + # scales that and 0 lifts it. The fixture's rec pass takes 3 matches + # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and + # the pass after it still runs. + FIX=.github/workflows/tests/rec-limit + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then + echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1 + fi + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(NLP_REC_LIMIT=1 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=2" ]; then + echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1 + fi + if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then + echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1 + fi + rm -f rec.err + echo "[rec-limit] a rec pass region past its match limit is stopped, with a message" + - name: Lazy kbb lookups from any pass, and << to a path (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 35c4fd53..33ea8a79 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -225,6 +225,34 @@ jobs: done echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs" + - name: A rec pass that never settles is stopped + shell: bash + run: | + set -eo pipefail + # A rec pass repeats while any rule matches, so a rule that kept + # matching without changing what it matched (parse-en-us pos50 on + # "much I have been to blame.") hung forever with nothing said. A + # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT + # scales that and 0 lifts it. The fixture's rec pass takes 3 matches + # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and + # the pass after it still runs. + FIX=.github/workflows/tests/rec-limit + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then + echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1 + fi + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(NLP_REC_LIMIT=1 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=2" ]; then + echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1 + fi + if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then + echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1 + fi + rm -f rec.err + echo "[rec-limit] a rec pass region past its match limit is stopped, with a message" + - name: Lazy kbb lookups from any pass, and << to a path shell: bash run: | diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 25c5fe77..8c3706a2 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -313,6 +313,34 @@ jobs: done echo "[zone-mode] one run of the zone passes per zone, zones kept as stubs" + - name: A rec pass that never settles is stopped + shell: bash + run: | + set -eo pipefail + # A rec pass repeats while any rule matches, so a rule that kept + # matching without changing what it matched (parse-en-us pos50 on + # "much I have been to blame.") hung forever with nothing said. A + # region may now take 1,000 matches plus 20 a node; NLP_REC_LIMIT + # scales that and 0 lifts it. The fixture's rec pass takes 3 matches + # in a 6-node region, so at NLP_REC_LIMIT=1 it is stopped at 2 and + # the pass after it still runs. + FIX=.github/workflows/tests/rec-limit + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=3" ] || grep -q "rec pass" rec.err; then + echo "FAIL(rec-limit default): stdout was: $out"; cat rec.err; exit 1 + fi + rm -rf "$FIX/logs" "$FIX"/input/*_log + out=$(NLP_REC_LIMIT=1 ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" 2> rec.err | tr -d '\r') + if [ "$out" != "matches=2" ]; then + echo "FAIL(rec-limit 1): stdout was: $out"; cat rec.err; exit 1 + fi + if ! grep -q "Error in rec pass 3 (cycle): rules matched 1 times in a region of 6 nodes" rec.err; then + echo "FAIL(rec-limit 1): no rec pass error on stderr:"; cat rec.err; exit 1 + fi + rm -f rec.err + echo "[rec-limit] a rec pass region past its match limit is stopped, with a message" + - name: Lazy kbb lookups from any pass, and << to a path shell: bash run: | diff --git a/.github/workflows/tests/rec-limit/input/text.txt b/.github/workflows/tests/rec-limit/input/text.txt new file mode 100644 index 00000000..3750c6e1 --- /dev/null +++ b/.github/workflows/tests/rec-limit/input/text.txt @@ -0,0 +1 @@ +one two three diff --git a/.github/workflows/tests/rec-limit/spec/analyzer.seq b/.github/workflows/tests/rec-limit/spec/analyzer.seq new file mode 100644 index 00000000..ab15c450 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/analyzer.seq @@ -0,0 +1,4 @@ +tokenize nil # +nlp number # number the words 0, 1, 2 +rec cycle # matches each word once, in turn: 3 matches in one region +nlp report # runs after the rec pass, stopped or not diff --git a/.github/workflows/tests/rec-limit/spec/cycle.nlp b/.github/workflows/tests/rec-limit/spec/cycle.nlp new file mode 100644 index 00000000..5173ba97 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/cycle.nlp @@ -0,0 +1,14 @@ +# A rec pass that takes one match per word. With NLP_REC_LIMIT=1 the +# region may take only one, so it is stopped after its second. +@NODES _ROOT + +@CHECK + if (N("i",1) != G("t") % 3) + fail(); +@POST + N("hits",1) = N("hits",1) + 1; + G("t")++; +@RULES +_xNIL <- + _xALPHA + @@ diff --git a/.github/workflows/tests/rec-limit/spec/number.nlp b/.github/workflows/tests/rec-limit/spec/number.nlp new file mode 100644 index 00000000..085b3c64 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/number.nlp @@ -0,0 +1,9 @@ +@NODES _ROOT + +@POST + N("i",1) = G("n"); + G("n")++; +@RULES +_xNIL <- + _xALPHA + @@ diff --git a/.github/workflows/tests/rec-limit/spec/report.nlp b/.github/workflows/tests/rec-limit/spec/report.nlp new file mode 100644 index 00000000..d7e1d2b7 --- /dev/null +++ b/.github/workflows/tests/rec-limit/spec/report.nlp @@ -0,0 +1,3 @@ +@CODE +cout() << "matches=" << G("t") << "\n"; +@@CODE diff --git a/include/Api/lite/parse.h b/include/Api/lite/parse.h index 1ea10eda..06460c85 100644 --- a/include/Api/lite/parse.h +++ b/include/Api/lite/parse.h @@ -138,6 +138,7 @@ class LITE_API Parse bool getCodeexit(); // @CODE exited the current pass (#392). _TCHAR *getAlgo(); // 05/31/00 AM. bool getIntern(); // 05/26/01 AM. + bool getRulesFile() {return rulesFile_;} long getCurrpass(); // REN. // 08/22/02 AM. long getRulepass(); // 02/03/05 AM. long getInputpass(); // 08/23/02 AM. @@ -193,6 +194,7 @@ class LITE_API Parse void setAlgo(_TCHAR *); // 05/31/00 AM. void setFbatchstart(bool); // 10/19/00 AM. void setIntern(bool); // 05/26/01 AM. + void setRulesFile(bool x) {rulesFile_ = x;} void setCurrpass(long);// REN. // 08/22/02 AM. void setRulepass(long); // 02/03/05 AM. void setInputpass(long); // 08/23/02 AM. @@ -238,6 +240,7 @@ class LITE_API Parse bool errOut(std::_t_strstream*,bool,bool=false); // 02/25/05 AM. // Echo an NLP++ error to stderr: region ("@CODE"), what it skipped. void nlpppError(const _TCHAR *region, const _TCHAR *effect); + void recLimitError(long ruleline, long matches, long nodes); inline void prettyPassnum(int, /*UP*/ _TCHAR *); inline void prettyParsedone(); @@ -603,6 +606,7 @@ class LITE_API Parse // If this parse interns an analyzers (eg, RFB or user's analyzer), // then record if any syntax errors were encountered. // 05/26/01 AM. bool intern_; // If interned analyzer successfully. // 05/26/01 AM. + bool rulesFile_; // If parsing a rules file with the RFB grammar. Dlist *blobs_; // Global list for blobs. // 02/27/03 AM. @@ -665,6 +669,7 @@ class LITE_API Parse long zoneNum_; // Zone being run, from 1; 0 when not in zone mode. bool zoneFirst_; // $isfirstzone. bool zoneLast_; // $islastzone. + std::set> recLimitErrs_; // (pass, rule line) reported. // Pass info to analyzer, to G("$datum") variable. // 03/13/03 AM. _TCHAR *datum_; // 03/13/03 AM. diff --git a/lite/parse.cpp b/lite/parse.cpp index 82b7c0af..4ae5eac6 100644 --- a/lite/parse.cpp +++ b/lite/parse.cpp @@ -89,6 +89,7 @@ verbose = false; // Default. // 12/26/98 AM. intern_ = true; // 05/26/01 AM. zoneNum_ = 0; zoneFirst_ = zoneLast_ = false; +rulesFile_ = false; currpass_ = 0; // 07/24/01 AM. rulepass_ = 0; // 02/03/05 AM. inputpass_ = 0; // 08/23/02 AM. @@ -2010,6 +2011,46 @@ if (input && *input) std::_t_cerr << _T(". Details in err.log.]") << std::endl; } +/******************************************** +* FN: RECLIMITERROR +* SUBJ: Report a rec pass region stopped at the match limit. +* NOTE: A rule that keeps matching without changing what it matched +* (eg, its loop guard is never set) repeated a rec pass forever, +* with nothing on the console. Once per input for each pass and +* rule. +********************************************/ + +void Parse::recLimitError(long ruleline, long matches, long nodes) +{ +if (!recLimitErrs_.insert(std::make_pair(currpass_, ruleline)).second) + return; + +_TCHAR *file = 0; +Dlist *seqs = ana ? ana->getSeqlist() : 0; +for (Delt *d = seqs ? seqs->getFirst() : 0; d; d = d->Right()) + if (d->getData()->getPassnum() == currpass_) + { + file = d->getData()->getRulesfilename(); + break; + } + +std::basic_ostringstream<_TCHAR> msg; +msg << _T("[Error in rec pass ") << currpass_; +if (file && *file) + msg << _T(" (") << file << _T(")"); +msg << _T(": rules matched ") << matches << _T(" times in a region of ") + << nodes << _T(" nodes without settling, so the region was stopped there.") + << _T(" Last rule to match: line ") << ruleline; +if (input && *input) + msg << _T(". Input: ") << input; +msg << _T(". NLP_REC_LIMIT sets the limit; 0 lifts it.]"); +std::_t_cerr << msg.str() << std::endl; + +std::_t_strstream gerrStr; +gerrStr << msg.str() << std::ends; +::errOut(&gerrStr,false,currpass_,ruleline); +} + /******************************************** * FN: NEWBLOB * CR: 02/27/03 AM. diff --git a/lite/rec.cpp b/lite/rec.cpp index f72536e8..8fda0e1f 100644 --- a/lite/rec.cpp +++ b/lite/rec.cpp @@ -455,6 +455,8 @@ if (!start || !rules || !parse) Delt *prule; // Current rules; also, the rule that matched. Node *node; bool matched = false; // If any rules matched. +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // 11/20/99 AM. Using this object to collect the entire parse state. Nlppp nlppp( // 11/20/99 AM. @@ -525,7 +527,12 @@ for (;;) // WHILE RECURSING.... } matched = true; // A RULE MATCHED IN THIS CYCLE. - + if (overRecLimit(&nlppp, matches, size)) // Region never settles. + { + delete (Tree *) nlppp.collect_; + return true; + } + // Optimization. Go back according to SPAN. // 02/04/05 AM. // start = recMoveback(&nlppp); // 02/04/05 AM. @@ -589,6 +596,8 @@ Slist *rules = 0; // Current rules list. Selt *prule = 0; // Current rules; also, the rule that matched. Node *node; bool matched = false; +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // 11/20/99 AM. Using this object to collect the entire parse state. Nlppp nlppp( // 11/20/99 AM. @@ -725,6 +734,13 @@ for (;;) // WHILE RECURSING..... nlppp.sem_ = 0; // Zero out. // 11/20/99 AM. nlppp.dsem_ = 0; // Zero out. // 11/20/99 AM. matched = true; // A RULE MATCHED IN THIS CYCLE. + if (overRecLimit(&nlppp, matches, size)) // Region never settles. + { + if (rules) + Slist::DeleteSlist(rules); + delete (Tree *) nlppp.collect_; + return true; + } node = matchRestart(&nlppp); // RECOPT1 // 04/27/06 AM. break; // RESTART AT BEGINNING OF PHRASE. } @@ -742,6 +758,62 @@ delete (Tree *) nlppp.collect_; return true; } +/******************************************** +* FN: RECLIMIT +* SUBJ: Scale of the most rule matches one region of a rec pass may take. +* NOTE: RecinfLoop only counts a match that rebuilds a node over the same +* span, and not one that regroups or excises, so a rule whose +* loop guard is never set (parse-en-us pos50, "much I have been +* to blame.") repeated a pass forever. overRecLimit counts every +* match. NLP_REC_LIMIT sets the scale (default 1000); 0 means no +* limit. +********************************************/ + +long Rec::recLimit() +{ +static long limit = -1; +if (limit < 0) + { + const char *s = getenv("NLP_REC_LIMIT"); + limit = (s && *s) ? atol(s) : 1000; + if (limit < 0) + limit = 0; + } +return limit; +} + +// Nodes in a region, start through end (or the end of the list). +long Rec::regionSize(Node *start, Node *end) +{ +long n = 0; +for (Node *x = start; x; x = x->Right()) + { + ++n; + if (x == end) + break; + } +return n; +} + +// A region may take recLimit() matches plus recLimit()/50 per node it +// started with: 1,000 plus 20 a node by default. Measured over every +// analyzer in the analyzers repo and a 738 KB novel, no region took more +// than 2.3 matches a node. The engine's own rules-file grammar is not +// limited: it reads a whole pass file as one region. +bool Rec::overRecLimit(Nlppp *nlppp, long &matches, long size) +{ +long scale = recLimit(); +if (!scale || nlppp->parse_->getRulesFile()) + return false; +long long limit = (long long) scale + (long long) scale * size / 50; +if (++matches <= limit) + return false; +nlppp->parse_->recLimitError(nlppp->rule_ ? nlppp->rule_->getLine() : 0, + matches - 1, size); +return true; +} + + /******************************************** * FN: RECINFLOOP * CR: 06/09/05 AM. @@ -1172,6 +1244,8 @@ if (!start || !pass->rules || !parse) //Delt *prule; // Current rules; also, the rule that matched. Node *node; bool matched = false; // If any rules matched. +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // Using this object to collect the entire parse state. // Reset the nlppp object. @@ -1293,6 +1367,8 @@ for (;;) // WHILE RECURSING.... } matched = true; // A RULE MATCHED IN THIS CYCLE. + if (overRecLimit(nlppp, matches, size)) // Region never settles. + return true; // RESET START AND END OF PHRASE. start = nlppp->start_ = nlppp->select_->Down(); end = nlppp->end_ = 0; @@ -1596,6 +1672,8 @@ if (!start || !pass->rules || !parse) //Delt *prule; // Current rules; also, the rule that matched. Node *node; bool matched = false; // If any rules matched. +long matches = 0; // Matches in this region, for recLimit(). +long size = regionSize(start, end); // Nodes in this region at the start. // Using this object to collect the entire parse state. // Reset the nlppp object. @@ -1710,6 +1788,8 @@ for (;;) // WHILE RECURSING.... } matched = true; // A RULE MATCHED IN THIS CYCLE. + if (overRecLimit(nlppp, matches, size)) // Region never settles. + return true; // RESET START AND END OF PHRASE. start = nlppp->start_ = nlppp->select_->Down(); end = nlppp->end_ = 0; diff --git a/lite/rec.h b/lite/rec.h index c4dabfef..9b1f88c0 100644 --- a/lite/rec.h +++ b/lite/rec.h @@ -119,6 +119,13 @@ class Rec : public Algo Node *select ); + // Most rule matches one region of a rec pass may take before the + // region is stopped. NLP_REC_LIMIT overrides it; 0 means no limit. + static long recLimit(); + // Count a match in the current region; true once over the limit. + static bool overRecLimit(Nlppp *nlppp, long &matches, long size); + static long regionSize(Node *start, Node *end); + static // RECINFLOOP // 05/10/13 AM. inline bool RecinfLoop( // 06/09/05 AM. Nlppp *nlppp, diff --git a/lite/rfa.cpp b/lite/rfa.cpp index d5e3786c..d26d1ef9 100644 --- a/lite/rfa.cpp +++ b/lite/rfa.cpp @@ -305,6 +305,7 @@ if (debug) // 10/13/99 AM. *gout << _T("Rules file name=") << file << std::endl; parse.setInput(file); // Input rules file for the parse. +parse.setRulesFile(true); // The engine's own grammar: no rec match limit. //delete [] file; // 11/19/98 AM. Chars::destroy(file); diff --git a/nlp/main.cpp b/nlp/main.cpp index ffea0e85..f3040476 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.46" +#define NLP_ENGINE_VERSION "4.2.47" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&, double &); void cmdHelpargs(_TCHAR *);