diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 9b0f0538..cfc7a00a 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1017,6 +1017,37 @@ jobs: fi echo "[element-pre] element-key conditions match like @PRE" + - name: Wildcards stop before later elements (#427) (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # With a wildcard pending, an optional element is absent if a later + # element (up to the next required one) matches first (#427), so + # the wildcard no longer runs past it. Both directions; ties keep + # the optional element. + FIX=.github/workflows/tests/smart-wildcard + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(smart-wildcard): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'right: [the big bears] wild=big three=0 bears=bears' \ + 'right: [the big three bears] wild=big three=three bears=bears' \ + 'present: [the big bears] three=0' \ + 'present: [the big three bears] three=three' \ + 'tie: [the big bears] opt=big' \ + 'left: [bears met]' \ + 'left: [bears met]') + if [ "$got" != "$want" ]; then + echo "FAIL(smart-wildcard): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[smart-wildcard] wildcards stop before later elements" + - name: factorial() up to 20! (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 48d4f7cf..2a23fe5a 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -915,6 +915,36 @@ jobs: fi echo "[element-pre] element-key conditions match like @PRE" + - name: Wildcards stop before later elements (#427) + shell: bash + run: | + set -eo pipefail + # With a wildcard pending, an optional element is absent if a later + # element (up to the next required one) matches first (#427), so + # the wildcard no longer runs past it. Both directions; ties keep + # the optional element. + FIX=.github/workflows/tests/smart-wildcard + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(smart-wildcard): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'right: [the big bears] wild=big three=0 bears=bears' \ + 'right: [the big three bears] wild=big three=three bears=bears' \ + 'present: [the big bears] three=0' \ + 'present: [the big three bears] three=three' \ + 'tie: [the big bears] opt=big' \ + 'left: [bears met]' \ + 'left: [bears met]') + if [ "$got" != "$want" ]; then + echo "FAIL(smart-wildcard): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[smart-wildcard] wildcards stop before later elements" + - name: factorial() up to 20! shell: bash run: | diff --git a/.github/workflows/tests/smart-wildcard/input/text.txt b/.github/workflows/tests/smart-wildcard/input/text.txt new file mode 100644 index 00000000..573df5d9 --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/input/text.txt @@ -0,0 +1,3 @@ +the big bears met three bears +the big three bears +bears big three bears met diff --git a/.github/workflows/tests/smart-wildcard/spec/analyzer.seq b/.github/workflows/tests/smart-wildcard/spec/analyzer.seq new file mode 100644 index 00000000..636b52e2 --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/analyzer.seq @@ -0,0 +1,7 @@ +tokenize nil # +nlp lines # one _line per input line +nlp nowhite # drop whitespace so the rules read like #427 +nlp right # optional element absent when a later one matches first +nlp present # optional element present when it comes first +nlp tie # same node: the optional element wins +nlp left # the same, matching leftward from a trigger diff --git a/.github/workflows/tests/smart-wildcard/spec/left.nlp b/.github/workflows/tests/smart-wildcard/spec/left.nlp new file mode 100644 index 00000000..4db2253f --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/left.nlp @@ -0,0 +1,14 @@ +############################################### +# FILE: left.nlp +# SUBJ: The same rule matching leftward from a trigger. On "bears big +# three bears met", leftward from "met", the optional "three" is +# absent because the required "bears" matches first. +############################################### + +@NODES _line + +@POST + "output.txt" << "left: [" << phrasetext() << "]\n"; + noop(); +@RULES +_xNIL <- bears _xWILD [opt match=(three)] _xWILD met [trig] @@ diff --git a/.github/workflows/tests/smart-wildcard/spec/lines.nlp b/.github/workflows/tests/smart-wildcard/spec/lines.nlp new file mode 100644 index 00000000..94dc4641 --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/lines.nlp @@ -0,0 +1,9 @@ +############################################### +# FILE: lines.nlp +# SUBJ: One _line node per input line. +############################################### + +@NODES _ROOT + +@RULES +_line <- _xWILD [plus fail=(\n)] @@ diff --git a/.github/workflows/tests/smart-wildcard/spec/nowhite.nlp b/.github/workflows/tests/smart-wildcard/spec/nowhite.nlp new file mode 100644 index 00000000..a87b4f5f --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/nowhite.nlp @@ -0,0 +1,12 @@ +############################################### +# FILE: nowhite.nlp +# SUBJ: Drop the spaces inside each line, so the rules can be written +# word by word, as in #427. +############################################### + +@NODES _line + +@POST + excise(1,1); +@RULES +_xNIL <- _xWHITE @@ diff --git a/.github/workflows/tests/smart-wildcard/spec/present.nlp b/.github/workflows/tests/smart-wildcard/spec/present.nlp new file mode 100644 index 00000000..73b6d2b5 --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/present.nlp @@ -0,0 +1,12 @@ +############################################### +# FILE: present.nlp +# SUBJ: The optional element still matches when it comes first. +############################################### + +@NODES _line + +@POST + "output.txt" << "present: [" << phrasetext() << "] three=" << N("$text", 3) << "\n"; + noop(); +@RULES +_xNIL <- the _xWILD _xWILD [opt match=(three)] bears @@ diff --git a/.github/workflows/tests/smart-wildcard/spec/right.nlp b/.github/workflows/tests/smart-wildcard/spec/right.nlp new file mode 100644 index 00000000..c7a64f94 --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/right.nlp @@ -0,0 +1,17 @@ +############################################### +# FILE: right.nlp +# SUBJ: #427: with a wildcard pending, an optional element is absent if a +# later element matches first. On "the big bears met three bears" +# the wildcard used to run past the first "bears" to reach "three". +# NOTE: Part of the smart-wildcard regression; expected output is in the +# CI step. +############################################### + +@NODES _line + +@POST + "output.txt" << "right: [" << phrasetext() << "] wild=" << N("$text", 2) + << " three=" << N("$text", 3) << " bears=" << N("$text", 4) << "\n"; + noop(); +@RULES +_xNIL <- the _xWILD _xWILD [opt match=(three)] _xWILD [one match=(bears)] @@ diff --git a/.github/workflows/tests/smart-wildcard/spec/tie.nlp b/.github/workflows/tests/smart-wildcard/spec/tie.nlp new file mode 100644 index 00000000..e15411e3 --- /dev/null +++ b/.github/workflows/tests/smart-wildcard/spec/tie.nlp @@ -0,0 +1,13 @@ +############################################### +# FILE: tie.nlp +# SUBJ: When the optional element and a later one both match the same +# node, the optional element wins, as before. +############################################### + +@NODES _line + +@POST + "output.txt" << "tie: [" << phrasetext() << "] opt=" << N("$text", 3) << "\n"; + noop(); +@RULES +_xNIL <- the _xWILD _xALPHA [opt] bears @@ diff --git a/include/Api/lite/Arun.h b/include/Api/lite/Arun.h index ac2f4704..1f045c9f 100644 --- a/include/Api/lite/Arun.h +++ b/include/Api/lite/Arun.h @@ -258,6 +258,7 @@ class LITE_API Arun static bool match_simple(NODE *,const ELT *,Nlppp *); // 05/23/00 AM. static int stop_ord(Nlppp *, int dir); // #449. static bool stop_match(NODE *, int ord, Nlppp *); // #449. + static bool later_matches(Nlppp *, int ord, int dir); // #427. static bool match_singlet(NODE *,const ELT *,Nlppp *); static bool match_tree(NODE *,const ELT *,Nlppp *); static bool match_list(NODE *,const ELT *,Nlppp *); diff --git a/lite/Arun.cpp b/lite/Arun.cpp index 106c7fc9..4d163a46 100644 --- a/lite/Arun.cpp +++ b/lite/Arun.cpp @@ -583,6 +583,9 @@ if (nlppp->wildord_) // If pending wildcards. { if ((matched = match_real_left(match,nlppp))) break; + // An optional element is absent if a later element matches first (#427). + if (!elt->min && later_matches(nlppp, ord, -1)) + break; ++count; if (limit && count > limit) break; @@ -687,6 +690,9 @@ if (nlppp->wildord_) // If pending wildcards. { if ((matched = match_real_right(match,nlppp))) // 05/23/00 AM. break; + // An optional element is absent if a later element matches first (#427). + if (!elt->min && later_matches(nlppp, ord, 1)) + break; ++count; if (limit && count > limit) break; @@ -1035,6 +1041,34 @@ return next; } +/******************************************** +* FN: LATER_MATCHES +* SUBJ: Whether the current node matches one of the elements after ORD in +* direction DIR, up to and including the first required one (#427). +* NOTE: Compiled twin of Pat::laterMatches: with a wildcard pending, an +* optional element is absent if a later element matches first. A bare +* wildcard among them ends the look. +********************************************/ + +bool Arun::later_matches(Nlppp *nlppp, int ord, int dir) +{ +NODE *node = (NODE *) nlppp->node_; +if (!node) + return false; +for (int k = ord + dir; k >= 1 && k <= nlppp->len_; k += dir) + { + const ELT *e = &(nlppp->elts_[k]); + if (e->wild) + return false; + if (stop_match(node, k, nlppp)) + return true; + if (e->min) + return false; // Elements past a required one don't count. + } +return false; +} + + /******************************************** * FN: STOP_MATCH * SUBJ: Whether NODE matches compiled rule element ORD, using the match @@ -3211,6 +3245,9 @@ if (nlppp->wildord_) // If pending wildcards. { if ((matched = special_real_left(match,micro,nlppp))) break; + // An optional element is absent if a later element matches first (#427). + if (!elt->min && later_matches(nlppp, ord, -1)) + break; ++count; if (limit && count > limit) break; @@ -3311,6 +3348,9 @@ if (nlppp->wildord_) // If pending wildcards. { if ((matched = special_real_right(match,micro,nlppp))) break; + // An optional element is absent if a later element matches first (#427). + if (!elt->min && later_matches(nlppp, ord, 1)) + break; ++count; if (limit && count > limit) break; diff --git a/lite/pat.cpp b/lite/pat.cpp index f3280376..4848184e 100644 --- a/lite/pat.cpp +++ b/lite/pat.cpp @@ -1448,7 +1448,8 @@ Node *real = 0; // First real node matched, moving left. //if (!matchRealLeft(curr,start,wildone,wildmin,wildmax, /*DU*/ real,node)) if (!matchRealLeft(ielt,nlppp->start_,wildone,wildmin,wildmax, /*DU*/ real,nlppp->node_, - preWildStop(ielt, curr->Left()))) // #449. + preWildStop(ielt, curr->Left()), // #449. + curr->Left())) // #427. { if (optionalElt(ielt)) // If current rule elt is optional, succeed. { @@ -1600,7 +1601,8 @@ long tcount = 0; // 02/04/05 AM. //if (!matchRealRight(curr,end,wildone,wildmin,wildmax, /*DU*/ real,node)) if (!matchRealRight(ielt,nlppp->end_,wildone,wildmin,wildmax,nlppp, /*DU*/ real,nlppp->node_,tcount, - preWildStop(ielt, curr->Right()))) // #449. + preWildStop(ielt, curr->Right()), // #449. + curr->Right())) // #427. { // nlppp->span_ += tcount; // 02/04/05 AM. if (optionalElt(ielt)) // If current rule elt is optional, succeed. @@ -1866,7 +1868,8 @@ bool Pat::matchRealLeft( Node* &real, // First real node matched (moving left). // (Allows for repetition.) Node* &node, // DN: Current node. UP: Last real node matched. - Ielt *stop // Stop before a match of this elt (#449). + Ielt *stop, // Stop before a match of this elt (#449). + Delt *next // Elements after an optional one (#427). ) { if (!node) // This can happen now, to handle wildcard at start of rule. @@ -1895,6 +1898,11 @@ for (;;) // Look for real match till done. return true; } + // While a wildcard is pending, an optional element is absent if a later + // element (up to the next required one) matches before it does (#427). + if (wildone && next && optionalElt(ielt) && laterMatches(next, test, false)) + return false; + if (!wildone) return false; // No wildcards pending. Fail. if (wildmax && count > wildmax) @@ -1932,7 +1940,8 @@ bool Pat::matchRealRight( // (Allows for repetition.) Node* &node, // DN: Current node. UP: Last real node matched. long &tot, // 02/04/05 AM. - Ielt *stop // Stop before a match of this elt (#449). + Ielt *stop, // Stop before a match of this elt (#449). + Delt *next // Elements after an optional one (#427). ) { if (!node) // This can happen now, to handle wildcard at end of rule. @@ -1970,6 +1979,11 @@ for (;;) // Look for real match till done. else // 02/04/05 AM. tot += tcount; // 02/04/05 AM. + // While a wildcard is pending, an optional element is absent if a later + // element (up to the next required one) matches before it does (#427). + if (wildone && next && optionalElt(ielt) && laterMatches(next, test, true)) + return false; + if (!wildone) return false; // No wildcards pending. Fail. if (wildmax && count > wildmax) @@ -2311,6 +2325,35 @@ return true; // 07/31/02 AM. } +/******************************************** +* FN: LATERMATCHES +* SUBJ: Whether NODE matches one of the elements from D onward, in the +* matching direction, up to and including the first required one (#427). +* NOTE: With a wildcard pending, an optional element used to be searched for +* past the elements after it: in _xWILD _xWILD [opt match=(three)] +* _xWILD [one match=(bears)] the wildcard ran past "bears" to reach a +* later "three". A bare wildcard among them ends the look: what it +* covers isn't known yet. +********************************************/ + +bool Pat::laterMatches(Delt *d, Node *node, bool right) +{ +if (!node) + return false; +for (; d; d = (right ? d->Right() : d->Left())) + { + Ielt *e = d->getData(); + if (!e || wildElt(e)) + return false; + if (Match(e, node)) + return true; + if (!optionalElt(e)) + return false; // Elements past a required one don't count. + } +return false; +} + + /******************************************** * FN: WILDFILLLEFT * CR: 10/30/98 AM. diff --git a/lite/pat.h b/lite/pat.h index ff39646c..ec216f17 100644 --- a/lite/pat.h +++ b/lite/pat.h @@ -275,7 +275,8 @@ class Pat : public Algo Node* &real, // First real node matched (moving left). // (Allows for repetition.) Node* &node, // DN: Current node. UP: Last real node matched. - Ielt *stop = 0 // Stop before a match of this elt (#449). + Ielt *stop = 0, // Stop before a match of this elt (#449). + Delt *next = 0 // Elements after an optional one (#427). ); static bool trueMatchLeft( @@ -288,6 +289,7 @@ class Pat : public Algo ); static Ielt *preWildStop(Ielt *ielt, Delt *next); // #449. + static bool laterMatches(Delt *d, Node *node, bool right); // #427. static void markFiredRule(Nlppp *nlppp, long ruleline); // #73. static bool wildFillLeft( @@ -310,7 +312,8 @@ class Pat : public Algo // (Allows for repetition.) Node* &node, // DN: Current node. UP: Last real node matched. long & tot, // 02/04/05 AM. - Ielt *stop = 0 // Stop before a match of this elt (#449). + Ielt *stop = 0, // Stop before a match of this elt (#449). + Delt *next = 0 // Elements after an optional one (#427). ); static bool trueMatchRight( diff --git a/nlp/main.cpp b/nlp/main.cpp index 75f47062..dfbb128c 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.26" +#define NLP_ENGINE_VERSION "4.2.27" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &); void cmdHelpargs(_TCHAR *);