From 3898bda7d787a9a9ec15d27959e3a63a7edbfc05 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Wed, 14 Oct 2026 22:35:44 -0400 Subject: [PATCH 1/2] Dictionary join=, split=, _name lines; bare entries load (#433 #434 #426 #417) (4.2.30) - #433: a .dict line with no attributes ("license plate") was a "missing equal sign" error that ended the read of the file, silently dropping every entry after it. It is now an entry (a phrase becomes _phrase), and any bad line is reported in cgerr.log and skipped. - #434: join=1 turns a phrase into one token of its words ("HOT-ROD" -> HOTROD), which is then looked up in turn. - #426: split="license plate" replaces a token with those words, with their own offsets when they spell the token. - #417: a "_bridge type=location" line puts attributes on every _bridge node the dictionary builds; an entry's own attributes win. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-linux.yml | 41 +++ .github/workflows/build-windows.yml | 40 +++ .../workflows/tests/dict-shape/input/text.txt | 3 + .../tests/dict-shape/kb/user/words.dict | 15 ++ .../tests/dict-shape/spec/analyzer.seq | 2 + .../workflows/tests/dict-shape/spec/show.nlp | 33 +++ cs/libconsh/cg.cpp | 14 +- lite/dicttok.cpp | 250 +++++++++++++++++- lite/dicttok.h | 11 + nlp/main.cpp | 2 +- 10 files changed, 397 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/tests/dict-shape/input/text.txt create mode 100644 .github/workflows/tests/dict-shape/kb/user/words.dict create mode 100644 .github/workflows/tests/dict-shape/spec/analyzer.seq create mode 100644 .github/workflows/tests/dict-shape/spec/show.nlp diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index deae83c1..6278c048 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1113,6 +1113,47 @@ jobs: fi echo "[element-eq] attr=value on an element tests the attribute" + - name: Dictionary join=, split=, _name lines and bare entries (#433 #434 #426 #417) (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # A bare dictionary line (no attributes) used to end the read of the + # file (#433). join=1 makes a phrase one token (#434), split="a b" + # makes a token several words (#426), and a "_bridge type=x" line puts + # attributes on every _bridge node (#417). + FIX=.github/workflows/tests/dict-shape + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(dict-shape): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'A alpha text=A at 0-0' \ + '_vehicle node text=HOT-ROD at 2-8 speed=fast [HOTROD 2-8]' \ + 'near alpha text=near at 10-13' \ + 'the alpha text=the at 15-17' \ + '_bridge node text=viaduct at 19-25 type=location kind=raised [viaduct 19-25]' \ + 'and alpha text=and at 27-29' \ + '_bridge node text=bridge at 31-36 type=location kind=generic [bridge 31-36]' \ + '. punct text=. at 37-37' \ + 'My alpha text=My at 39-40' \ + '_phrase node text=licenseplate at 42-53 [license 42-48, plate 49-53]' \ + ', punct text=, at 54-54' \ + '_weather node text=weathervconditions at 56-73 [weather 56-73, conditions 56-73]' \ + ', punct text=, at 74-74' \ + 'email alpha text=email at 76-81' \ + '. punct text=. at 82-82' \ + '_phrase node text=license plate at 84-96 [license 84-90, plate 92-96]' \ + '. punct text=. at 97-97') + if [ "$got" != "$want" ]; then + echo "FAIL(dict-shape): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[dict-shape] dictionary join, split, _name lines and bare entries" + - name: factorial() up to 20! (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 862e0f70..592cb528 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -1008,6 +1008,46 @@ jobs: fi echo "[element-eq] attr=value on an element tests the attribute" + - name: Dictionary join=, split=, _name lines and bare entries (#433 #434 #426 #417) + shell: bash + run: | + set -eo pipefail + # A bare dictionary line (no attributes) used to end the read of the + # file (#433). join=1 makes a phrase one token (#434), split="a b" + # makes a token several words (#426), and a "_bridge type=x" line puts + # attributes on every _bridge node (#417). + FIX=.github/workflows/tests/dict-shape + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + mkdir -p "$FIX/logs" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + OUT="$FIX/input/text.txt_log/output.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(dict-shape): no output.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + 'A alpha text=A at 0-0' \ + '_vehicle node text=HOT-ROD at 2-8 speed=fast [HOTROD 2-8]' \ + 'near alpha text=near at 10-13' \ + 'the alpha text=the at 15-17' \ + '_bridge node text=viaduct at 19-25 type=location kind=raised [viaduct 19-25]' \ + 'and alpha text=and at 27-29' \ + '_bridge node text=bridge at 31-36 type=location kind=generic [bridge 31-36]' \ + '. punct text=. at 37-37' \ + 'My alpha text=My at 39-40' \ + '_phrase node text=licenseplate at 42-53 [license 42-48, plate 49-53]' \ + ', punct text=, at 54-54' \ + '_weather node text=weathervconditions at 56-73 [weather 56-73, conditions 56-73]' \ + ', punct text=, at 74-74' \ + 'email alpha text=email at 76-81' \ + '. punct text=. at 82-82' \ + '_phrase node text=license plate at 84-96 [license 84-90, plate 92-96]' \ + '. punct text=. at 97-97') + if [ "$got" != "$want" ]; then + echo "FAIL(dict-shape): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[dict-shape] dictionary join, split, _name lines and bare entries" + - name: factorial() up to 20! shell: bash run: | diff --git a/.github/workflows/tests/dict-shape/input/text.txt b/.github/workflows/tests/dict-shape/input/text.txt new file mode 100644 index 00000000..573bc595 --- /dev/null +++ b/.github/workflows/tests/dict-shape/input/text.txt @@ -0,0 +1,3 @@ +A HOT-ROD near the viaduct and bridge. +My licenseplate, weathervconditions, e-mail. +license plate. diff --git a/.github/workflows/tests/dict-shape/kb/user/words.dict b/.github/workflows/tests/dict-shape/kb/user/words.dict new file mode 100644 index 00000000..9c593919 --- /dev/null +++ b/.github/workflows/tests/dict-shape/kb/user/words.dict @@ -0,0 +1,15 @@ +# Regression for the dictionary tokenizer (#433 #434 #426 #417). +# A bare entry (no attributes) used to end the read of the file. +license plate +# join=1: the phrase becomes one token, looked up in turn. +hot - rod join=1 +hotrod s=vehicle speed=fast +e - mail join=1 +# split=: one token becomes several words. +licenseplate split="license plate" +weathervconditions split="weather conditions" +weather conditions s=weather +# _name: attributes for every _bridge node; an entry's own win. +bridge s=bridge +viaduct s=bridge kind=raised +_bridge type=location kind=generic diff --git a/.github/workflows/tests/dict-shape/spec/analyzer.seq b/.github/workflows/tests/dict-shape/spec/analyzer.seq new file mode 100644 index 00000000..3e05f85f --- /dev/null +++ b/.github/workflows/tests/dict-shape/spec/analyzer.seq @@ -0,0 +1,2 @@ +dicttok nil # +nlp show # diff --git a/.github/workflows/tests/dict-shape/spec/show.nlp b/.github/workflows/tests/dict-shape/spec/show.nlp new file mode 100644 index 00000000..3b2a8061 --- /dev/null +++ b/.github/workflows/tests/dict-shape/spec/show.nlp @@ -0,0 +1,33 @@ +############################################### +# FILE: show.nlp +# SUBJ: Each top node: name, type, $text, offsets, and the +# attributes the dictionary gave it; then its children. +# NOTE: Part of the dict-shape regression; expected output is in +# the CI step. +############################################### + +@CODE +L("n") = pndown(pnroot()); +while (L("n")) { + if (pntype(L("n")) != "white") { + "output.txt" << pnname(L("n")) << " " << pntype(L("n")) << " text=" << pnvar(L("n"), "$text") + << " at " << pnvar(L("n"), "$ostart") << "-" << pnvar(L("n"), "$oend"); + if (pnvar(L("n"), "type")) "output.txt" << " type=" << pnvar(L("n"), "type"); + if (pnvar(L("n"), "kind")) "output.txt" << " kind=" << pnvar(L("n"), "kind"); + if (pnvar(L("n"), "speed")) "output.txt" << " speed=" << pnvar(L("n"), "speed"); + L("c") = pndown(L("n")); + L("sep") = " ["; + while (L("c")) { + if (pntype(L("c")) != "white") { + "output.txt" << L("sep") << pnname(L("c")) << " " << pnvar(L("c"), "$ostart") + << "-" << pnvar(L("c"), "$oend"); + L("sep") = ", "; + } + L("c") = pnnext(L("c")); + } + if (L("sep") == ", ") "output.txt" << "]"; + "output.txt" << "\n"; + } + L("n") = pnnext(L("n")); +} +@@CODE diff --git a/cs/libconsh/cg.cpp b/cs/libconsh/cg.cpp index 045ee76e..65c43728 100644 --- a/cs/libconsh/cg.cpp +++ b/cs/libconsh/cg.cpp @@ -3902,8 +3902,9 @@ bool CG::readDict(std::string file, std::vector kbfiles) strip_block_comments(buf, inBlock, '#'); if (unicu::isStrWhiteSpace(buf)) continue; - if (!parseDictLine(buf, ambigKB, file, lineCount)) - return false; + // A bad line is reported (cgerr) and skipped. It used to end the + // read, silently dropping every entry after it (#433). + parseDictLine(buf, ambigKB, file, lineCount); } allDictStream_.close(); @@ -4083,11 +4084,10 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i return false; } - if (eqSign == -1) { - sprintf(errout_, "%d 1 [missing equal sign - %s]", lineCount, filename.c_str()); - *cgerr << errout_ << std::endl; - return false; - } + // No attributes: the line is all words, eg "license plate" (#433). + // It used to be a "missing equal sign" error. + if (eqSign == -1) + eqSign = tokint + 1; // Loop through tokens diff --git a/lite/dicttok.cpp b/lite/dicttok.cpp index 8d5ce963..e53de6a3 100644 --- a/lite/dicttok.cpp +++ b/lite/dicttok.cpp @@ -266,6 +266,11 @@ bool DICTTok::ApplyDictFiles() { } CON *c = (CON *)con; + // A token the dictionary splits into words (#426): replace it with + // the words and go on from the first of them. + if (con && splitToken(node, con)) + continue; + int length = 0; CONCEPT *end = NULL; Node *endNode = MatchLongest(con,node,&end,length,0); @@ -273,6 +278,20 @@ bool DICTTok::ApplyDictFiles() { if (endNode && end && attrs) { CON *e = (CON *)end; + // A phrase whose words become one token (#434): look the joined + // token up in turn, so "hotrod s=vehicle" applies to it. + if (length > 1 && cg_->findVals(end, _T("join"))) { + VAL *jv = cg_->findVals(end, _T("join")); + long long jnum = 1; + if (cg_->isValNum(jv)) + cg_->popVal(jv, jnum); + Node *joined = jnum ? joinPhrase(node, endNode) : 0; + if (joined) { + node = joined; + continue; + } + } + // Construct phrase text from the end up std::string text = ""; CONCEPT *start = end; @@ -350,6 +369,15 @@ bool DICTTok::ApplyDictFiles() { // "New York" read "new york". // 10/13/26 DD. suggestedN->getData()->setText(pn->getText()); + // Attributes for every node of this name, from a dictionary + // line such as "_bridge type=location" (#417). The entry's + // own attributes, applied next, win over them. + if (suggName[0] == '_') { + CONCEPT *shared = cg_->findWordInMemory(suggName); + if (shared) + applyAttrs(suggestedN, shared); + } + findAttrs(suggestedN, con, name, true); node = suggestedN; pn = node->getData(); @@ -466,6 +494,206 @@ Node* DICTTok::MatchLongest(CONCEPT *con, Node *parentN, CONCEPT **end, return pn; } +/******************************************** +* FN: LOOKUPWORD +* SUBJ: A word's dictionary concept: as written, lowercase, then Capitalized. +********************************************/ + +CONCEPT *DICTTok::lookupWord(_TCHAR *name) { + _TCHAR buf[PATHSIZ]; + CONCEPT *con = cg_->findWordConcept(name); + if (!con) + con = cg_->findWordConcept(str_to_lower(name, buf)); + if (!con) + con = findCapitalized(cg_, buf); // #386. + return con; +} + +/******************************************** +* FN: MAKELITTOK +* SUBJ: A token whose text is NAME, not the input it spans (#426, #434). +* NOTE: Typed as the tokenizer would type NAME: all digits num, one ASCII +* punctuation character punct, otherwise alpha. +********************************************/ + +Node *DICTTok::makeLitTok(const _TCHAR *name, long start, long end, long ustart, + long uend, _TCHAR *text, long line) { + long len = (long) _tcslen(name); + if (!len || len >= MAXSTR) + return 0; + _TCHAR buf[MAXSTR]; + _tcscpy(buf, name); + _TCHAR *lcstr = 0; + Sym *sym = internTok(buf, len, htab_, lcstr); + if (!sym) + return 0; + enum Pntype typ = PNNUM; + for (const _TCHAR *p = name; *p; ++p) + if (*p < '0' || *p > '9') + typ = PNALPHA; + if (typ == PNALPHA && len == 1 && (_TUCHAR) name[0] < 0x80 && !alphabetic(name[0])) + typ = PNPUNCT; + Node *node = Pn::makeTnode(start, end, ustart, uend, typ, text, sym->getStr(), sym, line); + if (node) + node->getData()->setLiteral(true); + return node; +} + +/******************************************** +* FN: REPLACERANGE +* SUBJ: Put the sibling list NFIRST..NLAST in place of FIRST..LAST, and +* delete FIRST..LAST. +********************************************/ + +void DICTTok::replaceRange(Node *first, Node *last, Node *nfirst, + Node *nlast) { + Node *lefty = first->Left(); + Node *righty = last->Right(); + if (lefty) { + lefty->setRight(nfirst); + nfirst->setLeft(lefty); + } else { + Node *up = first->Up(); + nfirst->setUp(up); + if (up) + up->setDown(nfirst); + first->setUp(0); + } + if (righty) { + righty->setLeft(nlast); + nlast->setRight(righty); + } + first->setLeft(0); + last->setRight(0); + tree_->deleteSubsNC(first); +} + +/******************************************** +* FN: SPLITTOKEN +* SUBJ: Replace a token with the words of its split= value (#426). +* EX: licenseplate split="license plate" +* NOTE: The words are separated by whitespace in the value and become +* adjacent tokens (no whitespace node between them). When their +* letters spell the token, each word gets the offsets of its own +* letters; otherwise (clutter, a misspelling) each spans the whole +* token. Only tokens straight from the tokenizer are split, so a +* word is never split again. NODE becomes the first word. +********************************************/ + +bool DICTTok::splitToken(Node* &node, CONCEPT *con) { + Pn *pn = node->getData(); + if (pn->getType() == PNNODE || pn->getType() == PNWHITE || pn->getLiteral()) + return false; + VAL *vals = cg_->findVals(con, _T("split")); + if (!vals || !cg_->isValStr(vals)) + return false; + _TCHAR value[MAXSTR]; + value[0] = '\0'; + cg_->popSval(vals, value); + + std::vector words; + std::string spelled; + for (_TCHAR *p = value; *p; ) { + while (*p == ' ' || *p == '\t') + ++p; + _TCHAR *w = p; + while (*p && *p != ' ' && *p != '\t') + ++p; + if (p > w) { + words.push_back(std::string(w, p - w)); + spelled += words.back(); + } + } + if (words.empty()) + return false; + + // Do the words spell the token, ignoring ASCII case? + long blen = pn->getEnd() - pn->getStart() + 1; + _TCHAR *text = pn->getText(); + bool aligned = text && (long) spelled.size() == blen; + for (long i = 0; aligned && i < blen; ++i) { + _TCHAR a = text[i], b = spelled[i]; + if (a >= 'A' && a <= 'Z') a += 'a' - 'A'; + if (b >= 'A' && b <= 'Z') b += 'a' - 'A'; + if (a != b) + aligned = false; + } + + long bpos = pn->getStart(); + long upos = pn->getUstart(); + _TCHAR *tpos = text; + Node *first = 0, *prev = 0; + for (size_t i = 0; i < words.size(); ++i) { + const std::string &w = words[i]; + long wb = (long) w.size(); + long wu = 0; + for (size_t k = 0; k < w.size(); ++k) + if (((unsigned char) w[k] & 0xC0) != 0x80) + ++wu; + Node *piece; + if (aligned) { + piece = makeLitTok(w.c_str(), bpos, bpos + wb - 1, upos, upos + wu - 1, tpos, pn->getLine()); + bpos += wb; + upos += wu; + tpos += wb; + } else + piece = makeLitTok(w.c_str(), pn->getStart(), pn->getEnd(), pn->getUstart(), + pn->getUend(), text, pn->getLine()); + if (!piece) { + if (first) + tree_->deleteSubsNC(first); + return false; + } + if (prev) { + prev->setRight(piece); + piece->setLeft(prev); + replaceNum(piece, _T("SP"), 1); + } else { + first = piece; + copyAttrsToSuggested(pn, piece, node); // NL, SP, NOSP, TABS. + } + Pn *pp = piece->getData(); + findAttrs(piece, lookupWord(pp->getName()), pp->getName(), false); + prev = piece; + } + + replaceRange(node, node, first, prev); + node = first; + return true; +} + +/******************************************** +* FN: JOINPHRASE +* SUBJ: Replace a phrase's tokens with one token of its words (#434). +* EX: hot - rod join=1 turns "hot-rod" into the token "hotrod". +* NOTE: Whitespace and punctuation are dropped; the token keeps the case +* of the input and spans the phrase. Returns the token, or 0. +********************************************/ + +Node *DICTTok::joinPhrase(Node *node, Node *endNode) { + std::string joined; + for (Node *n = node; n; n = n->Right()) { + Pn *p = n->getData(); + if (p->getType() == PNNODE) + return 0; + if (p->getType() != PNWHITE && p->getType() != PNPUNCT) + joined += p->getName(); + if (n == endNode) + break; + } + Pn *pn = node->getData(); + Pn *pe = endNode->getData(); + Node *tok = makeLitTok(joined.c_str(), pn->getStart(), pe->getEnd(), + pn->getUstart(), pe->getUend(), pn->getText(), pn->getLine()); + if (!tok) + return 0; + copyAttrsToSuggested(pn, tok, endNode); // NL, SP, NOSP, TABS. + Pn *tp = tok->getData(); + findAttrs(tok, lookupWord(tp->getName()), tp->getName(), false); + replaceRange(node, endNode, tok, tok); + return tok; +} + bool DICTTok::copyAttrsToSuggested(Pn *pn, Node *suggestedPn, Node *endNode) { Dlist *dsem_ = pn->getDsem(); bool found = false; @@ -915,11 +1143,6 @@ void DICTTok::makeTextAttrs(Node *node, Node *last) { } inline bool DICTTok::findAttrs(Node *node, CONCEPT *con, _TCHAR *str, bool isSuggested) { - _TCHAR attrName[NAMESIZ]; - _TCHAR bufval[NAMESIZ]; - - ATTR *attrs = cg_->findAttrs(con); - bool reduceIt = false; bool lower = false; bool cap = false; bool upper = false; @@ -966,6 +1189,20 @@ inline bool DICTTok::findAttrs(Node *node, CONCEPT *con, _TCHAR *str, bool i } } + return applyAttrs(node, con); +} + +/******************************************** +* FN: APPLYATTRS +* SUBJ: Put a dictionary entry's attributes on a node. +* RET: True if the entry has an s= (suggested node) attribute. +********************************************/ + +bool DICTTok::applyAttrs(Node *node, CONCEPT *con) { + _TCHAR attrName[NAMESIZ]; + _TCHAR bufval[NAMESIZ]; + ATTR *attrs = cg_->findAttrs(con); + bool reduceIt = false; _TCHAR *val; while (attrs) { @@ -1002,7 +1239,8 @@ inline bool DICTTok::findAttrs(Node *node, CONCEPT *con, _TCHAR *str, bool i _TCHAR *strattr, *strval; parse_->internStr(attrName, strattr); - if (_tcscmp(_T("pos"),strattr) && _tcscmp(_T("s"),strattr) && _tcscmp(_T("l"),strattr)) { + if (_tcscmp(_T("pos"),strattr) && _tcscmp(_T("s"),strattr) && _tcscmp(_T("l"),strattr) + && _tcscmp(_T("join"),strattr) && _tcscmp(_T("split"),strattr)) { // This needs to loop if there is more than one value for the same attribute if (cg_->isValStr(vals)) { cg_->popSval(vals,bufval); diff --git a/lite/dicttok.h b/lite/dicttok.h index af65862a..f5316bc2 100644 --- a/lite/dicttok.h +++ b/lite/dicttok.h @@ -134,6 +134,17 @@ class DICTTok : public Algo VAL *val ); + // Dictionary entries that reshape tokens (#426, #434) and attributes + // shared by every node of a name (#417). + bool applyAttrs(Node *node, CONCEPT *con); + CONCEPT *lookupWord(_TCHAR *name); + Node *makeLitTok(const _TCHAR *name, long start, long end, long ustart, + long uend, _TCHAR *text, long line); + void replaceRange(Node *first, Node *last, Node *nfirst, + Node *nlast); + bool splitToken(Node* &node, CONCEPT *con); + Node *joinPhrase(Node *node, Node *endNode); + private: //Parse *parse; // The analyze instance that we're tokenizing for. bool bad_; // If unknown char seen. // 01/15/99 AM. diff --git a/nlp/main.cpp b/nlp/main.cpp index 26e60aaf..10c3a52d 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.29" +#define NLP_ENGINE_VERSION "4.2.30" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &); void cmdHelpargs(_TCHAR *); From 8f4af3c3d56613a1462a8881a5726b324293b92a Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Wed, 14 Oct 2026 22:39:50 -0400 Subject: [PATCH 2/2] dict-shape fixture: one input line, so CRLF checkouts keep the offsets Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-linux.yml | 6 +++--- .github/workflows/build-windows.yml | 6 +++--- .github/workflows/tests/dict-shape/input/text.txt | 4 +--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 6278c048..874a4916 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1139,14 +1139,14 @@ jobs: '_bridge node text=viaduct at 19-25 type=location kind=raised [viaduct 19-25]' \ 'and alpha text=and at 27-29' \ '_bridge node text=bridge at 31-36 type=location kind=generic [bridge 31-36]' \ - '. punct text=. at 37-37' \ - 'My alpha text=My at 39-40' \ + '; punct text=; at 37-37' \ + 'my alpha text=my at 39-40' \ '_phrase node text=licenseplate at 42-53 [license 42-48, plate 49-53]' \ ', punct text=, at 54-54' \ '_weather node text=weathervconditions at 56-73 [weather 56-73, conditions 56-73]' \ ', punct text=, at 74-74' \ 'email alpha text=email at 76-81' \ - '. punct text=. at 82-82' \ + '; punct text=; at 82-82' \ '_phrase node text=license plate at 84-96 [license 84-90, plate 92-96]' \ '. punct text=. at 97-97') if [ "$got" != "$want" ]; then diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 592cb528..f9411799 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -1033,14 +1033,14 @@ jobs: '_bridge node text=viaduct at 19-25 type=location kind=raised [viaduct 19-25]' \ 'and alpha text=and at 27-29' \ '_bridge node text=bridge at 31-36 type=location kind=generic [bridge 31-36]' \ - '. punct text=. at 37-37' \ - 'My alpha text=My at 39-40' \ + '; punct text=; at 37-37' \ + 'my alpha text=my at 39-40' \ '_phrase node text=licenseplate at 42-53 [license 42-48, plate 49-53]' \ ', punct text=, at 54-54' \ '_weather node text=weathervconditions at 56-73 [weather 56-73, conditions 56-73]' \ ', punct text=, at 74-74' \ 'email alpha text=email at 76-81' \ - '. punct text=. at 82-82' \ + '; punct text=; at 82-82' \ '_phrase node text=license plate at 84-96 [license 84-90, plate 92-96]' \ '. punct text=. at 97-97') if [ "$got" != "$want" ]; then diff --git a/.github/workflows/tests/dict-shape/input/text.txt b/.github/workflows/tests/dict-shape/input/text.txt index 573bc595..ec6f3324 100644 --- a/.github/workflows/tests/dict-shape/input/text.txt +++ b/.github/workflows/tests/dict-shape/input/text.txt @@ -1,3 +1 @@ -A HOT-ROD near the viaduct and bridge. -My licenseplate, weathervconditions, e-mail. -license plate. +A HOT-ROD near the viaduct and bridge; my licenseplate, weathervconditions, e-mail; license plate.