Skip to content

Dictionary join=, split=, _name lines; bare entries load (#433 #434 #426 #417) (4.2.30) #786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ddehilster merged 2 commits into from
Oct 15, 2026
Merged

Dictionary join=, split=, _name lines; bare entries load (#433 #434 #426 #417) (4.2.30) #786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3898bda
Dictionary join=, split=, _name lines; bare entries load (#433 #434 #…
ddehilster Oct 15, 2026
8f4af3c
dict-shape fixture: one input line, so CRLF checkouts keep the offsets
ddehilster Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions .github/workflows/build-linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1113,6 +1113,47 @@ jobs:
fi
echo "[element-eq] attr=value on an element tests the attribute"

- name: Dictionary join=, split=, _name lines and bare entries (#433 #434 #426 #417) (Non-20.04)
if: matrix.id != 'docker-ubuntu-20.04'
shell: bash
run: |
set -eo pipefail
# A bare dictionary line (no attributes) used to end the read of the
# file (#433). join=1 makes a phrase one token (#434), split="a b"
# makes a token several words (#426), and a "_bridge type=x" line puts
# attributes on every _bridge node (#417).
FIX=.github/workflows/tests/dict-shape
rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
mkdir -p "$FIX/logs"
./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
OUT="$FIX/input/text.txt_log/output.txt"
if [ ! -f "$OUT" ]; then
echo "FAIL(dict-shape): no output.txt -- the analyzer did not build"; exit 1
fi
got=$(tr -d '\r' < "$OUT")
want=$(printf '%s\n' \
'A alpha text=A at 0-0' \
'_vehicle node text=HOT-ROD at 2-8 speed=fast [HOTROD 2-8]' \
'near alpha text=near at 10-13' \
'the alpha text=the at 15-17' \
'_bridge node text=viaduct at 19-25 type=location kind=raised [viaduct 19-25]' \
'and alpha text=and at 27-29' \
'_bridge node text=bridge at 31-36 type=location kind=generic [bridge 31-36]' \
'; punct text=; at 37-37' \
'my alpha text=my at 39-40' \
'_phrase node text=licenseplate at 42-53 [license 42-48, plate 49-53]' \
', punct text=, at 54-54' \
'_weather node text=weathervconditions at 56-73 [weather 56-73, conditions 56-73]' \
', punct text=, at 74-74' \
'email alpha text=email at 76-81' \
'; punct text=; at 82-82' \
'_phrase node text=license plate at 84-96 [license 84-90, plate 92-96]' \
'. punct text=. at 97-97')
if [ "$got" != "$want" ]; then
echo "FAIL(dict-shape): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
fi
echo "[dict-shape] dictionary join, split, _name lines and bare entries"

- name: factorial() up to 20! (Non-20.04)
if: matrix.id != 'docker-ubuntu-20.04'
shell: bash
Expand Down
40 changes: 40 additions & 0 deletions .github/workflows/build-windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1008,6 +1008,46 @@ jobs:
fi
echo "[element-eq] attr=value on an element tests the attribute"

- name: Dictionary join=, split=, _name lines and bare entries (#433 #434 #426 #417)
shell: bash
run: |
set -eo pipefail
# A bare dictionary line (no attributes) used to end the read of the
# file (#433). join=1 makes a phrase one token (#434), split="a b"
# makes a token several words (#426), and a "_bridge type=x" line puts
# attributes on every _bridge node (#417).
FIX=.github/workflows/tests/dict-shape
rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
mkdir -p "$FIX/logs"
./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
OUT="$FIX/input/text.txt_log/output.txt"
if [ ! -f "$OUT" ]; then
echo "FAIL(dict-shape): no output.txt -- the analyzer did not build"; exit 1
fi
got=$(tr -d '\r' < "$OUT")
want=$(printf '%s\n' \
'A alpha text=A at 0-0' \
'_vehicle node text=HOT-ROD at 2-8 speed=fast [HOTROD 2-8]' \
'near alpha text=near at 10-13' \
'the alpha text=the at 15-17' \
'_bridge node text=viaduct at 19-25 type=location kind=raised [viaduct 19-25]' \
'and alpha text=and at 27-29' \
'_bridge node text=bridge at 31-36 type=location kind=generic [bridge 31-36]' \
'; punct text=; at 37-37' \
'my alpha text=my at 39-40' \
'_phrase node text=licenseplate at 42-53 [license 42-48, plate 49-53]' \
', punct text=, at 54-54' \
'_weather node text=weathervconditions at 56-73 [weather 56-73, conditions 56-73]' \
', punct text=, at 74-74' \
'email alpha text=email at 76-81' \
'; punct text=; at 82-82' \
'_phrase node text=license plate at 84-96 [license 84-90, plate 92-96]' \
'. punct text=. at 97-97')
if [ "$got" != "$want" ]; then
echo "FAIL(dict-shape): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
fi
echo "[dict-shape] dictionary join, split, _name lines and bare entries"

- name: factorial() up to 20!
shell: bash
run: |
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/tests/dict-shape/input/text.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A HOT-ROD near the viaduct and bridge; my licenseplate, weathervconditions, e-mail; license plate.
15 changes: 15 additions & 0 deletions .github/workflows/tests/dict-shape/kb/user/words.dict
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Regression for the dictionary tokenizer (#433 #434 #426 #417).
# A bare entry (no attributes) used to end the read of the file.
license plate
# join=1: the phrase becomes one token, looked up in turn.
hot - rod join=1
hotrod s=vehicle speed=fast
e - mail join=1
# split=: one token becomes several words.
licenseplate split="license plate"
weathervconditions split="weather conditions"
weather conditions s=weather
# _name: attributes for every _bridge node; an entry's own win.
bridge s=bridge
viaduct s=bridge kind=raised
_bridge type=location kind=generic
2 changes: 2 additions & 0 deletions .github/workflows/tests/dict-shape/spec/analyzer.seq
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dicttok nil #
nlp show #
33 changes: 33 additions & 0 deletions .github/workflows/tests/dict-shape/spec/show.nlp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
###############################################
# FILE: show.nlp
# SUBJ: Each top node: name, type, $text, offsets, and the
# attributes the dictionary gave it; then its children.
# NOTE: Part of the dict-shape regression; expected output is in
# the CI step.
###############################################

@CODE
L("n") = pndown(pnroot());
while (L("n")) {
if (pntype(L("n")) != "white") {
"output.txt" << pnname(L("n")) << " " << pntype(L("n")) << " text=" << pnvar(L("n"), "$text")
<< " at " << pnvar(L("n"), "$ostart") << "-" << pnvar(L("n"), "$oend");
if (pnvar(L("n"), "type")) "output.txt" << " type=" << pnvar(L("n"), "type");
if (pnvar(L("n"), "kind")) "output.txt" << " kind=" << pnvar(L("n"), "kind");
if (pnvar(L("n"), "speed")) "output.txt" << " speed=" << pnvar(L("n"), "speed");
L("c") = pndown(L("n"));
L("sep") = " [";
while (L("c")) {
if (pntype(L("c")) != "white") {
"output.txt" << L("sep") << pnname(L("c")) << " " << pnvar(L("c"), "$ostart")
<< "-" << pnvar(L("c"), "$oend");
L("sep") = ", ";
}
L("c") = pnnext(L("c"));
}
if (L("sep") == ", ") "output.txt" << "]";
"output.txt" << "\n";
}
L("n") = pnnext(L("n"));
}
@@CODE
14 changes: 7 additions & 7 deletions cs/libconsh/cg.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3902,8 +3902,9 @@ bool CG::readDict(std::string file, std::vector<std::filesystem::path> kbfiles)
strip_block_comments(buf, inBlock, '#');
if (unicu::isStrWhiteSpace(buf))
continue;
if (!parseDictLine(buf, ambigKB, file, lineCount))
return false;
// A bad line is reported (cgerr) and skipped. It used to end the
// read, silently dropping every entry after it (#433).
parseDictLine(buf, ambigKB, file, lineCount);
}

allDictStream_.close();
Expand Down Expand Up @@ -4083,11 +4084,10 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i
return false;
}

if (eqSign == -1) {
sprintf(errout_, "%d 1 [missing equal sign - %s]", lineCount, filename.c_str());
*cgerr << errout_ << std::endl;
return false;
}
// No attributes: the line is all words, eg "license plate" (#433).
// It used to be a "missing equal sign" error.
if (eqSign == -1)
eqSign = tokint + 1;

// Loop through tokens

Expand Down
Loading
Loading