diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index edcd71ea..5c9f221d 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -358,6 +358,35 @@ jobs: fi echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: String positions are characters (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # strindexof and strlastindexof answered in bytes, strlength in + # UTF-16 units, and strpiece and strchar in characters, so on text + # outside ASCII a position from one was wrong in another. A loop + # that cut at the first colon until none was left never ended on + # a curly apostrophe ahead of the colon. strpiece also refused any + # piece of 512 characters or more. All five count characters now, + # and strpiece has no length limit. + FIX=.github/workflows/tests/str-positions + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + timeout 120 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi + OUT="$FIX/input/text.txt_log/out.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(str positions): no out.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' 'len:36' 'colon:26' 'lasta:16' 'tail:TEIXEIRA' 'emoji-len:5' 'emoji-colon:3' 'emoji-lastb:2' 'emoji-piece:b' 'long:600' 'long-piece:599' 'label:TEIXEIRA' 'turns:1') + if [ "$got" != "$want" ]; then + echo "FAIL(str positions): out.txt was:"; echo "$got" + echo "wanted:"; echo "$want"; exit 1 + fi + echo "[str positions] every string position is a character" + - name: Run-mode globals ($dev / $silent, Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 93f13f67..23966386 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -206,6 +206,35 @@ jobs: fi echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: String positions are characters + timeout-minutes: 3 + shell: bash + run: | + set -eo pipefail + # strindexof and strlastindexof answered in bytes, strlength in + # UTF-16 units, and strpiece and strchar in characters, so on text + # outside ASCII a position from one was wrong in another. A loop + # that cut at the first colon until none was left never ended on + # a curly apostrophe ahead of the colon. strpiece also refused any + # piece of 512 characters or more. All five count characters now, + # and strpiece has no length limit. + FIX=.github/workflows/tests/str-positions + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi + OUT="$FIX/input/text.txt_log/out.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(str positions): no out.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' 'len:36' 'colon:26' 'lasta:16' 'tail:TEIXEIRA' 'emoji-len:5' 'emoji-colon:3' 'emoji-lastb:2' 'emoji-piece:b' 'long:600' 'long-piece:599' 'label:TEIXEIRA' 'turns:1') + if [ "$got" != "$want" ]; then + echo "FAIL(str positions): out.txt was:"; echo "$got" + echo "wanted:"; echo "$want"; exit 1 + fi + echo "[str positions] every string position is a character" + # INTEL MACS (#231). The steps above built and tested arm64 (Apple # Silicon). Build the same tree for x86_64 too and merge the two with # lipo, so nlpm.exe and the compile libraries are universal binaries: diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 3e880fd8..3c0204a9 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -294,6 +294,34 @@ jobs: fi echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path" + - name: String positions are characters + shell: bash + run: | + set -eo pipefail + # strindexof and strlastindexof answered in bytes, strlength in + # UTF-16 units, and strpiece and strchar in characters, so on text + # outside ASCII a position from one was wrong in another. A loop + # that cut at the first colon until none was left never ended on + # a curly apostrophe ahead of the colon. strpiece also refused any + # piece of 512 characters or more. All five count characters now, + # and strpiece has no length limit. + FIX=.github/workflows/tests/str-positions + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" + mkdir -p "$FIX/logs" + timeout 120 ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi + OUT="$FIX/input/text.txt_log/out.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(str positions): no out.txt -- the analyzer did not build"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' 'len:36' 'colon:26' 'lasta:16' 'tail:TEIXEIRA' 'emoji-len:5' 'emoji-colon:3' 'emoji-lastb:2' 'emoji-piece:b' 'long:600' 'long-piece:599' 'label:TEIXEIRA' 'turns:1') + if [ "$got" != "$want" ]; then + echo "FAIL(str positions): out.txt was:"; echo "$got" + echo "wanted:"; echo "$want"; exit 1 + fi + echo "[str positions] every string position is a character" + - name: Run-mode globals ($dev / $silent) shell: bash run: | diff --git a/.github/workflows/tests/str-positions/input/text.txt b/.github/workflows/tests/str-positions/input/text.txt new file mode 100644 index 00000000..f76f8d4b --- /dev/null +++ b/.github/workflows/tests/str-positions/input/text.txt @@ -0,0 +1 @@ +The fixture only does string arithmetic. diff --git a/.github/workflows/tests/str-positions/kb/user/.gitkeep b/.github/workflows/tests/str-positions/kb/user/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/tests/str-positions/spec/analyzer.seq b/.github/workflows/tests/str-positions/spec/analyzer.seq new file mode 100644 index 00000000..746bb4be --- /dev/null +++ b/.github/workflows/tests/str-positions/spec/analyzer.seq @@ -0,0 +1,2 @@ +tokenize nil # tokenize only -- fixture needs no dictionary +nlp positions # string positions are characters, everywhere diff --git a/.github/workflows/tests/str-positions/spec/positions.nlp b/.github/workflows/tests/str-positions/spec/positions.nlp new file mode 100644 index 00000000..67249c9b --- /dev/null +++ b/.github/workflows/tests/str-positions/spec/positions.nlp @@ -0,0 +1,66 @@ +############################################### +# FILE: positions.nlp +# SUBJ: A string position is a character, in every string function. +# NOTE: strindexof and strlastindexof answered in BYTES, while strpiece +# and strchar count characters (code points) and strlength counted +# UTF-16 units. On text with one curly apostrophe ahead of a colon +# the index pointed past the colon, so a loop that cut at the first +# colon until none was left never got there, and allocated a string +# a turn until the process was killed at 2 GB. strpiece also +# refused any piece of 512 characters or more, which is why that +# loop's first cut failed outright. +# The label loop below is that loop, capped at 20 turns so a +# regression fails here instead of hanging the job. +# Expected out.txt: +# len:36 +# colon:26 +# lasta:16 +# tail:TEIXEIRA +# emoji-len:5 +# emoji-colon:3 +# emoji-lastb:2 +# emoji-piece:b +# long:600 +# long-piece:599 +# label:TEIXEIRA +# turns:1 +############################################### + +@CODE + +L("s") = "Registrar’s Signature NAME: TEIXEIRA"; +L("i") = strindexof(L("s"),":"); +"out.txt" << "len:" << str(strlength(L("s"))) << "\n"; +"out.txt" << "colon:" << str(L("i")) << "\n"; +"out.txt" << "lasta:" << str(strlastindexof(L("s"),"a")) << "\n"; +"out.txt" << "tail:" << strtrim(strpiece(L("s"),L("i") + 1,-1)) << "\n"; + +# One character outside the Basic Multilingual Plane: two UTF-16 units, +# four bytes, one character. +L("e") = "a😀b:c"; +"out.txt" << "emoji-len:" << str(strlength(L("e"))) << "\n"; +"out.txt" << "emoji-colon:" << str(strindexof(L("e"),":")) << "\n"; +"out.txt" << "emoji-lastb:" << str(strlastindexof(L("e"),"b")) << "\n"; +"out.txt" << "emoji-piece:" << strpiece(L("e"),2,2) << "\n"; + +L("long") = ""; +L("n") = 0; +while (L("n") < 600) { + L("long") = L("long") + "x"; + L("n")++; +} +"out.txt" << "long:" << str(strlength(L("long"))) << "\n"; +"out.txt" << "long-piece:" << str(strlength(strpiece(L("long"),1,-1))) << "\n"; + +# The loop that hung: cut at the first colon until none is left. +L("v") = L("s"); +L("turns") = 0; +while (strcontains(":",L("v")) && L("turns") < 20) { + L("k") = strindexof(L("v"),":"); + L("v") = strtrim(strpiece(L("v"),L("k") + 1,-1)); + L("turns")++; +} +"out.txt" << "label:" << L("v") << "\n"; +"out.txt" << "turns:" << str(L("turns")) << "\n"; + +@@CODE diff --git a/lite/fn.cpp b/lite/fn.cpp index 0e3a0bb5..c632cbbd 100644 --- a/lite/fn.cpp +++ b/lite/fn.cpp @@ -6264,9 +6264,10 @@ else { } */ -icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(name1)); -const UChar *strBuf = ustr.getTerminatedBuffer(); -len = unicu::strLen(strBuf); +// Characters (code points), the unit strpiece and strindexof use. This +// counted UTF-16 units, so a character outside the Basic Multilingual Plane +// (an emoji) was two here and one everywhere else. // 10/25/26 DD. +len = str_chars(name1); sem = new RFASem(len); @@ -6313,7 +6314,12 @@ if (!str1) // 10/30/00 AM. return parse->errOut(true); // UNFIXED // 05/18/01 AM. } -_TCHAR buf[MAXSTR]; +// A piece is never longer than the string, so a buffer the string's size +// in bytes always holds it. This was MAXSTR (512), and every piece of 512 +// characters or more failed with "strpiece: Error." -- an empty answer to a +// loop that then cut the other way. // 10/25/26 DD. +long siz = (long)_tcslen(str1) + 1; +_TCHAR *buf = Chars::create(siz); #ifdef JUNK_ _TCHAR *c; @@ -6333,14 +6339,16 @@ if (!buf[0]) // FIX. // 02/09/01 AM. return true; // FIX. // 02/09/01 AM. #endif -if (!str_piece(str1,start,end,MAXSTR,/*UP*/buf)) // 11/18/02 AM. +if (!str_piece(str1,start,end,siz,/*UP*/buf)) // 10/25/26 DD. { + Chars::destroy(buf); _stprintf(Errbuf,_T("[strpiece: Error.]")); // 11/18/02 AM. return parse->errOut(true); // UNFIXED // 11/18/02 AM. } _TCHAR *str; parse->internStr(buf, /*UP*/ str); +Chars::destroy(buf); // Return as str type. sem = new RFASem(str, RSSTR); @@ -12342,7 +12350,7 @@ return true; * FN: FNSTRINDEXOF * CR: 08/03/26 DD. * SUBJ: Position of the first occurrence of a substring. -* RET: True (executed ok); sem = zero-based index, or -1 if absent. +* RET: True (executed ok); sem = zero-based CHARACTER index, or -1 if absent. * FORMS: strindexof(str, substr) * NOTE: strchr takes only a single character and returns the tail of the * string rather than a position, so getting an index used to mean @@ -12372,9 +12380,12 @@ if (!Arg::done((DELTS*)args, _T("strindexof"),parse)) long long at = -1LL; if (str && *str && sub && *sub) { + // In characters, so the answer can be handed to strpiece. It was + // hit - str, a BYTE offset: past the match on any text with a + // multi-byte character ahead of it. // 10/25/26 DD. _TCHAR *hit = _tcsstr(str, sub); if (hit) - at = (long long)(hit - str); + at = str_char_index(str, hit); } sem = new RFASem(at); return true; @@ -12385,7 +12396,7 @@ return true; * FN: FNSTRLASTINDEXOF * CR: 08/03/26 DD. * SUBJ: Position of the last occurrence of a substring. -* RET: True (executed ok); sem = zero-based index, or -1 if absent. +* RET: True (executed ok); sem = zero-based CHARACTER index, or -1 if absent. * FORMS: strlastindexof(str, substr) ********************************************/ @@ -12413,11 +12424,14 @@ long long at = -1LL; if (str && *str && sub && *sub) { _TCHAR *hit = _tcsstr(str, sub); + _TCHAR *last = 0; while (hit) { - at = (long long)(hit - str); + last = hit; hit = _tcsstr(hit + 1, sub); } + if (last) + at = str_char_index(str, last); // 10/25/26 DD. } sem = new RFASem(at); return true; diff --git a/lite/fnrun.cpp b/lite/fnrun.cpp index 2010a731..6574469f 100644 --- a/lite/fnrun.cpp +++ b/lite/fnrun.cpp @@ -7069,7 +7069,7 @@ long long Arun::strlength( { if (!name1 || !*name1) return 0; -return _tcsclen(name1); +return str_chars(name1); // characters, as interpreted strlength // 10/25/26 DD. } // VARIANT. // 01/23/01/00 AM. @@ -7095,7 +7095,7 @@ switch (sem->getType()) delete sem; if (!st || !*st) return 0; -return _tcsclen(st); +return str_chars(st); // 10/25/26 DD. } long long Arun::strlength( // 07/11/03 AM. @@ -7146,7 +7146,10 @@ if (!buf[0]) // FIX. // 02/09/01 AM. return 0; // FIX. // 02/09/01 AM. #endif -long long xlen = end - start + 1; // 07/16/03 AM. +// Sized in BYTES. It was end - start + 1, a count of CHARACTERS, and +// str_piece copies bytes, so a piece with a multi-byte character in it +// overran the buffer. // 10/25/26 DD. +long long xlen = (long long)_tcslen(str1); _TCHAR *buf = Chars::create(xlen+1); // 07/16/03 AM. if (!str_piece(str1,start,end,xlen+1,/*UP*/buf)) // 11/18/02 AM. diff --git a/lite/str.cpp b/lite/str.cpp index 59272c90..d00ab952 100644 --- a/lite/str.cpp +++ b/lite/str.cpp @@ -2054,12 +2054,15 @@ if (start < 0 || end < 0) if (end < 0) end += nchars; } -long len = end-start+1; - +// siz is the size of buf in BYTES. It used to be compared with the number +// of CHARACTERS asked for, which refused any piece of MAXSTR (512) or more +// from a caller with a 512-byte buffer, and let a caller that sized buf in +// characters overflow it with multi-byte text. The copy is bounded below, +// where its length in bytes is known. // 10/25/26 DD. if (end < start - || start < 0 - || len >= siz ) + || start < 0) return false; +long len = end-start+1; UChar32 c = 1; int32_t s = 0; @@ -2080,12 +2083,57 @@ while (c && i= siz) + return false; _tcsnccpy(buf,&(str[s]),len); buf[len] = '\0'; return true; } +/******************************************** +* FN: STR_CHARS +* CR: 10/25/26 DD. +* SUBJ: Length of a UTF-8 string in characters (code points). +* NOTE: The one unit every NLP++ string position is in: strlength, +* strpiece, strchar, strindexof and strlastindexof all count this way, +* so a position one of them returns can be handed to another. +* Before, strindexof counted bytes and strlength UTF-16 units, and +* on text outside ASCII the numbers did not line up. +********************************************/ + +long long str_chars(const _TCHAR *str) +{ +if (!str) + return 0; +long long n = 0; +for (const unsigned char *c = (const unsigned char *)str; *c; ++c) + if ((*c & 0xC0) != 0x80) // not a continuation byte + ++n; +return n; +} + + +/******************************************** +* FN: STR_CHAR_INDEX +* CR: 10/25/26 DD. +* SUBJ: Character position of a pointer into a UTF-8 string. +* RET: Characters from str up to, not including, at; -1 if at is not in str. +********************************************/ + +long long str_char_index(const _TCHAR *str, const _TCHAR *at) +{ +if (!str || !at || at < str) + return -1; +long long n = 0; +for (const unsigned char *c = (const unsigned char *)str; + c < (const unsigned char *)at && *c; ++c) + if ((*c & 0xC0) != 0x80) + ++n; +return n; +} + + /******************************************** * FN: STR_UNIQUE_CHARS diff --git a/lite/str.h b/lite/str.h index 11a8fd0d..f7795360 100644 --- a/lite/str.h +++ b/lite/str.h @@ -188,6 +188,8 @@ bool convert_to_float(_TCHAR *str1, _TCHAR *str2, /*UP*/ float &fnum); bool s_to_f(_TCHAR *str, /*UP*/ float *fnum); // 08/23/01 AM. bool str_piece(_TCHAR*,long,long,long,/*UP*/_TCHAR*); // 11/18/02 AM. +long long str_chars(const _TCHAR*); // 10/25/26 DD. +long long str_char_index(const _TCHAR*, const _TCHAR*); // 10/25/26 DD. bool str_unique_chars(_TCHAR*,/*UP*/_TCHAR*); // Should go in a morph module. diff --git a/nlp/main.cpp b/nlp/main.cpp index abd988fa..801da19d 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,7 +15,7 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.39" +#define NLP_ENGINE_VERSION "4.2.40" bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&); void cmdHelpargs(_TCHAR *);