String positions are characters, in every string function (4.2.40)

.github/workflows/build-linux.yml

@@ -358,6 +358,35 @@ jobs:
           fi
           echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path"
 
+      - name: String positions are characters (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # strindexof and strlastindexof answered in bytes, strlength in
+          # UTF-16 units, and strpiece and strchar in characters, so on text
+          # outside ASCII a position from one was wrong in another. A loop
+          # that cut at the first colon until none was left never ended on
+          # a curly apostrophe ahead of the colon. strpiece also refused any
+          # piece of 512 characters or more. All five count characters now,
+          # and strpiece has no length limit.
+          FIX=.github/workflows/tests/str-positions
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          timeout 120 ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi
+          OUT="$FIX/input/text.txt_log/out.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(str positions): no out.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'len:36' 'colon:26' 'lasta:16' 'tail:TEIXEIRA' 'emoji-len:5' 'emoji-colon:3' 'emoji-lastb:2' 'emoji-piece:b' 'long:600' 'long-piece:599' 'label:TEIXEIRA' 'turns:1')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(str positions): out.txt was:"; echo "$got"
+            echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[str positions] every string position is a character"
+
       - name: Run-mode globals ($dev / $silent, Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-macos.yml

@@ -206,6 +206,35 @@ jobs:
           fi
           echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path"
 
+      - name: String positions are characters
+        timeout-minutes: 3
+        shell: bash
+        run: |
+          set -eo pipefail
+          # strindexof and strlastindexof answered in bytes, strlength in
+          # UTF-16 units, and strpiece and strchar in characters, so on text
+          # outside ASCII a position from one was wrong in another. A loop
+          # that cut at the first colon until none was left never ended on
+          # a curly apostrophe ahead of the colon. strpiece also refused any
+          # piece of 512 characters or more. All five count characters now,
+          # and strpiece has no length limit.
+          FIX=.github/workflows/tests/str-positions
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi
+          OUT="$FIX/input/text.txt_log/out.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(str positions): no out.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'len:36' 'colon:26' 'lasta:16' 'tail:TEIXEIRA' 'emoji-len:5' 'emoji-colon:3' 'emoji-lastb:2' 'emoji-piece:b' 'long:600' 'long-piece:599' 'label:TEIXEIRA' 'turns:1')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(str positions): out.txt was:"; echo "$got"
+            echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[str positions] every string position is a character"
+
       # INTEL MACS (#231). The steps above built and tested arm64 (Apple
       # Silicon). Build the same tree for x86_64 too and merge the two with
       # lipo, so nlpm.exe and the compile libraries are universal binaries:

.github/workflows/build-windows.yml

@@ -294,6 +294,34 @@ jobs:
           fi
           echo "[lazy kbb] findconcept/getconcept/down reach a lazy file's keys; << writes to an absolute path"
 
+      - name: String positions are characters
+        shell: bash
+        run: |
+          set -eo pipefail
+          # strindexof and strlastindexof answered in bytes, strlength in
+          # UTF-16 units, and strpiece and strchar in characters, so on text
+          # outside ASCII a position from one was wrong in another. A loop
+          # that cut at the first colon until none was left never ended on
+          # a curly apostrophe ahead of the colon. strpiece also refused any
+          # piece of 512 characters or more. All five count characters now,
+          # and strpiece has no length limit.
+          FIX=.github/workflows/tests/str-positions
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          timeout 120 ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi
+          OUT="$FIX/input/text.txt_log/out.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(str positions): no out.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'len:36' 'colon:26' 'lasta:16' 'tail:TEIXEIRA' 'emoji-len:5' 'emoji-colon:3' 'emoji-lastb:2' 'emoji-piece:b' 'long:600' 'long-piece:599' 'label:TEIXEIRA' 'turns:1')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(str positions): out.txt was:"; echo "$got"
+            echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[str positions] every string position is a character"
+
       - name: Run-mode globals ($dev / $silent)
         shell: bash
         run: |

.github/workflows/tests/str-positions/input/text.txt

@@ -0,0 +1 @@
+The fixture only does string arithmetic.

.github/workflows/tests/str-positions/spec/analyzer.seq

@@ -0,0 +1,2 @@
+tokenize	nil	# tokenize only -- fixture needs no dictionary
+nlp	positions	# string positions are characters, everywhere

.github/workflows/tests/str-positions/spec/positions.nlp

@@ -0,0 +1,66 @@
+###############################################
+# FILE:     positions.nlp
+# SUBJ:     A string position is a character, in every string function.
+# NOTE:     strindexof and strlastindexof answered in BYTES, while strpiece
+#           and strchar count characters (code points) and strlength counted
+#           UTF-16 units. On text with one curly apostrophe ahead of a colon
+#           the index pointed past the colon, so a loop that cut at the first
+#           colon until none was left never got there, and allocated a string
+#           a turn until the process was killed at 2 GB. strpiece also
+#           refused any piece of 512 characters or more, which is why that
+#           loop's first cut failed outright.
+#           The label loop below is that loop, capped at 20 turns so a
+#           regression fails here instead of hanging the job.
+#           Expected out.txt:
+#           len:36
+#           colon:26
+#           lasta:16
+#           tail:TEIXEIRA
+#           emoji-len:5
+#           emoji-colon:3
+#           emoji-lastb:2
+#           emoji-piece:b
+#           long:600
+#           long-piece:599
+#           label:TEIXEIRA
+#           turns:1
+###############################################
+
+@CODE
+
+L("s") = "Registrar’s Signature NAME: TEIXEIRA";
+L("i") = strindexof(L("s"),":");
+"out.txt" << "len:"   << str(strlength(L("s"))) << "\n";
+"out.txt" << "colon:" << str(L("i")) << "\n";
+"out.txt" << "lasta:" << str(strlastindexof(L("s"),"a")) << "\n";
+"out.txt" << "tail:"  << strtrim(strpiece(L("s"),L("i") + 1,-1)) << "\n";
+
+# One character outside the Basic Multilingual Plane: two UTF-16 units,
+# four bytes, one character.
+L("e") = "a😀b:c";
+"out.txt" << "emoji-len:"   << str(strlength(L("e"))) << "\n";
+"out.txt" << "emoji-colon:" << str(strindexof(L("e"),":")) << "\n";
+"out.txt" << "emoji-lastb:" << str(strlastindexof(L("e"),"b")) << "\n";
+"out.txt" << "emoji-piece:" << strpiece(L("e"),2,2) << "\n";
+
+L("long") = "";
+L("n") = 0;
+while (L("n") < 600) {
+	L("long") = L("long") + "x";
+	L("n")++;
+}
+"out.txt" << "long:"       << str(strlength(L("long"))) << "\n";
+"out.txt" << "long-piece:" << str(strlength(strpiece(L("long"),1,-1))) << "\n";
+
+# The loop that hung: cut at the first colon until none is left.
+L("v") = L("s");
+L("turns") = 0;
+while (strcontains(":",L("v")) && L("turns") < 20) {
+	L("k") = strindexof(L("v"),":");
+	L("v") = strtrim(strpiece(L("v"),L("k") + 1,-1));
+	L("turns")++;
+}
+"out.txt" << "label:" << L("v") << "\n";
+"out.txt" << "turns:" << str(L("turns")) << "\n";
+
+@@CODE

lite/fn.cpp

@@ -6264,9 +6264,10 @@ else {
 }
 */
 
-icu::UnicodeString ustr = icu::UnicodeString::fromUTF8(icu::StringPiece(name1));
-const UChar *strBuf = ustr.getTerminatedBuffer();
-len = unicu::strLen(strBuf);
+// Characters (code points), the unit strpiece and strindexof use. This
+// counted UTF-16 units, so a character outside the Basic Multilingual Plane
+// (an emoji) was two here and one everywhere else.			// 10/25/26 DD.
+len = str_chars(name1);
 
 sem = new RFASem(len);
 
@@ -6313,7 +6314,12 @@ if (!str1)																		// 10/30/00 AM.
 	return parse->errOut(true); // UNFIXED 														// 05/18/01 AM.
 	}
 
-_TCHAR buf[MAXSTR];
+// A piece is never longer than the string, so a buffer the string's size
+// in bytes always holds it. This was MAXSTR (512), and every piece of 512
+// characters or more failed with "strpiece: Error." -- an empty answer to a
+// loop that then cut the other way.							// 10/25/26 DD.
+long siz = (long)_tcslen(str1) + 1;
+_TCHAR *buf = Chars::create(siz);
 
 #ifdef JUNK_
 _TCHAR *c;
@@ -6333,14 +6339,16 @@ if (!buf[0])														// FIX.	// 02/09/01 AM.
 	return true;													// FIX.	// 02/09/01 AM.
 #endif
 
-if (!str_piece(str1,start,end,MAXSTR,/*UP*/buf))					// 11/18/02 AM.
+if (!str_piece(str1,start,end,siz,/*UP*/buf))							// 10/25/26 DD.
 	{
+	Chars::destroy(buf);
 	_stprintf(Errbuf,_T("[strpiece: Error.]"));								// 11/18/02 AM.
 	return parse->errOut(true); // UNFIXED 											// 11/18/02 AM.
 	}
 
 _TCHAR *str;
 parse->internStr(buf, /*UP*/ str);
+Chars::destroy(buf);
 
 // Return as str type.
 sem = new RFASem(str, RSSTR);
@@ -12342,7 +12350,7 @@ return true;
 * FN:		FNSTRINDEXOF
 * CR:		08/03/26 DD.
 * SUBJ:	Position of the first occurrence of a substring.
-* RET:	True (executed ok); sem = zero-based index, or -1 if absent.
+* RET:	True (executed ok); sem = zero-based CHARACTER index, or -1 if absent.
 * FORMS:	strindexof(str, substr)
 * NOTE:	strchr takes only a single character and returns the tail of the
 *			string rather than a position, so getting an index used to mean
@@ -12372,9 +12380,12 @@ if (!Arg::done((DELTS*)args, _T("strindexof"),parse))
 long long at = -1LL;
 if (str && *str && sub && *sub)
 	{
+	// In characters, so the answer can be handed to strpiece. It was
+	// hit - str, a BYTE offset: past the match on any text with a
+	// multi-byte character ahead of it.						// 10/25/26 DD.
 	_TCHAR *hit = _tcsstr(str, sub);
 	if (hit)
-		at = (long long)(hit - str);
+		at = str_char_index(str, hit);
 	}
 sem = new RFASem(at);
 return true;
@@ -12385,7 +12396,7 @@ return true;
 * FN:		FNSTRLASTINDEXOF
 * CR:		08/03/26 DD.
 * SUBJ:	Position of the last occurrence of a substring.
-* RET:	True (executed ok); sem = zero-based index, or -1 if absent.
+* RET:	True (executed ok); sem = zero-based CHARACTER index, or -1 if absent.
 * FORMS:	strlastindexof(str, substr)
 ********************************************/
 
@@ -12413,11 +12424,14 @@ long long at = -1LL;
 if (str && *str && sub && *sub)
 	{
 	_TCHAR *hit = _tcsstr(str, sub);
+	_TCHAR *last = 0;
 	while (hit)
 		{
-		at = (long long)(hit - str);
+		last = hit;
 		hit = _tcsstr(hit + 1, sub);
 		}
+	if (last)
+		at = str_char_index(str, last);						// 10/25/26 DD.
 	}
 sem = new RFASem(at);
 return true;

lite/fnrun.cpp

@@ -7069,7 +7069,7 @@ long long Arun::strlength(
 {
 if (!name1 || !*name1)
 	return 0;
-return _tcsclen(name1);
+return str_chars(name1);		// characters, as interpreted strlength	// 10/25/26 DD.
 }
 
 // VARIANT.																// 01/23/01/00 AM.
@@ -7095,7 +7095,7 @@ switch (sem->getType())
 delete sem;
 if (!st || !*st)
 	return 0;
-return _tcsclen(st);
+return str_chars(st);			// 10/25/26 DD.
 }
 
 long long Arun::strlength(		// 07/11/03 AM.
@@ -7146,7 +7146,10 @@ if (!buf[0])														// FIX.	// 02/09/01 AM.
 	return 0;														// FIX.	// 02/09/01 AM.
 #endif
 
-long long xlen = end - start + 1;												// 07/16/03 AM.
+// Sized in BYTES. It was end - start + 1, a count of CHARACTERS, and
+// str_piece copies bytes, so a piece with a multi-byte character in it
+// overran the buffer.												// 10/25/26 DD.
+long long xlen = (long long)_tcslen(str1);
 _TCHAR *buf = Chars::create(xlen+1);										// 07/16/03 AM.
 
 if (!str_piece(str1,start,end,xlen+1,/*UP*/buf))					// 11/18/02 AM.

lite/str.cpp

@@ -2054,12 +2054,15 @@ if (start < 0 || end < 0)
 	if (end < 0)
 		end += nchars;
 	}
-long len = end-start+1;
-
+// siz is the size of buf in BYTES. It used to be compared with the number
+// of CHARACTERS asked for, which refused any piece of MAXSTR (512) or more
+// from a caller with a 512-byte buffer, and let a caller that sized buf in
+// characters overflow it with multi-byte text. The copy is bounded below,
+// where its length in bytes is known.						// 10/25/26 DD.
 if (end < start
- || start < 0
- || len >= siz )
+ || start < 0)
 	return false;
+long len = end-start+1;
 
 UChar32 c = 1;
 int32_t s = 0;
@@ -2080,12 +2083,57 @@ while (c && i<len) {
 	i++;
 }
 len = e-s;
+if (len >= siz)
+	return false;
 _tcsnccpy(buf,&(str[s]),len);
 buf[len] = '\0';
 return true;
 }
 
 
+/********************************************
+* FN:	   STR_CHARS
+* CR:	   10/25/26 DD.
+* SUBJ: Length of a UTF-8 string in characters (code points).
+* NOTE: The one unit every NLP++ string position is in: strlength,
+*		strpiece, strchar, strindexof and strlastindexof all count this way,
+*		so a position one of them returns can be handed to another.
+*		Before, strindexof counted bytes and strlength UTF-16 units, and
+*		on text outside ASCII the numbers did not line up.
+********************************************/
+
+long long str_chars(const _TCHAR *str)
+{
+if (!str)
+	return 0;
+long long n = 0;
+for (const unsigned char *c = (const unsigned char *)str; *c; ++c)
+	if ((*c & 0xC0) != 0x80)		// not a continuation byte
+		++n;
+return n;
+}
+
+
+/********************************************
+* FN:	   STR_CHAR_INDEX
+* CR:	   10/25/26 DD.
+* SUBJ: Character position of a pointer into a UTF-8 string.
+* RET:  Characters from str up to, not including, at; -1 if at is not in str.
+********************************************/
+
+long long str_char_index(const _TCHAR *str, const _TCHAR *at)
+{
+if (!str || !at || at < str)
+	return -1;
+long long n = 0;
+for (const unsigned char *c = (const unsigned char *)str;
+	  c < (const unsigned char *)at && *c; ++c)
+	if ((*c & 0xC0) != 0x80)
+		++n;
+return n;
+}
+
+
 
 /********************************************
 * FN:	   STR_UNIQUE_CHARS

lite/str.h

@@ -188,6 +188,8 @@ bool convert_to_float(_TCHAR *str1, _TCHAR *str2, /*UP*/ float &fnum);
 bool s_to_f(_TCHAR *str, /*UP*/ float *fnum);							// 08/23/01 AM.
 
 bool str_piece(_TCHAR*,long,long,long,/*UP*/_TCHAR*);					// 11/18/02 AM.
+long long str_chars(const _TCHAR*);										// 10/25/26 DD.
+long long str_char_index(const _TCHAR*, const _TCHAR*);				// 10/25/26 DD.
 bool str_unique_chars(_TCHAR*,/*UP*/_TCHAR*);
 
 // Should go in a morph module.

nlp/main.cpp

@@ -15,7 +15,7 @@ All rights reserved.
 #include "lite/nlp_engine.h"
 #include "version.h"
 
-#define NLP_ENGINE_VERSION "4.2.39"
+#define NLP_ENGINE_VERSION "4.2.40"
 
 bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &, _TCHAR *&);
 void cmdHelpargs(_TCHAR *);