-DICTSOURCE: nodes record the .dict file and line that built them (#432 #475) (4.2.33)

.github/workflows/build-linux.yml

@@ -1188,6 +1188,48 @@ jobs:
           [ "$(tr -d '\r' < "$DUMP" | grep -v '^$' | tail -1)" = quit ] || { echo "FAIL(fs-ops): dump.kb does not end in quit"; cat "$DUMP"; exit 1; }
           echo "[fs-ops] mkdir() and kbdumptree() work without the shell"
 
+      - name: -DICTSOURCE marks nodes with their dictionary file and line (#432) (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # With -DICTSOURCE, nodes dicttok builds from .dict entries get
+          # dictfile and dictline attributes (#432, #475); without it, none.
+          FIX=.github/workflows/tests/dict-source
+          OUT="$FIX/input/text.txt_log/output.txt"
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -s "$OUT" ]; then
+            echo "FAIL(dict-source): attributes without -DICTSOURCE:"; cat "$OUT"; exit 1
+          fi
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ -DICTSOURCE "$FIX/input/text.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(dict-source): no output.txt with -DICTSOURCE"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' \
+            '_vehicle words.dict:6' \
+            '  HOTROD words.dict:6' \
+            '_bridge words.dict:14' \
+            '  viaduct words.dict:14' \
+            '_bridge words.dict:13' \
+            '  bridge words.dict:13' \
+            '_phrase words.dict:3' \
+            '  license words.dict:9' \
+            '  plate words.dict:9' \
+            '_weather words.dict:11' \
+            '  weather words.dict:10' \
+            '  conditions words.dict:10' \
+            'email words.dict:7' \
+            '_phrase words.dict:3')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(dict-source): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[dict-source] -DICTSOURCE marks dictionary nodes with file and line"
+
       - name: factorial() up to 20! (Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-windows.yml

@@ -1081,6 +1081,47 @@ jobs:
           [ "$(tr -d '\r' < "$DUMP" | grep -v '^$' | tail -1)" = quit ] || { echo "FAIL(fs-ops): dump.kb does not end in quit"; cat "$DUMP"; exit 1; }
           echo "[fs-ops] mkdir() and kbdumptree() work without the shell"
 
+      - name: -DICTSOURCE marks nodes with their dictionary file and line (#432)
+        shell: bash
+        run: |
+          set -eo pipefail
+          # With -DICTSOURCE, nodes dicttok builds from .dict entries get
+          # dictfile and dictline attributes (#432, #475); without it, none.
+          FIX=.github/workflows/tests/dict-source
+          OUT="$FIX/input/text.txt_log/output.txt"
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -s "$OUT" ]; then
+            echo "FAIL(dict-source): attributes without -DICTSOURCE:"; cat "$OUT"; exit 1
+          fi
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output"
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ -DICTSOURCE "$FIX/input/text.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(dict-source): no output.txt with -DICTSOURCE"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' \
+            '_vehicle words.dict:6' \
+            '  HOTROD words.dict:6' \
+            '_bridge words.dict:14' \
+            '  viaduct words.dict:14' \
+            '_bridge words.dict:13' \
+            '  bridge words.dict:13' \
+            '_phrase words.dict:3' \
+            '  license words.dict:9' \
+            '  plate words.dict:9' \
+            '_weather words.dict:11' \
+            '  weather words.dict:10' \
+            '  conditions words.dict:10' \
+            'email words.dict:7' \
+            '_phrase words.dict:3')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(dict-source): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          echo "[dict-source] -DICTSOURCE marks dictionary nodes with file and line"
+
       - name: factorial() up to 20!
         shell: bash
         run: |

.github/workflows/tests/dict-source/input/text.txt

@@ -0,0 +1 @@
+A HOT-ROD near the viaduct and bridge; my licenseplate, weathervconditions, e-mail; license plate.

.github/workflows/tests/dict-source/kb/user/words.dict

@@ -0,0 +1,15 @@
+# Regression for the dictionary tokenizer (#433 #434 #426 #417).
+# A bare entry (no attributes) used to end the read of the file.
+license plate
+# join=1: the phrase becomes one token, looked up in turn.
+hot - rod join=1
+hotrod s=vehicle speed=fast
+e - mail join=1
+# split=: one token becomes several words.
+licenseplate split="license plate"
+weathervconditions split="weather conditions"
+weather conditions s=weather
+# _name: attributes for every _bridge node; an entry's own win.
+bridge s=bridge
+viaduct s=bridge kind=raised
+_bridge type=location kind=generic

.github/workflows/tests/dict-source/spec/analyzer.seq

@@ -0,0 +1,2 @@
+dicttok	nil	# 
+nlp	show	# 

.github/workflows/tests/dict-source/spec/show.nlp

@@ -0,0 +1,22 @@
+###############################################
+# FILE:     show.nlp
+# SUBJ:     Each node from a dictionary entry, and its children, with
+#           the entry's file and line (-DICTSOURCE, #432).
+# NOTE:     Part of the dict-source regression; expected output is in
+#           the CI step. Without -DICTSOURCE nothing is printed.
+###############################################
+
+@CODE
+L("n") = pndown(pnroot());
+while (L("n")) {
+    if (pnvar(L("n"), "dictfile"))
+        "output.txt" << pnname(L("n")) << " " << pnvar(L("n"), "dictfile") << ":" << pnvar(L("n"), "dictline") << "\n";
+    L("c") = pndown(L("n"));
+    while (L("c")) {
+        if (pnvar(L("c"), "dictfile"))
+            "output.txt" << "  " << pnname(L("c")) << " " << pnvar(L("c"), "dictfile") << ":" << pnvar(L("c"), "dictline") << "\n";
+        L("c") = pnnext(L("c"));
+    }
+    L("n") = pnnext(L("n"));
+}
+@@CODE

README.md

@@ -85,7 +85,11 @@ You can get help on nlp.exe:
                  [-OUT outdir] output directory
                  [-WORK workdir] working directory
                  [-DEV][-SILENT] -DEV generates logs, -SILENT suppresses logs/output files (off by default)
+                 [-DEBUG port] wait for a rule debugger on 127.0.0.1:port, then run under it
                  [infile [outfile]] when no -IN or -OUT specified
+                 [-STDIN] load once, then analyze one input file path per stdin line
+                 [-DICTSOURCE] nodes built from .dict entries get dictfile and dictline
+                          attributes naming the entry's file and line
 
       Directories in the nlp.exe files:
          data        nlp engine bootstrap grammar
@@ -106,6 +110,8 @@ Switch | Function
 -WORK | Working director where the library and executable files are
 -DEV / -SILENT | -DEV generates logs, -SILENT suppresses logs/output files (off by default)
 -DEBUG | Port to serve the debug protocol on. The engine waits for a client, then stops at pass boundaries, rule attempts and statements so the client can step and inspect. Used by the VS Code extension's debugger; see "What's New in Version 4".
+-STDIN | Load the analyzer once, then analyze one input file path per line of standard input. A blank line or end of input ends the run.
+-DICTSOURCE | Nodes that dicttok builds from `.dict` entries get `dictfile` and `dictline` attributes naming the entry's file and line, so a debugger or editor can jump to it. Off by default: it adds two attributes to every dictionary entry.
 [infile [outfile]] | when no -IN or -OUT specified
 
 # Calling NLP++ Analyzers from C++

cs/libconsh/cg.cpp

@@ -552,7 +552,7 @@ std::_t_ofstream f_main(TCHAR2A(o_main));
 
 // Use forward-slash separators in the generated `take` statements. The
 // take parser (cmd_take -> resolve_file in cs/libprim/io.cpp) does NOT
-// normalize backslashes to slashes on Linux; resolve_file_unix
+// normalize backslashes to slashes on Linux; resolve_file
 // concatenates verbatim and then fopen sees one filename with literal
 // '\' chars. Windows historically accepted forward slashes here, so
 // emitting them on both platforms preserves Windows behavior bit-for-bit
@@ -3864,6 +3864,10 @@ bool CG::readDicts(std::vector<std::filesystem::path> files, std::vector<std::fi
 	return true;
 }
 
+static bool dictSource = false;	// -DICTSOURCE (#432).
+void CG::setDictSource(bool on) { dictSource = on; }
+bool CG::getDictSource() { return dictSource; }
+
 bool CG::readDict(std::string file, std::vector<std::filesystem::path> kbfiles) {
 	_TCHAR buf[MAXMSG];
 	int lineCount = 0;
@@ -4200,6 +4204,17 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i
 			addSval(parentCon,_T("s"),_T("phrase"));
 		}
 
+		// -DICTSOURCE: where the entry is, for the debugger and editors
+		// (#432, #475). A word on several lines keeps its first. A lazily
+		// loaded *full.dict line has no line number, so only its file.
+		if (dictSource && parentCon && !findVals(parentCon, _T("dictfile"))) {
+			_TCHAR fname[MAXSTR];
+			_stprintf(fname, _T("%.*s"), (int)(MAXSTR - 1), filename.c_str());
+			addSval(parentCon,_T("dictfile"),fname);
+			if (lineCount > 0)
+				addVal(parentCon,_T("dictline"),(long long) lineCount);
+		}
+
 	}
 
 	return true;

cs/libconsh/cmd.cpp

@@ -1611,7 +1611,9 @@ if (args)
 
 // Handling relative filename.											// 07/01/03 AM.
 _TCHAR buf[FNAME_SIZE+1];														// 07/01/03 AM.
-if (!resolve_file(fil,cg->getAppdir(),/*DU*/buf))					// 07/01/03 AM.
+_TCHAR *appdir = cg->getAppdir();
+if (_tcslen(fil) + (appdir ? _tcslen(appdir) : 0) + 2 > FNAME_SIZE	// #323.
+ || !resolve_file(fil,appdir,/*DU*/buf))					// 07/01/03 AM.
 	{
 	std::_t_cerr << _T("[Bad filename=") << fil << _T("]") << std::endl;
 	return false;

cs/libprim/io.cpp

@@ -18,6 +18,7 @@ All rights reserved.
 *******************************************************************************/
 
 #include "StdAfx.h"
+#include <filesystem>		// #323.
 #include <fstream>											// Upgrade	// 01/24/01 AM.
 
 #include <stdio.h>
@@ -34,8 +35,6 @@ All rights reserved.
 #include "prim/io.h"
 
 // Local defs.
-bool resolve_file_win(_TCHAR*,_TCHAR*,_TCHAR*);		// 07/01/03 AM.
-bool resolve_file_unix(_TCHAR*,_TCHAR*,_TCHAR*);	// 07/01/03 AM.
 
 /**************************************************
 *						INDENT
@@ -120,91 +119,21 @@ while (num-- > 0)
 LIBPRIM_API bool
 resolve_file(_TCHAR *fname, _TCHAR *bname, /*DU*/ _TCHAR *buf)
 {
-#ifndef LINUX
-return resolve_file_win(fname,bname,buf);
-#else
-return resolve_file_unix(fname,bname,buf);
-#endif
-}
-
-
-// VARIANT.
-bool
-resolve_file_win(_TCHAR *fname, _TCHAR *bname, /*DU*/ _TCHAR *buf)
-{
+// One way for every platform (#323): a name with a root ("/x", "C:\\x",
+// "\\x") is used as is, any other is taken relative to BNAME.
 if (Str_empty(fname))
 	{
 	*buf = '\0';
 	return false;
 	}
-if ((_istalpha(*fname) && *(fname+1) == ':')	// eg, "c:"
- || (*fname == '\\') )	// Another absolute name in Windows.
-	{
-	_tcscpy(buf, fname);
-	return true;
-	}
-
-// Relative name
-
-if (Str_empty(bname))
-	{
-	// Keep it relative and hope for the best.
-	_tcscpy(buf, fname);
-	return true;
-	}
-
-// If basename ends in slash, catenate.
-_TCHAR *ptr = bname;
-while (*ptr)
-	++ptr;
---ptr;	// Back up to last char.
-if (*ptr == '\\')	// Base ends in slash.
-	{
-	_stprintf(buf,_T("%s%s"),bname,fname);
-	return true;
-	}
-_stprintf(buf,_T("%s\\%s"),bname,fname);
+std::filesystem::path p(fname);
+if (!p.has_root_path() && !Str_empty(bname))
+	p = std::filesystem::path(bname) / p;
+_tcscpy(buf, p.string().c_str());
 return true;
 }
 
 
-// VARIANT.
-bool
-resolve_file_unix(_TCHAR *fname, _TCHAR *bname, /*DU*/ _TCHAR *buf)
-{
-if (Str_empty(fname))
-	{
-	*buf = '\0';
-	return false;
-	}
-if (*fname == '/')	// Name starts with slash.  Absolute Linux/Unix.
-	{
-	_tcscpy(buf, fname);
-	return true;
-	}
-
-// Relative name
-
-if (Str_empty(bname))
-	{
-	// Keep it relative and hope for the best.
-	_tcscpy(buf, fname);
-	return true;
-	}
-
-// If basename ends in slash, catenate.
-_TCHAR *ptr = bname;
-while (*ptr)
-	++ptr;
---ptr;	// Back up to last char.
-if (*ptr == '/')	// Base ends in slash.
-	{
-	_stprintf(buf,_T("%s%s"),bname,fname);
-	return true;
-	}
-_stprintf(buf,_T("%s/%s"),bname,fname);
-return true;
-}
 
 /********************************************
 * FN:		RUN_SILENT

include/Api/consh/cg.h

@@ -224,6 +224,12 @@ class LIBCONSH_API CG
 	// Loaded dictionary only: never searches *full files or logs a miss.
 	CONCEPT *findWordInMemory(_TCHAR*);								// 10/13/26 DD.
 
+	// -DICTSOURCE: give each dictionary entry dictfile and dictline
+	// attributes, so the nodes dicttok builds say where they came from.
+	// Static: set once per process, and no change to the object layout.
+	static void setDictSource(bool on);							// #432.
+	static bool getDictSource();									// #432.
+
 	CONCEPT *matchDictKB(std::string dictFilename, std::vector<std::filesystem::path> kbfiles);
 	std::string removeExtension(const std::string& filename);
 	bool openDict(std::vector<std::filesystem::path>& files);

include/Api/lite/nlp_engine.h

@@ -59,6 +59,10 @@ class LITE_API NLP_ENGINE
         bool silent=false
  	);
 
+	// -DICTSOURCE (#432): dictionary entries record their file and line on
+	// the nodes dicttok builds. Set before the analyzer is loaded.
+	static void setDictSource(bool on);
+
 	~NLP_ENGINE();
 
     _TCHAR *m_analyzer;

lite/ana.cpp

@@ -906,6 +906,9 @@ genSeqfile(fout);																// 03/08/00 AM.
 //delete fout;																	// 03/08/00 AM.
 // should probably have had: delete outStr;							// 03/08/00 AM.
 
+// Close it first: copying a file still open for writing could leave its
+// last, unflushed lines out of the sequence file (#323).
+fout.close();
 copy_file(fname, this->getSeqfile());	// Copy to sequence file.
 }