diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 54f30d6b..ed463738 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1188,6 +1188,48 @@ jobs: [ "$(tr -d '\r' < "$DUMP" | grep -v '^$' | tail -1)" = quit ] || { echo "FAIL(fs-ops): dump.kb does not end in quit"; cat "$DUMP"; exit 1; } echo "[fs-ops] mkdir() and kbdumptree() work without the shell" + - name: -DICTSOURCE marks nodes with their dictionary file and line (#432) (Non-20.04) + if: matrix.id != 'docker-ubuntu-20.04' + shell: bash + run: | + set -eo pipefail + # With -DICTSOURCE, nodes dicttok builds from .dict entries get + # dictfile and dictline attributes (#432, #475); without it, none. + FIX=.github/workflows/tests/dict-source + OUT="$FIX/input/text.txt_log/output.txt" + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + if [ -s "$OUT" ]; then + echo "FAIL(dict-source): attributes without -DICTSOURCE:"; cat "$OUT"; exit 1 + fi + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + mkdir -p "$FIX/logs" + ./bin/nlp -ANA "$FIX" -WORK ./ -DICTSOURCE "$FIX/input/text.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(dict-source): no output.txt with -DICTSOURCE"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + '_vehicle words.dict:6' \ + ' HOTROD words.dict:6' \ + '_bridge words.dict:14' \ + ' viaduct words.dict:14' \ + '_bridge words.dict:13' \ + ' bridge words.dict:13' \ + '_phrase words.dict:3' \ + ' license words.dict:9' \ + ' plate words.dict:9' \ + '_weather words.dict:11' \ + ' weather words.dict:10' \ + ' conditions words.dict:10' \ + 'email words.dict:7' \ + '_phrase words.dict:3') + if [ "$got" != "$want" ]; then + echo "FAIL(dict-source): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[dict-source] -DICTSOURCE marks dictionary nodes with file and line" + - name: factorial() up to 20! (Non-20.04) if: matrix.id != 'docker-ubuntu-20.04' shell: bash diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index d5a6d84c..70383d3f 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -1081,6 +1081,47 @@ jobs: [ "$(tr -d '\r' < "$DUMP" | grep -v '^$' | tail -1)" = quit ] || { echo "FAIL(fs-ops): dump.kb does not end in quit"; cat "$DUMP"; exit 1; } echo "[fs-ops] mkdir() and kbdumptree() work without the shell" + - name: -DICTSOURCE marks nodes with their dictionary file and line (#432) + shell: bash + run: | + set -eo pipefail + # With -DICTSOURCE, nodes dicttok builds from .dict entries get + # dictfile and dictline attributes (#432, #475); without it, none. + FIX=.github/workflows/tests/dict-source + OUT="$FIX/input/text.txt_log/output.txt" + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + mkdir -p "$FIX/logs" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt" + if [ -s "$OUT" ]; then + echo "FAIL(dict-source): attributes without -DICTSOURCE:"; cat "$OUT"; exit 1 + fi + rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" + mkdir -p "$FIX/logs" + ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ -DICTSOURCE "$FIX/input/text.txt" + if [ ! -f "$OUT" ]; then + echo "FAIL(dict-source): no output.txt with -DICTSOURCE"; exit 1 + fi + got=$(tr -d '\r' < "$OUT") + want=$(printf '%s\n' \ + '_vehicle words.dict:6' \ + ' HOTROD words.dict:6' \ + '_bridge words.dict:14' \ + ' viaduct words.dict:14' \ + '_bridge words.dict:13' \ + ' bridge words.dict:13' \ + '_phrase words.dict:3' \ + ' license words.dict:9' \ + ' plate words.dict:9' \ + '_weather words.dict:11' \ + ' weather words.dict:10' \ + ' conditions words.dict:10' \ + 'email words.dict:7' \ + '_phrase words.dict:3') + if [ "$got" != "$want" ]; then + echo "FAIL(dict-source): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1 + fi + echo "[dict-source] -DICTSOURCE marks dictionary nodes with file and line" + - name: factorial() up to 20! shell: bash run: | diff --git a/.github/workflows/tests/dict-source/input/text.txt b/.github/workflows/tests/dict-source/input/text.txt new file mode 100644 index 00000000..ec6f3324 --- /dev/null +++ b/.github/workflows/tests/dict-source/input/text.txt @@ -0,0 +1 @@ +A HOT-ROD near the viaduct and bridge; my licenseplate, weathervconditions, e-mail; license plate. diff --git a/.github/workflows/tests/dict-source/kb/user/words.dict b/.github/workflows/tests/dict-source/kb/user/words.dict new file mode 100644 index 00000000..9c593919 --- /dev/null +++ b/.github/workflows/tests/dict-source/kb/user/words.dict @@ -0,0 +1,15 @@ +# Regression for the dictionary tokenizer (#433 #434 #426 #417). +# A bare entry (no attributes) used to end the read of the file. +license plate +# join=1: the phrase becomes one token, looked up in turn. +hot - rod join=1 +hotrod s=vehicle speed=fast +e - mail join=1 +# split=: one token becomes several words. +licenseplate split="license plate" +weathervconditions split="weather conditions" +weather conditions s=weather +# _name: attributes for every _bridge node; an entry's own win. +bridge s=bridge +viaduct s=bridge kind=raised +_bridge type=location kind=generic diff --git a/.github/workflows/tests/dict-source/spec/analyzer.seq b/.github/workflows/tests/dict-source/spec/analyzer.seq new file mode 100644 index 00000000..3e05f85f --- /dev/null +++ b/.github/workflows/tests/dict-source/spec/analyzer.seq @@ -0,0 +1,2 @@ +dicttok nil # +nlp show # diff --git a/.github/workflows/tests/dict-source/spec/show.nlp b/.github/workflows/tests/dict-source/spec/show.nlp new file mode 100644 index 00000000..2c31c884 --- /dev/null +++ b/.github/workflows/tests/dict-source/spec/show.nlp @@ -0,0 +1,22 @@ +############################################### +# FILE: show.nlp +# SUBJ: Each node from a dictionary entry, and its children, with +# the entry's file and line (-DICTSOURCE, #432). +# NOTE: Part of the dict-source regression; expected output is in +# the CI step. Without -DICTSOURCE nothing is printed. +############################################### + +@CODE +L("n") = pndown(pnroot()); +while (L("n")) { + if (pnvar(L("n"), "dictfile")) + "output.txt" << pnname(L("n")) << " " << pnvar(L("n"), "dictfile") << ":" << pnvar(L("n"), "dictline") << "\n"; + L("c") = pndown(L("n")); + while (L("c")) { + if (pnvar(L("c"), "dictfile")) + "output.txt" << " " << pnname(L("c")) << " " << pnvar(L("c"), "dictfile") << ":" << pnvar(L("c"), "dictline") << "\n"; + L("c") = pnnext(L("c")); + } + L("n") = pnnext(L("n")); +} +@@CODE diff --git a/README.md b/README.md index 06e5c7cf..f9132dce 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,11 @@ You can get help on nlp.exe: [-OUT outdir] output directory [-WORK workdir] working directory [-DEV][-SILENT] -DEV generates logs, -SILENT suppresses logs/output files (off by default) + [-DEBUG port] wait for a rule debugger on 127.0.0.1:port, then run under it [infile [outfile]] when no -IN or -OUT specified + [-STDIN] load once, then analyze one input file path per stdin line + [-DICTSOURCE] nodes built from .dict entries get dictfile and dictline + attributes naming the entry's file and line Directories in the nlp.exe files: data nlp engine bootstrap grammar @@ -106,6 +110,8 @@ Switch | Function -WORK | Working director where the library and executable files are -DEV / -SILENT | -DEV generates logs, -SILENT suppresses logs/output files (off by default) -DEBUG | Port to serve the debug protocol on. The engine waits for a client, then stops at pass boundaries, rule attempts and statements so the client can step and inspect. Used by the VS Code extension's debugger; see "What's New in Version 4". +-STDIN | Load the analyzer once, then analyze one input file path per line of standard input. A blank line or end of input ends the run. +-DICTSOURCE | Nodes that dicttok builds from `.dict` entries get `dictfile` and `dictline` attributes naming the entry's file and line, so a debugger or editor can jump to it. Off by default: it adds two attributes to every dictionary entry. [infile [outfile]] | when no -IN or -OUT specified # Calling NLP++ Analyzers from C++ diff --git a/cs/libconsh/cg.cpp b/cs/libconsh/cg.cpp index 5fe3e17f..79209a87 100644 --- a/cs/libconsh/cg.cpp +++ b/cs/libconsh/cg.cpp @@ -552,7 +552,7 @@ std::_t_ofstream f_main(TCHAR2A(o_main)); // Use forward-slash separators in the generated `take` statements. The // take parser (cmd_take -> resolve_file in cs/libprim/io.cpp) does NOT -// normalize backslashes to slashes on Linux; resolve_file_unix +// normalize backslashes to slashes on Linux; resolve_file // concatenates verbatim and then fopen sees one filename with literal // '\' chars. Windows historically accepted forward slashes here, so // emitting them on both platforms preserves Windows behavior bit-for-bit @@ -3864,6 +3864,10 @@ bool CG::readDicts(std::vector files, std::vector kbfiles) { _TCHAR buf[MAXMSG]; int lineCount = 0; @@ -4200,6 +4204,17 @@ bool CG::parseDictLine(_TCHAR *buf, CONCEPT *ambigKB, const std::string &file, i addSval(parentCon,_T("s"),_T("phrase")); } + // -DICTSOURCE: where the entry is, for the debugger and editors + // (#432, #475). A word on several lines keeps its first. A lazily + // loaded *full.dict line has no line number, so only its file. + if (dictSource && parentCon && !findVals(parentCon, _T("dictfile"))) { + _TCHAR fname[MAXSTR]; + _stprintf(fname, _T("%.*s"), (int)(MAXSTR - 1), filename.c_str()); + addSval(parentCon,_T("dictfile"),fname); + if (lineCount > 0) + addVal(parentCon,_T("dictline"),(long long) lineCount); + } + } return true; diff --git a/cs/libconsh/cmd.cpp b/cs/libconsh/cmd.cpp index d95b97d0..7344ee5c 100644 --- a/cs/libconsh/cmd.cpp +++ b/cs/libconsh/cmd.cpp @@ -1611,7 +1611,9 @@ if (args) // Handling relative filename. // 07/01/03 AM. _TCHAR buf[FNAME_SIZE+1]; // 07/01/03 AM. -if (!resolve_file(fil,cg->getAppdir(),/*DU*/buf)) // 07/01/03 AM. +_TCHAR *appdir = cg->getAppdir(); +if (_tcslen(fil) + (appdir ? _tcslen(appdir) : 0) + 2 > FNAME_SIZE // #323. + || !resolve_file(fil,appdir,/*DU*/buf)) // 07/01/03 AM. { std::_t_cerr << _T("[Bad filename=") << fil << _T("]") << std::endl; return false; diff --git a/cs/libprim/io.cpp b/cs/libprim/io.cpp index ca895ccc..5e2bdab9 100644 --- a/cs/libprim/io.cpp +++ b/cs/libprim/io.cpp @@ -18,6 +18,7 @@ All rights reserved. *******************************************************************************/ #include "StdAfx.h" +#include // #323. #include // Upgrade // 01/24/01 AM. #include @@ -34,8 +35,6 @@ All rights reserved. #include "prim/io.h" // Local defs. -bool resolve_file_win(_TCHAR*,_TCHAR*,_TCHAR*); // 07/01/03 AM. -bool resolve_file_unix(_TCHAR*,_TCHAR*,_TCHAR*); // 07/01/03 AM. /************************************************** * INDENT @@ -120,91 +119,21 @@ while (num-- > 0) LIBPRIM_API bool resolve_file(_TCHAR *fname, _TCHAR *bname, /*DU*/ _TCHAR *buf) { -#ifndef LINUX -return resolve_file_win(fname,bname,buf); -#else -return resolve_file_unix(fname,bname,buf); -#endif -} - - -// VARIANT. -bool -resolve_file_win(_TCHAR *fname, _TCHAR *bname, /*DU*/ _TCHAR *buf) -{ +// One way for every platform (#323): a name with a root ("/x", "C:\\x", +// "\\x") is used as is, any other is taken relative to BNAME. if (Str_empty(fname)) { *buf = '\0'; return false; } -if ((_istalpha(*fname) && *(fname+1) == ':') // eg, "c:" - || (*fname == '\\') ) // Another absolute name in Windows. - { - _tcscpy(buf, fname); - return true; - } - -// Relative name - -if (Str_empty(bname)) - { - // Keep it relative and hope for the best. - _tcscpy(buf, fname); - return true; - } - -// If basename ends in slash, catenate. -_TCHAR *ptr = bname; -while (*ptr) - ++ptr; ---ptr; // Back up to last char. -if (*ptr == '\\') // Base ends in slash. - { - _stprintf(buf,_T("%s%s"),bname,fname); - return true; - } -_stprintf(buf,_T("%s\\%s"),bname,fname); +std::filesystem::path p(fname); +if (!p.has_root_path() && !Str_empty(bname)) + p = std::filesystem::path(bname) / p; +_tcscpy(buf, p.string().c_str()); return true; } -// VARIANT. -bool -resolve_file_unix(_TCHAR *fname, _TCHAR *bname, /*DU*/ _TCHAR *buf) -{ -if (Str_empty(fname)) - { - *buf = '\0'; - return false; - } -if (*fname == '/') // Name starts with slash. Absolute Linux/Unix. - { - _tcscpy(buf, fname); - return true; - } - -// Relative name - -if (Str_empty(bname)) - { - // Keep it relative and hope for the best. - _tcscpy(buf, fname); - return true; - } - -// If basename ends in slash, catenate. -_TCHAR *ptr = bname; -while (*ptr) - ++ptr; ---ptr; // Back up to last char. -if (*ptr == '/') // Base ends in slash. - { - _stprintf(buf,_T("%s%s"),bname,fname); - return true; - } -_stprintf(buf,_T("%s/%s"),bname,fname); -return true; -} /******************************************** * FN: RUN_SILENT diff --git a/include/Api/consh/cg.h b/include/Api/consh/cg.h index bceeac07..a3239265 100644 --- a/include/Api/consh/cg.h +++ b/include/Api/consh/cg.h @@ -224,6 +224,12 @@ class LIBCONSH_API CG // Loaded dictionary only: never searches *full files or logs a miss. CONCEPT *findWordInMemory(_TCHAR*); // 10/13/26 DD. + // -DICTSOURCE: give each dictionary entry dictfile and dictline + // attributes, so the nodes dicttok builds say where they came from. + // Static: set once per process, and no change to the object layout. + static void setDictSource(bool on); // #432. + static bool getDictSource(); // #432. + CONCEPT *matchDictKB(std::string dictFilename, std::vector kbfiles); std::string removeExtension(const std::string& filename); bool openDict(std::vector& files); diff --git a/include/Api/lite/nlp_engine.h b/include/Api/lite/nlp_engine.h index 250ac186..2c910ca3 100644 --- a/include/Api/lite/nlp_engine.h +++ b/include/Api/lite/nlp_engine.h @@ -59,6 +59,10 @@ class LITE_API NLP_ENGINE bool silent=false ); + // -DICTSOURCE (#432): dictionary entries record their file and line on + // the nodes dicttok builds. Set before the analyzer is loaded. + static void setDictSource(bool on); + ~NLP_ENGINE(); _TCHAR *m_analyzer; diff --git a/lite/ana.cpp b/lite/ana.cpp index d6e703bd..8f6fc5d1 100644 --- a/lite/ana.cpp +++ b/lite/ana.cpp @@ -906,6 +906,9 @@ genSeqfile(fout); // 03/08/00 AM. //delete fout; // 03/08/00 AM. // should probably have had: delete outStr; // 03/08/00 AM. +// Close it first: copying a file still open for writing could leave its +// last, unflushed lines out of the sequence file (#323). +fout.close(); copy_file(fname, this->getSeqfile()); // Copy to sequence file. } diff --git a/lite/dicttok.cpp b/lite/dicttok.cpp index e53de6a3..987cdde5 100644 --- a/lite/dicttok.cpp +++ b/lite/dicttok.cpp @@ -285,7 +285,7 @@ bool DICTTok::ApplyDictFiles() { long long jnum = 1; if (cg_->isValNum(jv)) cg_->popVal(jv, jnum); - Node *joined = jnum ? joinPhrase(node, endNode) : 0; + Node *joined = jnum ? joinPhrase(node, endNode, end) : 0; if (joined) { node = joined; continue; @@ -654,6 +654,7 @@ bool DICTTok::splitToken(Node* &node, CONCEPT *con) { } Pn *pp = piece->getData(); findAttrs(piece, lookupWord(pp->getName()), pp->getName(), false); + markSource(piece, con); // The split= entry, if the word has none. prev = piece; } @@ -670,7 +671,7 @@ bool DICTTok::splitToken(Node* &node, CONCEPT *con) { * of the input and spans the phrase. Returns the token, or 0. ********************************************/ -Node *DICTTok::joinPhrase(Node *node, Node *endNode) { +Node *DICTTok::joinPhrase(Node *node, Node *endNode, CONCEPT *entry) { std::string joined; for (Node *n = node; n; n = n->Right()) { Pn *p = n->getData(); @@ -690,10 +691,41 @@ Node *DICTTok::joinPhrase(Node *node, Node *endNode) { copyAttrsToSuggested(pn, tok, endNode); // NL, SP, NOSP, TABS. Pn *tp = tok->getData(); findAttrs(tok, lookupWord(tp->getName()), tp->getName(), false); + markSource(tok, entry); // The join= entry, if the word has none. replaceRange(node, endNode, tok, tok); return tok; } +/******************************************** +* FN: MARKSOURCE +* SUBJ: Under -DICTSOURCE, give a token made by join= or split= the file +* and line of the entry that made it, unless its own word has an +* entry (#432). +********************************************/ + +void DICTTok::markSource(Node *node, CONCEPT *entry) { + if (!CG::getDictSource() || !node || !entry) + return; + if (Ivar::nodeVarhasval(node->getData(), _T("dictfile"))) + return; + _TCHAR buf[MAXSTR]; + _TCHAR *key, *val; + VAL *vals = cg_->findVals(entry, _T("dictfile")); + if (vals && cg_->isValStr(vals)) { + cg_->popSval(vals, buf); + parse_->internStr(_T("dictfile"), key); + parse_->internStr(buf, val); + replaceStr(node, key, val); + } + vals = cg_->findVals(entry, _T("dictline")); + if (vals && cg_->isValNum(vals)) { + long long num = 0; + cg_->popVal(vals, num); + parse_->internStr(_T("dictline"), key); + replaceNum(node, key, num); + } +} + bool DICTTok::copyAttrsToSuggested(Pn *pn, Node *suggestedPn, Node *endNode) { Dlist *dsem_ = pn->getDsem(); bool found = false; diff --git a/lite/dicttok.h b/lite/dicttok.h index f5316bc2..0850d1da 100644 --- a/lite/dicttok.h +++ b/lite/dicttok.h @@ -143,7 +143,8 @@ class DICTTok : public Algo void replaceRange(Node *first, Node *last, Node *nfirst, Node *nlast); bool splitToken(Node* &node, CONCEPT *con); - Node *joinPhrase(Node *node, Node *endNode); + Node *joinPhrase(Node *node, Node *endNode, CONCEPT *entry); + void markSource(Node *node, CONCEPT *entry); // -DICTSOURCE (#432). private: //Parse *parse; // The analyze instance that we're tokenizing for. diff --git a/lite/io.cpp b/lite/io.cpp index 5a41294b..8e684db7 100644 --- a/lite/io.cpp +++ b/lite/io.cpp @@ -45,39 +45,16 @@ All rights reserved. void copy_file(const _TCHAR *iname, const _TCHAR *oname) { - -//ifstream inFile(iname, std::ios::in | std::ios::nocreate); -std::_t_ifstream inFile(CTCHAR2CA(iname), std::ios::in); // Upgrade. // 01/24/01 AM. -if (!inFile) - { - std::_t_strstream gerrStr; - gerrStr << _T("Could not open input file '") << iname << _T("'.") << std::ends; - errOut(&gerrStr,false); - return; // 06/15/99 AM. - } - -std::_t_ofstream outFile(CTCHAR2CA(oname), std::ios::out); -if (!outFile) +// A byte-for-byte copy (#323). Was a character loop through text-mode streams. +std::error_code ec; +std::filesystem::copy_file(std::filesystem::path(iname), std::filesystem::path(oname), + std::filesystem::copy_options::overwrite_existing, ec); +if (ec) { std::_t_strstream gerrStr; - gerrStr << _T("Could not open output file '") << oname << _T("'.") << std::ends; + gerrStr << _T("Could not copy '") << iname << _T("' to '") << oname << _T("': ") + << ec.message().c_str() << _T(".") << std::ends; errOut(&gerrStr,false); - return; // 06/15/99 AM. - } - -// Echo input file to output. -// << and >> ignore whitespace! -_TCHAR ch; - -// Changing handling of EOF. // 12/16/01 AM. -//while ((ch = inFile.get()) != EOF) // 12/16/01 AM. -// outFile.put(ch); // 12/16/01 AM. -for (;;) // 12/18/01 AM. - { - ch = inFile.get(); // 12/18/01 AM. - if (inFile.eof()) // 12/18/01 AM. - break; // 12/18/01 AM. - outFile.put(ch); // 12/18/01 AM. } } diff --git a/lite/nlp_engine.cpp b/lite/nlp_engine.cpp index b0fe6dce..8a2fff05 100644 --- a/lite/nlp_engine.cpp +++ b/lite/nlp_engine.cpp @@ -800,6 +800,10 @@ int NLP_ENGINE::close(_TCHAR *analyzer) return 0; } +void NLP_ENGINE::setDictSource(bool on) { + CG::setDictSource(on); // #432. +} + int NLP_ENGINE::createDir(_TCHAR *dirPath) { std::error_code ec; // #323. if (!std::filesystem::exists(std::filesystem::path(dirPath), ec)) { diff --git a/nlp/main.cpp b/nlp/main.cpp index 7682fba7..39411872 100644 --- a/nlp/main.cpp +++ b/nlp/main.cpp @@ -15,9 +15,9 @@ All rights reserved. #include "lite/nlp_engine.h" #include "version.h" -#define NLP_ENGINE_VERSION "4.2.31" +#define NLP_ENGINE_VERSION "4.2.33" -bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &); +bool cmdReadArgs(int, _TCHAR *argv[], _TCHAR *&, _TCHAR *&, _TCHAR *&, _TCHAR *&, bool &, bool &, bool &, bool &, bool &, bool &, int &, bool &, bool &); void cmdHelpargs(_TCHAR *); #include "lite/nlpdebug.h" @@ -46,14 +46,17 @@ int _tmain( bool compileAna = false; // Compile only the analyzer to C++. int debugPort = 0; // -debug : rule-level debug server. 0 = off. bool stream = false; // -STDIN: read input file paths from stdin. + bool dictSource = false; // -DICTSOURCE: nodes record their .dict file and line. ///////////////////////////////////////////////// // GET APP INFORMATION ///////////////////////////////////////////////// // Get analyzer name, input and output filenames from command line. - if (!cmdReadArgs(argc, argv, analyzerpath, input, output, workdir, develop, compile, compiled, silent, compileKB, compileAna, debugPort, stream)) + if (!cmdReadArgs(argc, argv, analyzerpath, input, output, workdir, develop, compile, compiled, silent, compileKB, compileAna, debugPort, stream, dictSource)) exit(1); + if (dictSource) + NLP_ENGINE::setDictSource(true); // #432. // Rule-level debugger. Blocks until a client attaches, so it is only started // for an actual analysis run -- attaching to a compile would wait forever on @@ -129,7 +132,8 @@ bool cmdReadArgs( bool &compileKB, // true - compile only the KB to C++. bool &compileAna, // true - compile only the analyzer to C++. int &debugPort, // -debug : rule-level debug server. 0 = off. - bool &stream // -STDIN: read input file paths from stdin. + bool &stream, // -STDIN: read input file paths from stdin. + bool &dictSource // -DICTSOURCE: nodes record their .dict file and line. ) { _TCHAR *ptr; @@ -277,6 +281,10 @@ bool cmdReadArgs( { stream = true; } + else if (!strcmp_i(ptr, _T("dictsource"))) // #432. + { + dictSource = true; + } else { // Used to be ignored silently, so a typo like -ANALYZER ran with @@ -417,6 +425,8 @@ void cmdHelpargs(_TCHAR *name) << _T(" [-STDIN] load once, then analyze one input file path per stdin line;") << std::endl << _T(" a blank line or EOF ends the run. Each input is analyzed") << std::endl << _T(" when the next line arrives, so $islastfile can be set.") << std::endl + << _T(" [-DICTSOURCE] nodes built from .dict entries get dictfile and dictline") << std::endl + << _T(" attributes naming the entry's file and line") << std::endl << std::endl << _T("Directories in the nlp.exe files:") << std::endl << _T(" data nlp engine bootstrap grammar") << std::endl