From e7b47f8299d471c20e51bd4aa7ca2c862c7324a5 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:22:12 -0400 Subject: [PATCH 01/12] Show the parse trees a failed run left behind (#785) A run counts as failed when the engine exits with an error or err.log holds a located error -- and the engine has usually written the trees of every pass it got through. The failure branch refreshed only the output and log views, so the sequence view's per-pass tree icons kept whatever state the previous run left: passes that did produce trees offered none on a first run, and the output view listed no trees. Now a failed run refreshes the sequence and text views, switches the output view to its tree listing when any trees exist, and logs how far they go. Checked with a pass that hits a run-time error: 14 trees written, run reported failed. Closes #785. Co-Authored-By: Claude Opus 5.5 --- src/nlp.ts | 14 +++++++++++++- src/output/analyzerOutput.ts | 14 ++++++++++++++ src/output/outputTest.ts | 6 +++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/nlp.ts b/src/nlp.ts index a1591ace..12845aa3 100644 --- a/src/nlp.ts +++ b/src/nlp.ts @@ -14,7 +14,7 @@ import { ReformatType as reformatType } from './format/types'; import { formatRuleText } from './format/rulesRegion'; import { refreshEngineDiagnostics } from './language/engineDiagnostics'; import * as telemetry from './telemetry/telemetry'; -import { analyzerOutputLines } from './output/analyzerOutput'; +import { analyzerOutputLines, lastTreePass } from './output/analyzerOutput'; let analyzerOutputChannel: vscode.OutputChannel | undefined; @@ -201,6 +201,18 @@ export class NLPFile extends TextFile { else if (!logView.makeAna(false)) logView.loadAnalyzerOuts(false); + // #785: the passes that ran before the error left their parse + // trees, which are the best clue to what went wrong. Refresh the + // sequence so each of those passes offers its tree, and list the + // trees in the output view. + const logDir = filestr + visualText.LOG_SUFFIX; + const lastTree = fs.existsSync(logDir) && dirfuncs.isDir(logDir) ? lastTreePass(fs.readdirSync(logDir)) : 0; + if (lastTree > 0) { + outputView.setType(outputFileType.TREE); + logView.addMessage(`Parse trees for passes up to ${lastTree} are in the output view, and on each of those passes in the sequence`, logLineType.ANALYER_OUTPUT, vscode.Uri.file(filestr)); + } + vscode.commands.executeCommand('sequenceView.refreshAll'); + vscode.commands.executeCommand('textView.refreshAll'); vscode.commands.executeCommand('outputView.refreshAll'); vscode.commands.executeCommand('logView.refreshAll'); resolve('Failed'); diff --git a/src/output/analyzerOutput.ts b/src/output/analyzerOutput.ts index 3188ccd4..591e2577 100644 --- a/src/output/analyzerOutput.ts +++ b/src/output/analyzerOutput.ts @@ -26,3 +26,17 @@ export function analyzerOutputLines(stdout: string): string[] { lines.shift(); return lines; } + +/** + * The highest pass that left a parse tree (`anaNNN.tree`) among a log directory's + * file names, or 0 if none did. After a failed run these are the passes that + * completed before the error (#785). + */ +export function lastTreePass(fileNames: string[]): number { + let last = 0; + for (const name of fileNames) { + const m = /^ana(\d+)\.tree$/i.exec(name); + if (m) last = Math.max(last, Number(m[1])); + } + return last; +} diff --git a/src/output/outputTest.ts b/src/output/outputTest.ts index e94aadaf..033dac81 100644 --- a/src/output/outputTest.ts +++ b/src/output/outputTest.ts @@ -1,6 +1,6 @@ // Tests for analyzerOutputLines. Runs with plain Node via `npm run test:output`. -import { analyzerOutputLines } from "./analyzerOutput"; +import { analyzerOutputLines, lastTreePass } from "./analyzerOutput"; let passed = 0; let failed = 0; @@ -50,6 +50,10 @@ check("analyzer brackets kept", check("no output", analyzerOutputLines(oldEngine.replace("21\r\n34\r\n", "")), []); check("empty", analyzerOutputLines(""), []); +// #785: the last pass that wrote a tree before a run stopped. +check("last tree pass", [String(lastTreePass(["ana001.tree", "ana012.tree", "ana003.tree", "ana014.kbb", "final.tree", "err.log"]))], ["12"]); +check("no trees", [String(lastTreePass(["err.log", "dbg.log"]))], ["0"]); + console.log(`analyzer output: ${passed} passed, ${failed} failed`); if (failed) process.exit(1); From 946608807c03ffd884d6bb9090dea1a4fc842f7f Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:24:12 -0400 Subject: [PATCH 02/12] Generate rules from a selection in a parse tree (#475) "Generate Rule General/Exact" worked from text (.txt), highlighted text (.txxt) and pass files, but not from the place a rule is usually designed: a pass's parse tree. findSelectedTreeStr had no .tree branch, so the command did nothing there, and the editor menu did not offer it. In a .tree file the selected lines are parsed directly (parseTreeLines, as Generate Path does) and the top-level nodes become the rule's elements. The rule goes into the pass the tree belongs to (anaNNN.tree -> pass NNN, as Generate Path does); final.tree falls back to the current pass. Both commands are now in the .tree editor menu. Closes #475. Co-Authored-By: Claude Opus 5.5 --- package.json | 4 ++-- src/treeFile.ts | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 758d01e4..5cb0fa4c 100644 --- a/package.json +++ b/package.json @@ -2018,12 +2018,12 @@ "group": "nlp" }, { - "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat", + "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat || resourceExtname == .tree", "command": "nlp.generateRule", "group": "nlp" }, { - "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat", + "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat || resourceExtname == .tree", "command": "nlp.generateExactRule", "group": "nlp" }, diff --git a/src/treeFile.ts b/src/treeFile.ts index 989d28d9..ce04f26f 100644 --- a/src/treeFile.ts +++ b/src/treeFile.ts @@ -321,6 +321,11 @@ export class TreeFile extends TextFile { this.absoluteRangeFromSelection(this.HighlightFile, editor.selection); } this.findTreeFileLines(); + } else if (this.getFileType() == nlpFileType.TREE) { + // #475: in a parse tree the selected lines ARE the tree. + this.setFile(editor.document.uri); + this.parseTreeLines(editor); + this.selectedTreeStr = this.selectedLines.map(l => l.node).join(' '); } return this.selectedTreeStr.length ? true : false; } @@ -339,10 +344,18 @@ export class TreeFile extends TextFile { editor.insertSnippet(snippet, range); } else { - const passFilePath = visualText.analyzer.getPassPath(); - const passName = visualText.analyzer.seqFile.base(passFilePath.fsPath); - const passItem = visualText.analyzer.seqFile.findPass('nlp', passName); - this.treeFile = this.anaFile(passItem.passNum).fsPath; + let passFilePath = visualText.analyzer.getPassPath(); + if (type == nlpFileType.TREE) { + // A pass's tree (anaNNN.tree) adds the rule to that pass, as + // "Generate Path" does; final.tree to the current pass. + const treePass = this.getPassFromPath(editor); + if (treePass.fsPath.length > 2) + passFilePath = treePass; + } else { + const passName = visualText.analyzer.seqFile.base(passFilePath.fsPath); + const passItem = visualText.analyzer.seqFile.findPass('nlp', passName); + this.treeFile = this.anaFile(passItem.passNum).fsPath; + } if (this.findSelectedTreeStr(editor)) { let ruleStr = this.ruleFromLines(genType); From 16f832a46e1ed7b4dee931315fbb8de31d83cc29 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:28:52 -0400 Subject: [PATCH 03/12] Settings: declare analyzer.runMode, say what each setting does (#328) Going through every setting the extension declares and every one it reads turned up two defects and a set of descriptions that did not say what the setting does: - analyzer.runMode (the status-bar run mode: interpreted / compiled / compiled-analyzer / compiled-kb) was read and written but never declared. VS Code refuses to write an unregistered setting, so the mode was lost on every reload; the refusal is an async rejection the surrounding try/catch could not catch. Declared as an enum, the write's rejection is now handled, and the integration test that guards written settings covers it. - createtmpfile.tmpDir, another extension's setting copied in with a snippet, chose where partial trees were written. They go to the system temp folder, which is what it fell back to. - update.auto, textView.fast, analyzer.current, engine.visualtext, engine.analyzers, user.name and logs.clear get descriptions that say what they do and which ones the extension sets itself. Every declared setting is read somewhere. Keys are not renamed: moving the un-prefixed ones (update.*, user.name, ...) under nlp.* would silently reset everyone's values. Closes #328. Co-Authored-By: Claude Opus 5.5 --- package.json | 22 +++++++++++++++------- src/status.ts | 6 ++++-- src/test/suite/extension.test.ts | 1 + src/treeFile.ts | 5 +---- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 5cb0fa4c..add5aa0d 100644 --- a/package.json +++ b/package.json @@ -3433,13 +3433,21 @@ "type": "boolean", "scope": "application", "default": true, - "description": "Auto update on load or reload" + "markdownDescription": "Check for and install updates to the NLP++ engine, VisualText files and example analyzers each time the extension starts. Turn off to update only by command." }, "textView.fast": { "type": "boolean", "scope": "application", "default": false, - "description": "Load textview fast. Attributes for text files are lost such as has logs, etc." + "markdownDescription": "Fill the Text view without checking each file for logs and outputs. Faster for large input folders, but files no longer show whether they have been run." + }, + "analyzer.runMode": { + "type": "string", + "scope": "resource", + "enum": ["interpreted", "compiled", "compiled-analyzer", "compiled-kb"], + "enumDescriptions": ["Run the analyzer and its KB as source (the default).", "Run the compiled analyzer and compiled KB.", "Run the compiled analyzer with the KB as source.", "Run the analyzer as source with the compiled KB."], + "default": "interpreted", + "markdownDescription": "How the analyzer runs. Set from the run-mode button in the status bar and remembered per workspace folder." }, "analyzer.regressionTerminal": { "type": "boolean", @@ -3463,7 +3471,7 @@ "type": "string", "scope": "application", "default": "", - "description": "User name for top pass file comments" + "markdownDescription": "Your name, written in the `AUTH:` line of the header comment of new pass files." }, "analyzer.directory": { "type": "string", @@ -3475,25 +3483,25 @@ "type": "string", "scope": "resource", "default": "", - "description": "Current Analyzer" + "markdownDescription": "The analyzer that was open last. Set by the extension when you open an analyzer; you do not normally need to change it." }, "engine.visualtext": { "type": "string", "scope": "application", "default": "", - "description": "VisualText files version" + "markdownDescription": "Version of the downloaded VisualText files (help, libraries, templates). Set by the updater." }, "engine.analyzers": { "type": "string", "scope": "application", "default": "", - "description": "Analyzers version" + "markdownDescription": "Version of the downloaded example analyzers. Set by the updater." }, "logs.clear": { "type": "boolean", "scope": "application", "default": true, - "description": "Clear logs before new operations" + "markdownDescription": "Clear the NLP++ log view before each run or other operation." }, "compile.mode": { "type": "string", diff --git a/src/status.ts b/src/status.ts index 9041e77b..9dc49f29 100644 --- a/src/status.ts +++ b/src/status.ts @@ -347,11 +347,13 @@ export class NLPStatusBar { if (folder.fsPath.length) { vscode.workspace .getConfiguration('analyzer', folder) - .update('runMode', value, vscode.ConfigurationTarget.WorkspaceFolder); + .update('runMode', value, vscode.ConfigurationTarget.WorkspaceFolder) + .then(undefined, () => { /* not saved; the mode still applies this session */ }); } else { vscode.workspace .getConfiguration('analyzer') - .update('runMode', value, vscode.ConfigurationTarget.Global); + .update('runMode', value, vscode.ConfigurationTarget.Global) + .then(undefined, () => { /* not saved; the mode still applies this session */ }); } } catch { // ignore persistence failures; state still lives in memory diff --git a/src/test/suite/extension.test.ts b/src/test/suite/extension.test.ts index d88e5dc0..4a941455 100644 --- a/src/test/suite/extension.test.ts +++ b/src/test/suite/extension.test.ts @@ -124,6 +124,7 @@ const WRITTEN_SETTINGS: Array<[string, string]> = [ ["analyzer", "directory"], ["analyzer", "current"], ["textView", "fast"], + ["analyzer", "runMode"], ]; export async function configurationTests(): Promise { diff --git a/src/treeFile.ts b/src/treeFile.ts index ce04f26f..1f332b05 100644 --- a/src/treeFile.ts +++ b/src/treeFile.ts @@ -533,10 +533,7 @@ ${ruleStr} } openTemporaryFile(filepath: string, content: string) { - const tempDir = path.resolve( - vscode.workspace - .getConfiguration('createtmpfile') - .get('tmpDir') || os.tmpdir()); + const tempDir = os.tmpdir(); const filePath = vscode.Uri.file(path.join(tempDir, filepath)); fs.writeFileSync(filePath.fsPath, content); vscode.workspace.openTextDocument(filePath).then(document => { From bc58e77b2f4f4d47cfbfa4df699eb534f70e46f6 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:34:06 -0400 Subject: [PATCH 04/12] Sort & Count lines, and Count Words in Folder, for corpus study (#613) "Sort & Unique text lines" sorted and de-duplicated but threw the counts away, and nothing counted words across a corpus. - Sort & Count text lines (editor menu, beside Sort & Unique): each distinct line with how often it occurs, most frequent first, as "countline" in a new editor -- the text itself is untouched. Works on the selection or the whole file. - Count Words in Folder (Text view, on a folder): word frequencies over every file under it, skipping the engine's _log folders, with a header giving the total, distinct and file counts. A word is a run of letters and digits in any script, inner apostrophes and hyphens kept, lowercased. Counting is pure (src/output/counts.ts) and unit-tested; an integration test runs Sort & Count in VS Code. Closes #613. Co-Authored-By: Claude Opus 5.5 --- package.json | 20 +++++++++++++++ src/command.ts | 16 ++++++++++++ src/output/counts.ts | 44 ++++++++++++++++++++++++++++++++ src/output/outputTest.ts | 8 ++++++ src/test/suite/extension.test.ts | 13 ++++++++++ src/test/suite/index.ts | 3 ++- src/textView.ts | 25 ++++++++++++++++++ 7 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 src/output/counts.ts diff --git a/package.json b/package.json index add5aa0d..15886112 100644 --- a/package.json +++ b/package.json @@ -589,6 +589,16 @@ "title": "Sort & Unique text lines", "description": "Sort and unique text lines" }, + { + "command": "nlp.sortCount", + "title": "Sort & Count text lines", + "description": "How often each line occurs, most frequent first, in a new editor" + }, + { + "command": "textView.countWords", + "title": "Count Words in Folder", + "description": "Word frequencies over every text file in the folder, in a new editor" + }, { "command": "nlp.lowerCase", "title": "Change to Lowercase", @@ -1992,6 +2002,11 @@ "command": "nlp.sortText", "group": "nlp" }, + { + "when": "resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .dict", + "command": "nlp.sortCount", + "group": "nlp" + }, { "when": "resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .dict || resourceExtname == .kbb", "command": "nlp.lowerCase", @@ -3258,6 +3273,11 @@ "when": "view == textView && viewItem =~ /.*dir.*/", "group": "inline@01" }, + { + "command": "textView.countWords", + "when": "view == textView && viewItem =~ /.*dir.*/", + "group": "count" + }, { "command": "textView.runRegressionItem", "when": "view == textView && viewItem =~ /file|dir/", diff --git a/src/command.ts b/src/command.ts index b3dce621..d14778e6 100644 --- a/src/command.ts +++ b/src/command.ts @@ -2,6 +2,7 @@ import * as vscode from 'vscode'; import { TreeFile, generateType } from './treeFile'; import { NLPFile, reformatType } from './nlp'; import { TextFile } from './textFile'; +import { countLines, formatCounts } from './output/counts'; export let nlpCommands: NLPCommands; export class NLPCommands { @@ -29,6 +30,7 @@ export class NLPCommands { ctx.subscriptions.push(vscode.commands.registerCommand('nlp.passTree', this.passTree)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.displayMatchedRules', this.displayMatchedRulesNLP)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.sortText', this.sortText)); + ctx.subscriptions.push(vscode.commands.registerCommand('nlp.sortCount', this.sortCount)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.lowerCase', this.lowerCase)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.removeAccents', this.removeAccents)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.runPython', this.runPython)); @@ -84,6 +86,20 @@ export class NLPCommands { } } + // #613: how often each line occurs, most frequent first, in a new editor + // beside the text so the corpus itself is left as it is. + async sortCount() { + const editor = vscode.window.activeTextEditor; + if (!editor) return; + const doc = editor.document; + const range = editor.selection.isEmpty + ? new vscode.Range(0, 0, doc.lineCount, 0) + : new vscode.Range(editor.selection.start.line, 0, editor.selection.end.line + 1, 0); + const counts = countLines(doc.getText(range).split(/\r?\n/)); + const result = await vscode.workspace.openTextDocument({ content: formatCounts(counts), language: 'plaintext' }); + await vscode.window.showTextDocument(result, { viewColumn: vscode.ViewColumn.Beside }); + } + lowerCase() { if (vscode.window.activeTextEditor) { const editor = vscode.window.activeTextEditor; diff --git a/src/output/counts.ts b/src/output/counts.ts new file mode 100644 index 00000000..b38d0e04 --- /dev/null +++ b/src/output/counts.ts @@ -0,0 +1,44 @@ +// Frequency counts for corpus study (#613). PURE: no 'vscode' import. + +export interface Count { + key: string; + count: number; +} + +// Highest count first; ties alphabetically, ignoring case. +function sorted(map: Map): Count[] { + return [...map].map(([key, count]) => ({ key, count })) + .sort((a, b) => b.count - a.count || a.key.toLowerCase().localeCompare(b.key.toLowerCase()) || a.key.localeCompare(b.key)); +} + +// Each distinct line and how many times it occurs. Blank lines are skipped; +// lines are compared exactly, after trimming trailing whitespace. +export function countLines(lines: string[]): Count[] { + const map = new Map(); + for (const raw of lines) { + const line = raw.replace(/\s+$/, ""); + if (!line.length) continue; + map.set(line, (map.get(line) ?? 0) + 1); + } + return sorted(map); +} + +// Each distinct word and how many times it occurs, lowercased. A word is a run +// of letters and digits, with inner apostrophes and hyphens kept ("don't", +// "x-ray"); everything else separates words. +export function countWords(texts: string[]): Count[] { + const map = new Map(); + const word = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu; + for (const text of texts) { + for (const m of text.matchAll(word)) { + const w = m[0].toLowerCase(); + map.set(w, (map.get(w) ?? 0) + 1); + } + } + return sorted(map); +} + +// "countkey" lines, ready to open in an editor. +export function formatCounts(counts: Count[]): string { + return counts.map((c) => `${c.count}\t${c.key}`).join("\n") + (counts.length ? "\n" : ""); +} diff --git a/src/output/outputTest.ts b/src/output/outputTest.ts index 033dac81..656d0293 100644 --- a/src/output/outputTest.ts +++ b/src/output/outputTest.ts @@ -1,6 +1,7 @@ // Tests for analyzerOutputLines. Runs with plain Node via `npm run test:output`. import { analyzerOutputLines, lastTreePass } from "./analyzerOutput"; +import { countLines, countWords, formatCounts } from "./counts"; let passed = 0; let failed = 0; @@ -54,6 +55,13 @@ check("empty", analyzerOutputLines(""), []); check("last tree pass", [String(lastTreePass(["ana001.tree", "ana012.tree", "ana003.tree", "ana014.kbb", "final.tree", "err.log"]))], ["12"]); check("no trees", [String(lastTreePass(["err.log", "dbg.log"]))], ["0"]); +// #613: sort and count for corpus study. +check("count lines", formatCounts(countLines(["b", "a", "b", "", "c ", "c", "B"])).split("\n"), ["2\tb", "2\tc", "1\ta", "1\tB", ""]); +check("count words", formatCounts(countWords(["The cat, the hat.", "THE x-ray didn't 42 42"])).split("\n"), + ["3\tthe", "2\t42", "1\tcat", "1\tdidn't", "1\that", "1\tx-ray", ""]); +check("count words: accents and other scripts", countWords(["café Café 東京"]).map((c) => `${c.count}:${c.key}`), ["2:café", "1:東京"]); +check("count nothing", [formatCounts(countLines(["", " "]))], [""]); + console.log(`analyzer output: ${passed} passed, ${failed} failed`); if (failed) process.exit(1); diff --git a/src/test/suite/extension.test.ts b/src/test/suite/extension.test.ts index 4a941455..7f4d2fc8 100644 --- a/src/test/suite/extension.test.ts +++ b/src/test/suite/extension.test.ts @@ -528,3 +528,16 @@ export async function analyzerSwitchTests(): Promise { check("switch: dirty file stays open", await until(() => isOpen(bFile)) && isOpen(aFile)); await vscode.commands.executeCommand("workbench.action.revertAndCloseActiveEditor"); } + +// ---- sort & count (#613) ----------------------------------------------------- + +export async function sortCountTests(): Promise { + const doc = await vscode.workspace.openTextDocument({ content: "cat\ndog\ncat\n\nbird\ncat\ndog\n", language: "plaintext" }); + await vscode.window.showTextDocument(doc); + await vscode.commands.executeCommand("nlp.sortCount"); + const result = vscode.window.activeTextEditor?.document; + check("sort & count: opens a new editor", !!result && result !== doc); + eq("sort & count: counts, most frequent first", result?.getText(), "3\tcat\n2\tdog\n1\tbird\n"); + eq("sort & count: leaves the text alone", doc.getText(), "cat\ndog\ncat\n\nbird\ncat\ndog\n"); + await vscode.commands.executeCommand("workbench.action.closeAllEditors"); +} diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index e2e6baa0..a0b623cd 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -12,7 +12,7 @@ import { configurationTests, providerTests, languageFeatureTests, - crossPassTests, debugSetupTests, analyzerSwitchTests, + crossPassTests, debugSetupTests, analyzerSwitchTests, sortCountTests, } from "./extension.test"; export async function run(): Promise { @@ -25,6 +25,7 @@ export async function run(): Promise { ["language features", languageFeatureTests], ["cross-pass resolution", crossPassTests], ["debugger setup", debugSetupTests], + ["sort and count", sortCountTests], ["analyzer switching", analyzerSwitchTests], ]; diff --git a/src/textView.ts b/src/textView.ts index 53bcc383..c234cee7 100644 --- a/src/textView.ts +++ b/src/textView.ts @@ -10,6 +10,7 @@ import { fileOperation, fileOpRefresh } from './fileOps'; import { anaSubDir } from './analyzer'; import { regressionRunner } from './regression'; import * as fs from 'fs'; +import { countWords, formatCounts } from './output/counts'; export interface TextItem { uri: vscode.Uri; @@ -323,6 +324,7 @@ export class TextView { ctx.subscriptions.push(vscode.commands.registerCommand('textView.analyzerCurrent', () => this.analyzerCurrent())); ctx.subscriptions.push(vscode.commands.registerCommand('textView.analyze', (textItem) => this.analyze(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.analyzeDir', (textItem) => this.analyzeDir(textItem))); + ctx.subscriptions.push(vscode.commands.registerCommand('textView.countWords', (textItem) => this.countWords(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.runRegressionItem', (textItem) => this.runRegressionItem(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.blessRegressionItem', (textItem) => this.blessRegressionItem(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.openText', () => this.openText())); @@ -572,6 +574,29 @@ export class TextView { return bytes.toFixed(dp) + ' ' + units[u]; } + // #613: word frequencies over every text file in a folder, most frequent + // first. The engine's per-file _log folders are skipped. + async countWords(textItem: TextItem) { + const files: string[] = []; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (!dirfuncs.directoryIsLog(full)) walk(full); + } else if (entry.isFile()) { + files.push(full); + } + } + }; + walk(textItem.uri.fsPath); + const texts = files.map(f => fs.readFileSync(f, 'utf8')); + const counts = countWords(texts); + const total = counts.reduce((n, c) => n + c.count, 0); + const header = `# ${path.basename(textItem.uri.fsPath)}: ${total} words, ${counts.length} distinct, in ${files.length} files\n`; + const doc = await vscode.workspace.openTextDocument({ content: header + formatCounts(counts), language: 'plaintext' }); + await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.Beside }); + } + analyzeDir(textItem: TextItem) { if (textItem.uri.fsPath.length) { From 7bcac28e8f2206054bb6facc0ee3a3e11a07bfc7 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:36:21 -0400 Subject: [PATCH 05/12] Output files: copy or move anywhere; add a pass file to the sequence (#658, #794) #658: the output view could copy a file only to another analyzer's kb/ or input/ folder. "Copy or Move File To..." (output item context menu) asks where with a save dialog, starting in the analyzer's input folder, then whether to copy or move. #794: a pass file a run produced -- a generated rule file, a RUG pass -- had to be found again through Select Library Pass to get it into the sequence. Output .nlp/.pat files now carry "Add to Analyzer Sequence": pick the pass to insert after; if a pass of that name is already in the sequence, either replace it (it keeps its place, only its file changes) or add this one under the next free name (name2, name3, ...). Closes #658. Closes #794. Co-Authored-By: Claude Opus 5.5 --- package.json | 19 ++++++++++++ src/outputView.ts | 78 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 15886112..ff556ac2 100644 --- a/package.json +++ b/package.json @@ -1626,6 +1626,15 @@ "dark": "resources/dark/rename.svg" } }, + { + "command": "outputView.copyOrMoveTo", + "title": "Copy or Move File To..." + }, + { + "command": "outputView.addToSequence", + "title": "Add to Analyzer Sequence", + "icon": "$(add)" + }, { "command": "outputView.explore", "title": "Explore Output Folder", @@ -2628,6 +2637,16 @@ "when": "view == outputView", "group": "navigation" }, + { + "command": "outputView.copyOrMoveTo", + "when": "view == outputView", + "group": "move" + }, + { + "command": "outputView.addToSequence", + "when": "view == outputView && viewItem =~ /nlp/", + "group": "move" + }, { "command": "outputView.editTest", "when": "view == outputView && viewItem =~ /.*test.*/", diff --git a/src/outputView.ts b/src/outputView.ts index 7c5e2976..4b10dd7d 100644 --- a/src/outputView.ts +++ b/src/outputView.ts @@ -29,7 +29,8 @@ export class OutputTreeDataProvider implements vscode.TreeDataProvider this.deleteTest(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.editTest', (resource) => this.editTest(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.rename', (resource) => this.rename(resource))); + ctx.subscriptions.push(vscode.commands.registerCommand('outputView.copyOrMoveTo', (resource) => this.copyOrMoveTo(resource))); + ctx.subscriptions.push(vscode.commands.registerCommand('outputView.addToSequence', (resource) => this.addToSequence(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.kb', () => this.loadKB())); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.matches', () => this.loadTxxt())); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.trees', () => this.loadTrees())); @@ -364,6 +367,79 @@ export class OutputView { } } + // #658: put an output file anywhere, as a copy or by moving it. + async copyOrMoveTo(outputItem: OutputItem) { + const from = outputItem.uri; + const target = await vscode.window.showSaveDialog({ + title: 'Copy or move output file to', + defaultUri: vscode.Uri.file(path.join(visualText.analyzer.getInputDirectory().fsPath, path.basename(from.fsPath))), + }); + if (!target || target.fsPath === from.fsPath) + return; + const how = await vscode.window.showQuickPick(['Copy', 'Move'], { title: path.basename(target.fsPath), placeHolder: 'Copy the file, or move it?' }); + if (!how) + return; + try { + fs.mkdirSync(path.dirname(target.fsPath), { recursive: true }); + fs.copyFileSync(from.fsPath, target.fsPath); + if (how === 'Move') + fs.unlinkSync(from.fsPath); + } catch (err) { + vscode.window.showErrorMessage(`Could not ${how.toLowerCase()} ${path.basename(from.fsPath)}: ${err instanceof Error ? err.message : err}`); + return; + } + vscode.commands.executeCommand('outputView.refreshAll'); + vscode.commands.executeCommand('textView.refreshAll'); + vscode.commands.executeCommand('kbView.refreshAll'); + } + + // #794: a pass file produced by a run (rule generation, a RUG pass) goes into + // the sequence after the pass the user picks. If a pass of that name is + // already in the sequence, they choose to replace it or add this one under a + // free name. + async addToSequence(outputItem: OutputItem) { + const seqFile = visualText.analyzer.seqFile; + const passes = seqFile.getPasses(); + if (!passes.length) { + vscode.window.showWarningMessage('Open an analyzer first'); + return; + } + const items = passes.map((pass, index) => ({ + label: pass.passNum ? `${pass.passNum} ${pass.name}` : pass.name, + description: pass.isFolder() ? 'folder' : pass.typeStr, + index, + })); + const after = await vscode.window.showQuickPick(items, { title: `Add ${path.basename(outputItem.uri.fsPath)} to the sequence`, placeHolder: 'Insert after which pass?' }); + if (!after) + return; + + const specDir = visualText.analyzer.getSpecDirectory().fsPath; + const name = path.basename(outputItem.uri.fsPath); + const existing = seqFile.findPassByFilename(name); + if (existing) { + const replace = `Replace pass ${existing} (${path.parse(name).name})`; + const choice = await vscode.window.showQuickPick([replace, 'Add it as a new pass under another name'], { title: `A pass named ${path.parse(name).name} is already in the sequence` }); + if (!choice) + return; + if (choice === replace) { + // The pass keeps its place in the sequence; only its file changes. + fs.copyFileSync(outputItem.uri.fsPath, path.join(specDir, name)); + vscode.commands.executeCommand('sequenceView.refreshAll'); + return; + } + const { name: base, ext } = path.parse(name); + let n = 2; + while (fs.existsSync(path.join(specDir, `${base}${n}${ext}`)) || seqFile.findPassByFilename(`${base}${n}${ext}`)) + n++; + const renamed = path.join(specDir, `${base}${n}${ext}`); + fs.copyFileSync(outputItem.uri.fsPath, renamed); + seqFile.insertPass(after.index, vscode.Uri.file(renamed)); + } else { + seqFile.insertPass(after.index, outputItem.uri); + } + vscode.commands.executeCommand('sequenceView.refreshAll'); + } + copytoKB(outputItem: OutputItem) { const kbDir = visualText.analyzer.anaSubDirPath(anaSubDir.KB); this.copyFileToAnalyzer(outputItem.uri, kbDir, 'Copy file to another analyzer', 'Copy file to the KB directory of:'); From c88d3312a92e986a6addb7cdae884ed3ce48c8c9 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:43:57 -0400 Subject: [PATCH 06/12] Browse the outputs of every file in a folder (#688) The output view showed one input file's outputs at a time. After analyzing a folder, each file's _log folder had to be reached by selecting the files one by one. Selecting a folder in the Text view now lists every input file under it that has outputs, each expanding to its own outputs in the current mode (all / matches / trees). Selecting a file goes back to that file's flat list. KB and orphan-pass modes are per analyzer and unchanged. The file actions (delete, rename, copy, add test) are not offered on the input-file rows, so they cannot act on the input itself. Also fixed on the way: - The matches and trees modes reused the _log folder the "all" mode had last found, and listed nothing if "all" had not run first. Each mode now finds the folder itself. The trees mode no longer lists final.tree twice. - fileHasTest set the log folder instead of the test folder, so it always answered false. Closes #688. Co-Authored-By: Claude Opus 5.5 --- package.json | 12 +++--- src/outputView.ts | 105 ++++++++++++++++++++++++++++++++++------------ src/textView.ts | 4 +- 3 files changed, 87 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index ff556ac2..a511259e 100644 --- a/package.json +++ b/package.json @@ -2634,12 +2634,12 @@ }, { "command": "outputView.rename", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "navigation" }, { "command": "outputView.copyOrMoveTo", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "move" }, { @@ -3349,17 +3349,17 @@ }, { "command": "outputView.copytoText", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@0" }, { "command": "outputView.copytoKB", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@1" }, { "command": "outputView.addTest", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@2" }, { @@ -3369,7 +3369,7 @@ }, { "command": "outputView.deleteOutput", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@4" }, { diff --git a/src/outputView.ts b/src/outputView.ts index 4b10dd7d..fee3c283 100644 --- a/src/outputView.ts +++ b/src/outputView.ts @@ -11,6 +11,7 @@ export enum outputFileType { ALL, TXXT, TREE, KB, NLP } interface OutputItem { uri: vscode.Uri; + input?: boolean; // an input file whose outputs are its children (#688) } export class OutputTreeDataProvider implements vscode.TreeDataProvider { @@ -25,6 +26,21 @@ export class OutputTreeDataProvider implements vscode.TreeDataProvider ({ uri })); + if (visualText.hasWorkspaceFolder() && outputView.browsingFolder()) + return outputView.inputsWithOutput().map(uri => ({ uri, input: true })); if (visualText.hasWorkspaceFolder()) { const children: OutputItem[] = new Array(); for (const folder of outputView.getOutputFiles()) { @@ -76,6 +96,7 @@ export class OutputView { this.outputView = vscode.window.createTreeView('outputView', { treeDataProvider: outputViewProvider }); ctx.subscriptions.push(this.outputView); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.refreshAll', () => outputViewProvider.refresh())); + ctx.subscriptions.push(vscode.commands.registerCommand('outputView.showFolder', (folder?: vscode.Uri) => this.showFolder(folder))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.copytoKB', (resource) => this.copytoKB(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.copytoText', (resource) => this.copytoText(resource))); @@ -284,7 +305,7 @@ export class OutputView { this.testDirectory = vscode.Uri.file(''); if (filepath.length == 0) return false; - this.logDirectory = vscode.Uri.file(filepath + visualText.TEST_SUFFIX); + this.testDirectory = vscode.Uri.file(filepath + visualText.TEST_SUFFIX); if (!fs.existsSync(this.testDirectory.fsPath)) return false; const stats = fs.lstatSync(this.testDirectory.fsPath); @@ -311,38 +332,68 @@ export class OutputView { } } } - else if (this.type == outputFileType.TXXT) { - const matchFiles = dirfuncs.getFiles(this.logDirectory, ['.txxt']); - this.outputFiles = this.outputFiles.concat(matchFiles); - } - else if (this.type == outputFileType.TREE) { - const finalTree = vscode.Uri.file(path.join(this.logDirectory.fsPath, 'final.tree')); - if (fs.existsSync(finalTree.fsPath)) { - this.outputFiles.push(finalTree); - } - const matchFiles = dirfuncs.getFiles(this.logDirectory, ['.tree']); - this.outputFiles = this.outputFiles.concat(matchFiles); - } else { + // ALL, TXXT and TREE list one input file's _log folder. Each mode + // finds that folder itself: TXXT and TREE used to reuse the one ALL + // had last set, and listed nothing if ALL had not run first. const textPath = visualText.analyzer.getTextPath().fsPath; - this.outputFiles = []; - if (textPath.length && this.fileHasLog(textPath)) { - const finalTree = vscode.Uri.file(path.join(this.logDirectory.fsPath, 'final.tree')); - if (fs.existsSync(finalTree.fsPath)) { - this.outputFiles.push(finalTree); - } - const candidates = dirfuncs.getFiles(this.logDirectory); - for (const cand of candidates) { - const ext = path.parse(cand.fsPath).ext; - if (ext.localeCompare('.tree') != 0 && ext.localeCompare('.txxt') != 0) - this.outputFiles.push(cand); - } - } + if (textPath.length && this.fileHasLog(textPath)) + this.outputFiles = this.filesForLog(this.logDirectory); } } return this.outputFiles; } + // The outputs in one input file's _log folder that the current mode shows. + filesForLog(logDir: vscode.Uri): vscode.Uri[] { + const files: vscode.Uri[] = []; + const finalTree = vscode.Uri.file(path.join(logDir.fsPath, 'final.tree')); + if (this.type == outputFileType.TXXT) + return dirfuncs.getFiles(logDir, ['.txxt']); + if (fs.existsSync(finalTree.fsPath)) + files.push(finalTree); + if (this.type == outputFileType.TREE) + return files.concat(dirfuncs.getFiles(logDir, ['.tree']).filter(f => f.fsPath !== finalTree.fsPath)); + for (const cand of dirfuncs.getFiles(logDir)) { + const ext = path.parse(cand.fsPath).ext; + if (ext.localeCompare('.tree') != 0 && ext.localeCompare('.txxt') != 0) + files.push(cand); + } + return files; + } + + // #688: with a folder selected in the Text view, the view lists every input + // file under it that has outputs, each expanding to its own outputs. + public folder: vscode.Uri | undefined; + + public showFolder(folder: vscode.Uri | undefined) { + this.folder = folder; + vscode.commands.executeCommand('outputView.refreshAll'); + } + + public browsingFolder(): boolean { + return !!this.folder && this.type != outputFileType.KB && this.type != outputFileType.NLP; + } + + // Input files under the selected folder that have a _log folder, depth first. + inputsWithOutput(): vscode.Uri[] { + const found: vscode.Uri[] = []; + const walk = (dir: string) => { + let entries: fs.Dirent[]; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (!dirfuncs.directoryIsLog(full)) walk(full); + } else if (entry.isFile() && dirfuncs.isDir(full + visualText.LOG_SUFFIX)) { + found.push(vscode.Uri.file(full)); + } + } + }; + if (this.folder) walk(this.folder.fsPath); + return found; + } + private openFile(resource: vscode.Uri): void { const textFile = new TextFile(resource.fsPath); textFile.cleanZeroZero(); diff --git a/src/textView.ts b/src/textView.ts index c234cee7..eeb01ba6 100644 --- a/src/textView.ts +++ b/src/textView.ts @@ -683,10 +683,12 @@ export class TextView { this.folderUri = undefined; vscode.window.showTextDocument(textItem.uri); visualText.analyzer.saveCurrentFile(textItem.uri); - vscode.commands.executeCommand('outputView.refreshAll'); + vscode.commands.executeCommand('outputView.showFolder', undefined); vscode.commands.executeCommand('status.update'); } else { + // #688: the output view lists the outputs of every file in the folder. this.folderUri = textItem.uri; + vscode.commands.executeCommand('outputView.showFolder', textItem.uri); } } From c387d49d6ee03390232cad3aa1240a7b2e6331cf Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:46:51 -0400 Subject: [PATCH 07/12] Show the updater's progress in the status bar at startup (#230) At activation the updater checks, downloads and unpacks the engine, ICU libraries, VisualText files and example analyzers. It reported only to the log view, so a first start that downloaded the engine looked like an extension that did nothing for a minute. While the updater runs, the status bar now shows a spinning "NLP++" item naming the step in progress -- "Downloading the NLP++ engine", "Unpacking the example analyzers" -- and it goes away when the queue finishes or the user stops it. Status-bar progress rather than a notification, so an ordinary start that finds nothing to do is not interrupted. Closes #230. Co-Authored-By: Claude Opus 5.5 --- src/visualText.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/visualText.ts b/src/visualText.ts index 9403c2ac..454e88c5 100644 --- a/src/visualText.ts +++ b/src/visualText.ts @@ -128,6 +128,10 @@ export class VisualText { private latestExtIndex: number = 0; private lastestEngineIndex: number = 0; private updaterID: number = 0; + // #230: the status-bar progress shown while the updater runs at startup. + private updaterProgress: { report: (message: string) => void; done: () => void } | undefined; + private static readonly OP_WORDS = ['', 'Checking the', 'Checking the version of the', 'Downloading the', 'Unpacking the', 'Removing the old', '', '']; + private static readonly COMP_WORDS = ['', 'ICU libraries', 'ICU libraries', 'ICU libraries', 'NLP++ engine', 'engine files', 'compiler files', 'example analyzers', 'VisualText files']; private testItem: TestItem = { fileCount: 0, matchFiles: 0, matchLines: 0, misFiles: 0, misLines: 0 }; @@ -277,6 +281,14 @@ export class VisualText { this.debugMessage('Checking for updates or repairs...', logLineType.UPDATER); this.pushCheckVersions(); this.startTimer(); + vscode.window.withProgress({ location: vscode.ProgressLocation.Window, title: 'NLP++' }, progress => + new Promise(resolve => { + progress.report({ message: 'checking for updates' }); + this.updaterProgress = { + report: (message: string) => progress.report({ message }), + done: () => { this.updaterProgress = undefined; resolve(); }, + }; + })); } } @@ -518,6 +530,7 @@ export class VisualText { else visualText.debugMessage('UPDATE CHECK COMPLETE', logLineType.UPDATER); visualText.stopAll = false; + visualText.updaterProgress?.done(); vscode.commands.executeCommand('status.update'); return; } else { @@ -526,6 +539,10 @@ export class VisualText { if (visualText.stopAll) return; + const opWords = VisualText.OP_WORDS[op.operation], compWords = VisualText.COMP_WORDS[op.component]; + if (opWords && compWords) + visualText.updaterProgress?.report(`${opWords} ${compWords}`); + if (visualText.debug) visualText.debugMessage(visualText.statusStrs[op.status] + ' ' + visualText.opStrs[op.operation] + ' ' + visualText.compStrs[op.component], logLineType.UPDATER); From c9f6d44161a9dcf6030a04e8c7d161a2f6dd552c Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:49:18 -0400 Subject: [PATCH 08/12] Load the engine of the extension version that is running (#481) The engine is downloaded into the extension's own folder, ~/.vscode/extensions/dehilster.nlp-/nlp-engine. Which was decided in three places: at activation the newest version found on disk; then, only if auto-update was on, the running extension's version; and a lexicographic sort in getLatestExtPath that also wrote it. With several versions on disk -- after an update VS Code has not cleaned up yet, or a downgrade, or with update.auto off -- a new version could run an older version's engine, or download into another version's folder. Removing the older versions was the workaround the issue describes. (This machine had seven.) The version is now the running extension's, set once at activation and never replaced by a directory scan. getExtensionPath() (settings init, the example-analyzers path, engine updates) prefers the running version's folder, falling back to the newest installed and then to the code's own location as before. getLatestExtPath is gone. The folder layout is unchanged, so F5 debugging still uses the installed engine of the same version. Closes #481. Co-Authored-By: Claude Opus 5.5 --- src/visualText.ts | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/visualText.ts b/src/visualText.ts index 454e88c5..f96ae948 100644 --- a/src/visualText.ts +++ b/src/visualText.ts @@ -151,6 +151,12 @@ export class VisualText { visualText.readConfig(); visualText.readState(); + // #481: the engine lives in the folder of the extension that is + // RUNNING. It used to be the newest version found on disk unless the + // auto-updater happened to run and correct it, so with auto-update off, + // after a downgrade, or with an old copy still on disk, a new version + // could load (or download into) another version's engine. + visualText.version = vscode.extensions.getExtension(visualText.EXTENSION_NAME)?.packageJSON.version ?? ''; visualText.getExtensionDirs(); visualText.initSettings(); } @@ -268,9 +274,8 @@ export class VisualText { this.platform = os.platform(); const plat = this.platform == 'darwin' ? 'mac' : this.platform; this.homeDir = os.homedir(); - const rootPath = this.getLatestExtPath(this.extensionParentDirectory()); - if (rootPath) this.extensionDir = vscode.Uri.file(rootPath); - this.version = vscode.extensions.getExtension(this.EXTENSION_NAME)?.packageJSON.version; + this.version = vscode.extensions.getExtension(this.EXTENSION_NAME)?.packageJSON.version ?? this.version; + this.extensionDir = this.extensionDirectory(); if (preInfoFlag) { this.debugMessage('Platform: ' + plat, logLineType.UPDATER); @@ -482,18 +487,6 @@ export class VisualText { op.local = path.join(engDir, lib); } - getLatestExtPath(dir: vscode.Uri): string { - const files = dirfuncs.getFiles(dir); - for (const file of files.reverse()) { - const filename = path.basename(file.fsPath); - if (filename.startsWith(this.EXTENSION_NAME)) { - this.version = this.versionFromPath(file); - return file.fsPath; - } - } - return ''; - } - updaterTimer() { let op = visualText.opsQueue[0]; let allDone = true; @@ -1248,6 +1241,9 @@ export class VisualText { } getExtensionPath(): vscode.Uri { + // The running version's folder first (#481), then the newest installed. + const running = this.extensionDirectory(); + if (this.version.length && fs.existsSync(running.fsPath)) return running; const latest = this.extensionItems[this.latestExtIndex]; if (latest) return latest.uri; From f6ac011f5cedd6b4a90969e073a4ab4e0b79f49b Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 18:58:52 -0400 Subject: [PATCH 09/12] Partial trees keep "[" tokens; partial and full trees color alike (#295) #295 reported wrong colorization in the partial tree ("View selected tree"), with no repro. Checked against today's tree grammar: 1,230 partial trees cut from 15 real final.tree files exactly as findTreeFileLines cuts them, 30,469 lines, tokenized with vscode-textmate beside the full trees they came from -- every line gets the same scopes in both. The grammar colors a tree line by line, so a partial tree cannot color differently any more. What the check did find is in the cut itself. findTreeFileLines split each line on its first "[" to read the offsets, and the engine writes a literal bracket token as " [ [2,2,2,2,0,0,punct]": the token was taken for the field list and the line dropped, so the partial tree of "x [y] z" lost its "[". The cut now goes through the shared, tested parseTreeLine (src/trace/treeParse.ts), which already knew that case, via a new treeLinesInRange -- tested on the engine's own output for that text. Closes #295. Co-Authored-By: Claude Opus 5.5 --- src/trace/traceTest.ts | 13 ++++++++++++- src/trace/treeParse.ts | 14 ++++++++++++++ src/treeFile.ts | 24 +++++------------------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/trace/traceTest.ts b/src/trace/traceTest.ts index e8a7e627..6ab0d28e 100644 --- a/src/trace/traceTest.ts +++ b/src/trace/traceTest.ts @@ -11,7 +11,7 @@ import * as fs from "fs"; import * as path from "path"; -import { parseTreeLine, parseTreeFile, nodeAtOffset, walkTree, parseNodeFlags, newFiredSpan } from "./treeParse"; +import { parseTreeLine, parseTreeFile, nodeAtOffset, walkTree, parseNodeFlags, newFiredSpan, treeLinesInRange } from "./treeParse"; import { loadTrace, findLatestLogDir, parseSequence, sequencePassNames, passesForSource, firedRuleLines, nodesBuiltBy, @@ -83,6 +83,17 @@ function eq(name: string, actual: T, expected: T): void { check("line: banner is not a node", parseTreeLine("PAT OUTPUT TREE:") === undefined); } +// ---- partial tree (#295) ----------------------------------------------------- +// The engine's own output for the text "x [y] z": the bracket tokens are nodes. +{ + const tree = ["", "FINAL OUTPUT TREE:", "", "_ROOT [0,6,0,6,0,0,node,un]", " x [0,0,0,0,0,0,alpha]", + " \_ [1,1,1,1,0,0,white]", " [ [2,2,2,2,0,0,punct]", " y [3,3,3,3,0,0,alpha]", + " ] [4,4,4,4,0,0,punct]", " \_ [5,5,5,5,0,0,white]", " z [6,6,6,6,0,0,alpha]"]; + eq("partial: '[y]' keeps both brackets", JSON.stringify(treeLinesInRange(tree, 2, 4)), JSON.stringify([6, 7, 8])); + eq("partial: whole text includes the root", treeLinesInRange(tree, 0, 6).length, 8); + eq("partial: banner lines never included", treeLinesInRange(tree, 0, 6).includes(1), false); +} + // ---- fired spans (#810) ---------------------------------------------------- // treeFile.writeFiredText slices the input at each fired span. The fired spans // below are read back from the display reported in #810, where an older engine diff --git a/src/trace/treeParse.ts b/src/trace/treeParse.ts index 4c1ff9a5..99c2a1cf 100644 --- a/src/trace/treeParse.ts +++ b/src/trace/treeParse.ts @@ -241,3 +241,17 @@ export function nodeAtOffset(root: TraceNode, offset: number): TraceNode | undef }); return best; } + +// Indices of the node lines whose text span (UTF-16, the ustart/uend pair) lies +// within [from, to] -- the lines of a "partial tree" for a selection. Read through +// parseTreeLine so a literal "[" token is a node like any other: splitting each +// line on its first "[" mistook that token for the start of the fields and +// silently left it out of every partial tree that should hold it (#295). +export function treeLinesInRange(lines: string[], from: number, to: number): number[] { + const out: number[] = []; + lines.forEach((line, i) => { + const node = parseTreeLine(line); + if (node && node.ustart >= from && node.uend <= to) out.push(i); + }); + return out; +} diff --git a/src/treeFile.ts b/src/treeFile.ts index 1f332b05..a72e1c9a 100644 --- a/src/treeFile.ts +++ b/src/treeFile.ts @@ -9,7 +9,7 @@ import { SequenceFile } from './sequence'; import { FindFile, FindItem } from './findFile'; import { findView } from './findView'; import { dirfuncs } from './dirfuncs'; -import { parseNodeFlags, newFiredSpan } from './trace/treeParse'; +import { parseNodeFlags, newFiredSpan, treeLinesInRange } from './trace/treeParse'; import * as os from 'os'; export enum generateType { GENERAL, EXACT } @@ -605,27 +605,13 @@ ${ruleStr} findTreeFileLines() { const file = new TextFile(this.treeFile); const sep = file.getSeparatorNormalized(); - let from = 0; - let to = 0; this.selectedLines = []; this.selectedTreeStr = ''; - for (const line of file.getLines()) { - from = 0; - to = 0; - - const tokens = line.split('['); - if (tokens.length > 1) { - const toks = tokens[1].split(/[,\]]/); - if (toks.length > 4) { - from = +toks[2]; - to = +toks[3]; - if (from >= this.selStart && to <= this.selEnd) { - this.selectedLines.push(this.parseTreeLine(line)); - this.selectedTreeStr = this.selectedTreeStr.concat(line, sep); - } - } - } + const lines = file.getLines(); + for (const i of treeLinesInRange(lines, this.selStart, this.selEnd)) { + this.selectedLines.push(this.parseTreeLine(lines[i])); + this.selectedTreeStr = this.selectedTreeStr.concat(lines[i], sep); } } From 6e32dd98bcc3e978795ee5dbba8fd6e1ba92e175 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 19:06:43 -0400 Subject: [PATCH 10/12] Mod files load on macOS and Linux (#98) #98 asked for a manifest that bundles .nlp passes and KB files. That is the .nlm mod file: sections under "" headers, unpacked by the extension into kb/user, spec and input, with the passes inserted into the sequence after a pass the user picks (Mod Load / Create / Add in the KB, sequence and text views). The engine needs no change -- the extension does the unpacking. But the format only worked on Windows. Headers use backslashes, and sections were classified by looking for "\kb\user\", "\spec\" and "\input\" in a joined path. On macOS and Linux nothing matched, so every section was "unknown" and its pass never went into the sequence, and path.join kept the backslashes, writing a file literally named "kb\user\en-numbers.dict" in the analyzer folder. Building a mod file there recorded the path with "/". Paths are now split on either separator (src/session/modPaths.ts, tested), and headers are always written with backslashes so a mod file made anywhere loads in older versions too. Checked against the shipped languages/English/en-numbers.nlm: kb, input and spec sections all recognized. Closes #98. Co-Authored-By: Claude Opus 5.5 --- src/modFile.ts | 35 +++++++++++++---------------------- src/session/modPaths.ts | 31 +++++++++++++++++++++++++++++++ src/session/sessionTest.ts | 10 ++++++++++ 3 files changed, 54 insertions(+), 22 deletions(-) create mode 100644 src/session/modPaths.ts diff --git a/src/modFile.ts b/src/modFile.ts index 13ec3ec0..9518bc48 100644 --- a/src/modFile.ts +++ b/src/modFile.ts @@ -1,5 +1,6 @@ import * as vscode from 'vscode'; import * as path from 'path'; +import { modHeaderPath, modParts, modSection } from './session/modPaths'; import * as fs from 'fs'; import { visualText } from './visualText'; import { TextFile } from './textFile'; @@ -22,9 +23,6 @@ export let modFile: ModFile; export class ModFile extends TextFile { public readonly MODFILE_HEADER = ' { this.setFile(filePath); - let filepath = ''; for (const line of this.getLines()) { if (line.indexOf(this.MODFILE_HEADER) == 0) { const tokens = line.split(/[\<\t\s\>]/); const relFilePath = tokens[2]; - filepath = path.join(visualText.getAnalyzerDir().fsPath,relFilePath); - if (filepath.includes(this.MODFILE_SPEC)) { + if (modSection(relFilePath) == 'spec') { const seq = new SequenceFile; const items: vscode.QuickPickItem[] = []; seq.choicePasses(visualText.analyzer.getSpecDirectory().fsPath,items,''); @@ -220,17 +210,18 @@ export class ModFile extends TextFile { let header = ''; const filepath = filePath.fsPath; let dir = ''; - if (filepath.includes(this.MODFILE_KB)) { + const section = modSection(filepath); + if (section == 'kb') { dir = visualText.analyzer.anaSubDirPath(anaSubDir.KB); } - else if (filepath.includes(this.MODFILE_SPEC)) { + else if (section == 'spec') { dir = visualText.analyzer.anaSubDirPath(anaSubDir.SPEC); } - else if (filepath.includes(this.MODFILE_INPUT)) { + else if (section == 'input') { dir = visualText.analyzer.anaSubDirPath(anaSubDir.INPUT); } const name = path.basename(filePath.fsPath); - const finalPath = path.join(dir,name); + const finalPath = modHeaderPath(dir, name); header = '\n' + this.MODFILE_HEADER + ' ' + finalPath + '>\n'; return header; } diff --git a/src/session/modPaths.ts b/src/session/modPaths.ts new file mode 100644 index 00000000..3c380ba0 --- /dev/null +++ b/src/session/modPaths.ts @@ -0,0 +1,31 @@ +// Paths inside .nlm mod files. PURE: no 'vscode' import. +// +// A mod file bundles analyzer files under "" headers. +// The format was written on Windows: headers use backslashes, and the code +// classified paths by looking for "\kb\user\", "\spec\" and "\input\". On macOS +// and Linux no path contains those, so every section was "unknown" -- its pass +// never inserted into the sequence -- and path.join kept the backslashes, writing +// a file literally named "kb\user\x.dict". + +export type ModSection = "kb" | "spec" | "input"; + +// The segments of a header path, whichever separator it was written with. +export function modParts(relPath: string): string[] { + return relPath.split(/[\\/]+/).filter((p) => p.length); +} + +// Which analyzer folder a path (relative, as in a header, or absolute) belongs to. +export function modSection(anyPath: string): ModSection | undefined { + const p = "/" + modParts(anyPath).join("/").toLowerCase() + "/"; + if (p.includes("/kb/user/")) return "kb"; + if (p.includes("/spec/")) return "spec"; + if (p.includes("/input/")) return "input"; + return undefined; +} + +// The path a header records for a file in an analyzer folder. Always written +// with backslashes, so a mod file made anywhere still loads in older versions, +// which only understood those. +export function modHeaderPath(dir: string, name: string): string { + return [...modParts(dir), name].join("\\"); +} diff --git a/src/session/sessionTest.ts b/src/session/sessionTest.ts index 929ea36a..b196c648 100644 --- a/src/session/sessionTest.ts +++ b/src/session/sessionTest.ts @@ -2,6 +2,7 @@ import * as path from "path"; import { analyzerFiles, isInside, pushRecent, samePath } from "./session"; +import { modHeaderPath, modParts, modSection } from "./modPaths"; let passed = 0; let failed = 0; @@ -50,5 +51,14 @@ eq("same: trailing separator", samePath(ana + path.sep, ana), true); eq("same: prefix sibling differs", samePath(ana, other), false); eq("same: empty never matches", samePath("", ""), false); +// ---- mod file paths ------------------------------------------------------------ +eq("mod: header with backslashes", modParts("kb\\user\\en-numbers.dict"), ["kb", "user", "en-numbers.dict"]); +eq("mod: header with slashes", modParts("spec/numbers.nlp"), ["spec", "numbers.nlp"]); +eq("mod: kb section from a Windows header", modSection("kb\\user\\en-numbers.dict"), "kb"); +eq("mod: spec section from a POSIX absolute path", modSection("/home/me/ana/spec/numbers.nlp"), "spec"); +eq("mod: input section from a Windows absolute path", modSection("C:\\ana\\input\\numbers.txt"), "input"); +eq("mod: kb only under kb/user", modSection("/ana/kb/numbers.kb"), undefined); +eq("mod: header written with backslashes", modHeaderPath("kb/user", "x.dict"), "kb\\user\\x.dict"); + console.log(`\nsession tests: ${passed} passed, ${failed} failed`); if (failed > 0) process.exit(1); From c58804b5d617e5809148f36ba89316f63c27b0f5 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 19:15:33 -0400 Subject: [PATCH 11/12] Clear the issue backlog (4.3.12) Version bump and CHANGELOG for the twelve issue fixes on this branch. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a73be0..e62a7392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ All notable changes to the [VSCode NLP++ extension](http://vscode.visualtext.org Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. +### 4.3.12 +Clears the issue backlog: twelve requests and bugs, some from 2020. + +- **Generate a rule from a parse tree** (#475). Select lines in a pass's `.tree` file and run Generate Rule General or Exact from the editor menu. The selected top-level nodes become the rule's elements, and the rule goes into the pass the tree belongs to. +- **Add a generated pass to the sequence** (#794). An `.nlp` or `.pat` file in the Output view now has "Add to Analyzer Sequence". You pick the pass to insert it after. If a pass of that name is already in the sequence, you can replace it, and it keeps its place, or add the new one under the next free name. +- **Copy or move an output file anywhere** (#658). "Copy or Move File To..." on an output file opens a save dialog, starting in the analyzer's input folder. +- **Browse the outputs of a whole folder** (#688). Select a folder in the Text view, and the Output view lists every file under it that has outputs, each expanding to its own. The matches and trees buttons also work now even when "all" was not clicked first. +- **Sort & Count for corpus study** (#613). "Sort & Count text lines" counts each distinct line, and "Count Words in Folder" in the Text view counts words across every file in the folder. Both open the counts, most frequent first, in a new editor and leave the text alone. +- **The parse trees of a failed run are reachable** (#785). The passes that ran before an error still offer their trees in the sequence, and the Output view lists them. +- **Startup shows what the updater is doing** (#230). While it checks, downloads or unpacks the engine and its files, the status bar shows which step it is on. +- **The engine of the running version is used** (#481). With several versions of the extension on disk, a new version could use an older version's engine. That was the reason for the advice to delete old versions, which is no longer needed. +- **The run mode is remembered** (#328). The status-bar run mode (interpreted or compiled) was lost on every reload, because its setting was never declared. Several settings now say what they do. +- **Partial trees keep `[` tokens** (#295). "View selected tree" dropped a literal `[` from the tree. A partial tree now colors exactly like the full one. +- **Mod files work on macOS and Linux** (#98). `.nlm` files, which bundle passes and KB files into one file, found none of their sections outside Windows, and created files with backslashes in their names. +- **writekb help** (#626) was already in the help pages, under Knowledge Base Functions. + ### 4.3.11 Publishes everything since 4.3.4 to the VS Code Marketplace. diff --git a/package-lock.json b/package-lock.json index 6e4c5e5a..7a186140 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nlp", - "version": "4.3.11", + "version": "4.3.12", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "nlp", - "version": "4.3.11", + "version": "4.3.12", "license": "MIT", "dependencies": { "@vscode/debugadapter": "^1.68.0", diff --git a/package.json b/package.json index a511259e..ca918816 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "NLP", "description": "NLP++: the only computer programming language built for NLP — create and visually debug analyzers directly in VS Code.", "icon": "resources/NLPppLogo.png", - "version": "4.3.11", + "version": "4.3.12", "license": "MIT", "publisher": "dehilster", "enableApiProposals": false, From fea03088f667f77a09204033d476333238cddc75 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 19:33:59 -0400 Subject: [PATCH 12/12] KB generator passes: copy what they write into kb/user, run again (#272) A pass that builds a knowledge base -- SaveKB("mykb.kbb", ...), or a .dict written with << -- puts the file in the run's _log folder, where the analyzer's kb/user never sees it; using it meant copying it by hand and running again. - "Toggle KB Generator" on a pass in the sequence marks it (shown as "KB generator"); the marks live in the analyzer's state.json. - With nlp.analyzer.regenerateKB on (off by default), after a successful run in which a marked pass has changed since its output was last copied, the .kbb/.dict files named in that pass's own text are copied from the log folder into kb/user and the text is queued once more. The second run finds nothing changed, so it does not repeat. The re-run waits for the current queue, since a second queue entry for the same file would share the first one's status. Checked with the engine: a pass writing words.dict -> run 1 leaves the attribute empty; with words.dict copied into kb/user, run 2 reads it (animal=1). File-name extraction and change detection are pure and unit-tested (src/session/kbGen.ts). Closes #272. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- package.json | 15 +++++++++ src/analyzer.ts | 39 ++++++++++++++++++++++- src/nlp.ts | 63 ++++++++++++++++++++++++++++++++++++++ src/sequenceView.ts | 16 ++++++++++ src/session/kbGen.ts | 24 +++++++++++++++ src/session/sessionTest.ts | 10 ++++++ 7 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 src/session/kbGen.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e62a7392..ef101153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the [VSCode NLP++ extension](http://vscode.visualtext.org Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. ### 4.3.12 -Clears the issue backlog: twelve requests and bugs, some from 2020. +Clears the issue backlog: thirteen requests and bugs, some from 2020. - **Generate a rule from a parse tree** (#475). Select lines in a pass's `.tree` file and run Generate Rule General or Exact from the editor menu. The selected top-level nodes become the rule's elements, and the rule goes into the pass the tree belongs to. - **Add a generated pass to the sequence** (#794). An `.nlp` or `.pat` file in the Output view now has "Add to Analyzer Sequence". You pick the pass to insert it after. If a pass of that name is already in the sequence, you can replace it, and it keeps its place, or add the new one under the next free name. @@ -17,6 +17,7 @@ Clears the issue backlog: twelve requests and bugs, some from 2020. - **The run mode is remembered** (#328). The status-bar run mode (interpreted or compiled) was lost on every reload, because its setting was never declared. Several settings now say what they do. - **Partial trees keep `[` tokens** (#295). "View selected tree" dropped a literal `[` from the tree. A partial tree now colors exactly like the full one. - **Mod files work on macOS and Linux** (#98). `.nlm` files, which bundle passes and KB files into one file, found none of their sections outside Windows, and created files with backslashes in their names. +- **Passes that build a knowledge base can feed the run** (#272). Right-click a pass in the sequence and choose "Toggle KB Generator" to mark it; the sequence shows "KB generator" beside it. With the new setting `nlp.analyzer.regenerateKB` on (it is off by default), after a run in which a marked pass has changed, the `.kbb` and `.dict` files it writes (the file names in its own text, such as `SaveKB("mykb.kbb", ...)`) are copied from the run's log folder into `kb/user`, and the text runs once more, so the result uses the knowledge base the pass just built. The second run copies nothing, so it does not repeat. - **writekb help** (#626) was already in the help pages, under Knowledge Base Functions. ### 4.3.11 diff --git a/package.json b/package.json index ca918816..49dd3bd4 100644 --- a/package.json +++ b/package.json @@ -1635,6 +1635,10 @@ "title": "Add to Analyzer Sequence", "icon": "$(add)" }, + { + "command": "sequenceView.toggleKbGenerator", + "title": "Toggle KB Generator" + }, { "command": "outputView.explore", "title": "Explore Output Folder", @@ -2622,6 +2626,11 @@ } ], "view/item/context": [ + { + "command": "sequenceView.toggleKbGenerator", + "when": "view == sequenceView && viewItem =~ /filenotok/", + "group": "kb" + }, { "command": "helpView.copyPath", "when": "view == helpView && viewItem == helpFile", @@ -3420,6 +3429,12 @@ "default": true, "markdownDescription": "Send anonymous usage counts (feature usage, run mode, errors) to help improve the extension. Never includes file contents, analyzer/KB source, file names, or paths. This is layered on top of VS Code's global `#telemetry.telemetryLevel#` setting — if that is off, nothing is sent regardless of this setting." }, + "nlp.analyzer.regenerateKB": { + "type": "boolean", + "scope": "resource", + "default": false, + "markdownDescription": "After a run in which a pass marked as a **KB generator** (right-click a pass in the sequence) has changed, copy the `.kbb` and `.dict` files that pass writes into the analyzer's `kb/user` folder and run the text again, so the run uses the knowledge base it just built." + }, "nlp.analyzer.rememberOpenFiles": { "type": "boolean", "scope": "application", diff --git a/src/analyzer.ts b/src/analyzer.ts index 70e42db6..687da55d 100644 --- a/src/analyzer.ts +++ b/src/analyzer.ts @@ -30,6 +30,10 @@ export class Analyzer { // Files open in the editor when the user last switched away from this // analyzer, relative to its directory (#131). private openFiles: string[] = []; + // #272: passes marked as KB generators, and when each one's output was + // last copied into kb/user (the pass file's mtime at the time). + private kbGenerators: string[] = []; + private kbGenCopied: Record = {}; private passNum: number = 0;; private loaded: boolean = false; @@ -44,8 +48,14 @@ export class Analyzer { readState() { this.openFiles = []; + this.kbGenerators = []; + this.kbGenCopied = {}; if (this.jsonState.jsonParse(this.analyzerDir, "state")) { const parse = this.jsonState.json.visualText[0]; + if (Array.isArray(parse.kbGenerators)) + this.kbGenerators = parse.kbGenerators.filter((f: unknown) => typeof f === "string"); + if (parse.kbGenCopied && typeof parse.kbGenCopied === "object") + this.kbGenCopied = parse.kbGenCopied; if (Array.isArray(parse.openFiles)) this.openFiles = parse.openFiles.filter((f: unknown) => typeof f === "string"); if (parse.currentTextFile) { @@ -292,13 +302,40 @@ export class Analyzer { "type": "state", "currentTextFile": this.currentTextFile.fsPath, "currentPassFile": this.currentPassFile.fsPath, - "openFiles": this.openFiles + "openFiles": this.openFiles, + "kbGenerators": this.kbGenerators, + "kbGenCopied": this.kbGenCopied } ] } this.jsonState.saveFile(this.analyzerDir.fsPath, "state", stateJsonDefault); } + isKbGenerator(passName: string): boolean { + return this.kbGenerators.includes(passName); + } + + getKbGenerators(): string[] { + return this.kbGenerators; + } + + toggleKbGenerator(passName: string) { + if (this.isKbGenerator(passName)) + this.kbGenerators = this.kbGenerators.filter(n => n !== passName); + else + this.kbGenerators.push(passName); + this.saveAnalyzerState(); + } + + kbGeneratorCopied(passName: string): number | undefined { + return this.kbGenCopied[passName]; + } + + setKbGeneratorCopied(passName: string, modifiedMs: number) { + this.kbGenCopied[passName] = modifiedMs; + this.saveAnalyzerState(); + } + getOpenFiles(): string[] { return this.openFiles; } diff --git a/src/nlp.ts b/src/nlp.ts index 12845aa3..e27c3c1c 100644 --- a/src/nlp.ts +++ b/src/nlp.ts @@ -15,6 +15,7 @@ import { formatRuleText } from './format/rulesRegion'; import { refreshEngineDiagnostics } from './language/engineDiagnostics'; import * as telemetry from './telemetry/telemetry'; import { analyzerOutputLines, lastTreePass } from './output/analyzerOutput'; +import { generatorChanged, kbOutputsOf } from './session/kbGen'; let analyzerOutputChannel: vscode.OutputChannel | undefined; @@ -51,6 +52,9 @@ export class NLPFile extends TextFile { public timerStatus: anaQueueStatus = anaQueueStatus.UNKNOWN; private timerID = 0; private stopAllFlag: boolean = false; + // #272: a text to run again once the current queue is done, because a KB + // generator pass just rebuilt part of kb/user. + private kbRerun: vscode.Uri | undefined; constructor(filepath: string = '', separateLines: boolean = true, text: string = '') { super(); @@ -315,6 +319,13 @@ export class NLPFile extends TextFile { vscode.commands.executeCommand('logView.refreshAll'); visualText.nlp.setAnalyzerStatus(filepath, analyzerStatus.DONE); nlpStatusBar.resetAnalyzerButton(); + if (visualText.nlp.copyKbGeneratorOutput(filepath)) { + // Queue the second run for when this queue finishes; running + // it now would share the queue entry of this one. + visualText.nlp.kbRerun = filepath; + if (visualText.nlp.timerID == 0) + visualText.nlp.startKbRerun(); + } resolve('Processed'); } }).pid; @@ -499,11 +510,63 @@ export class NLPFile extends TextFile { } shutDown() { + const stopped = visualText.nlp.stopAllFlag; clearInterval(visualText.nlp.timerID); visualText.debugMessage('Analyzing done', logLineType.ANALYER_OUTPUT); visualText.nlp.stopAllFlag = false; visualText.nlp.timerID = 0; visualText.nlp.anaQueue = []; + if (stopped) + visualText.nlp.kbRerun = undefined; + else if (visualText.nlp.kbRerun) + visualText.nlp.startKbRerun(); + } + + private startKbRerun() { + const uri = visualText.nlp.kbRerun; + visualText.nlp.kbRerun = undefined; + if (!uri) return; + visualText.nlp.addAnalyzer(uri, analyzerType.FILE); + visualText.nlp.startAnalyzer(); + } + + // #272: after a successful run of one text, copy into kb/user the .kbb/.dict + // files that each changed KB generator pass wrote to the run's log folder. + // Returns true when something was copied, i.e. the text should run again. + // A generator counts as changed until its output has been copied once since + // its last edit, so the second run copies nothing and does not repeat. + copyKbGeneratorOutput(textFile: vscode.Uri): boolean { + if (!vscode.workspace.getConfiguration('nlp').get('analyzer.regenerateKB', false)) + return false; + const analyzer = visualText.analyzer; + const logDir = textFile.fsPath + visualText.LOG_SUFFIX; + if (dirfuncs.isDir(textFile.fsPath) || !dirfuncs.isDir(logDir)) + return false; + const kbUser = analyzer.getAnalyzerDirectory(path.join('kb', 'user')).fsPath; + const copied: string[] = []; + for (const name of analyzer.getKbGenerators()) { + const pass = analyzer.seqFile.getPasses().find(p => p.name == name); + if (!pass || !fs.existsSync(pass.uri.fsPath)) + continue; + const modified = fs.statSync(pass.uri.fsPath).mtimeMs; + if (!generatorChanged(modified, analyzer.kbGeneratorCopied(name))) + continue; + const outputs = kbOutputsOf(fs.readFileSync(pass.uri.fsPath, 'utf8')).filter(f => fs.existsSync(path.join(logDir, f))); + if (!outputs.length) + continue; + fs.mkdirSync(kbUser, { recursive: true }); + for (const f of outputs) { + fs.copyFileSync(path.join(logDir, f), path.join(kbUser, f)); + copied.push(f); + } + analyzer.setKbGeneratorCopied(name, modified); + } + if (!copied.length) + return false; + logView.addMessage(`KB generator output copied to kb/user: ${copied.join(', ')}. Running ${path.basename(textFile.fsPath)} again with it.`, logLineType.ANALYER_OUTPUT, textFile); + vscode.commands.executeCommand('kbView.refreshAll'); + vscode.commands.executeCommand('logView.refreshAll'); + return true; } insertRule(ruleStr: string) { diff --git a/src/sequenceView.ts b/src/sequenceView.ts index a2d785e6..44e5254e 100644 --- a/src/sequenceView.ts +++ b/src/sequenceView.ts @@ -137,6 +137,8 @@ export class PassTree implements vscode.TreeDataProvider { conVal = conVal + 'hasLog'; if (treeFile.hasFileType(passItem.uri, passItem.passNum, nlpFileType.KBB)) conVal = conVal + 'hasKB'; + if (visualText.analyzer.isKbGenerator(passItem.name)) + conVal = conVal + 'kbgen'; if (debugConVal) label = row.toString() + ' ' + conVal; tooltip = passTooltip(passItem, passItem.uri.fsPath); if (passItem.fileExists()) @@ -232,6 +234,7 @@ export class PassTree implements vscode.TreeDataProvider { resourceUri: seqItem.uri, tooltip: seqItem.tooltip, label: seqItem.label, + description: seqItem.contextValue?.includes('kbgen') ? 'KB generator' : undefined, contextValue: seqItem.contextValue, collapsibleState: collapse, iconPath: { @@ -246,6 +249,18 @@ export class PassTree implements vscode.TreeDataProvider { } } + // #272: mark or unmark a pass whose output (.kbb/.dict) should be copied + // into kb/user, and the text run again, when the pass changes. + toggleKbGenerator(seqItem: SequenceItem): void { + visualText.analyzer.toggleKbGenerator(seqItem.name); + if (visualText.analyzer.isKbGenerator(seqItem.name) && !vscode.workspace.getConfiguration('nlp').get('analyzer.regenerateKB', false)) { + vscode.window.showInformationMessage(`${seqItem.name} is marked as a KB generator. Turn on "NLP: Analyzer Regenerate KB" in Settings to have runs use what it writes.`, 'Open Setting').then(choice => { + if (choice) vscode.commands.executeCommand('workbench.action.openSettings', 'nlp.analyzer.regenerateKB'); + }); + } + vscode.commands.executeCommand('sequenceView.refreshAll'); + } + moveUp(seqItem: SequenceItem): void { this.moveSequence(seqItem, moveDirection.UP); } @@ -682,6 +697,7 @@ export class SequenceView { this.sequenceView = vscode.window.createTreeView('sequenceView', { treeDataProvider }); ctx.subscriptions.push(this.sequenceView); + ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.toggleKbGenerator', (seqItem: SequenceItem) => treeDataProvider.toggleKbGenerator(seqItem))); ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.openPass', (seqItem) => this.openPass(seqItem))); ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.openTree', (seqItem) => this.openTree(seqItem))); ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.displayMatchedRules', (seqItem) => this.displayMatchedRules(seqItem))); diff --git a/src/session/kbGen.ts b/src/session/kbGen.ts new file mode 100644 index 00000000..b596ddc5 --- /dev/null +++ b/src/session/kbGen.ts @@ -0,0 +1,24 @@ +// KB generator passes (#272). PURE: no 'vscode' import. +// +// A KB generator is a pass the user marks in the sequence because it writes a +// knowledge base or dictionary -- SaveKB("mykb.kbb", ...), or a .dict written +// with <<. The engine puts such a file in the run's _log folder, where the +// analyzer's own kb/user never sees it. With regeneration on, after a run in +// which a generator pass changed, the extension copies those files into kb/user +// and runs the text again, so the run uses the dictionary it just built. + +// The .kbb and .dict files a pass writes: the names in its string literals. +// Only plain file names count (no folders), since that is what lands in the log +// folder; the pass's own text is the only reliable record of what it writes. +export function kbOutputsOf(passSource: string): string[] { + const out: string[] = []; + for (const m of passSource.matchAll(/"([^"\\/:*?<>|\r\n]+\.(?:kbb|dict))"/gi)) { + if (!out.includes(m[1])) out.push(m[1]); + } + return out; +} + +// Whether a generator pass has changed since its output was last copied in. +export function generatorChanged(modifiedMs: number, lastCopiedMs: number | undefined): boolean { + return lastCopiedMs === undefined || modifiedMs > lastCopiedMs; +} diff --git a/src/session/sessionTest.ts b/src/session/sessionTest.ts index b196c648..4d78a2b6 100644 --- a/src/session/sessionTest.ts +++ b/src/session/sessionTest.ts @@ -3,6 +3,7 @@ import * as path from "path"; import { analyzerFiles, isInside, pushRecent, samePath } from "./session"; import { modHeaderPath, modParts, modSection } from "./modPaths"; +import { generatorChanged, kbOutputsOf } from "./kbGen"; let passed = 0; let failed = 0; @@ -60,5 +61,14 @@ eq("mod: input section from a Windows absolute path", modSection("C:\\ana\\input eq("mod: kb only under kb/user", modSection("/ana/kb/numbers.kb"), undefined); eq("mod: header written with backslashes", modHeaderPath("kb/user", "x.dict"), "kb\\user\\x.dict"); +// ---- KB generator passes (#272) -------------------------------------------------- +eq("kbgen: SaveKB target", kbOutputsOf('@CODE\nSaveKB("mykb.kbb",G("kb"),2);\n@@CODE'), ["mykb.kbb"]); +eq("kbgen: dict written with <<, once", kbOutputsOf('L("f") = "words.dict"; "words.dict" << x; "out.txt" << y;'), ["words.dict"]); +eq("kbgen: paths are not plain names", kbOutputsOf('SaveKB("kb/user/x.kbb", G("kb"), 2);'), []); +eq("kbgen: nothing written", kbOutputsOf("@RULES\n_x <- a @@"), []); +eq("kbgen: never copied counts as changed", generatorChanged(5, undefined), true); +eq("kbgen: unchanged since the copy", generatorChanged(5, 5), false); +eq("kbgen: edited after the copy", generatorChanged(6, 5), true); + console.log(`\nsession tests: ${passed} passed, ${failed} failed`); if (failed > 0) process.exit(1);