diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a73be0..ef101153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to the [VSCode NLP++ extension](http://vscode.visualtext.org Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. +### 4.3.12 +Clears the issue backlog: thirteen requests and bugs, some from 2020. + +- **Generate a rule from a parse tree** (#475). Select lines in a pass's `.tree` file and run Generate Rule General or Exact from the editor menu. The selected top-level nodes become the rule's elements, and the rule goes into the pass the tree belongs to. +- **Add a generated pass to the sequence** (#794). An `.nlp` or `.pat` file in the Output view now has "Add to Analyzer Sequence". You pick the pass to insert it after. If a pass of that name is already in the sequence, you can replace it, and it keeps its place, or add the new one under the next free name. +- **Copy or move an output file anywhere** (#658). "Copy or Move File To..." on an output file opens a save dialog, starting in the analyzer's input folder. +- **Browse the outputs of a whole folder** (#688). Select a folder in the Text view, and the Output view lists every file under it that has outputs, each expanding to its own. The matches and trees buttons also work now even when "all" was not clicked first. +- **Sort & Count for corpus study** (#613). "Sort & Count text lines" counts each distinct line, and "Count Words in Folder" in the Text view counts words across every file in the folder. Both open the counts, most frequent first, in a new editor and leave the text alone. +- **The parse trees of a failed run are reachable** (#785). The passes that ran before an error still offer their trees in the sequence, and the Output view lists them. +- **Startup shows what the updater is doing** (#230). While it checks, downloads or unpacks the engine and its files, the status bar shows which step it is on. +- **The engine of the running version is used** (#481). With several versions of the extension on disk, a new version could use an older version's engine. That was the reason for the advice to delete old versions, which is no longer needed. +- **The run mode is remembered** (#328). The status-bar run mode (interpreted or compiled) was lost on every reload, because its setting was never declared. Several settings now say what they do. +- **Partial trees keep `[` tokens** (#295). "View selected tree" dropped a literal `[` from the tree. A partial tree now colors exactly like the full one. +- **Mod files work on macOS and Linux** (#98). `.nlm` files, which bundle passes and KB files into one file, found none of their sections outside Windows, and created files with backslashes in their names. +- **Passes that build a knowledge base can feed the run** (#272). Right-click a pass in the sequence and choose "Toggle KB Generator" to mark it; the sequence shows "KB generator" beside it. With the new setting `nlp.analyzer.regenerateKB` on (it is off by default), after a run in which a marked pass has changed, the `.kbb` and `.dict` files it writes (the file names in its own text, such as `SaveKB("mykb.kbb", ...)`) are copied from the run's log folder into `kb/user`, and the text runs once more, so the result uses the knowledge base the pass just built. The second run copies nothing, so it does not repeat. +- **writekb help** (#626) was already in the help pages, under Knowledge Base Functions. + ### 4.3.11 Publishes everything since 4.3.4 to the VS Code Marketplace. diff --git a/package-lock.json b/package-lock.json index 6e4c5e5a..7a186140 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nlp", - "version": "4.3.11", + "version": "4.3.12", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "nlp", - "version": "4.3.11", + "version": "4.3.12", "license": "MIT", "dependencies": { "@vscode/debugadapter": "^1.68.0", diff --git a/package.json b/package.json index 758d01e4..49dd3bd4 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "NLP", "description": "NLP++: the only computer programming language built for NLP — create and visually debug analyzers directly in VS Code.", "icon": "resources/NLPppLogo.png", - "version": "4.3.11", + "version": "4.3.12", "license": "MIT", "publisher": "dehilster", "enableApiProposals": false, @@ -589,6 +589,16 @@ "title": "Sort & Unique text lines", "description": "Sort and unique text lines" }, + { + "command": "nlp.sortCount", + "title": "Sort & Count text lines", + "description": "How often each line occurs, most frequent first, in a new editor" + }, + { + "command": "textView.countWords", + "title": "Count Words in Folder", + "description": "Word frequencies over every text file in the folder, in a new editor" + }, { "command": "nlp.lowerCase", "title": "Change to Lowercase", @@ -1616,6 +1626,19 @@ "dark": "resources/dark/rename.svg" } }, + { + "command": "outputView.copyOrMoveTo", + "title": "Copy or Move File To..." + }, + { + "command": "outputView.addToSequence", + "title": "Add to Analyzer Sequence", + "icon": "$(add)" + }, + { + "command": "sequenceView.toggleKbGenerator", + "title": "Toggle KB Generator" + }, { "command": "outputView.explore", "title": "Explore Output Folder", @@ -1992,6 +2015,11 @@ "command": "nlp.sortText", "group": "nlp" }, + { + "when": "resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .dict", + "command": "nlp.sortCount", + "group": "nlp" + }, { "when": "resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .dict || resourceExtname == .kbb", "command": "nlp.lowerCase", @@ -2018,12 +2046,12 @@ "group": "nlp" }, { - "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat", + "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat || resourceExtname == .tree", "command": "nlp.generateRule", "group": "nlp" }, { - "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat", + "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat || resourceExtname == .tree", "command": "nlp.generateExactRule", "group": "nlp" }, @@ -2598,6 +2626,11 @@ } ], "view/item/context": [ + { + "command": "sequenceView.toggleKbGenerator", + "when": "view == sequenceView && viewItem =~ /filenotok/", + "group": "kb" + }, { "command": "helpView.copyPath", "when": "view == helpView && viewItem == helpFile", @@ -2610,9 +2643,19 @@ }, { "command": "outputView.rename", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "navigation" }, + { + "command": "outputView.copyOrMoveTo", + "when": "view == outputView && viewItem != input", + "group": "move" + }, + { + "command": "outputView.addToSequence", + "when": "view == outputView && viewItem =~ /nlp/", + "group": "move" + }, { "command": "outputView.editTest", "when": "view == outputView && viewItem =~ /.*test.*/", @@ -3258,6 +3301,11 @@ "when": "view == textView && viewItem =~ /.*dir.*/", "group": "inline@01" }, + { + "command": "textView.countWords", + "when": "view == textView && viewItem =~ /.*dir.*/", + "group": "count" + }, { "command": "textView.runRegressionItem", "when": "view == textView && viewItem =~ /file|dir/", @@ -3310,17 +3358,17 @@ }, { "command": "outputView.copytoText", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@0" }, { "command": "outputView.copytoKB", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@1" }, { "command": "outputView.addTest", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@2" }, { @@ -3330,7 +3378,7 @@ }, { "command": "outputView.deleteOutput", - "when": "view == outputView", + "when": "view == outputView && viewItem != input", "group": "inline@4" }, { @@ -3381,6 +3429,12 @@ "default": true, "markdownDescription": "Send anonymous usage counts (feature usage, run mode, errors) to help improve the extension. Never includes file contents, analyzer/KB source, file names, or paths. This is layered on top of VS Code's global `#telemetry.telemetryLevel#` setting — if that is off, nothing is sent regardless of this setting." }, + "nlp.analyzer.regenerateKB": { + "type": "boolean", + "scope": "resource", + "default": false, + "markdownDescription": "After a run in which a pass marked as a **KB generator** (right-click a pass in the sequence) has changed, copy the `.kbb` and `.dict` files that pass writes into the analyzer's `kb/user` folder and run the text again, so the run uses the knowledge base it just built." + }, "nlp.analyzer.rememberOpenFiles": { "type": "boolean", "scope": "application", @@ -3433,13 +3487,21 @@ "type": "boolean", "scope": "application", "default": true, - "description": "Auto update on load or reload" + "markdownDescription": "Check for and install updates to the NLP++ engine, VisualText files and example analyzers each time the extension starts. Turn off to update only by command." }, "textView.fast": { "type": "boolean", "scope": "application", "default": false, - "description": "Load textview fast. Attributes for text files are lost such as has logs, etc." + "markdownDescription": "Fill the Text view without checking each file for logs and outputs. Faster for large input folders, but files no longer show whether they have been run." + }, + "analyzer.runMode": { + "type": "string", + "scope": "resource", + "enum": ["interpreted", "compiled", "compiled-analyzer", "compiled-kb"], + "enumDescriptions": ["Run the analyzer and its KB as source (the default).", "Run the compiled analyzer and compiled KB.", "Run the compiled analyzer with the KB as source.", "Run the analyzer as source with the compiled KB."], + "default": "interpreted", + "markdownDescription": "How the analyzer runs. Set from the run-mode button in the status bar and remembered per workspace folder." }, "analyzer.regressionTerminal": { "type": "boolean", @@ -3463,7 +3525,7 @@ "type": "string", "scope": "application", "default": "", - "description": "User name for top pass file comments" + "markdownDescription": "Your name, written in the `AUTH:` line of the header comment of new pass files." }, "analyzer.directory": { "type": "string", @@ -3475,25 +3537,25 @@ "type": "string", "scope": "resource", "default": "", - "description": "Current Analyzer" + "markdownDescription": "The analyzer that was open last. Set by the extension when you open an analyzer; you do not normally need to change it." }, "engine.visualtext": { "type": "string", "scope": "application", "default": "", - "description": "VisualText files version" + "markdownDescription": "Version of the downloaded VisualText files (help, libraries, templates). Set by the updater." }, "engine.analyzers": { "type": "string", "scope": "application", "default": "", - "description": "Analyzers version" + "markdownDescription": "Version of the downloaded example analyzers. Set by the updater." }, "logs.clear": { "type": "boolean", "scope": "application", "default": true, - "description": "Clear logs before new operations" + "markdownDescription": "Clear the NLP++ log view before each run or other operation." }, "compile.mode": { "type": "string", diff --git a/src/analyzer.ts b/src/analyzer.ts index 70e42db6..687da55d 100644 --- a/src/analyzer.ts +++ b/src/analyzer.ts @@ -30,6 +30,10 @@ export class Analyzer { // Files open in the editor when the user last switched away from this // analyzer, relative to its directory (#131). private openFiles: string[] = []; + // #272: passes marked as KB generators, and when each one's output was + // last copied into kb/user (the pass file's mtime at the time). + private kbGenerators: string[] = []; + private kbGenCopied: Record = {}; private passNum: number = 0;; private loaded: boolean = false; @@ -44,8 +48,14 @@ export class Analyzer { readState() { this.openFiles = []; + this.kbGenerators = []; + this.kbGenCopied = {}; if (this.jsonState.jsonParse(this.analyzerDir, "state")) { const parse = this.jsonState.json.visualText[0]; + if (Array.isArray(parse.kbGenerators)) + this.kbGenerators = parse.kbGenerators.filter((f: unknown) => typeof f === "string"); + if (parse.kbGenCopied && typeof parse.kbGenCopied === "object") + this.kbGenCopied = parse.kbGenCopied; if (Array.isArray(parse.openFiles)) this.openFiles = parse.openFiles.filter((f: unknown) => typeof f === "string"); if (parse.currentTextFile) { @@ -292,13 +302,40 @@ export class Analyzer { "type": "state", "currentTextFile": this.currentTextFile.fsPath, "currentPassFile": this.currentPassFile.fsPath, - "openFiles": this.openFiles + "openFiles": this.openFiles, + "kbGenerators": this.kbGenerators, + "kbGenCopied": this.kbGenCopied } ] } this.jsonState.saveFile(this.analyzerDir.fsPath, "state", stateJsonDefault); } + isKbGenerator(passName: string): boolean { + return this.kbGenerators.includes(passName); + } + + getKbGenerators(): string[] { + return this.kbGenerators; + } + + toggleKbGenerator(passName: string) { + if (this.isKbGenerator(passName)) + this.kbGenerators = this.kbGenerators.filter(n => n !== passName); + else + this.kbGenerators.push(passName); + this.saveAnalyzerState(); + } + + kbGeneratorCopied(passName: string): number | undefined { + return this.kbGenCopied[passName]; + } + + setKbGeneratorCopied(passName: string, modifiedMs: number) { + this.kbGenCopied[passName] = modifiedMs; + this.saveAnalyzerState(); + } + getOpenFiles(): string[] { return this.openFiles; } diff --git a/src/command.ts b/src/command.ts index b3dce621..d14778e6 100644 --- a/src/command.ts +++ b/src/command.ts @@ -2,6 +2,7 @@ import * as vscode from 'vscode'; import { TreeFile, generateType } from './treeFile'; import { NLPFile, reformatType } from './nlp'; import { TextFile } from './textFile'; +import { countLines, formatCounts } from './output/counts'; export let nlpCommands: NLPCommands; export class NLPCommands { @@ -29,6 +30,7 @@ export class NLPCommands { ctx.subscriptions.push(vscode.commands.registerCommand('nlp.passTree', this.passTree)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.displayMatchedRules', this.displayMatchedRulesNLP)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.sortText', this.sortText)); + ctx.subscriptions.push(vscode.commands.registerCommand('nlp.sortCount', this.sortCount)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.lowerCase', this.lowerCase)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.removeAccents', this.removeAccents)); ctx.subscriptions.push(vscode.commands.registerCommand('nlp.runPython', this.runPython)); @@ -84,6 +86,20 @@ export class NLPCommands { } } + // #613: how often each line occurs, most frequent first, in a new editor + // beside the text so the corpus itself is left as it is. + async sortCount() { + const editor = vscode.window.activeTextEditor; + if (!editor) return; + const doc = editor.document; + const range = editor.selection.isEmpty + ? new vscode.Range(0, 0, doc.lineCount, 0) + : new vscode.Range(editor.selection.start.line, 0, editor.selection.end.line + 1, 0); + const counts = countLines(doc.getText(range).split(/\r?\n/)); + const result = await vscode.workspace.openTextDocument({ content: formatCounts(counts), language: 'plaintext' }); + await vscode.window.showTextDocument(result, { viewColumn: vscode.ViewColumn.Beside }); + } + lowerCase() { if (vscode.window.activeTextEditor) { const editor = vscode.window.activeTextEditor; diff --git a/src/modFile.ts b/src/modFile.ts index 13ec3ec0..9518bc48 100644 --- a/src/modFile.ts +++ b/src/modFile.ts @@ -1,5 +1,6 @@ import * as vscode from 'vscode'; import * as path from 'path'; +import { modHeaderPath, modParts, modSection } from './session/modPaths'; import * as fs from 'fs'; import { visualText } from './visualText'; import { TextFile } from './textFile'; @@ -22,9 +23,6 @@ export let modFile: ModFile; export class ModFile extends TextFile { public readonly MODFILE_HEADER = ' { this.setFile(filePath); - let filepath = ''; for (const line of this.getLines()) { if (line.indexOf(this.MODFILE_HEADER) == 0) { const tokens = line.split(/[\<\t\s\>]/); const relFilePath = tokens[2]; - filepath = path.join(visualText.getAnalyzerDir().fsPath,relFilePath); - if (filepath.includes(this.MODFILE_SPEC)) { + if (modSection(relFilePath) == 'spec') { const seq = new SequenceFile; const items: vscode.QuickPickItem[] = []; seq.choicePasses(visualText.analyzer.getSpecDirectory().fsPath,items,''); @@ -220,17 +210,18 @@ export class ModFile extends TextFile { let header = ''; const filepath = filePath.fsPath; let dir = ''; - if (filepath.includes(this.MODFILE_KB)) { + const section = modSection(filepath); + if (section == 'kb') { dir = visualText.analyzer.anaSubDirPath(anaSubDir.KB); } - else if (filepath.includes(this.MODFILE_SPEC)) { + else if (section == 'spec') { dir = visualText.analyzer.anaSubDirPath(anaSubDir.SPEC); } - else if (filepath.includes(this.MODFILE_INPUT)) { + else if (section == 'input') { dir = visualText.analyzer.anaSubDirPath(anaSubDir.INPUT); } const name = path.basename(filePath.fsPath); - const finalPath = path.join(dir,name); + const finalPath = modHeaderPath(dir, name); header = '\n' + this.MODFILE_HEADER + ' ' + finalPath + '>\n'; return header; } diff --git a/src/nlp.ts b/src/nlp.ts index a1591ace..e27c3c1c 100644 --- a/src/nlp.ts +++ b/src/nlp.ts @@ -14,7 +14,8 @@ import { ReformatType as reformatType } from './format/types'; import { formatRuleText } from './format/rulesRegion'; import { refreshEngineDiagnostics } from './language/engineDiagnostics'; import * as telemetry from './telemetry/telemetry'; -import { analyzerOutputLines } from './output/analyzerOutput'; +import { analyzerOutputLines, lastTreePass } from './output/analyzerOutput'; +import { generatorChanged, kbOutputsOf } from './session/kbGen'; let analyzerOutputChannel: vscode.OutputChannel | undefined; @@ -51,6 +52,9 @@ export class NLPFile extends TextFile { public timerStatus: anaQueueStatus = anaQueueStatus.UNKNOWN; private timerID = 0; private stopAllFlag: boolean = false; + // #272: a text to run again once the current queue is done, because a KB + // generator pass just rebuilt part of kb/user. + private kbRerun: vscode.Uri | undefined; constructor(filepath: string = '', separateLines: boolean = true, text: string = '') { super(); @@ -201,6 +205,18 @@ export class NLPFile extends TextFile { else if (!logView.makeAna(false)) logView.loadAnalyzerOuts(false); + // #785: the passes that ran before the error left their parse + // trees, which are the best clue to what went wrong. Refresh the + // sequence so each of those passes offers its tree, and list the + // trees in the output view. + const logDir = filestr + visualText.LOG_SUFFIX; + const lastTree = fs.existsSync(logDir) && dirfuncs.isDir(logDir) ? lastTreePass(fs.readdirSync(logDir)) : 0; + if (lastTree > 0) { + outputView.setType(outputFileType.TREE); + logView.addMessage(`Parse trees for passes up to ${lastTree} are in the output view, and on each of those passes in the sequence`, logLineType.ANALYER_OUTPUT, vscode.Uri.file(filestr)); + } + vscode.commands.executeCommand('sequenceView.refreshAll'); + vscode.commands.executeCommand('textView.refreshAll'); vscode.commands.executeCommand('outputView.refreshAll'); vscode.commands.executeCommand('logView.refreshAll'); resolve('Failed'); @@ -303,6 +319,13 @@ export class NLPFile extends TextFile { vscode.commands.executeCommand('logView.refreshAll'); visualText.nlp.setAnalyzerStatus(filepath, analyzerStatus.DONE); nlpStatusBar.resetAnalyzerButton(); + if (visualText.nlp.copyKbGeneratorOutput(filepath)) { + // Queue the second run for when this queue finishes; running + // it now would share the queue entry of this one. + visualText.nlp.kbRerun = filepath; + if (visualText.nlp.timerID == 0) + visualText.nlp.startKbRerun(); + } resolve('Processed'); } }).pid; @@ -487,11 +510,63 @@ export class NLPFile extends TextFile { } shutDown() { + const stopped = visualText.nlp.stopAllFlag; clearInterval(visualText.nlp.timerID); visualText.debugMessage('Analyzing done', logLineType.ANALYER_OUTPUT); visualText.nlp.stopAllFlag = false; visualText.nlp.timerID = 0; visualText.nlp.anaQueue = []; + if (stopped) + visualText.nlp.kbRerun = undefined; + else if (visualText.nlp.kbRerun) + visualText.nlp.startKbRerun(); + } + + private startKbRerun() { + const uri = visualText.nlp.kbRerun; + visualText.nlp.kbRerun = undefined; + if (!uri) return; + visualText.nlp.addAnalyzer(uri, analyzerType.FILE); + visualText.nlp.startAnalyzer(); + } + + // #272: after a successful run of one text, copy into kb/user the .kbb/.dict + // files that each changed KB generator pass wrote to the run's log folder. + // Returns true when something was copied, i.e. the text should run again. + // A generator counts as changed until its output has been copied once since + // its last edit, so the second run copies nothing and does not repeat. + copyKbGeneratorOutput(textFile: vscode.Uri): boolean { + if (!vscode.workspace.getConfiguration('nlp').get('analyzer.regenerateKB', false)) + return false; + const analyzer = visualText.analyzer; + const logDir = textFile.fsPath + visualText.LOG_SUFFIX; + if (dirfuncs.isDir(textFile.fsPath) || !dirfuncs.isDir(logDir)) + return false; + const kbUser = analyzer.getAnalyzerDirectory(path.join('kb', 'user')).fsPath; + const copied: string[] = []; + for (const name of analyzer.getKbGenerators()) { + const pass = analyzer.seqFile.getPasses().find(p => p.name == name); + if (!pass || !fs.existsSync(pass.uri.fsPath)) + continue; + const modified = fs.statSync(pass.uri.fsPath).mtimeMs; + if (!generatorChanged(modified, analyzer.kbGeneratorCopied(name))) + continue; + const outputs = kbOutputsOf(fs.readFileSync(pass.uri.fsPath, 'utf8')).filter(f => fs.existsSync(path.join(logDir, f))); + if (!outputs.length) + continue; + fs.mkdirSync(kbUser, { recursive: true }); + for (const f of outputs) { + fs.copyFileSync(path.join(logDir, f), path.join(kbUser, f)); + copied.push(f); + } + analyzer.setKbGeneratorCopied(name, modified); + } + if (!copied.length) + return false; + logView.addMessage(`KB generator output copied to kb/user: ${copied.join(', ')}. Running ${path.basename(textFile.fsPath)} again with it.`, logLineType.ANALYER_OUTPUT, textFile); + vscode.commands.executeCommand('kbView.refreshAll'); + vscode.commands.executeCommand('logView.refreshAll'); + return true; } insertRule(ruleStr: string) { diff --git a/src/output/analyzerOutput.ts b/src/output/analyzerOutput.ts index 3188ccd4..591e2577 100644 --- a/src/output/analyzerOutput.ts +++ b/src/output/analyzerOutput.ts @@ -26,3 +26,17 @@ export function analyzerOutputLines(stdout: string): string[] { lines.shift(); return lines; } + +/** + * The highest pass that left a parse tree (`anaNNN.tree`) among a log directory's + * file names, or 0 if none did. After a failed run these are the passes that + * completed before the error (#785). + */ +export function lastTreePass(fileNames: string[]): number { + let last = 0; + for (const name of fileNames) { + const m = /^ana(\d+)\.tree$/i.exec(name); + if (m) last = Math.max(last, Number(m[1])); + } + return last; +} diff --git a/src/output/counts.ts b/src/output/counts.ts new file mode 100644 index 00000000..b38d0e04 --- /dev/null +++ b/src/output/counts.ts @@ -0,0 +1,44 @@ +// Frequency counts for corpus study (#613). PURE: no 'vscode' import. + +export interface Count { + key: string; + count: number; +} + +// Highest count first; ties alphabetically, ignoring case. +function sorted(map: Map): Count[] { + return [...map].map(([key, count]) => ({ key, count })) + .sort((a, b) => b.count - a.count || a.key.toLowerCase().localeCompare(b.key.toLowerCase()) || a.key.localeCompare(b.key)); +} + +// Each distinct line and how many times it occurs. Blank lines are skipped; +// lines are compared exactly, after trimming trailing whitespace. +export function countLines(lines: string[]): Count[] { + const map = new Map(); + for (const raw of lines) { + const line = raw.replace(/\s+$/, ""); + if (!line.length) continue; + map.set(line, (map.get(line) ?? 0) + 1); + } + return sorted(map); +} + +// Each distinct word and how many times it occurs, lowercased. A word is a run +// of letters and digits, with inner apostrophes and hyphens kept ("don't", +// "x-ray"); everything else separates words. +export function countWords(texts: string[]): Count[] { + const map = new Map(); + const word = /[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu; + for (const text of texts) { + for (const m of text.matchAll(word)) { + const w = m[0].toLowerCase(); + map.set(w, (map.get(w) ?? 0) + 1); + } + } + return sorted(map); +} + +// "countkey" lines, ready to open in an editor. +export function formatCounts(counts: Count[]): string { + return counts.map((c) => `${c.count}\t${c.key}`).join("\n") + (counts.length ? "\n" : ""); +} diff --git a/src/output/outputTest.ts b/src/output/outputTest.ts index e94aadaf..656d0293 100644 --- a/src/output/outputTest.ts +++ b/src/output/outputTest.ts @@ -1,6 +1,7 @@ // Tests for analyzerOutputLines. Runs with plain Node via `npm run test:output`. -import { analyzerOutputLines } from "./analyzerOutput"; +import { analyzerOutputLines, lastTreePass } from "./analyzerOutput"; +import { countLines, countWords, formatCounts } from "./counts"; let passed = 0; let failed = 0; @@ -50,6 +51,17 @@ check("analyzer brackets kept", check("no output", analyzerOutputLines(oldEngine.replace("21\r\n34\r\n", "")), []); check("empty", analyzerOutputLines(""), []); +// #785: the last pass that wrote a tree before a run stopped. +check("last tree pass", [String(lastTreePass(["ana001.tree", "ana012.tree", "ana003.tree", "ana014.kbb", "final.tree", "err.log"]))], ["12"]); +check("no trees", [String(lastTreePass(["err.log", "dbg.log"]))], ["0"]); + +// #613: sort and count for corpus study. +check("count lines", formatCounts(countLines(["b", "a", "b", "", "c ", "c", "B"])).split("\n"), ["2\tb", "2\tc", "1\ta", "1\tB", ""]); +check("count words", formatCounts(countWords(["The cat, the hat.", "THE x-ray didn't 42 42"])).split("\n"), + ["3\tthe", "2\t42", "1\tcat", "1\tdidn't", "1\that", "1\tx-ray", ""]); +check("count words: accents and other scripts", countWords(["café Café 東京"]).map((c) => `${c.count}:${c.key}`), ["2:café", "1:東京"]); +check("count nothing", [formatCounts(countLines(["", " "]))], [""]); + console.log(`analyzer output: ${passed} passed, ${failed} failed`); if (failed) process.exit(1); diff --git a/src/outputView.ts b/src/outputView.ts index 7c5e2976..fee3c283 100644 --- a/src/outputView.ts +++ b/src/outputView.ts @@ -11,6 +11,7 @@ export enum outputFileType { ALL, TXXT, TREE, KB, NLP } interface OutputItem { uri: vscode.Uri; + input?: boolean; // an input file whose outputs are its children (#688) } export class OutputTreeDataProvider implements vscode.TreeDataProvider { @@ -25,11 +26,27 @@ export class OutputTreeDataProvider implements vscode.TreeDataProvider ({ uri })); + if (visualText.hasWorkspaceFolder() && outputView.browsingFolder()) + return outputView.inputsWithOutput().map(uri => ({ uri, input: true })); if (visualText.hasWorkspaceFolder()) { const children: OutputItem[] = new Array(); for (const folder of outputView.getOutputFiles()) { @@ -75,6 +96,7 @@ export class OutputView { this.outputView = vscode.window.createTreeView('outputView', { treeDataProvider: outputViewProvider }); ctx.subscriptions.push(this.outputView); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.refreshAll', () => outputViewProvider.refresh())); + ctx.subscriptions.push(vscode.commands.registerCommand('outputView.showFolder', (folder?: vscode.Uri) => this.showFolder(folder))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.copytoKB', (resource) => this.copytoKB(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.copytoText', (resource) => this.copytoText(resource))); @@ -85,6 +107,8 @@ export class OutputView { ctx.subscriptions.push(vscode.commands.registerCommand('outputView.deleteTest', (resource) => this.deleteTest(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.editTest', (resource) => this.editTest(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.rename', (resource) => this.rename(resource))); + ctx.subscriptions.push(vscode.commands.registerCommand('outputView.copyOrMoveTo', (resource) => this.copyOrMoveTo(resource))); + ctx.subscriptions.push(vscode.commands.registerCommand('outputView.addToSequence', (resource) => this.addToSequence(resource))); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.kb', () => this.loadKB())); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.matches', () => this.loadTxxt())); ctx.subscriptions.push(vscode.commands.registerCommand('outputView.trees', () => this.loadTrees())); @@ -281,7 +305,7 @@ export class OutputView { this.testDirectory = vscode.Uri.file(''); if (filepath.length == 0) return false; - this.logDirectory = vscode.Uri.file(filepath + visualText.TEST_SUFFIX); + this.testDirectory = vscode.Uri.file(filepath + visualText.TEST_SUFFIX); if (!fs.existsSync(this.testDirectory.fsPath)) return false; const stats = fs.lstatSync(this.testDirectory.fsPath); @@ -308,38 +332,68 @@ export class OutputView { } } } - else if (this.type == outputFileType.TXXT) { - const matchFiles = dirfuncs.getFiles(this.logDirectory, ['.txxt']); - this.outputFiles = this.outputFiles.concat(matchFiles); - } - else if (this.type == outputFileType.TREE) { - const finalTree = vscode.Uri.file(path.join(this.logDirectory.fsPath, 'final.tree')); - if (fs.existsSync(finalTree.fsPath)) { - this.outputFiles.push(finalTree); - } - const matchFiles = dirfuncs.getFiles(this.logDirectory, ['.tree']); - this.outputFiles = this.outputFiles.concat(matchFiles); - } else { + // ALL, TXXT and TREE list one input file's _log folder. Each mode + // finds that folder itself: TXXT and TREE used to reuse the one ALL + // had last set, and listed nothing if ALL had not run first. const textPath = visualText.analyzer.getTextPath().fsPath; - this.outputFiles = []; - if (textPath.length && this.fileHasLog(textPath)) { - const finalTree = vscode.Uri.file(path.join(this.logDirectory.fsPath, 'final.tree')); - if (fs.existsSync(finalTree.fsPath)) { - this.outputFiles.push(finalTree); - } - const candidates = dirfuncs.getFiles(this.logDirectory); - for (const cand of candidates) { - const ext = path.parse(cand.fsPath).ext; - if (ext.localeCompare('.tree') != 0 && ext.localeCompare('.txxt') != 0) - this.outputFiles.push(cand); - } - } + if (textPath.length && this.fileHasLog(textPath)) + this.outputFiles = this.filesForLog(this.logDirectory); } } return this.outputFiles; } + // The outputs in one input file's _log folder that the current mode shows. + filesForLog(logDir: vscode.Uri): vscode.Uri[] { + const files: vscode.Uri[] = []; + const finalTree = vscode.Uri.file(path.join(logDir.fsPath, 'final.tree')); + if (this.type == outputFileType.TXXT) + return dirfuncs.getFiles(logDir, ['.txxt']); + if (fs.existsSync(finalTree.fsPath)) + files.push(finalTree); + if (this.type == outputFileType.TREE) + return files.concat(dirfuncs.getFiles(logDir, ['.tree']).filter(f => f.fsPath !== finalTree.fsPath)); + for (const cand of dirfuncs.getFiles(logDir)) { + const ext = path.parse(cand.fsPath).ext; + if (ext.localeCompare('.tree') != 0 && ext.localeCompare('.txxt') != 0) + files.push(cand); + } + return files; + } + + // #688: with a folder selected in the Text view, the view lists every input + // file under it that has outputs, each expanding to its own outputs. + public folder: vscode.Uri | undefined; + + public showFolder(folder: vscode.Uri | undefined) { + this.folder = folder; + vscode.commands.executeCommand('outputView.refreshAll'); + } + + public browsingFolder(): boolean { + return !!this.folder && this.type != outputFileType.KB && this.type != outputFileType.NLP; + } + + // Input files under the selected folder that have a _log folder, depth first. + inputsWithOutput(): vscode.Uri[] { + const found: vscode.Uri[] = []; + const walk = (dir: string) => { + let entries: fs.Dirent[]; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (!dirfuncs.directoryIsLog(full)) walk(full); + } else if (entry.isFile() && dirfuncs.isDir(full + visualText.LOG_SUFFIX)) { + found.push(vscode.Uri.file(full)); + } + } + }; + if (this.folder) walk(this.folder.fsPath); + return found; + } + private openFile(resource: vscode.Uri): void { const textFile = new TextFile(resource.fsPath); textFile.cleanZeroZero(); @@ -364,6 +418,79 @@ export class OutputView { } } + // #658: put an output file anywhere, as a copy or by moving it. + async copyOrMoveTo(outputItem: OutputItem) { + const from = outputItem.uri; + const target = await vscode.window.showSaveDialog({ + title: 'Copy or move output file to', + defaultUri: vscode.Uri.file(path.join(visualText.analyzer.getInputDirectory().fsPath, path.basename(from.fsPath))), + }); + if (!target || target.fsPath === from.fsPath) + return; + const how = await vscode.window.showQuickPick(['Copy', 'Move'], { title: path.basename(target.fsPath), placeHolder: 'Copy the file, or move it?' }); + if (!how) + return; + try { + fs.mkdirSync(path.dirname(target.fsPath), { recursive: true }); + fs.copyFileSync(from.fsPath, target.fsPath); + if (how === 'Move') + fs.unlinkSync(from.fsPath); + } catch (err) { + vscode.window.showErrorMessage(`Could not ${how.toLowerCase()} ${path.basename(from.fsPath)}: ${err instanceof Error ? err.message : err}`); + return; + } + vscode.commands.executeCommand('outputView.refreshAll'); + vscode.commands.executeCommand('textView.refreshAll'); + vscode.commands.executeCommand('kbView.refreshAll'); + } + + // #794: a pass file produced by a run (rule generation, a RUG pass) goes into + // the sequence after the pass the user picks. If a pass of that name is + // already in the sequence, they choose to replace it or add this one under a + // free name. + async addToSequence(outputItem: OutputItem) { + const seqFile = visualText.analyzer.seqFile; + const passes = seqFile.getPasses(); + if (!passes.length) { + vscode.window.showWarningMessage('Open an analyzer first'); + return; + } + const items = passes.map((pass, index) => ({ + label: pass.passNum ? `${pass.passNum} ${pass.name}` : pass.name, + description: pass.isFolder() ? 'folder' : pass.typeStr, + index, + })); + const after = await vscode.window.showQuickPick(items, { title: `Add ${path.basename(outputItem.uri.fsPath)} to the sequence`, placeHolder: 'Insert after which pass?' }); + if (!after) + return; + + const specDir = visualText.analyzer.getSpecDirectory().fsPath; + const name = path.basename(outputItem.uri.fsPath); + const existing = seqFile.findPassByFilename(name); + if (existing) { + const replace = `Replace pass ${existing} (${path.parse(name).name})`; + const choice = await vscode.window.showQuickPick([replace, 'Add it as a new pass under another name'], { title: `A pass named ${path.parse(name).name} is already in the sequence` }); + if (!choice) + return; + if (choice === replace) { + // The pass keeps its place in the sequence; only its file changes. + fs.copyFileSync(outputItem.uri.fsPath, path.join(specDir, name)); + vscode.commands.executeCommand('sequenceView.refreshAll'); + return; + } + const { name: base, ext } = path.parse(name); + let n = 2; + while (fs.existsSync(path.join(specDir, `${base}${n}${ext}`)) || seqFile.findPassByFilename(`${base}${n}${ext}`)) + n++; + const renamed = path.join(specDir, `${base}${n}${ext}`); + fs.copyFileSync(outputItem.uri.fsPath, renamed); + seqFile.insertPass(after.index, vscode.Uri.file(renamed)); + } else { + seqFile.insertPass(after.index, outputItem.uri); + } + vscode.commands.executeCommand('sequenceView.refreshAll'); + } + copytoKB(outputItem: OutputItem) { const kbDir = visualText.analyzer.anaSubDirPath(anaSubDir.KB); this.copyFileToAnalyzer(outputItem.uri, kbDir, 'Copy file to another analyzer', 'Copy file to the KB directory of:'); diff --git a/src/sequenceView.ts b/src/sequenceView.ts index a2d785e6..44e5254e 100644 --- a/src/sequenceView.ts +++ b/src/sequenceView.ts @@ -137,6 +137,8 @@ export class PassTree implements vscode.TreeDataProvider { conVal = conVal + 'hasLog'; if (treeFile.hasFileType(passItem.uri, passItem.passNum, nlpFileType.KBB)) conVal = conVal + 'hasKB'; + if (visualText.analyzer.isKbGenerator(passItem.name)) + conVal = conVal + 'kbgen'; if (debugConVal) label = row.toString() + ' ' + conVal; tooltip = passTooltip(passItem, passItem.uri.fsPath); if (passItem.fileExists()) @@ -232,6 +234,7 @@ export class PassTree implements vscode.TreeDataProvider { resourceUri: seqItem.uri, tooltip: seqItem.tooltip, label: seqItem.label, + description: seqItem.contextValue?.includes('kbgen') ? 'KB generator' : undefined, contextValue: seqItem.contextValue, collapsibleState: collapse, iconPath: { @@ -246,6 +249,18 @@ export class PassTree implements vscode.TreeDataProvider { } } + // #272: mark or unmark a pass whose output (.kbb/.dict) should be copied + // into kb/user, and the text run again, when the pass changes. + toggleKbGenerator(seqItem: SequenceItem): void { + visualText.analyzer.toggleKbGenerator(seqItem.name); + if (visualText.analyzer.isKbGenerator(seqItem.name) && !vscode.workspace.getConfiguration('nlp').get('analyzer.regenerateKB', false)) { + vscode.window.showInformationMessage(`${seqItem.name} is marked as a KB generator. Turn on "NLP: Analyzer Regenerate KB" in Settings to have runs use what it writes.`, 'Open Setting').then(choice => { + if (choice) vscode.commands.executeCommand('workbench.action.openSettings', 'nlp.analyzer.regenerateKB'); + }); + } + vscode.commands.executeCommand('sequenceView.refreshAll'); + } + moveUp(seqItem: SequenceItem): void { this.moveSequence(seqItem, moveDirection.UP); } @@ -682,6 +697,7 @@ export class SequenceView { this.sequenceView = vscode.window.createTreeView('sequenceView', { treeDataProvider }); ctx.subscriptions.push(this.sequenceView); + ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.toggleKbGenerator', (seqItem: SequenceItem) => treeDataProvider.toggleKbGenerator(seqItem))); ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.openPass', (seqItem) => this.openPass(seqItem))); ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.openTree', (seqItem) => this.openTree(seqItem))); ctx.subscriptions.push(vscode.commands.registerCommand('sequenceView.displayMatchedRules', (seqItem) => this.displayMatchedRules(seqItem))); diff --git a/src/session/kbGen.ts b/src/session/kbGen.ts new file mode 100644 index 00000000..b596ddc5 --- /dev/null +++ b/src/session/kbGen.ts @@ -0,0 +1,24 @@ +// KB generator passes (#272). PURE: no 'vscode' import. +// +// A KB generator is a pass the user marks in the sequence because it writes a +// knowledge base or dictionary -- SaveKB("mykb.kbb", ...), or a .dict written +// with <<. The engine puts such a file in the run's _log folder, where the +// analyzer's own kb/user never sees it. With regeneration on, after a run in +// which a generator pass changed, the extension copies those files into kb/user +// and runs the text again, so the run uses the dictionary it just built. + +// The .kbb and .dict files a pass writes: the names in its string literals. +// Only plain file names count (no folders), since that is what lands in the log +// folder; the pass's own text is the only reliable record of what it writes. +export function kbOutputsOf(passSource: string): string[] { + const out: string[] = []; + for (const m of passSource.matchAll(/"([^"\\/:*?<>|\r\n]+\.(?:kbb|dict))"/gi)) { + if (!out.includes(m[1])) out.push(m[1]); + } + return out; +} + +// Whether a generator pass has changed since its output was last copied in. +export function generatorChanged(modifiedMs: number, lastCopiedMs: number | undefined): boolean { + return lastCopiedMs === undefined || modifiedMs > lastCopiedMs; +} diff --git a/src/session/modPaths.ts b/src/session/modPaths.ts new file mode 100644 index 00000000..3c380ba0 --- /dev/null +++ b/src/session/modPaths.ts @@ -0,0 +1,31 @@ +// Paths inside .nlm mod files. PURE: no 'vscode' import. +// +// A mod file bundles analyzer files under "" headers. +// The format was written on Windows: headers use backslashes, and the code +// classified paths by looking for "\kb\user\", "\spec\" and "\input\". On macOS +// and Linux no path contains those, so every section was "unknown" -- its pass +// never inserted into the sequence -- and path.join kept the backslashes, writing +// a file literally named "kb\user\x.dict". + +export type ModSection = "kb" | "spec" | "input"; + +// The segments of a header path, whichever separator it was written with. +export function modParts(relPath: string): string[] { + return relPath.split(/[\\/]+/).filter((p) => p.length); +} + +// Which analyzer folder a path (relative, as in a header, or absolute) belongs to. +export function modSection(anyPath: string): ModSection | undefined { + const p = "/" + modParts(anyPath).join("/").toLowerCase() + "/"; + if (p.includes("/kb/user/")) return "kb"; + if (p.includes("/spec/")) return "spec"; + if (p.includes("/input/")) return "input"; + return undefined; +} + +// The path a header records for a file in an analyzer folder. Always written +// with backslashes, so a mod file made anywhere still loads in older versions, +// which only understood those. +export function modHeaderPath(dir: string, name: string): string { + return [...modParts(dir), name].join("\\"); +} diff --git a/src/session/sessionTest.ts b/src/session/sessionTest.ts index 929ea36a..4d78a2b6 100644 --- a/src/session/sessionTest.ts +++ b/src/session/sessionTest.ts @@ -2,6 +2,8 @@ import * as path from "path"; import { analyzerFiles, isInside, pushRecent, samePath } from "./session"; +import { modHeaderPath, modParts, modSection } from "./modPaths"; +import { generatorChanged, kbOutputsOf } from "./kbGen"; let passed = 0; let failed = 0; @@ -50,5 +52,23 @@ eq("same: trailing separator", samePath(ana + path.sep, ana), true); eq("same: prefix sibling differs", samePath(ana, other), false); eq("same: empty never matches", samePath("", ""), false); +// ---- mod file paths ------------------------------------------------------------ +eq("mod: header with backslashes", modParts("kb\\user\\en-numbers.dict"), ["kb", "user", "en-numbers.dict"]); +eq("mod: header with slashes", modParts("spec/numbers.nlp"), ["spec", "numbers.nlp"]); +eq("mod: kb section from a Windows header", modSection("kb\\user\\en-numbers.dict"), "kb"); +eq("mod: spec section from a POSIX absolute path", modSection("/home/me/ana/spec/numbers.nlp"), "spec"); +eq("mod: input section from a Windows absolute path", modSection("C:\\ana\\input\\numbers.txt"), "input"); +eq("mod: kb only under kb/user", modSection("/ana/kb/numbers.kb"), undefined); +eq("mod: header written with backslashes", modHeaderPath("kb/user", "x.dict"), "kb\\user\\x.dict"); + +// ---- KB generator passes (#272) -------------------------------------------------- +eq("kbgen: SaveKB target", kbOutputsOf('@CODE\nSaveKB("mykb.kbb",G("kb"),2);\n@@CODE'), ["mykb.kbb"]); +eq("kbgen: dict written with <<, once", kbOutputsOf('L("f") = "words.dict"; "words.dict" << x; "out.txt" << y;'), ["words.dict"]); +eq("kbgen: paths are not plain names", kbOutputsOf('SaveKB("kb/user/x.kbb", G("kb"), 2);'), []); +eq("kbgen: nothing written", kbOutputsOf("@RULES\n_x <- a @@"), []); +eq("kbgen: never copied counts as changed", generatorChanged(5, undefined), true); +eq("kbgen: unchanged since the copy", generatorChanged(5, 5), false); +eq("kbgen: edited after the copy", generatorChanged(6, 5), true); + console.log(`\nsession tests: ${passed} passed, ${failed} failed`); if (failed > 0) process.exit(1); diff --git a/src/status.ts b/src/status.ts index 9041e77b..9dc49f29 100644 --- a/src/status.ts +++ b/src/status.ts @@ -347,11 +347,13 @@ export class NLPStatusBar { if (folder.fsPath.length) { vscode.workspace .getConfiguration('analyzer', folder) - .update('runMode', value, vscode.ConfigurationTarget.WorkspaceFolder); + .update('runMode', value, vscode.ConfigurationTarget.WorkspaceFolder) + .then(undefined, () => { /* not saved; the mode still applies this session */ }); } else { vscode.workspace .getConfiguration('analyzer') - .update('runMode', value, vscode.ConfigurationTarget.Global); + .update('runMode', value, vscode.ConfigurationTarget.Global) + .then(undefined, () => { /* not saved; the mode still applies this session */ }); } } catch { // ignore persistence failures; state still lives in memory diff --git a/src/test/suite/extension.test.ts b/src/test/suite/extension.test.ts index d88e5dc0..7f4d2fc8 100644 --- a/src/test/suite/extension.test.ts +++ b/src/test/suite/extension.test.ts @@ -124,6 +124,7 @@ const WRITTEN_SETTINGS: Array<[string, string]> = [ ["analyzer", "directory"], ["analyzer", "current"], ["textView", "fast"], + ["analyzer", "runMode"], ]; export async function configurationTests(): Promise { @@ -527,3 +528,16 @@ export async function analyzerSwitchTests(): Promise { check("switch: dirty file stays open", await until(() => isOpen(bFile)) && isOpen(aFile)); await vscode.commands.executeCommand("workbench.action.revertAndCloseActiveEditor"); } + +// ---- sort & count (#613) ----------------------------------------------------- + +export async function sortCountTests(): Promise { + const doc = await vscode.workspace.openTextDocument({ content: "cat\ndog\ncat\n\nbird\ncat\ndog\n", language: "plaintext" }); + await vscode.window.showTextDocument(doc); + await vscode.commands.executeCommand("nlp.sortCount"); + const result = vscode.window.activeTextEditor?.document; + check("sort & count: opens a new editor", !!result && result !== doc); + eq("sort & count: counts, most frequent first", result?.getText(), "3\tcat\n2\tdog\n1\tbird\n"); + eq("sort & count: leaves the text alone", doc.getText(), "cat\ndog\ncat\n\nbird\ncat\ndog\n"); + await vscode.commands.executeCommand("workbench.action.closeAllEditors"); +} diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index e2e6baa0..a0b623cd 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -12,7 +12,7 @@ import { configurationTests, providerTests, languageFeatureTests, - crossPassTests, debugSetupTests, analyzerSwitchTests, + crossPassTests, debugSetupTests, analyzerSwitchTests, sortCountTests, } from "./extension.test"; export async function run(): Promise { @@ -25,6 +25,7 @@ export async function run(): Promise { ["language features", languageFeatureTests], ["cross-pass resolution", crossPassTests], ["debugger setup", debugSetupTests], + ["sort and count", sortCountTests], ["analyzer switching", analyzerSwitchTests], ]; diff --git a/src/textView.ts b/src/textView.ts index 53bcc383..eeb01ba6 100644 --- a/src/textView.ts +++ b/src/textView.ts @@ -10,6 +10,7 @@ import { fileOperation, fileOpRefresh } from './fileOps'; import { anaSubDir } from './analyzer'; import { regressionRunner } from './regression'; import * as fs from 'fs'; +import { countWords, formatCounts } from './output/counts'; export interface TextItem { uri: vscode.Uri; @@ -323,6 +324,7 @@ export class TextView { ctx.subscriptions.push(vscode.commands.registerCommand('textView.analyzerCurrent', () => this.analyzerCurrent())); ctx.subscriptions.push(vscode.commands.registerCommand('textView.analyze', (textItem) => this.analyze(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.analyzeDir', (textItem) => this.analyzeDir(textItem))); + ctx.subscriptions.push(vscode.commands.registerCommand('textView.countWords', (textItem) => this.countWords(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.runRegressionItem', (textItem) => this.runRegressionItem(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.blessRegressionItem', (textItem) => this.blessRegressionItem(textItem))); ctx.subscriptions.push(vscode.commands.registerCommand('textView.openText', () => this.openText())); @@ -572,6 +574,29 @@ export class TextView { return bytes.toFixed(dp) + ' ' + units[u]; } + // #613: word frequencies over every text file in a folder, most frequent + // first. The engine's per-file _log folders are skipped. + async countWords(textItem: TextItem) { + const files: string[] = []; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (!dirfuncs.directoryIsLog(full)) walk(full); + } else if (entry.isFile()) { + files.push(full); + } + } + }; + walk(textItem.uri.fsPath); + const texts = files.map(f => fs.readFileSync(f, 'utf8')); + const counts = countWords(texts); + const total = counts.reduce((n, c) => n + c.count, 0); + const header = `# ${path.basename(textItem.uri.fsPath)}: ${total} words, ${counts.length} distinct, in ${files.length} files\n`; + const doc = await vscode.workspace.openTextDocument({ content: header + formatCounts(counts), language: 'plaintext' }); + await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.Beside }); + } + analyzeDir(textItem: TextItem) { if (textItem.uri.fsPath.length) { @@ -658,10 +683,12 @@ export class TextView { this.folderUri = undefined; vscode.window.showTextDocument(textItem.uri); visualText.analyzer.saveCurrentFile(textItem.uri); - vscode.commands.executeCommand('outputView.refreshAll'); + vscode.commands.executeCommand('outputView.showFolder', undefined); vscode.commands.executeCommand('status.update'); } else { + // #688: the output view lists the outputs of every file in the folder. this.folderUri = textItem.uri; + vscode.commands.executeCommand('outputView.showFolder', textItem.uri); } } diff --git a/src/trace/traceTest.ts b/src/trace/traceTest.ts index e8a7e627..6ab0d28e 100644 --- a/src/trace/traceTest.ts +++ b/src/trace/traceTest.ts @@ -11,7 +11,7 @@ import * as fs from "fs"; import * as path from "path"; -import { parseTreeLine, parseTreeFile, nodeAtOffset, walkTree, parseNodeFlags, newFiredSpan } from "./treeParse"; +import { parseTreeLine, parseTreeFile, nodeAtOffset, walkTree, parseNodeFlags, newFiredSpan, treeLinesInRange } from "./treeParse"; import { loadTrace, findLatestLogDir, parseSequence, sequencePassNames, passesForSource, firedRuleLines, nodesBuiltBy, @@ -83,6 +83,17 @@ function eq(name: string, actual: T, expected: T): void { check("line: banner is not a node", parseTreeLine("PAT OUTPUT TREE:") === undefined); } +// ---- partial tree (#295) ----------------------------------------------------- +// The engine's own output for the text "x [y] z": the bracket tokens are nodes. +{ + const tree = ["", "FINAL OUTPUT TREE:", "", "_ROOT [0,6,0,6,0,0,node,un]", " x [0,0,0,0,0,0,alpha]", + " \_ [1,1,1,1,0,0,white]", " [ [2,2,2,2,0,0,punct]", " y [3,3,3,3,0,0,alpha]", + " ] [4,4,4,4,0,0,punct]", " \_ [5,5,5,5,0,0,white]", " z [6,6,6,6,0,0,alpha]"]; + eq("partial: '[y]' keeps both brackets", JSON.stringify(treeLinesInRange(tree, 2, 4)), JSON.stringify([6, 7, 8])); + eq("partial: whole text includes the root", treeLinesInRange(tree, 0, 6).length, 8); + eq("partial: banner lines never included", treeLinesInRange(tree, 0, 6).includes(1), false); +} + // ---- fired spans (#810) ---------------------------------------------------- // treeFile.writeFiredText slices the input at each fired span. The fired spans // below are read back from the display reported in #810, where an older engine diff --git a/src/trace/treeParse.ts b/src/trace/treeParse.ts index 4c1ff9a5..99c2a1cf 100644 --- a/src/trace/treeParse.ts +++ b/src/trace/treeParse.ts @@ -241,3 +241,17 @@ export function nodeAtOffset(root: TraceNode, offset: number): TraceNode | undef }); return best; } + +// Indices of the node lines whose text span (UTF-16, the ustart/uend pair) lies +// within [from, to] -- the lines of a "partial tree" for a selection. Read through +// parseTreeLine so a literal "[" token is a node like any other: splitting each +// line on its first "[" mistook that token for the start of the fields and +// silently left it out of every partial tree that should hold it (#295). +export function treeLinesInRange(lines: string[], from: number, to: number): number[] { + const out: number[] = []; + lines.forEach((line, i) => { + const node = parseTreeLine(line); + if (node && node.ustart >= from && node.uend <= to) out.push(i); + }); + return out; +} diff --git a/src/treeFile.ts b/src/treeFile.ts index 989d28d9..a72e1c9a 100644 --- a/src/treeFile.ts +++ b/src/treeFile.ts @@ -9,7 +9,7 @@ import { SequenceFile } from './sequence'; import { FindFile, FindItem } from './findFile'; import { findView } from './findView'; import { dirfuncs } from './dirfuncs'; -import { parseNodeFlags, newFiredSpan } from './trace/treeParse'; +import { parseNodeFlags, newFiredSpan, treeLinesInRange } from './trace/treeParse'; import * as os from 'os'; export enum generateType { GENERAL, EXACT } @@ -321,6 +321,11 @@ export class TreeFile extends TextFile { this.absoluteRangeFromSelection(this.HighlightFile, editor.selection); } this.findTreeFileLines(); + } else if (this.getFileType() == nlpFileType.TREE) { + // #475: in a parse tree the selected lines ARE the tree. + this.setFile(editor.document.uri); + this.parseTreeLines(editor); + this.selectedTreeStr = this.selectedLines.map(l => l.node).join(' '); } return this.selectedTreeStr.length ? true : false; } @@ -339,10 +344,18 @@ export class TreeFile extends TextFile { editor.insertSnippet(snippet, range); } else { - const passFilePath = visualText.analyzer.getPassPath(); - const passName = visualText.analyzer.seqFile.base(passFilePath.fsPath); - const passItem = visualText.analyzer.seqFile.findPass('nlp', passName); - this.treeFile = this.anaFile(passItem.passNum).fsPath; + let passFilePath = visualText.analyzer.getPassPath(); + if (type == nlpFileType.TREE) { + // A pass's tree (anaNNN.tree) adds the rule to that pass, as + // "Generate Path" does; final.tree to the current pass. + const treePass = this.getPassFromPath(editor); + if (treePass.fsPath.length > 2) + passFilePath = treePass; + } else { + const passName = visualText.analyzer.seqFile.base(passFilePath.fsPath); + const passItem = visualText.analyzer.seqFile.findPass('nlp', passName); + this.treeFile = this.anaFile(passItem.passNum).fsPath; + } if (this.findSelectedTreeStr(editor)) { let ruleStr = this.ruleFromLines(genType); @@ -520,10 +533,7 @@ ${ruleStr} } openTemporaryFile(filepath: string, content: string) { - const tempDir = path.resolve( - vscode.workspace - .getConfiguration('createtmpfile') - .get('tmpDir') || os.tmpdir()); + const tempDir = os.tmpdir(); const filePath = vscode.Uri.file(path.join(tempDir, filepath)); fs.writeFileSync(filePath.fsPath, content); vscode.workspace.openTextDocument(filePath).then(document => { @@ -595,27 +605,13 @@ ${ruleStr} findTreeFileLines() { const file = new TextFile(this.treeFile); const sep = file.getSeparatorNormalized(); - let from = 0; - let to = 0; this.selectedLines = []; this.selectedTreeStr = ''; - for (const line of file.getLines()) { - from = 0; - to = 0; - - const tokens = line.split('['); - if (tokens.length > 1) { - const toks = tokens[1].split(/[,\]]/); - if (toks.length > 4) { - from = +toks[2]; - to = +toks[3]; - if (from >= this.selStart && to <= this.selEnd) { - this.selectedLines.push(this.parseTreeLine(line)); - this.selectedTreeStr = this.selectedTreeStr.concat(line, sep); - } - } - } + const lines = file.getLines(); + for (const i of treeLinesInRange(lines, this.selStart, this.selEnd)) { + this.selectedLines.push(this.parseTreeLine(lines[i])); + this.selectedTreeStr = this.selectedTreeStr.concat(lines[i], sep); } } diff --git a/src/visualText.ts b/src/visualText.ts index 9403c2ac..f96ae948 100644 --- a/src/visualText.ts +++ b/src/visualText.ts @@ -128,6 +128,10 @@ export class VisualText { private latestExtIndex: number = 0; private lastestEngineIndex: number = 0; private updaterID: number = 0; + // #230: the status-bar progress shown while the updater runs at startup. + private updaterProgress: { report: (message: string) => void; done: () => void } | undefined; + private static readonly OP_WORDS = ['', 'Checking the', 'Checking the version of the', 'Downloading the', 'Unpacking the', 'Removing the old', '', '']; + private static readonly COMP_WORDS = ['', 'ICU libraries', 'ICU libraries', 'ICU libraries', 'NLP++ engine', 'engine files', 'compiler files', 'example analyzers', 'VisualText files']; private testItem: TestItem = { fileCount: 0, matchFiles: 0, matchLines: 0, misFiles: 0, misLines: 0 }; @@ -147,6 +151,12 @@ export class VisualText { visualText.readConfig(); visualText.readState(); + // #481: the engine lives in the folder of the extension that is + // RUNNING. It used to be the newest version found on disk unless the + // auto-updater happened to run and correct it, so with auto-update off, + // after a downgrade, or with an old copy still on disk, a new version + // could load (or download into) another version's engine. + visualText.version = vscode.extensions.getExtension(visualText.EXTENSION_NAME)?.packageJSON.version ?? ''; visualText.getExtensionDirs(); visualText.initSettings(); } @@ -264,9 +274,8 @@ export class VisualText { this.platform = os.platform(); const plat = this.platform == 'darwin' ? 'mac' : this.platform; this.homeDir = os.homedir(); - const rootPath = this.getLatestExtPath(this.extensionParentDirectory()); - if (rootPath) this.extensionDir = vscode.Uri.file(rootPath); - this.version = vscode.extensions.getExtension(this.EXTENSION_NAME)?.packageJSON.version; + this.version = vscode.extensions.getExtension(this.EXTENSION_NAME)?.packageJSON.version ?? this.version; + this.extensionDir = this.extensionDirectory(); if (preInfoFlag) { this.debugMessage('Platform: ' + plat, logLineType.UPDATER); @@ -277,6 +286,14 @@ export class VisualText { this.debugMessage('Checking for updates or repairs...', logLineType.UPDATER); this.pushCheckVersions(); this.startTimer(); + vscode.window.withProgress({ location: vscode.ProgressLocation.Window, title: 'NLP++' }, progress => + new Promise(resolve => { + progress.report({ message: 'checking for updates' }); + this.updaterProgress = { + report: (message: string) => progress.report({ message }), + done: () => { this.updaterProgress = undefined; resolve(); }, + }; + })); } } @@ -470,18 +487,6 @@ export class VisualText { op.local = path.join(engDir, lib); } - getLatestExtPath(dir: vscode.Uri): string { - const files = dirfuncs.getFiles(dir); - for (const file of files.reverse()) { - const filename = path.basename(file.fsPath); - if (filename.startsWith(this.EXTENSION_NAME)) { - this.version = this.versionFromPath(file); - return file.fsPath; - } - } - return ''; - } - updaterTimer() { let op = visualText.opsQueue[0]; let allDone = true; @@ -518,6 +523,7 @@ export class VisualText { else visualText.debugMessage('UPDATE CHECK COMPLETE', logLineType.UPDATER); visualText.stopAll = false; + visualText.updaterProgress?.done(); vscode.commands.executeCommand('status.update'); return; } else { @@ -526,6 +532,10 @@ export class VisualText { if (visualText.stopAll) return; + const opWords = VisualText.OP_WORDS[op.operation], compWords = VisualText.COMP_WORDS[op.component]; + if (opWords && compWords) + visualText.updaterProgress?.report(`${opWords} ${compWords}`); + if (visualText.debug) visualText.debugMessage(visualText.statusStrs[op.status] + ' ' + visualText.opStrs[op.operation] + ' ' + visualText.compStrs[op.component], logLineType.UPDATER); @@ -1231,6 +1241,9 @@ export class VisualText { } getExtensionPath(): vscode.Uri { + // The running version's folder first (#481), then the newest installed. + const running = this.extensionDirectory(); + if (this.version.length && fs.existsSync(running.fsPath)) return running; const latest = this.extensionItems[this.latestExtIndex]; if (latest) return latest.uri;