Clear the issue backlog (4.3.12)

CHANGELOG.md

@@ -3,6 +3,23 @@ All notable changes to the [VSCode NLP++ extension](http://vscode.visualtext.org
 
 Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file.
 
+### 4.3.12
+Clears the issue backlog: thirteen requests and bugs, some from 2020.
+
+- **Generate a rule from a parse tree** (#475). Select lines in a pass's `.tree` file and run Generate Rule General or Exact from the editor menu. The selected top-level nodes become the rule's elements, and the rule goes into the pass the tree belongs to.
+- **Add a generated pass to the sequence** (#794). An `.nlp` or `.pat` file in the Output view now has "Add to Analyzer Sequence". You pick the pass to insert it after. If a pass of that name is already in the sequence, you can replace it, and it keeps its place, or add the new one under the next free name.
+- **Copy or move an output file anywhere** (#658). "Copy or Move File To..." on an output file opens a save dialog, starting in the analyzer's input folder.
+- **Browse the outputs of a whole folder** (#688). Select a folder in the Text view, and the Output view lists every file under it that has outputs, each expanding to its own. The matches and trees buttons also work now even when "all" was not clicked first.
+- **Sort & Count for corpus study** (#613). "Sort & Count text lines" counts each distinct line, and "Count Words in Folder" in the Text view counts words across every file in the folder. Both open the counts, most frequent first, in a new editor and leave the text alone.
+- **The parse trees of a failed run are reachable** (#785). The passes that ran before an error still offer their trees in the sequence, and the Output view lists them.
+- **Startup shows what the updater is doing** (#230). While it checks, downloads or unpacks the engine and its files, the status bar shows which step it is on.
+- **The engine of the running version is used** (#481). With several versions of the extension on disk, a new version could use an older version's engine. That was the reason for the advice to delete old versions, which is no longer needed.
+- **The run mode is remembered** (#328). The status-bar run mode (interpreted or compiled) was lost on every reload, because its setting was never declared. Several settings now say what they do.
+- **Partial trees keep `[` tokens** (#295). "View selected tree" dropped a literal `[` from the tree. A partial tree now colors exactly like the full one.
+- **Mod files work on macOS and Linux** (#98). `.nlm` files, which bundle passes and KB files into one file, found none of their sections outside Windows, and created files with backslashes in their names.
+- **Passes that build a knowledge base can feed the run** (#272). Right-click a pass in the sequence and choose "Toggle KB Generator" to mark it; the sequence shows "KB generator" beside it. With the new setting `nlp.analyzer.regenerateKB` on (it is off by default), after a run in which a marked pass has changed, the `.kbb` and `.dict` files it writes (the file names in its own text, such as `SaveKB("mykb.kbb", ...)`) are copied from the run's log folder into `kb/user`, and the text runs once more, so the result uses the knowledge base the pass just built. The second run copies nothing, so it does not repeat.
+- **writekb help** (#626) was already in the help pages, under Knowledge Base Functions.
+
 ### 4.3.11
 Publishes everything since 4.3.4 to the VS Code Marketplace.
 

package.json

@@ -3,7 +3,7 @@
     "displayName": "NLP",
     "description": "NLP++: the only computer programming language built for NLP — create and visually debug analyzers directly in VS Code.",
     "icon": "resources/NLPppLogo.png",
-    "version": "4.3.11",
+    "version": "4.3.12",
     "license": "MIT",
     "publisher": "dehilster",
     "enableApiProposals": false,
@@ -589,6 +589,16 @@
                 "title": "Sort & Unique text lines",
                 "description": "Sort and unique text lines"
             },
+            {
+                "command": "nlp.sortCount",
+                "title": "Sort & Count text lines",
+                "description": "How often each line occurs, most frequent first, in a new editor"
+            },
+            {
+                "command": "textView.countWords",
+                "title": "Count Words in Folder",
+                "description": "Word frequencies over every text file in the folder, in a new editor"
+            },
             {
                 "command": "nlp.lowerCase",
                 "title": "Change to Lowercase",
@@ -1616,6 +1626,19 @@
                     "dark": "resources/dark/rename.svg"
                 }
             },
+            {
+                "command": "outputView.copyOrMoveTo",
+                "title": "Copy or Move File To..."
+            },
+            {
+                "command": "outputView.addToSequence",
+                "title": "Add to Analyzer Sequence",
+                "icon": "$(add)"
+            },
+            {
+                "command": "sequenceView.toggleKbGenerator",
+                "title": "Toggle KB Generator"
+            },
             {
                 "command": "outputView.explore",
                 "title": "Explore Output Folder",
@@ -1992,6 +2015,11 @@
                     "command": "nlp.sortText",
                     "group": "nlp"
                 },
+                {
+                    "when": "resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .dict",
+                    "command": "nlp.sortCount",
+                    "group": "nlp"
+                },
                 {
                     "when": "resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .dict || resourceExtname == .kbb",
                     "command": "nlp.lowerCase",
@@ -2018,12 +2046,12 @@
                     "group": "nlp"
                 },
                 {
-                    "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat",
+                    "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat || resourceExtname == .tree",
                     "command": "nlp.generateRule",
                     "group": "nlp"
                 },
                 {
-                    "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat",
+                    "when": "resourceExtname == .txxt || resourceExtname == .txt || resourceExtname == .nlp || resourceExtname == .pat || resourceExtname == .tree",
                     "command": "nlp.generateExactRule",
                     "group": "nlp"
                 },
@@ -2598,6 +2626,11 @@
                 }
             ],
             "view/item/context": [
+                {
+                    "command": "sequenceView.toggleKbGenerator",
+                    "when": "view == sequenceView && viewItem =~ /filenotok/",
+                    "group": "kb"
+                },
                 {
                     "command": "helpView.copyPath",
                     "when": "view == helpView && viewItem == helpFile",
@@ -2610,9 +2643,19 @@
                 },
                 {
                     "command": "outputView.rename",
-                    "when": "view == outputView",
+                    "when": "view == outputView && viewItem != input",
                     "group": "navigation"
                 },
+                {
+                    "command": "outputView.copyOrMoveTo",
+                    "when": "view == outputView && viewItem != input",
+                    "group": "move"
+                },
+                {
+                    "command": "outputView.addToSequence",
+                    "when": "view == outputView && viewItem =~ /nlp/",
+                    "group": "move"
+                },
                 {
                     "command": "outputView.editTest",
                     "when": "view == outputView && viewItem =~ /.*test.*/",
@@ -3258,6 +3301,11 @@
                     "when": "view == textView && viewItem =~ /.*dir.*/",
                     "group": "inline@01"
                 },
+                {
+                    "command": "textView.countWords",
+                    "when": "view == textView && viewItem =~ /.*dir.*/",
+                    "group": "count"
+                },
                 {
                     "command": "textView.runRegressionItem",
                     "when": "view == textView && viewItem =~ /file|dir/",
@@ -3310,17 +3358,17 @@
                 },
                 {
                     "command": "outputView.copytoText",
-                    "when": "view == outputView",
+                    "when": "view == outputView && viewItem != input",
                     "group": "inline@0"
                 },
                 {
                     "command": "outputView.copytoKB",
-                    "when": "view == outputView",
+                    "when": "view == outputView && viewItem != input",
                     "group": "inline@1"
                 },
                 {
                     "command": "outputView.addTest",
-                    "when": "view == outputView",
+                    "when": "view == outputView && viewItem != input",
                     "group": "inline@2"
                 },
                 {
@@ -3330,7 +3378,7 @@
                 },
                 {
                     "command": "outputView.deleteOutput",
-                    "when": "view == outputView",
+                    "when": "view == outputView && viewItem != input",
                     "group": "inline@4"
                 },
                 {
@@ -3381,6 +3429,12 @@
                     "default": true,
                     "markdownDescription": "Send anonymous usage counts (feature usage, run mode, errors) to help improve the extension. Never includes file contents, analyzer/KB source, file names, or paths. This is layered on top of VS Code's global `#telemetry.telemetryLevel#` setting — if that is off, nothing is sent regardless of this setting."
                 },
+                "nlp.analyzer.regenerateKB": {
+                    "type": "boolean",
+                    "scope": "resource",
+                    "default": false,
+                    "markdownDescription": "After a run in which a pass marked as a **KB generator** (right-click a pass in the sequence) has changed, copy the `.kbb` and `.dict` files that pass writes into the analyzer's `kb/user` folder and run the text again, so the run uses the knowledge base it just built."
+                },
                 "nlp.analyzer.rememberOpenFiles": {
                     "type": "boolean",
                     "scope": "application",
@@ -3433,13 +3487,21 @@
                     "type": "boolean",
                     "scope": "application",
                     "default": true,
-                    "description": "Auto update on load or reload"
+                    "markdownDescription": "Check for and install updates to the NLP++ engine, VisualText files and example analyzers each time the extension starts. Turn off to update only by command."
                 },
                 "textView.fast": {
                     "type": "boolean",
                     "scope": "application",
                     "default": false,
-                    "description": "Load textview fast. Attributes for text files are lost such as has logs, etc."
+                    "markdownDescription": "Fill the Text view without checking each file for logs and outputs. Faster for large input folders, but files no longer show whether they have been run."
+                },
+                "analyzer.runMode": {
+                    "type": "string",
+                    "scope": "resource",
+                    "enum": ["interpreted", "compiled", "compiled-analyzer", "compiled-kb"],
+                    "enumDescriptions": ["Run the analyzer and its KB as source (the default).", "Run the compiled analyzer and compiled KB.", "Run the compiled analyzer with the KB as source.", "Run the analyzer as source with the compiled KB."],
+                    "default": "interpreted",
+                    "markdownDescription": "How the analyzer runs. Set from the run-mode button in the status bar and remembered per workspace folder."
                 },
                 "analyzer.regressionTerminal": {
                     "type": "boolean",
@@ -3463,7 +3525,7 @@
                     "type": "string",
                     "scope": "application",
                     "default": "",
-                    "description": "User name for top pass file comments"
+                    "markdownDescription": "Your name, written in the `AUTH:` line of the header comment of new pass files."
                 },
                 "analyzer.directory": {
                     "type": "string",
@@ -3475,25 +3537,25 @@
                     "type": "string",
                     "scope": "resource",
                     "default": "",
-                    "description": "Current Analyzer"
+                    "markdownDescription": "The analyzer that was open last. Set by the extension when you open an analyzer; you do not normally need to change it."
                 },
                 "engine.visualtext": {
                     "type": "string",
                     "scope": "application",
                     "default": "",
-                    "description": "VisualText files version"
+                    "markdownDescription": "Version of the downloaded VisualText files (help, libraries, templates). Set by the updater."
                 },
                 "engine.analyzers": {
                     "type": "string",
                     "scope": "application",
                     "default": "",
-                    "description": "Analyzers version"
+                    "markdownDescription": "Version of the downloaded example analyzers. Set by the updater."
                 },
                 "logs.clear": {
                     "type": "boolean",
                     "scope": "application",
                     "default": true,
-                    "description": "Clear logs before new operations"
+                    "markdownDescription": "Clear the NLP++ log view before each run or other operation."
                 },
                 "compile.mode": {
                     "type": "string",

src/analyzer.ts

@@ -30,6 +30,10 @@ export class Analyzer {
     // Files open in the editor when the user last switched away from this
     // analyzer, relative to its directory (#131).
     private openFiles: string[] = [];
+    // #272: passes marked as KB generators, and when each one's output was
+    // last copied into kb/user (the pass file's mtime at the time).
+    private kbGenerators: string[] = [];
+    private kbGenCopied: Record<string, number> = {};
     private passNum: number = 0;;
     private loaded: boolean = false;
 
@@ -44,8 +48,14 @@ export class Analyzer {
 
     readState() {
         this.openFiles = [];
+        this.kbGenerators = [];
+        this.kbGenCopied = {};
         if (this.jsonState.jsonParse(this.analyzerDir, "state")) {
             const parse = this.jsonState.json.visualText[0];
+            if (Array.isArray(parse.kbGenerators))
+                this.kbGenerators = parse.kbGenerators.filter((f: unknown) => typeof f === "string");
+            if (parse.kbGenCopied && typeof parse.kbGenCopied === "object")
+                this.kbGenCopied = parse.kbGenCopied;
             if (Array.isArray(parse.openFiles))
                 this.openFiles = parse.openFiles.filter((f: unknown) => typeof f === "string");
             if (parse.currentTextFile) {
@@ -292,13 +302,40 @@ export class Analyzer {
                     "type": "state",
                     "currentTextFile": this.currentTextFile.fsPath,
                     "currentPassFile": this.currentPassFile.fsPath,
-                    "openFiles": this.openFiles
+                    "openFiles": this.openFiles,
+                    "kbGenerators": this.kbGenerators,
+                    "kbGenCopied": this.kbGenCopied
                 }
             ]
         }
         this.jsonState.saveFile(this.analyzerDir.fsPath, "state", stateJsonDefault);
     }
 
+    isKbGenerator(passName: string): boolean {
+        return this.kbGenerators.includes(passName);
+    }
+
+    getKbGenerators(): string[] {
+        return this.kbGenerators;
+    }
+
+    toggleKbGenerator(passName: string) {
+        if (this.isKbGenerator(passName))
+            this.kbGenerators = this.kbGenerators.filter(n => n !== passName);
+        else
+            this.kbGenerators.push(passName);
+        this.saveAnalyzerState();
+    }
+
+    kbGeneratorCopied(passName: string): number | undefined {
+        return this.kbGenCopied[passName];
+    }
+
+    setKbGeneratorCopied(passName: string, modifiedMs: number) {
+        this.kbGenCopied[passName] = modifiedMs;
+        this.saveAnalyzerState();
+    }
+
     getOpenFiles(): string[] {
         return this.openFiles;
     }

src/command.ts

@@ -2,6 +2,7 @@ import * as vscode from 'vscode';
 import { TreeFile, generateType } from './treeFile';
 import { NLPFile, reformatType } from './nlp';
 import { TextFile } from './textFile';
+import { countLines, formatCounts } from './output/counts';
 
 export let nlpCommands: NLPCommands;
 export class NLPCommands {
@@ -29,6 +30,7 @@ export class NLPCommands {
         ctx.subscriptions.push(vscode.commands.registerCommand('nlp.passTree', this.passTree));
         ctx.subscriptions.push(vscode.commands.registerCommand('nlp.displayMatchedRules', this.displayMatchedRulesNLP));
         ctx.subscriptions.push(vscode.commands.registerCommand('nlp.sortText', this.sortText));
+        ctx.subscriptions.push(vscode.commands.registerCommand('nlp.sortCount', this.sortCount));
         ctx.subscriptions.push(vscode.commands.registerCommand('nlp.lowerCase', this.lowerCase));
         ctx.subscriptions.push(vscode.commands.registerCommand('nlp.removeAccents', this.removeAccents));
         ctx.subscriptions.push(vscode.commands.registerCommand('nlp.runPython', this.runPython));
@@ -84,6 +86,20 @@ export class NLPCommands {
         }
     }
 
+    // #613: how often each line occurs, most frequent first, in a new editor
+    // beside the text so the corpus itself is left as it is.
+    async sortCount() {
+        const editor = vscode.window.activeTextEditor;
+        if (!editor) return;
+        const doc = editor.document;
+        const range = editor.selection.isEmpty
+            ? new vscode.Range(0, 0, doc.lineCount, 0)
+            : new vscode.Range(editor.selection.start.line, 0, editor.selection.end.line + 1, 0);
+        const counts = countLines(doc.getText(range).split(/\r?\n/));
+        const result = await vscode.workspace.openTextDocument({ content: formatCounts(counts), language: 'plaintext' });
+        await vscode.window.showTextDocument(result, { viewColumn: vscode.ViewColumn.Beside });
+    }
+
     lowerCase() {
         if (vscode.window.activeTextEditor) {
             const editor = vscode.window.activeTextEditor;