diff --git a/.vscode/settings-light.json b/.vscode/settings-light.json index dcd111a6..35a1b0a6 100644 --- a/.vscode/settings-light.json +++ b/.vscode/settings-light.json @@ -8,37 +8,43 @@ } }, { - "scope": "variable.parameter.nlp", + "scope": [ + "variable.parameter.nlp", + "keyword.control.kb" + ], "settings": { "foreground": "#696969" } }, { - "scope": "keyword.attribute.nlp", + "scope": "entity.other.attribute-name.nlp", "settings": { "foreground": "#e655f3" } }, { - "scope": "constant.numeric.tree", + "scope": [ + "constant.numeric.tree", + "constant.numeric.kbb" + ], "settings": { "foreground": "#5596f0" } }, { - "scope": "keyword.fired.tree", + "scope": "keyword.other.fired.tree", "settings": { "foreground": "#dc1b1b" } }, { - "scope": "keyword.rewrite.tree", + "scope": "entity.name.type.tree", "settings": { "foreground": "#000000" } }, { - "scope": "keyword.node.tree", + "scope": "string.unquoted.token.tree", "settings": { "foreground": "#4e8e3c" } @@ -50,25 +56,25 @@ } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#909228" } }, { - "scope": "keyword.concept.kbb", + "scope": "entity.name.type.level1.kbb", "settings": { "foreground": "#7e7e7e" } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#2eb657" } }, { - "scope": "keyword.concept.kbb2", + "scope": "entity.name.type.level3.kbb", "settings": { "foreground": "#e049d9" } diff --git a/.vscode/settings.json b/.vscode/settings.json index db4932c3..07d58a21 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,20 +15,23 @@ } }, { - "scope": "variable.parameter.nlp", + "scope": [ + "variable.parameter.nlp", + "keyword.control.kb" + ], "settings": { "foreground": "#696969", "fontStyle": "bold" } }, { - "scope": "keyword.attribute.nlp", + "scope": "entity.other.attribute-name.nlp", "settings": { "foreground": "#e655f3" } }, { - "scope": "keyword.constants.nlp", + "scope": "constant.language.nlp", "settings": { "foreground": "#0a7b83", "fontStyle": "bold" @@ -44,27 +47,30 @@ } }, { - "scope": "constant.numeric.tree", + "scope": [ + "constant.numeric.tree", + "constant.numeric.kbb" + ], "settings": { "foreground": "#5596f0" } }, { - "scope": "keyword.fired.tree", + "scope": "keyword.other.fired.tree", "settings": { "foreground": "#dc1b1b", "fontStyle": "bold" } }, { - "scope": "keyword.rewrite.tree", + "scope": "entity.name.type.tree", "settings": { "foreground": "#000000", "fontStyle": "bold" } }, { - "scope": "keyword.node.tree", + "scope": "string.unquoted.token.tree", "settings": { "foreground": "#4e8e3c", "fontStyle": "bold" @@ -98,28 +104,28 @@ } }, { - "scope": "keyword.other.kbb", + "scope": "entity.name.type.level0.kbb", "settings": { "foreground": "#5d00d6", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb", + "scope": "entity.name.type.level1.kbb", "settings": { "foreground": "#7e7e7e", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#2eb657", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb2", + "scope": "entity.name.type.level3.kbb", "settings": { "foreground": "#e049d9", "fontStyle": "bold" @@ -143,20 +149,23 @@ } }, { - "scope": "variable.parameter.nlp", + "scope": [ + "variable.parameter.nlp", + "keyword.control.kb" + ], "settings": { "foreground": "#f8fa87", "fontStyle": "bold" } }, { - "scope": "keyword.attribute.nlp", + "scope": "entity.other.attribute-name.nlp", "settings": { "foreground": "#f293fb" } }, { - "scope": "keyword.constants.nlp", + "scope": "constant.language.nlp", "settings": { "foreground": "#4ec9b0", "fontStyle": "bold" @@ -172,27 +181,30 @@ } }, { - "scope": "constant.numeric.tree", + "scope": [ + "constant.numeric.tree", + "constant.numeric.kbb" + ], "settings": { "foreground": "#93bffb" } }, { - "scope": "keyword.fired.tree", + "scope": "keyword.other.fired.tree", "settings": { "foreground": "#dc1b1b", "fontStyle": "bold" } }, { - "scope": "keyword.rewrite.tree", + "scope": "entity.name.type.tree", "settings": { "foreground": "#f8fa87", "fontStyle": "bold" } }, { - "scope": "keyword.node.tree", + "scope": "string.unquoted.token.tree", "settings": { "foreground": "#4e8e3c", "fontStyle": "bold" @@ -211,28 +223,28 @@ } }, { - "scope": "keyword.other.kbb", + "scope": "entity.name.type.level0.kbb", "settings": { "foreground": "#896eff", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb", + "scope": "entity.name.type.level1.kbb", "settings": { "foreground": "#e3ee50", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#2eb657", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb2", + "scope": "entity.name.type.level3.kbb", "settings": { "foreground": "#e049d9", "fontStyle": "bold" diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d45d64..9d32ceb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the [VSCode NLP++ extension](http://vscode.visualtext.org Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. +### 4.3.10 +NLP++ files are colored by any VS Code theme, not only through the colors this extension writes into your settings. + +- **The grammars use standard TextMate scope names** (#107). They used to name things after the color they should get rather than what they are. Constants like `_xWILD` were marked as keywords, a dictionary headword as a comment, and a few scopes, such as `literal.nlp`, had names no theme recognizes. Themes, and tools such as GitHub and Shiki that read these grammars, now color `@RULES`, constants, rule modifiers, node names and dictionary keys and values like the equivalent parts of other languages. The full list of renames is in the [grammar README](https://github.com/VisualText/nlpplus-tmbundle#scope-names). +- **Your analyzer colors are unchanged.** The extension's color rules in each analyzer workspace's `.vscode/settings.json` are rewritten to the new names the first time the workspace opens, so everything you saw before keeps its color. A settings file the extension cannot read, for example one with comments in it, is left alone. + ### 4.3.9 Each analyzer keeps its own open files, and the analyzers you used last are a click away. diff --git a/grammars b/grammars index 877abbac..ab0b1389 160000 --- a/grammars +++ b/grammars @@ -1 +1 @@ -Subproject commit 877abbaceb7847a0f001c04ee8b3c6eeefd146ee +Subproject commit ab0b1389aff2f3b931743e0d43b97f07dc543ad3 diff --git a/package-lock.json b/package-lock.json index 656b27d6..a3ffcbe1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nlp", - "version": "4.3.9", + "version": "4.3.10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "nlp", - "version": "4.3.9", + "version": "4.3.10", "license": "MIT", "dependencies": { "@vscode/debugadapter": "^1.68.0", diff --git a/package.json b/package.json index 4e0996a9..b53f6563 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "NLP", "description": "NLP++: the only computer programming language built for NLP — create and visually debug analyzers directly in VS Code.", "icon": "resources/NLPppLogo.png", - "version": "4.3.9", + "version": "4.3.10", "license": "MIT", "publisher": "dehilster", "enableApiProposals": false, diff --git a/src/language/langTest.ts b/src/language/langTest.ts index 7fa64356..6f880efa 100644 --- a/src/language/langTest.ts +++ b/src/language/langTest.ts @@ -13,6 +13,9 @@ import { parseKbConcepts } from "./kbConcepts"; import { regionKindAt, RegionKind } from "./completion"; import { findEnclosingCall } from "./signature"; import { foldingRanges } from "./folding"; +import { RENAMED_SCOPES, migrateScope, migrateTokenColors } from "./scopeMigration"; +import * as fs from "fs"; +import * as path from "path"; import { RuleLocator, visualColumn } from "./ruleLocator"; import { classifyTokens } from "./semanticTokens"; import { findUnknownCalls, levenshtein } from "./quickfix"; @@ -185,6 +188,51 @@ _verb <- verb @@ eq("column: two tabs", visualColumn("\t\tx", 2, 4), 9); } +// ---- scope migration (#107) ------------------------------------------------ +{ + // The map must match the grammars it describes: no old name may survive in + // them unless it maps to itself, and every new name must exist in one. + const dir = path.join(__dirname, "..", "..", "grammars", "syntaxes"); + const names = new Set(); + const walk = (o: unknown): void => { + if (Array.isArray(o)) o.forEach(walk); + else if (o && typeof o === "object") { + for (const [k, v] of Object.entries(o)) { + if ((k === "name" || k === "contentName") && typeof v === "string") names.add(v); + else walk(v); + } + } + }; + for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".json"))) walk(JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"))); + for (const [oldName, renamed] of Object.entries(RENAMED_SCOPES)) { + check(`scopes: ${oldName} gone from grammars`, renamed.includes(oldName) || !names.has(oldName)); + for (const n of renamed) check(`scopes: ${n} exists in a grammar`, names.has(n)); + } + + eq("scopes: one-to-one", JSON.stringify(migrateScope("keyword.region.nlp")), JSON.stringify({ scope: "keyword.control.directive.region.nlp", changed: true })); + eq("scopes: one-to-many keeps all", JSON.stringify(migrateScope("comment.line.dict").scope), + JSON.stringify(["entity.name.tag.dict", "comment.line.dict", "comment.line.kb", "comment.line.kbb", "comment.line.tree"])); + eq("scopes: list, deduplicated", JSON.stringify(migrateScope(["keyword.operator.nlp", "keyword.equals.nlp", "keyword.operator.assignment.nlp"]).scope), + JSON.stringify(["keyword.operator.nlp", "keyword.operator.assignment.nlp"])); + eq("scopes: comma string", JSON.stringify(migrateScope("keyword.fired.tree, keyword.rewrite.tree").scope), + JSON.stringify(["keyword.other.fired.tree", "entity.name.type.tree"])); + eq("scopes: unknown untouched", migrateScope("source.js keyword").changed, false); + + const settings: Record = { + "files.eol": "lf", + "editor.tokenColorCustomizations": { + "textMateRules": [{ scope: "keyword.constants.nlp", settings: { foreground: "#0a7b83" } }], + "[*Dark*]": { textMateRules: [{ scope: "keyword.fired.tree", settings: { foreground: "#ff0000" } }, { name: "no scope" }] }, + }, + }; + check("scopes: settings migrated", migrateTokenColors(settings)); + const block = settings["editor.tokenColorCustomizations"] as { textMateRules: { scope: string }[]; "[*Dark*]": { textMateRules: { scope?: string }[] } }; + eq("scopes: top-level rule", block.textMateRules[0].scope, "constant.language.nlp"); + eq("scopes: per-theme rule", block["[*Dark*]"].textMateRules[0].scope, "keyword.other.fired.tree"); + eq("scopes: second run is a no-op", migrateTokenColors(settings), false); + eq("scopes: no block", migrateTokenColors({ "files.eol": "lf" }), false); +} + // ---- engine errors --------------------------------------------------------- { const log = [ diff --git a/src/language/nlpxxData.ts b/src/language/nlpxxData.ts index 530cce78..eff3fe54 100644 --- a/src/language/nlpxxData.ts +++ b/src/language/nlpxxData.ts @@ -27,7 +27,7 @@ export const KEYWORDS = [ "cap", "cout", "else", "gp", "group", "if", "inc", "return", "while", ]; -// Rule-element modifiers inside @RULES (grammar: keyword.attribute.nlp). +// Rule-element modifiers inside @RULES (grammar: entity.other.attribute-name.nlp). export const RULE_KEYWORDS = [ "attr", "attrs", "base", "da", "deacc", "deaccent", "except", "excepts", "fail", "fails", "gp", "group", "layer", "layers", "look", "lookahead", diff --git a/src/language/scopeMigration.ts b/src/language/scopeMigration.ts new file mode 100644 index 00000000..8c246e57 --- /dev/null +++ b/src/language/scopeMigration.ts @@ -0,0 +1,92 @@ +// Renamed TextMate scopes, and the migration of color rules keyed to the old ones (#107). +// +// PURE MODULE: no 'vscode' import. The grammars (grammars/, VisualText/nlpplus-tmbundle) +// used to name scopes for the color they got rather than for what they are, and +// colorizeAnalyzer() copied textMateRules for those names into every analyzer +// workspace's .vscode/settings.json. The grammars now use standard names; this +// rewrites old rules so they keep coloring exactly what they colored before. +// +// An old name can map to several new ones: "comment.line.dict" was the dictionary +// headword AND the line comment of .kb, .kbb and .tree files, so a rule for it +// becomes a rule for all of their new names. + +export const RENAMED_SCOPES: Record = { + "keyword.region.nlp": ["keyword.control.directive.region.nlp"], + "keyword.constants.nlp": ["constant.language.nlp"], + "keyword.attribute.nlp": ["entity.other.attribute-name.nlp"], + "keyword.equals.nlp": ["keyword.operator.assignment.nlp"], + "variable.rewrite.nlp": ["entity.name.type.rewrite.nlp"], + "variable.rewrite-symbol.nlp": ["meta.attribute.rewrite.nlp"], + "literal.nlp": ["meta.literal.nlp"], + "literal.seq": ["meta.literal.seq"], + "literal.tree": ["meta.literal.tree"], + "constant.character.escape": ["constant.character.escape.nlp"], + "punctuation.paren.open": ["punctuation.paren.open.nlp"], + "punctuation.paren.close": ["punctuation.paren.close.nlp"], + "punctuation.square.open": ["punctuation.square.open.nlp"], + "punctuation.square.close": ["punctuation.square.close.nlp"], + "string.double.nlp": ["string.quoted.double.nlp"], + "string.double.dict": ["string.quoted.double.dict"], + "string.double.kb": ["string.quoted.double.kb"], + "string.double.kbb": ["string.quoted.double.kbb"], + "string.double.seq": ["string.quoted.double.seq"], + "string.double.tree": ["string.quoted.double.tree"], + "keyword.rewrite.tree": ["entity.name.type.tree"], + "keyword.node.tree": ["string.unquoted.token.tree"], + "keyword.attribute.tree": ["storage.type.tree"], + "keyword.fired.tree": ["keyword.other.fired.tree"], + "keyword.other.kbb": ["entity.name.type.level0.kbb"], + "keyword.concept.kbb": ["entity.name.type.level1.kbb"], + "keyword.concept.kbb1": ["entity.name.type.level2.kbb"], + "keyword.concept.kbb2": ["entity.name.type.level3.kbb"], + "comment.line.dict": ["entity.name.tag.dict", "comment.line.dict", "comment.line.kb", "comment.line.kbb", "comment.line.tree"], + "entity.nothing.dict": ["meta.whitespace.dict"], + "keyword.region.dict": ["support.type.property-name.dict", "meta.whitespace.dict"], + "entity.name.function.dict": ["string.unquoted.value.dict"], + "constant.numeric.tree": ["constant.numeric.tree", "constant.numeric.kbb"], + "meta.expression.tree": ["meta.expression.tree", "meta.expression.txxt"], + "variable.parameter.nlp": ["variable.parameter.nlp", "keyword.control.kb"], +}; + +// A textMateRules "scope" is a string, a comma-separated selector list, or an array. +// Returns the migrated scope and whether anything changed. Selectors this does not +// recognize (descendant selectors, other languages) are kept as they are. +export function migrateScope(scope: string | string[]): { scope: string | string[]; changed: boolean } { + const list = Array.isArray(scope) ? scope : scope.split(",").map((s) => s.trim()).filter((s) => s.length); + const out: string[] = []; + let changed = false; + for (const s of list) { + const renamed = RENAMED_SCOPES[s]; + if (renamed) changed = true; + for (const n of renamed ?? [s]) if (!out.includes(n)) out.push(n); + } + if (!changed) return { scope, changed: false }; + return { scope: out.length === 1 ? out[0] : out, changed: true }; +} + +// Rewrite every textMateRules scope in a settings object, in place: the top-level +// editor.tokenColorCustomizations block and its per-theme "[Theme]" blocks. +// Returns true when something changed. +export function migrateTokenColors(settings: Record): boolean { + const block = settings["editor.tokenColorCustomizations"]; + if (!block || typeof block !== "object") return false; + let changed = false; + const fixRules = (holder: unknown) => { + const rules = (holder as { textMateRules?: unknown })?.textMateRules; + if (!Array.isArray(rules)) return; + for (const rule of rules) { + if (!rule || typeof rule !== "object" || !("scope" in rule)) continue; + const r = rule as { scope: string | string[] }; + const m = migrateScope(r.scope); + if (m.changed) { + r.scope = m.scope; + changed = true; + } + } + }; + fixRules(block); + for (const [key, value] of Object.entries(block as Record)) { + if (key.startsWith("[")) fixRules(value); + } + return changed; +} diff --git a/src/visualText.ts b/src/visualText.ts index 9f67f6ff..9403c2ac 100644 --- a/src/visualText.ts +++ b/src/visualText.ts @@ -13,6 +13,7 @@ import { ModFile } from './modFile'; import { SequenceFile } from './sequence'; import { TextFile } from './textFile'; import * as telemetry from './telemetry/telemetry'; +import { migrateTokenColors } from './language/scopeMigration'; import { analyzerFiles, isInside, pushRecent, samePath } from './session/session'; export enum upStat { UNKNOWN, START, RUNNING, CANCEL, FAILED, DONE } @@ -1663,6 +1664,11 @@ export class VisualText { ...settingsObj2 }; this.jsonState.saveFile(toDir.fsPath, "settings", mergedObj); + } else if (migrateTokenColors(this.jsonState.json)) { + // Rules copied here before the grammars took standard scope + // names (#107) would silently stop applying; rename them. + this.jsonState.saveFile(toDir.fsPath, "settings", this.jsonState.json); + this.debugMessage('Updated colorization scope names in ' + toFile, logLineType.UPDATER); } } } else