From ae1136148a5db2ffaaf6452c7ce29c571d214291 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Tue, 13 Oct 2026 13:37:13 -0400 Subject: [PATCH] Standard TextMate scope names; migrate workspace color rules (4.3.10) Bumps grammars/ to VisualText/nlpplus-tmbundle#3, which renames every invented scope to its TextMate-convention equivalent (keyword.constants -> constant.language, keyword.attribute -> entity.other.attribute-name, literal.* -> meta.literal.*, string.double -> string.quoted.double, the dictionary's comment/keyword/function borrowings, kbb concept levels, copied suffixes). Tokens are unchanged; only names differ -- checked over 16.3M tokens in 3,372 real files. colorizeAnalyzer() has been copying textMateRules keyed to the old names into every analyzer workspace. src/language/scopeMigration.ts maps each old name to every new name it used to cover (comment.line.dict was the dict headword and the .kb/.kbb/.tree comment) and rewrites those rules once, so existing colors are preserved exactly. The shipped templates were regenerated through the same function. Dry run over the 16 analyzer workspaces on this machine: all parse, all migrate. langTest checks the map against the grammars themselves: no old name survives unless it maps to itself, every new name exists. Closes #107. Co-Authored-By: Claude Opus 5.5 --- .vscode/settings-light.json | 26 ++++++---- .vscode/settings.json | 56 +++++++++++++-------- CHANGELOG.md | 6 +++ grammars | 2 +- package-lock.json | 4 +- package.json | 2 +- src/language/langTest.ts | 48 ++++++++++++++++++ src/language/nlpxxData.ts | 2 +- src/language/scopeMigration.ts | 92 ++++++++++++++++++++++++++++++++++ src/visualText.ts | 6 +++ 10 files changed, 207 insertions(+), 37 deletions(-) create mode 100644 src/language/scopeMigration.ts diff --git a/.vscode/settings-light.json b/.vscode/settings-light.json index dcd111a6..35a1b0a6 100644 --- a/.vscode/settings-light.json +++ b/.vscode/settings-light.json @@ -8,37 +8,43 @@ } }, { - "scope": "variable.parameter.nlp", + "scope": [ + "variable.parameter.nlp", + "keyword.control.kb" + ], "settings": { "foreground": "#696969" } }, { - "scope": "keyword.attribute.nlp", + "scope": "entity.other.attribute-name.nlp", "settings": { "foreground": "#e655f3" } }, { - "scope": "constant.numeric.tree", + "scope": [ + "constant.numeric.tree", + "constant.numeric.kbb" + ], "settings": { "foreground": "#5596f0" } }, { - "scope": "keyword.fired.tree", + "scope": "keyword.other.fired.tree", "settings": { "foreground": "#dc1b1b" } }, { - "scope": "keyword.rewrite.tree", + "scope": "entity.name.type.tree", "settings": { "foreground": "#000000" } }, { - "scope": "keyword.node.tree", + "scope": "string.unquoted.token.tree", "settings": { "foreground": "#4e8e3c" } @@ -50,25 +56,25 @@ } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#909228" } }, { - "scope": "keyword.concept.kbb", + "scope": "entity.name.type.level1.kbb", "settings": { "foreground": "#7e7e7e" } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#2eb657" } }, { - "scope": "keyword.concept.kbb2", + "scope": "entity.name.type.level3.kbb", "settings": { "foreground": "#e049d9" } diff --git a/.vscode/settings.json b/.vscode/settings.json index db4932c3..07d58a21 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,20 +15,23 @@ } }, { - "scope": "variable.parameter.nlp", + "scope": [ + "variable.parameter.nlp", + "keyword.control.kb" + ], "settings": { "foreground": "#696969", "fontStyle": "bold" } }, { - "scope": "keyword.attribute.nlp", + "scope": "entity.other.attribute-name.nlp", "settings": { "foreground": "#e655f3" } }, { - "scope": "keyword.constants.nlp", + "scope": "constant.language.nlp", "settings": { "foreground": "#0a7b83", "fontStyle": "bold" @@ -44,27 +47,30 @@ } }, { - "scope": "constant.numeric.tree", + "scope": [ + "constant.numeric.tree", + "constant.numeric.kbb" + ], "settings": { "foreground": "#5596f0" } }, { - "scope": "keyword.fired.tree", + "scope": "keyword.other.fired.tree", "settings": { "foreground": "#dc1b1b", "fontStyle": "bold" } }, { - "scope": "keyword.rewrite.tree", + "scope": "entity.name.type.tree", "settings": { "foreground": "#000000", "fontStyle": "bold" } }, { - "scope": "keyword.node.tree", + "scope": "string.unquoted.token.tree", "settings": { "foreground": "#4e8e3c", "fontStyle": "bold" @@ -98,28 +104,28 @@ } }, { - "scope": "keyword.other.kbb", + "scope": "entity.name.type.level0.kbb", "settings": { "foreground": "#5d00d6", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb", + "scope": "entity.name.type.level1.kbb", "settings": { "foreground": "#7e7e7e", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#2eb657", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb2", + "scope": "entity.name.type.level3.kbb", "settings": { "foreground": "#e049d9", "fontStyle": "bold" @@ -143,20 +149,23 @@ } }, { - "scope": "variable.parameter.nlp", + "scope": [ + "variable.parameter.nlp", + "keyword.control.kb" + ], "settings": { "foreground": "#f8fa87", "fontStyle": "bold" } }, { - "scope": "keyword.attribute.nlp", + "scope": "entity.other.attribute-name.nlp", "settings": { "foreground": "#f293fb" } }, { - "scope": "keyword.constants.nlp", + "scope": "constant.language.nlp", "settings": { "foreground": "#4ec9b0", "fontStyle": "bold" @@ -172,27 +181,30 @@ } }, { - "scope": "constant.numeric.tree", + "scope": [ + "constant.numeric.tree", + "constant.numeric.kbb" + ], "settings": { "foreground": "#93bffb" } }, { - "scope": "keyword.fired.tree", + "scope": "keyword.other.fired.tree", "settings": { "foreground": "#dc1b1b", "fontStyle": "bold" } }, { - "scope": "keyword.rewrite.tree", + "scope": "entity.name.type.tree", "settings": { "foreground": "#f8fa87", "fontStyle": "bold" } }, { - "scope": "keyword.node.tree", + "scope": "string.unquoted.token.tree", "settings": { "foreground": "#4e8e3c", "fontStyle": "bold" @@ -211,28 +223,28 @@ } }, { - "scope": "keyword.other.kbb", + "scope": "entity.name.type.level0.kbb", "settings": { "foreground": "#896eff", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb", + "scope": "entity.name.type.level1.kbb", "settings": { "foreground": "#e3ee50", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb1", + "scope": "entity.name.type.level2.kbb", "settings": { "foreground": "#2eb657", "fontStyle": "bold" } }, { - "scope": "keyword.concept.kbb2", + "scope": "entity.name.type.level3.kbb", "settings": { "foreground": "#e049d9", "fontStyle": "bold" diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d45d64..9d32ceb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the [VSCode NLP++ extension](http://vscode.visualtext.org Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. +### 4.3.10 +NLP++ files are colored by any VS Code theme, not only through the colors this extension writes into your settings. + +- **The grammars use standard TextMate scope names** (#107). They used to name things after the color they should get rather than what they are. Constants like `_xWILD` were marked as keywords, a dictionary headword as a comment, and a few scopes, such as `literal.nlp`, had names no theme recognizes. Themes, and tools such as GitHub and Shiki that read these grammars, now color `@RULES`, constants, rule modifiers, node names and dictionary keys and values like the equivalent parts of other languages. The full list of renames is in the [grammar README](https://github.com/VisualText/nlpplus-tmbundle#scope-names). +- **Your analyzer colors are unchanged.** The extension's color rules in each analyzer workspace's `.vscode/settings.json` are rewritten to the new names the first time the workspace opens, so everything you saw before keeps its color. A settings file the extension cannot read, for example one with comments in it, is left alone. + ### 4.3.9 Each analyzer keeps its own open files, and the analyzers you used last are a click away. diff --git a/grammars b/grammars index 877abbac..ab0b1389 160000 --- a/grammars +++ b/grammars @@ -1 +1 @@ -Subproject commit 877abbaceb7847a0f001c04ee8b3c6eeefd146ee +Subproject commit ab0b1389aff2f3b931743e0d43b97f07dc543ad3 diff --git a/package-lock.json b/package-lock.json index 656b27d6..a3ffcbe1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nlp", - "version": "4.3.9", + "version": "4.3.10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "nlp", - "version": "4.3.9", + "version": "4.3.10", "license": "MIT", "dependencies": { "@vscode/debugadapter": "^1.68.0", diff --git a/package.json b/package.json index 4e0996a9..b53f6563 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "NLP", "description": "NLP++: the only computer programming language built for NLP — create and visually debug analyzers directly in VS Code.", "icon": "resources/NLPppLogo.png", - "version": "4.3.9", + "version": "4.3.10", "license": "MIT", "publisher": "dehilster", "enableApiProposals": false, diff --git a/src/language/langTest.ts b/src/language/langTest.ts index 7fa64356..6f880efa 100644 --- a/src/language/langTest.ts +++ b/src/language/langTest.ts @@ -13,6 +13,9 @@ import { parseKbConcepts } from "./kbConcepts"; import { regionKindAt, RegionKind } from "./completion"; import { findEnclosingCall } from "./signature"; import { foldingRanges } from "./folding"; +import { RENAMED_SCOPES, migrateScope, migrateTokenColors } from "./scopeMigration"; +import * as fs from "fs"; +import * as path from "path"; import { RuleLocator, visualColumn } from "./ruleLocator"; import { classifyTokens } from "./semanticTokens"; import { findUnknownCalls, levenshtein } from "./quickfix"; @@ -185,6 +188,51 @@ _verb <- verb @@ eq("column: two tabs", visualColumn("\t\tx", 2, 4), 9); } +// ---- scope migration (#107) ------------------------------------------------ +{ + // The map must match the grammars it describes: no old name may survive in + // them unless it maps to itself, and every new name must exist in one. + const dir = path.join(__dirname, "..", "..", "grammars", "syntaxes"); + const names = new Set(); + const walk = (o: unknown): void => { + if (Array.isArray(o)) o.forEach(walk); + else if (o && typeof o === "object") { + for (const [k, v] of Object.entries(o)) { + if ((k === "name" || k === "contentName") && typeof v === "string") names.add(v); + else walk(v); + } + } + }; + for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".json"))) walk(JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"))); + for (const [oldName, renamed] of Object.entries(RENAMED_SCOPES)) { + check(`scopes: ${oldName} gone from grammars`, renamed.includes(oldName) || !names.has(oldName)); + for (const n of renamed) check(`scopes: ${n} exists in a grammar`, names.has(n)); + } + + eq("scopes: one-to-one", JSON.stringify(migrateScope("keyword.region.nlp")), JSON.stringify({ scope: "keyword.control.directive.region.nlp", changed: true })); + eq("scopes: one-to-many keeps all", JSON.stringify(migrateScope("comment.line.dict").scope), + JSON.stringify(["entity.name.tag.dict", "comment.line.dict", "comment.line.kb", "comment.line.kbb", "comment.line.tree"])); + eq("scopes: list, deduplicated", JSON.stringify(migrateScope(["keyword.operator.nlp", "keyword.equals.nlp", "keyword.operator.assignment.nlp"]).scope), + JSON.stringify(["keyword.operator.nlp", "keyword.operator.assignment.nlp"])); + eq("scopes: comma string", JSON.stringify(migrateScope("keyword.fired.tree, keyword.rewrite.tree").scope), + JSON.stringify(["keyword.other.fired.tree", "entity.name.type.tree"])); + eq("scopes: unknown untouched", migrateScope("source.js keyword").changed, false); + + const settings: Record = { + "files.eol": "lf", + "editor.tokenColorCustomizations": { + "textMateRules": [{ scope: "keyword.constants.nlp", settings: { foreground: "#0a7b83" } }], + "[*Dark*]": { textMateRules: [{ scope: "keyword.fired.tree", settings: { foreground: "#ff0000" } }, { name: "no scope" }] }, + }, + }; + check("scopes: settings migrated", migrateTokenColors(settings)); + const block = settings["editor.tokenColorCustomizations"] as { textMateRules: { scope: string }[]; "[*Dark*]": { textMateRules: { scope?: string }[] } }; + eq("scopes: top-level rule", block.textMateRules[0].scope, "constant.language.nlp"); + eq("scopes: per-theme rule", block["[*Dark*]"].textMateRules[0].scope, "keyword.other.fired.tree"); + eq("scopes: second run is a no-op", migrateTokenColors(settings), false); + eq("scopes: no block", migrateTokenColors({ "files.eol": "lf" }), false); +} + // ---- engine errors --------------------------------------------------------- { const log = [ diff --git a/src/language/nlpxxData.ts b/src/language/nlpxxData.ts index 530cce78..eff3fe54 100644 --- a/src/language/nlpxxData.ts +++ b/src/language/nlpxxData.ts @@ -27,7 +27,7 @@ export const KEYWORDS = [ "cap", "cout", "else", "gp", "group", "if", "inc", "return", "while", ]; -// Rule-element modifiers inside @RULES (grammar: keyword.attribute.nlp). +// Rule-element modifiers inside @RULES (grammar: entity.other.attribute-name.nlp). export const RULE_KEYWORDS = [ "attr", "attrs", "base", "da", "deacc", "deaccent", "except", "excepts", "fail", "fails", "gp", "group", "layer", "layers", "look", "lookahead", diff --git a/src/language/scopeMigration.ts b/src/language/scopeMigration.ts new file mode 100644 index 00000000..8c246e57 --- /dev/null +++ b/src/language/scopeMigration.ts @@ -0,0 +1,92 @@ +// Renamed TextMate scopes, and the migration of color rules keyed to the old ones (#107). +// +// PURE MODULE: no 'vscode' import. The grammars (grammars/, VisualText/nlpplus-tmbundle) +// used to name scopes for the color they got rather than for what they are, and +// colorizeAnalyzer() copied textMateRules for those names into every analyzer +// workspace's .vscode/settings.json. The grammars now use standard names; this +// rewrites old rules so they keep coloring exactly what they colored before. +// +// An old name can map to several new ones: "comment.line.dict" was the dictionary +// headword AND the line comment of .kb, .kbb and .tree files, so a rule for it +// becomes a rule for all of their new names. + +export const RENAMED_SCOPES: Record = { + "keyword.region.nlp": ["keyword.control.directive.region.nlp"], + "keyword.constants.nlp": ["constant.language.nlp"], + "keyword.attribute.nlp": ["entity.other.attribute-name.nlp"], + "keyword.equals.nlp": ["keyword.operator.assignment.nlp"], + "variable.rewrite.nlp": ["entity.name.type.rewrite.nlp"], + "variable.rewrite-symbol.nlp": ["meta.attribute.rewrite.nlp"], + "literal.nlp": ["meta.literal.nlp"], + "literal.seq": ["meta.literal.seq"], + "literal.tree": ["meta.literal.tree"], + "constant.character.escape": ["constant.character.escape.nlp"], + "punctuation.paren.open": ["punctuation.paren.open.nlp"], + "punctuation.paren.close": ["punctuation.paren.close.nlp"], + "punctuation.square.open": ["punctuation.square.open.nlp"], + "punctuation.square.close": ["punctuation.square.close.nlp"], + "string.double.nlp": ["string.quoted.double.nlp"], + "string.double.dict": ["string.quoted.double.dict"], + "string.double.kb": ["string.quoted.double.kb"], + "string.double.kbb": ["string.quoted.double.kbb"], + "string.double.seq": ["string.quoted.double.seq"], + "string.double.tree": ["string.quoted.double.tree"], + "keyword.rewrite.tree": ["entity.name.type.tree"], + "keyword.node.tree": ["string.unquoted.token.tree"], + "keyword.attribute.tree": ["storage.type.tree"], + "keyword.fired.tree": ["keyword.other.fired.tree"], + "keyword.other.kbb": ["entity.name.type.level0.kbb"], + "keyword.concept.kbb": ["entity.name.type.level1.kbb"], + "keyword.concept.kbb1": ["entity.name.type.level2.kbb"], + "keyword.concept.kbb2": ["entity.name.type.level3.kbb"], + "comment.line.dict": ["entity.name.tag.dict", "comment.line.dict", "comment.line.kb", "comment.line.kbb", "comment.line.tree"], + "entity.nothing.dict": ["meta.whitespace.dict"], + "keyword.region.dict": ["support.type.property-name.dict", "meta.whitespace.dict"], + "entity.name.function.dict": ["string.unquoted.value.dict"], + "constant.numeric.tree": ["constant.numeric.tree", "constant.numeric.kbb"], + "meta.expression.tree": ["meta.expression.tree", "meta.expression.txxt"], + "variable.parameter.nlp": ["variable.parameter.nlp", "keyword.control.kb"], +}; + +// A textMateRules "scope" is a string, a comma-separated selector list, or an array. +// Returns the migrated scope and whether anything changed. Selectors this does not +// recognize (descendant selectors, other languages) are kept as they are. +export function migrateScope(scope: string | string[]): { scope: string | string[]; changed: boolean } { + const list = Array.isArray(scope) ? scope : scope.split(",").map((s) => s.trim()).filter((s) => s.length); + const out: string[] = []; + let changed = false; + for (const s of list) { + const renamed = RENAMED_SCOPES[s]; + if (renamed) changed = true; + for (const n of renamed ?? [s]) if (!out.includes(n)) out.push(n); + } + if (!changed) return { scope, changed: false }; + return { scope: out.length === 1 ? out[0] : out, changed: true }; +} + +// Rewrite every textMateRules scope in a settings object, in place: the top-level +// editor.tokenColorCustomizations block and its per-theme "[Theme]" blocks. +// Returns true when something changed. +export function migrateTokenColors(settings: Record): boolean { + const block = settings["editor.tokenColorCustomizations"]; + if (!block || typeof block !== "object") return false; + let changed = false; + const fixRules = (holder: unknown) => { + const rules = (holder as { textMateRules?: unknown })?.textMateRules; + if (!Array.isArray(rules)) return; + for (const rule of rules) { + if (!rule || typeof rule !== "object" || !("scope" in rule)) continue; + const r = rule as { scope: string | string[] }; + const m = migrateScope(r.scope); + if (m.changed) { + r.scope = m.scope; + changed = true; + } + } + }; + fixRules(block); + for (const [key, value] of Object.entries(block as Record)) { + if (key.startsWith("[")) fixRules(value); + } + return changed; +} diff --git a/src/visualText.ts b/src/visualText.ts index 9f67f6ff..9403c2ac 100644 --- a/src/visualText.ts +++ b/src/visualText.ts @@ -13,6 +13,7 @@ import { ModFile } from './modFile'; import { SequenceFile } from './sequence'; import { TextFile } from './textFile'; import * as telemetry from './telemetry/telemetry'; +import { migrateTokenColors } from './language/scopeMigration'; import { analyzerFiles, isInside, pushRecent, samePath } from './session/session'; export enum upStat { UNKNOWN, START, RUNNING, CANCEL, FAILED, DONE } @@ -1663,6 +1664,11 @@ export class VisualText { ...settingsObj2 }; this.jsonState.saveFile(toDir.fsPath, "settings", mergedObj); + } else if (migrateTokenColors(this.jsonState.json)) { + // Rules copied here before the grammars took standard scope + // names (#107) would silently stop applying; rename them. + this.jsonState.saveFile(toDir.fsPath, "settings", this.jsonState.json); + this.debugMessage('Updated colorization scope names in ' + toFile, logLineType.UPDATER); } } } else