From 49340cfa7249f48542357b692dfbde9a5aaa893d Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Thu, 17 Sep 2026 17:37:57 -0700 Subject: [PATCH 1/2] A page for trying VisualText's analyzers, without an account /studio/try/ is a second page in the same build: pick one of VisualText's own analyzers, type text, press Run, and see the pass sequence, the knowledge base, what the run wrote, the parse trees, and -- with Log files on -- each pass's own tree and the text marked with what its rules matched. It is drawn with @visualtext/analyzer-views and loads no Monaco (5.5 kB). It is read-only, and that is the point. Running an analyzer is running its author's code, and docs/ARCHITECTURE.md says the per-run sandbox strangers would need is not built. This page sidesteps that rather than solving it: the analyzers are copied in when the image is built, from a pinned VisualText/analyzers release -- the same analyzers.zip VisualText downloads as its examples -- and the only thing a visitor supplies is the text. The visitor never supplies code. Two things have to stay true for that, and neither is enforced by the server: the page stays read-only, and /api/run -- which still accepts any files a caller sends -- is not reachable anonymously without rate limits in front. Both are written down rather than assumed. Eight of the twenty analyzers in the bundle, each showing something the others do not, 259 files and 1.45 MB in all. All twenty-one in the repository were run through /api/run first; the survey, and why the rest are left out, is in docs/TRY-PAGE.md so nobody repeats it. parse-en-us is refused because it calls interactive(), which nlp_run.BLOCKED lists with the desktop app's popups although at both its call sites it only reads state -- left alone here, and written up as an open question. Also: - scripts/analyzer-files.mjs holds what travels with an analyzer, shared with copy-samples.mjs so the two cannot drift on what an analyzer is. - src/theme.css holds the palette and plain controls, shared with the editor page so a colour is changed once. - src/try/result.ts keeps the two translations that are easy to get quietly wrong -- the run server's `size` against the views' `bytes`, and readOutput()'s { output } wrapper against what takes -- in one place, with tests. - smoke-test.sh proves the try page and its analyzers are in the image, and that one of them runs in debug mode, keeps a tree after every pass and can have one fetched back. - update.sh tracks the analyzers release as it tracks visualtext-files, so a new one makes the app out of date. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 5 + README.md | 25 +++ docs/ARCHITECTURE.md | 30 +++ docs/TRY-PAGE.md | 241 ++++++++++++++++++++++ studio/Dockerfile | 27 ++- studio/deploy/INSTALL.md | 49 +++++ studio/deploy/smoke-test.sh | 43 ++++ studio/deploy/update.sh | 15 +- studio/package.json | 7 +- studio/scripts/analyzer-files.mjs | 62 ++++++ studio/scripts/copy-analyzers.mjs | 110 ++++++++++ studio/scripts/copy-samples.mjs | 40 +--- studio/src/styles.css | 48 +---- studio/src/theme.css | 59 ++++++ studio/src/try/main.ts | 321 ++++++++++++++++++++++++++++++ studio/src/try/result.ts | 51 +++++ studio/src/try/try.css | 82 ++++++++ studio/test/try.test.ts | 89 +++++++++ studio/try/index.html | 65 ++++++ studio/vite.config.ts | 8 + 20 files changed, 1290 insertions(+), 87 deletions(-) create mode 100644 docs/TRY-PAGE.md create mode 100644 studio/scripts/analyzer-files.mjs create mode 100644 studio/scripts/copy-analyzers.mjs create mode 100644 studio/src/theme.css create mode 100644 studio/src/try/main.ts create mode 100644 studio/src/try/result.ts create mode 100644 studio/src/try/try.css create mode 100644 studio/test/try.test.ts create mode 100644 studio/try/index.html diff --git a/.gitignore b/.gitignore index d45b9cf..067a866 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,11 @@ stopgap/vsix/ # analyzer-templates checkout, which has no license to commit it under. studio/public/analyzers/ +# The try page's analyzers as it fetches them -- written by +# studio/scripts/copy-analyzers.mjs at build time from a VisualText/analyzers +# release. Build output, and a second copy of another repository besides. +studio/public/try/ + __pycache__/ .venv/ diff --git a/README.md b/README.md index c6ee8cc..a84ae96 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ see the parse tree — with no install, no C++ toolchain, and no VS Code. |---|---|---| | **1. Stopgap** | `openvscode-server` container with the real `dehilster.nlp` extension baked in | **in this repo** — see [stopgap/](stopgap/) | | **2. Studio** | Purpose-built web app: Monaco front end + an API server running the engine | **live** — in [studio/](studio/): the editor, the run server, analyzers opened from GitHub and committed back, deployed at studio.visualtext.org/studio/ for invited people | +| **2b. Try page** | A read-only page at `/try/`: run VisualText's own analyzers on your own text, with no install and no account | **in this repo** — see [docs/TRY-PAGE.md](docs/TRY-PAGE.md) | | **Shared views** | `@visualtext/analyzer-views`: the sequence, knowledge base, output files, parse trees, code, log and values as web components, for the studio and any other page that shows an analyzer | **released** — see [analyzer-views/](analyzer-views/) | | **3. Client-side** | Emscripten/WASM build of the engine for zero-server demos | speculative | @@ -94,6 +95,30 @@ Deploying beside the phase-1 editor, at studio.visualtext.org/studio/ — behind password, or with GitHub sign-in for invited people: [studio/deploy/INSTALL.md](studio/deploy/INSTALL.md). +## Trying an analyzer, without an account (`/try/`) + +The same build serves a second page at `/try/`: pick one of VisualText's own analyzers, type +text, press **Run**, and see the passes, the knowledge base, what the run wrote and the parse +trees — with each pass's own tree and matched rules when **Log files** is on. It is drawn with +the same `@visualtext/analyzer-views` elements as the editor, and loads no Monaco. + +It is read-only. The analyzers are copied in when the site is built, from a pinned +[`VisualText/analyzers`](https://github.com/VisualText/analyzers) release, and the only thing +a visitor supplies is the text. That is what lets this page be opened to people who are not +invited while the editor is not — **the visitor never supplies code**. It is also a promise +the page has to keep: accepting a pasted grammar would put it back behind the sandbox that +[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md#the-run-server-is-not-a-sandbox) describes as not +built. + +```bash +cd studio +VISUALTEXT_ANALYZERS=../../analyzers npm run analyzers # write the catalog +npm run dev # http://localhost:5173/try/ +``` + +[docs/TRY-PAGE.md](docs/TRY-PAGE.md) has which analyzers it offers and why, which ones are +left out and why, how the catalog is built, and what to watch for when changing it. + ## Quick start (phase 1) Requires Docker and a checkout of [`vscode-nlp`](https://github.com/VisualText/vscode-nlp) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fa29d46..2fcca92 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -195,6 +195,36 @@ With sign-in, the invited list — not a sign-up — is what decides who gets th `NLP_STUDIO_REQUIRE_SIGN_IN=1` makes the server refuse to start rather than run open when the proxy asks for no password. +#### Trying an analyzer without that sandbox (implemented) + +`/studio/try/` (`studio/try/`, `studio/src/try/`) lets anyone who reaches the page run one of +VisualText's analyzers on their own text. It does not need the per-run sandbox above, +because it does not run the visitor's code: the analyzers are copied in at build time by +`studio/scripts/copy-analyzers.mjs` from a pinned `VisualText/analyzers` release — the same +`analyzers.zip` VisualText downloads as its examples — and the page offers no way to edit +the NLP++. The text is the only thing a visitor supplies, and it is data the rules read, not +rules the engine runs. + +That is the whole of the argument, so the two halves of it have to hold: + +- **The page must stay read-only.** The moment it accepts a `.nlp` file, a pasted grammar or + an analyzer named by URL, the code being run is a stranger's again and everything in *The + run server is not a sandbox* applies. `/api/run` itself still takes arbitrary `files` — the + restraint is in the page, not the server — so opening this to the internet means rate + limits in front of `/api/run` and, for anything more than these analyzers, the sandbox. +- **The grammars must be ones we would run ourselves.** They are picked by name in + `copy-analyzers.mjs`, not discovered, and each one is confirmed to run before it is listed. + +Which analyzers it offers and why, which are left out and why, and what to watch for when +changing it: [TRY-PAGE.md](TRY-PAGE.md). + +What it shows is the extension's own views, through `@visualtext/analyzer-views`: the pass +sequence, the knowledge base (`.dict` and `.kbb`, never `.kb`), what the run wrote, the parse +trees, and with **Log files** on — the engine's `-DEV` — each pass's own tree and the text +marked with what that pass's rules matched. The marks are made in the browser by +`ruleMatches()` over *the text that ran*, which the page keeps beside the result rather than +reading back out of the box, so editing the text after a run cannot silently move them. + ### Keeping work (implemented) Edits are saved in the browser as they are typed (`studio/src/drafts.ts`). Each analyzer has diff --git a/docs/TRY-PAGE.md b/docs/TRY-PAGE.md new file mode 100644 index 0000000..b4dcc49 --- /dev/null +++ b/docs/TRY-PAGE.md @@ -0,0 +1,241 @@ +# The try page + +`/studio/try/` is a page where someone who has never installed anything picks one of +VisualText's analyzers, types some text, presses **Run**, and sees what the analyzer did +with it: the pass sequence, the knowledge base, the files the run wrote, the parse trees, +and — with **Log files** ticked — each pass's own tree and the text marked with what that +pass's rules matched. + +It is a second page in the phase-2 studio, not a separate service. It is built from the +same `studio/` project into the same `dist/`, shipped in the same container, and it calls +the same `/studio/api/`. If `/studio/` works, it works. + +It is deliberately **read-only**: the analyzers are chosen and copied in when the image is +built, and the only thing a visitor can change is the text. That is the whole reason it can +be opened to people who are not invited, and [§ Why it needs no sandbox](#why-it-needs-no-sandbox) +is the argument in full. Read that before changing what the page accepts. + +Related documents: + +| | | +|---|---| +| [ARCHITECTURE.md](ARCHITECTURE.md#trying-an-analyzer-without-that-sandbox-implemented) | why this design, next to *The run server is not a sandbox* | +| [studio/deploy/INSTALL.md](../studio/deploy/INSTALL.md#the-try-page) | operating it on studio.visualtext.org, and what opening it wider needs | +| [analyzer-views/README.md](../analyzer-views/README.md) | the seven elements the page is drawn with | + +## Why it needs no sandbox + +Running an analyzer is running its author's code. NLP++ has file functions that take any +path (`openfile`, `readfile`, `mkdir`, …) and they cannot be refused — the knowledge-base +library every template ships uses them. `ARCHITECTURE.md` says plainly that before +strangers can run *their own* analyzers, the operating system has to contain the run +process, and that this is not built. + +The try page sidesteps that rather than solving it. **The visitor never supplies code.** +The analyzers come from a pinned `VisualText/analyzers` release, chosen by name in +`studio/scripts/copy-analyzers.mjs`, and the page offers no way to edit the NLP++. The text +is data the rules read, not rules the engine runs. + +Two things have to stay true for that to hold, and neither is enforced by the server: + +1. **The page stays read-only.** The moment it accepts a pasted grammar, an uploaded + `.nlp`, or an analyzer named by URL, the code being run is a stranger's again and + everything in *The run server is not a sandbox* applies. +2. **`/api/run` is still the general endpoint.** It accepts any `files` a caller sends — + the restraint lives in the page, not in the server. Making the page reachable + anonymously without also rate-limiting and fronting `/api/run` would put a remote code + execution endpoint on the host that serves visualtext.org. + +## Where the analyzers come from + +`studio/scripts/copy-analyzers.mjs` runs before `npm run dev` and `npm run build`. It reads +a source tree of [VisualText/analyzers](https://github.com/VisualText/analyzers), copies the +picked analyzers' `spec/`, `kb/` and `input/` into `studio/public/try/analyzers//`, +and writes an `index.json` beside them. The page fetches those as ordinary static files: +no GitHub API at run time, no token, no rate limit, and nothing user-controlled in the +grammar. + +The source is either of two things, and the distinction matters: + +- **A release's `analyzers.zip`, unpacked** — what the Dockerfile uses. This is the same + bundle VisualText itself downloads as its example analyzers, so the page offers exactly + the set a VisualText user has. `VISUALTEXT_ANALYZERS_TAG` names the release and ends up + on the image as the `org.visualtext.analyzers` label. **The zip holds only the five + bundled entries** (`corporate/`, `files/`, `parse-en-us/`, `nlp-tutorials/`, + `nlpfix-analyzers/`), so anything outside them cannot be picked. +- **A git checkout**, for working on this locally. Three of the five entries are + submodules, so clone with `--recurse-submodules` or the tutorials and the NLPFix + analyzers will not be there. `VISUALTEXT_ANALYZERS` points at it; the default is + `../../analyzers`, a sibling of this repo. + +Only `spec/`, `kb/` and `input/` travel — never `output/`, `tmp/` or an engine run's +`*_log/`, which are a previous run's leavings and would otherwise be read back as though +this page's run had written them. Files over 1 MB are skipped; see +`studio/scripts/analyzer-files.mjs`, which both copy scripts share so they cannot drift on +what an analyzer is. + +`studio/public/try/` is generated and git-ignored. It is not committed: it is build output, +and a second copy of another repository besides. + +## Which analyzers, and why + +Eight of the twenty in the bundle, each showing something the others do not. Together they +come to 259 files and 1.45 MB — small enough to be fetched into a tab. + +| On the page | From | Files | Size | Passes | Shows | +|---|---|---|---|---|---| +| **Corporate** | `corporate` | 43 | 132K | 30 | Companies, money and events, with pronouns resolved back to what they refer to. The showcase analyzer. | +| **Regions** | `nlp-tutorials/tutorial-07` | 10 | 38K | 5 | Where NLP++ code, functions and rules live in a pass — and what a code-only pass matches. | +| **Dates and Times** | `nlpfix-analyzers/date-time` | 48 | 273K | 17 | Dates and times in many formats, where a regular expression needs rewriting for each. Four `.dict` files, and the only one that writes an `output.json`. | +| **Formatting** | `nlpfix-analyzers/formatting` | 66 | 225K | 26 | Recovers headings, lists and tables from text that lost its formatting. | +| **Entities** | `nlpfix-analyzers/nlp` | 45 | 661K | 17 | Gathers what a text says about each person or thing into one record. Writes eight files. | +| **Variables** | `nlp-tutorials/tutorial-02` | 17 | 49K | 12 | The five NLP++ variables — N, S, X, G and L — on a short résumé. | +| **Pronouns** | `nlp-tutorials/tutorial-08` | 19 | 51K | 14 | Builds a knowledge base from the text as it reads, then resolves pronouns with it. | +| **Ambiguity** | `nlp-tutorials/tutorial-15` | 11 | 53K | 4 | Choosing between readings of the same words, with a `.dict` and a `.kbb` file. | + +Two things about this set that are worth knowing before changing it. + +**Most of these analyzers have an empty knowledge base panel.** `` +lists `.dict` and `.kbb` files and never the engine's own `.kb` files, which is correct and +is the rule the package exists to keep. But only *Dates and Times*, *Entities* and +*Ambiguity* ship any — Corporate, for all that it is the knowledge-base showcase, has six +`.kb` files and no `.dict` or `.kbb`, so its panel hides itself. That is the truth about +those analyzers, not a bug in the view. + +**`` is not the answer here.** The reference implementation this page was +modelled on leads with the fields an `output.json` filled. Of VisualText's analyzers only +*Dates and Times* writes one at all; the rest write `.txt` and `.kbb` files. So the right +column leads with **what the run wrote** and the parse trees, and `` appears +only when there is an `output.json` to show. + +### What is left out, and why + +| Left out | Why | +|---|---| +| `parse-en-us` | **Refused by the run server**: it calls `interactive()`, which `nlp_run.BLOCKED` lists alongside the desktop app's popups. See [§ Open questions](#open-questions) — this one is probably worth revisiting. 4.6 MB besides. | +| `nlp-tutorials/tutorial-01` | 26 MB, over the run server's 8 MB `MAX_FILES_BYTES`, so it could be listed but never run. All of it is `kb/user/attr*.kb` and `word.kb` — engine `.kb` dumps, which are not listed in the knowledge base anyway — for a sequence of one `dicttokz` pass. The weakest demo in the set regardless. | +| `business` | Runs fine, and is one of the few analyzers with a visible knowledge base (2 `.dict`, 1 `.kbb`) — but it is **not in `analyzers.zip`**, only in the repository, so a build from a release cannot see it. It also covers much the same ground as Corporate. | +| `files` | Walks a directory of files keeping a knowledge base across them. On a page that runs one text at a time, the point of it does not come across. | +| the other tutorials | They run, but each overlaps one of the eight above. `tutorial-13-b` reports 14 problems, which would exercise `` nicely but is not something to ship as a demo. | + +### The survey this came from + +Every analyzer in the repository, run through `POST /api/run` with `develop: true`, engine +NLPPlus 2.2.38. Recorded here so nobody has to do it again. `out` is files written, `prob` +problems reported, `trees` parse trees kept. + +| Analyzer | MB | Status | out | prob | trees | +|---|---|---|---|---|---| +| `business` | 0.06 | ok | 4 | 0 | 26 | +| `corporate` | 0.13 | ok | 5 | 0 | 29 | +| `files` | 0.14 | ok | 1 | 0 | 8 | +| `nlp-tutorials/tutorial-01` | 26.08 | **too big** — over the 8 MB limit | | | | +| `nlp-tutorials/tutorial-02` | 0.05 | ok | 3 | 0 | 13 | +| `nlp-tutorials/tutorial-03` | 0.05 | ok | 3 | 0 | 13 | +| `nlp-tutorials/tutorial-04` | 0.04 | ok | 1 | 2 | 5 | +| `nlp-tutorials/tutorial-05` | 0.04 | ok | 0 | 1 | 10 | +| `nlp-tutorials/tutorial-06` | 0.05 | ok | 3 | 0 | 14 | +| `nlp-tutorials/tutorial-07` | 0.04 | ok | 2 | 0 | 6 | +| `nlp-tutorials/tutorial-08` | 0.05 | ok | 5 | 0 | 15 | +| `nlp-tutorials/tutorial-09` | 0.04 | ok | 0 | 0 | 13 | +| `nlp-tutorials/tutorial-11` | 0.04 | ok | 1 | 0 | 5 | +| `nlp-tutorials/tutorial-13/tutorial-13-a` | 0.04 | ok | 1 | 0 | 4 | +| `nlp-tutorials/tutorial-13/tutorial-13-b` | 0.05 | ok | 0 | 14 | 20 | +| `nlp-tutorials/tutorial-14` | 0.04 | ok | 1 | 0 | 5 | +| `nlp-tutorials/tutorial-15` | 0.05 | ok | 2 | 0 | 5 | +| `nlpfix-analyzers/date-time` | 0.27 | ok | 3 | 0 | 18 | +| `nlpfix-analyzers/formatting` | 0.22 | ok | 3 | 0 | 27 | +| `nlpfix-analyzers/nlp` | 1.82 | ok | 8 | 0 | 18 | +| `parse-en-us` | 4.58 | **rejected** — `interactive()` | | | | + +## How the page is put together + +| File | | +|---|---| +| `studio/try/index.html` | the page: a top bar and three columns | +| `studio/src/try/main.ts` | loading an analyzer, running it, and wiring the views' events | +| `studio/src/try/result.ts` | pure: turning a run result into what the views take, with tests in `studio/test/try.test.ts` | +| `studio/src/try/try.css` | the three-column layout | +| `studio/src/theme.css` | the palette and plain controls, shared with the studio page so a colour is changed once | +| `studio/scripts/copy-analyzers.mjs` | the catalog, at build time | +| `studio/scripts/analyzer-files.mjs` | what travels with an analyzer, shared with `copy-samples.mjs` | + +The layout is three columns that fill the window, each scrolling on its own so the page +never does: the sequence over the knowledge base on the left; whatever was clicked — a +pass, a dictionary, a tree, an output file — in the middle; the text and the results on the +right. Under 900px they stack. + +Every list is a `@visualtext/analyzer-views` element, which is the point: a pass is numbered +and a knowledge base filtered here exactly as in the VS Code extension, because it is the +same code. Nothing about an analyzer is drawn by hand. + +### Four things that will bite + +- **Pass numbering.** A switched-off pass (`/nlp foo`) keeps its number and writes no tree; + a `folder` or `stub` takes no number at all. `passes()` gets this right, and both cases + are live in the shipped set — `business` has the first, `corporate` the second — so + anything that renumbers by itself will mislabel every tree and every problem. `passTrees()` + also keeps `final.tree` away from ``: it was written after the last pass but + belongs to none, and handing it over puts the icons on whichever pass is numbered `null`. +- **Mark the matches over the text that ran.** `ruleMatches(tree, text)` is given + `state.ranText`, a copy kept beside the result — not the contents of the box, which may + have been edited since. The page has no way to notice that going wrong. +- **A pass that is all `@CODE` matches nothing**, and its tree is a header alone, so its + matches are the plain text. That is the truth about that pass. *Variables* and *Pronouns* + have several. Do not "fix" it. +- **`size` and `bytes`.** The run server calls a tree's size `size`; the views call it + `bytes`. `viewTrees()` is the one place that translates, and there is a test pinning it. + Similarly `readOutput()` answers a `{ output }` wrapper while `` wants the + parsed value itself — pass the wrapper and every field is listed as `output.something`. + +Trees are not sent with a run's result; they get large. The server keeps them for half an +hour and the page fetches one when it is opened (`GET /api/run/tree`). A tree too large to +keep comes back in `trees.skipped`, and `` says so rather than offering a button +that does nothing. + +Every outcome of a run is a status **and a sentence**, including "no run server", and the +page shows the sentence. A page that says nothing when the server is down is the bug that +keeps coming back. + +## Building and verifying + +```bash +# a source of analyzers: a checkout ... +git clone --recurse-submodules https://github.com/VisualText/analyzers.git ../analyzers +# ... or a release, as the Dockerfile does +curl -fsSL -o /tmp/analyzers.zip \ + https://github.com/VisualText/analyzers/releases/latest/download/analyzers.zip +unzip -q /tmp/analyzers.zip -d /tmp/analyzers +export VISUALTEXT_ANALYZERS=/tmp/analyzers VISUALTEXT_ANALYZERS_TAG=v1.9.7 + +cd studio +npm run analyzers # write public/try/analyzers/ and its index.json +npm run dev # http://localhost:5173/try/ +npm test # includes test/try.test.ts +``` + +`npm run build` writes both pages into `dist/`; `python server/app.py --dist dist` serves +the site and the API from one address, and the try page is then at `/try/`. + +The checks that matter run in `studio/deploy/smoke-test.sh`, against the image, under the +restrictions it runs live with: that `/try/` is the try page, that its `index.json` lists +analyzers and says which release they came from, and that one of them runs in debug mode, +keeps a tree after every pass, and can have one of those trees fetched back. Extend that +rather than testing by hand. + +## Open questions + +- **`interactive()` costs us the English parser.** It is in `nlp_run.BLOCKED` with + `exittopopup` and `getpopupdata`, but at both of its call sites in `parse-en-us` it is a + read-only predicate — `ini.nlp` uses it to decide whether to keep a worklog, `buff_out.nlp` + to choose `out.xml` over `cbuf()` — and in both the `false` branch is the *safer* one, with + no `openfile` and an in-memory buffer. Unblocking it would very likely make `parse-en-us` + runnable, and it is the analyzer most people would want to try. It needs someone to decide + that deliberately, and to confirm the run, before the list changes. +- **`VisualText/analyzers` has no LICENSE file.** This page redistributes its contents to + every visitor. `.gitignore` already declines to commit the copy, for the same reason it + declines to commit `analyzer-templates`, but the underlying question is the owner's to + settle. +- **Rate limiting is not written.** Nothing here limits how often `/api/run` can be called. + `--max-runs` caps concurrency and the timeout caps a single run, but a `limit_req` in front + is a prerequisite for opening the page to the internet, not an improvement on it. diff --git a/studio/Dockerfile b/studio/Dockerfile index c76a187..3f21f08 100644 --- a/studio/Dockerfile +++ b/studio/Dockerfile @@ -14,8 +14,14 @@ # can be labelled with what went into it. ARG VISUALTEXT_FILES_TAG=latest +# Which VisualText/analyzers release the try page's analyzers come from: the same +# analyzers.zip VisualText downloads as its examples, so /try/ offers the set a +# VisualText user has. update.sh pins the tag it resolved. +ARG VISUALTEXT_ANALYZERS_TAG=latest + FROM node:22-slim AS site ARG VISUALTEXT_FILES_TAG +ARG VISUALTEXT_ANALYZERS_TAG RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl unzip \ @@ -36,6 +42,21 @@ RUN set -eu; \ rm /tmp/visualtext.zip; \ test -d /visualtext/analyzer-templates +# VisualText's own analyzers, for the try page. The zip holds the five bundled entries +# (corporate, files, parse-en-us, nlp-tutorials, nlpfix-analyzers); copy-analyzers.mjs +# picks from them. +RUN set -eu; \ + if [ "$VISUALTEXT_ANALYZERS_TAG" = latest ]; then \ + url="https://github.com/VisualText/analyzers/releases/latest/download/analyzers.zip"; \ + else \ + url="https://github.com/VisualText/analyzers/releases/download/$VISUALTEXT_ANALYZERS_TAG/analyzers.zip"; \ + fi; \ + curl -fsSL --retry 3 --retry-delay 2 -o /tmp/analyzers.zip "$url"; \ + mkdir -p /analyzers; \ + unzip -q /tmp/analyzers.zip -d /analyzers; \ + rm /tmp/analyzers.zip; \ + test -d /analyzers/nlp-tutorials + WORKDIR /src/analyzer-views COPY analyzer-views/package.json analyzer-views/package-lock.json ./ RUN npm ci --no-audit --no-fund @@ -45,12 +66,15 @@ WORKDIR /src/studio COPY studio/package.json studio/package-lock.json ./ RUN npm ci --no-audit --no-fund COPY studio/ ./ -ENV ANALYZER_TEMPLATES=/visualtext/analyzer-templates +ENV ANALYZER_TEMPLATES=/visualtext/analyzer-templates \ + VISUALTEXT_ANALYZERS=/analyzers \ + VISUALTEXT_ANALYZERS_TAG=${VISUALTEXT_ANALYZERS_TAG} RUN npm run build FROM python:3.12-slim ARG VISUALTEXT_FILES_TAG +ARG VISUALTEXT_ANALYZERS_TAG ARG NLP_STUDIO_COMMIT=unknown ARG BUILD_DATE=unknown @@ -73,6 +97,7 @@ COPY --from=site /src/studio/dist dist # docker image inspect nlp-studio:app --format '{{json .Config.Labels}}' LABEL org.visualtext.nlp-studio="${NLP_STUDIO_COMMIT}" \ org.visualtext.visualtext-files="${VISUALTEXT_FILES_TAG}" \ + org.visualtext.analyzers="${VISUALTEXT_ANALYZERS_TAG}" \ org.visualtext.built-at="${BUILD_DATE}" USER studio diff --git a/studio/deploy/INSTALL.md b/studio/deploy/INSTALL.md index 3b27854..3e0e0ba 100644 --- a/studio/deploy/INSTALL.md +++ b/studio/deploy/INSTALL.md @@ -65,6 +65,15 @@ restrictions as the live container: the sample analyzer must run to its known ou Nothing is public yet: the container listens on loopback only. +The container runs with `network_mode: host`, so port 3001 has to be free before it +starts — nothing warns you if another service already answers there, and the health +check above would be that service's answer rather than the studio's. Check first: + + ss -ltnp | grep ':3001 ' || echo "3001 is free" + +If something else holds it, give the app another port in `docker-compose.yml` (the +`--port` in its `command`) and in the nginx `proxy_pass`; the two must agree. + ### 3. Route /studio/ in nginx, behind the password — as root The `/studio/` locations are their own file, installed beside the phase-1 one. Put the @@ -93,11 +102,13 @@ as for phase 1. # authenticated first -- this is what would populate a cache curl -sk -o /dev/null -w '%{http_code}\n' -u "$U" $S/studio/ # 200 curl -sk -u "$U" $S/studio/api/health # {"ok": true, ...} + curl -sk -o /dev/null -w '%{http_code}\n' -u "$U" $S/studio/try/ # 200 # anonymous IMMEDIATELY after: 401, never a cached 200 curl -sk -o /dev/null -w '%{http_code}\n' $S/studio/ # 401 curl -sk -o /dev/null -w '%{http_code}\n' $S/studio/api/health # 401 curl -sk -o /dev/null -w '%{http_code}\n' -X POST $S/studio/api/run # 401 + curl -sk -o /dev/null -w '%{http_code}\n' $S/studio/try/ # 401 # phase 1 is unchanged curl -sk -o /dev/null -w '%{http_code}\n' -u "$U" $S/ # 302 (to ?folder=/home/examples) @@ -201,6 +212,44 @@ signs everyone out. The app can keep its sign-in settings; behind the password, people sign in with both. +## The try page + +`/studio/try/` is a second page in the same container, served from the same `dist` and +using the same `/studio/api/`. It lists eight of VisualText's own analyzers, baked into +the image at build time from a pinned `VisualText/analyzers` release, and lets whoever +opens it run one on text they type. Nothing about it is separately installed or started: +if `/studio/` works, it works. + +What it offers and how the catalog is built: [docs/TRY-PAGE.md](../../docs/TRY-PAGE.md). + +Which release went in is on the image: + + docker image inspect nlp-studio:app --format '{{index .Config.Labels "org.visualtext.analyzers"}}' + +`update.sh` treats that release like the others — when VisualText/analyzers publishes a +new one, the app is out of date and the next run rebuilds with it. + +### Before opening it to people who are not invited + +Today the try page sits behind whatever guards `/studio/` — the site's password, or +GitHub sign-in. It is the one page here that *could* be opened wider, because it does not +run a visitor's code: the analyzers are ours, fixed at build time, and the page offers no +way to edit the NLP++. That argument is the only thing separating it from +`docs/ARCHITECTURE.md`, *The run server is not a sandbox*, so before it is opened: + +- **`/api/run` is still the general endpoint.** It accepts any `files` a caller sends. A + location that lets anonymous requests reach `/studio/api/run` is a remote code + execution endpoint on the host that serves visualtext.org, whatever the page does. Open + `/studio/try/` and the static files under it; do not open `/studio/api/run` without the + per-run sandbox that `ARCHITECTURE.md` describes as not built. +- **Rate limits.** A run costs a CPU second and a process. `limit_req` on the run endpoint, + sized against `--max-runs`, before it is reachable from the internet. +- **The page must stay read-only.** If it ever accepts a pasted grammar or an analyzer + chosen by URL, the reasoning above stops holding and the sandbox becomes a prerequisite + again. +- **Verify anonymously**, in the order §4 uses: authenticated first, anonymous immediately + after. A cached 200 is the failure to look for. + ## Keeping it current The **Deploy NLP-Studio** workflow — the one that updates phase 1 on every upstream diff --git a/studio/deploy/smoke-test.sh b/studio/deploy/smoke-test.sh index 57244b7..4768749 100755 --- a/studio/deploy/smoke-test.sh +++ b/studio/deploy/smoke-test.sh @@ -59,6 +59,16 @@ index="$(curl -s "$BASE/analyzers/index.json")" echo "$index" | grep -q '"hello-studio"' || fail "analyzers/index.json does not list hello-studio" echo " page and analyzer list: ok ($(echo "$index" | grep -c '"origin"') analyzers)" +# The try page (try/index.html) and the analyzers baked in for it. They are what the +# page can run, so an image that lost them would serve an empty picker. +curl -s "$BASE/try/" | grep -q 'Try an analyzer' || fail "/try/ is not the try page" +try_index="$(curl -s "$BASE/try/analyzers/index.json")" +try_count="$(echo "$try_index" | grep -c '"pinned"')" +[ "$try_count" -gt 0 ] || fail "try/analyzers/index.json lists no analyzers" +echo "$try_index" | grep -qE '"(release|commit)": "[^n]' \ + || fail "the try analyzers say no release or commit they came from" +echo " try page and its analyzers: ok ($try_count analyzers)" + # The runs, from inside the container as its own user, so the sample's files come # from the image that is being tested. docker exec -i "$NAME" python - "$PORT" <<'PY' || fail "runs" @@ -94,4 +104,37 @@ if os.path.exists(target): print(f" a run cannot write to /app: ok (that run: {r['status']})") PY +# The try page's own path: one of VisualText's analyzers, run in debug mode, keeps a +# tree after every pass, and each of those trees can be fetched back. That is what puts +# the two icons on a pass, so if this breaks the page loses the feature silently. +docker exec -i "$NAME" python - "$PORT" <<'PY' || fail "try-page run" +import json, pathlib, sys, urllib.parse, urllib.request + +base = f"http://127.0.0.1:{sys.argv[1]}" +index = json.load(urllib.request.urlopen(f"{base}/try/analyzers/index.json", timeout=30))["analyzers"] +entry = min(index, key=lambda a: len(a["files"])) # the quickest one to run +root = pathlib.Path("dist/try/analyzers") / entry["name"] +files = {f: (root / f).read_text(encoding="utf-8", errors="replace") for f in entry["files"]} +text = files.get(entry["input"], "") + +body = json.dumps({"files": files, "text": text, "develop": True}).encode("utf-8") +req = urllib.request.Request(base + "/api/run", data=body, headers={"Content-Type": "application/json"}) +with urllib.request.urlopen(req, timeout=120) as res: + r = json.load(res) +if r["status"] != "ok": + sys.exit(f"{entry['name']} did not run: {json.dumps(r)[:600]}") + +trees = r.get("trees") or {} +per_pass = [t for t in trees.get("files", []) if t["pass"] is not None] +if not per_pass: + sys.exit(f"a debug run kept no per-pass tree: {json.dumps(trees)[:400]}") + +name = per_pass[0]["name"] +query = urllib.parse.urlencode({"run": trees["run"], "name": name}) +tree = urllib.request.urlopen(f"{base}/api/run/tree?{query}", timeout=30).read().decode("utf-8", "replace") +if not tree.strip(): + sys.exit(f"{name} came back empty") +print(f" try run: ok, {entry['name']} in {r['ms']} ms, {len(per_pass)} pass trees, {name} fetched") +PY + echo "smoke-test: PASS" diff --git a/studio/deploy/update.sh b/studio/deploy/update.sh index 1c3d7c3..72fd66f 100755 --- a/studio/deploy/update.sh +++ b/studio/deploy/update.sh @@ -3,7 +3,7 @@ # Keep the NLP Studio app (studio.visualtext.org/studio/) current. # # The image is built from this checkout's studio/ and analyzer-views/, plus the analyzer templates of -# the latest visualtext-files release, and is rebuilt when either moves: a new +# the latest visualtext-files and analyzers releases, and is rebuilt when any moves: a new # commit here (after a deliberate git pull on the server) or a new release. Same # shape as stopgap/scripts/update-studio.sh: build a candidate, smoke-test it away # from the live container, only then swap, and roll back if it does not answer. @@ -72,6 +72,12 @@ WANT_VT_FILES="$(curl -fsSL --retry 3 --retry-delay 2 "${auth[@]}" \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["tag_name"])')" \ || die "cannot reach GitHub (visualtext-files)" +WANT_ANALYZERS="$(curl -fsSL --retry 3 --retry-delay 2 "${auth[@]}" \ + -H 'Accept: application/vnd.github+json' \ + https://api.github.com/repos/VisualText/analyzers/releases/latest \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["tag_name"])')" \ + || die "cannot reach GitHub (analyzers)" + WANT_COMMIT="$(git -C "$REPO" rev-parse --short=12 HEAD)" || die "$REPO is not a git checkout" # Edits on the server that were never committed are labelled as such, so --check # and the image labels do not claim a commit the image does not match. @@ -85,10 +91,11 @@ label() { docker image inspect "$1" --format "{{index .Config.Labels \"$2\"}}" 2 if docker image inspect "$IMAGE_LIVE" >/dev/null 2>&1; then HAVE_COMMIT="$(label "$IMAGE_LIVE" org.visualtext.nlp-studio)" HAVE_VT_FILES="$(label "$IMAGE_LIVE" org.visualtext.visualtext-files)" + HAVE_ANALYZERS="$(label "$IMAGE_LIVE" org.visualtext.analyzers)" else say "no $IMAGE_LIVE image yet -- treating everything as out of date" fi -: "${HAVE_COMMIT:=none}" "${HAVE_VT_FILES:=none}" +: "${HAVE_COMMIT:=none}" "${HAVE_VT_FILES:=none}" "${HAVE_ANALYZERS:=none}" drift=0 report() { # report <name> <have> <want> @@ -101,6 +108,7 @@ report() { # report <name> <have> <want> } report nlp-studio "$HAVE_COMMIT" "$WANT_COMMIT" report visualtext-files "$HAVE_VT_FILES" "$WANT_VT_FILES" +report analyzers "$HAVE_ANALYZERS" "$WANT_ANALYZERS" if [ "$MODE" = check ]; then [ "$drift" -eq 0 ] && { say "up to date"; exit 0; } @@ -120,6 +128,7 @@ docker build \ --pull \ --build-arg "NLP_STUDIO_COMMIT=$WANT_COMMIT" \ --build-arg "VISUALTEXT_FILES_TAG=$WANT_VT_FILES" \ + --build-arg "VISUALTEXT_ANALYZERS_TAG=$WANT_ANALYZERS" \ --build-arg "BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ -t "$IMAGE_NEW" \ -f "$STUDIO/Dockerfile" \ @@ -181,6 +190,6 @@ say "app is up (200 after ~$((i * 2))s)" docker image rm "$IMAGE_NEW" >/dev/null 2>&1 || true # the tag; :app holds the image docker image prune -f >/dev/null 2>&1 || true -say "updated: nlp-studio $HAVE_COMMIT -> $WANT_COMMIT, visualtext-files $HAVE_VT_FILES -> $WANT_VT_FILES" +say "updated: nlp-studio $HAVE_COMMIT -> $WANT_COMMIT, visualtext-files $HAVE_VT_FILES -> $WANT_VT_FILES, analyzers $HAVE_ANALYZERS -> $WANT_ANALYZERS" say "rollback if needed: docker tag $IMAGE_PREV $IMAGE_LIVE && ${COMPOSE[*]} up -d $SERVICE" say "=== done ===" diff --git a/studio/package.json b/studio/package.json index 5e5cb14..d51866f 100644 --- a/studio/package.json +++ b/studio/package.json @@ -5,9 +5,9 @@ "type": "module", "scripts": { "views": "npm --prefix ../analyzer-views run build", - "predev": "npm run views && node scripts/copy-samples.mjs", + "predev": "npm run views && npm run samples && npm run analyzers", "dev": "vite", - "prebuild": "npm run views && node scripts/copy-samples.mjs", + "prebuild": "npm run views && npm run samples && npm run analyzers", "build": "tsc --noEmit && vite build", "preview": "vite preview", "pretest": "npm run views", @@ -15,7 +15,8 @@ "test:server": "python -m unittest discover -s server", "server": "python server/app.py", "selftest": "python scripts/selftest.py dist", - "samples": "node scripts/copy-samples.mjs" + "samples": "node scripts/copy-samples.mjs", + "analyzers": "node scripts/copy-analyzers.mjs" }, "dependencies": { "@shikijs/core": "4.4.3", diff --git a/studio/scripts/analyzer-files.mjs b/studio/scripts/analyzer-files.mjs new file mode 100644 index 0000000..15eb4a7 --- /dev/null +++ b/studio/scripts/analyzer-files.mjs @@ -0,0 +1,62 @@ +// Which of an analyzer's files travel to a page, and how they are copied. +// +// Shared by copy-samples.mjs (the studio's own samples and the templates) and +// copy-analyzers.mjs (VisualText/analyzers, for the try page), so the two cannot +// drift on what an analyzer is or what goes out with it. +// +// PURE apart from fs: no network, no process state. + +import fs from "node:fs"; +import path from "node:path"; + +// Only the analyzer's own material. An engine run leaves output/, tmp/ and a +// <pass>_log/ behind; none of that describes the analyzer, and output/ would be +// read back as if the page's own run had written it. +export const TRAVELS = ["spec", "kb", "input"]; + +// The run server refuses a request over 8 MB of files (nlp_run.MAX_FILES_BYTES), +// so anything larger could be listed but never run. +export const MAX_FILE_BYTES = 1_000_000; + +export const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, ""); + +// An analyzer is any folder holding spec/analyzer.seq. +export const isAnalyzer = (dir) => fs.existsSync(path.join(dir, "spec", "analyzer.seq")); + +export function filesUnder(dir, rel = "") { + const found = []; + for (const entry of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) { + const r = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + if (entry.name.endsWith("_log") || entry.name === "output" || entry.name === "tmp") continue; + found.push(...filesUnder(dir, r)); + } else if (fs.statSync(path.join(dir, r)).size <= MAX_FILE_BYTES) { + found.push(r); + } + } + return found; +} + +// Copy one analyzer's files under <out>/<slug>/, and describe it for the index. +// Returns null for a folder that is not an analyzer. +export function copyAnalyzer(from, title, origin, out, extra = {}) { + if (!isAnalyzer(from)) return null; + const name = slug(title); + const files = TRAVELS.filter((d) => fs.existsSync(path.join(from, d))) + .flatMap((d) => filesUnder(from, d)) + .sort(); + let bytes = 0; + for (const f of files) { + const dest = path.join(out, name, f); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.copyFileSync(path.join(from, f), dest); + bytes += fs.statSync(path.join(from, f)).size; + } + return { name, title, origin, files, bytes, ...extra }; +} + +// The file whose text the page starts with. Analyzers here name it input/text.txt; +// where they do not, take the first, so the choice is the same on every build. +export function defaultInput(files) { + return files.find((f) => f === "input/text.txt") ?? files.find((f) => f.startsWith("input/")) ?? null; +} diff --git a/studio/scripts/copy-analyzers.mjs b/studio/scripts/copy-analyzers.mjs new file mode 100644 index 0000000..7cf5bf2 --- /dev/null +++ b/studio/scripts/copy-analyzers.mjs @@ -0,0 +1,110 @@ +// Put VisualText's own analyzers where the try page can fetch them, and list them. +// +// node scripts/copy-analyzers.mjs (runs before `npm run dev` and `npm run build`) +// +// Source: VisualText/analyzers, at ../../analyzers or VISUALTEXT_ANALYZERS. Either +// +// the analyzers.zip of one of its releases, unpacked -- what the Dockerfile uses, and +// what VisualText itself downloads as its example analyzers, so the page shows exactly +// the set a VisualText user has; VISUALTEXT_ANALYZERS_TAG names the release, or +// +// a git checkout, for working on this locally. Three of the five entries are +// submodules, so clone with --recurse-submodules or the tutorials and the NLPFix +// analyzers are not there. +// +// The zip holds the five bundled entries only, so anything outside them (business/) +// cannot be picked here. +// +// Output: public/try/analyzers/<slug>/... and public/try/analyzers/index.json, which +// the try page fetches as static files. Nothing is read from GitHub at run time and +// nothing the visitor types becomes NLP++: the grammars that run are these, fixed at +// build time, which is what lets the page be opened up without a per-run sandbox. +// What travels is scripts/analyzer-files.mjs, shared with copy-samples.mjs. +// +// Without the checkout the try page still builds and says it has no analyzers. + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { copyAnalyzer, defaultInput } from "./analyzer-files.mjs"; + +const studio = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const out = path.join(studio, "public", "try", "analyzers"); +const repo = process.env.VISUALTEXT_ANALYZERS || path.resolve(studio, "..", "..", "analyzers"); + +// Eight of the twenty in the bundle, each showing something the others do not, and each +// confirmed to run through the run server. What is left out, and why: +// +// parse-en-us refused: it calls interactive(), which nlp_run.BLOCKED +// lists with the desktop app's popups. 4.6 MB besides. +// nlp-tutorials/tutorial-01 26 MB of engine .kb dumps, over the run server's 8 MB +// limit, for a sequence of one pass. +// the remaining tutorials they run, but each overlaps one of the eight below. +const PICKED = [ + ["corporate", "Corporate", + "Companies, money and events, with pronouns resolved back to what they refer to."], + ["nlp-tutorials/tutorial-07", "Regions", + "Where NLP++ code, functions and rules live in a pass — and what a code-only pass matches."], + ["nlpfix-analyzers/date-time", "Dates and Times", + "Dates and times in many formats, where a regular expression needs rewriting for each."], + ["nlpfix-analyzers/formatting", "Formatting", + "Recovers headings, lists and tables from text that lost its formatting."], + ["nlpfix-analyzers/nlp", "Entities", + "Gathers what a text says about each person or thing into one record."], + ["nlp-tutorials/tutorial-02", "Variables", + "The five NLP++ variables — N, S, X, G and L — on a short résumé."], + ["nlp-tutorials/tutorial-08", "Pronouns", + "Builds a knowledge base from the text as it reads, then resolves pronouns with it."], + ["nlp-tutorials/tutorial-15", "Ambiguity", + "Choosing between readings of the same words, with a dictionary and a .kbb file."], +]; + +// The release the analyzers came from, when the build was given one. +const release = process.env.VISUALTEXT_ANALYZERS_TAG || null; + +// The commit each analyzer's files came from, so the page can say what it is showing. +// A submodule is its own repository, so ask in the analyzer's own folder. An unpacked +// release zip is not a checkout and has none; there the release tag says what it is. +function commitOf(dir) { + try { + // The zip keeps the submodules' .git pointer files, so git is asked and complains + // on stderr before failing; the answer is the release tag, not a commit. + return execFileSync("git", ["-C", dir, "rev-parse", "HEAD"], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); + } catch { + return null; + } +} + +fs.rmSync(out, { recursive: true, force: true }); +fs.mkdirSync(out, { recursive: true }); + +const analyzers = []; +if (fs.existsSync(repo)) { + for (const [folder, title, shows] of PICKED) { + const from = path.join(repo, folder); + // "pinned", not "source": in analyzers.ts a source means "fetch this through the + // studio server from GitHub", and these are static files. + const a = copyAnalyzer(from, title, "analyzer", out, { + shows, + pinned: { repo: "VisualText/analyzers", folder, commit: commitOf(from), release }, + }); + if (!a) { + console.warn(`copy-analyzers: no spec/analyzer.seq at ${folder} ` + + "-- clone with --recurse-submodules"); + continue; + } + a.input = defaultInput(a.files); + if (!a.input) console.warn(`copy-analyzers: ${folder} has no input file; the page starts empty`); + analyzers.push(a); + } +} else { + console.log(`copy-analyzers: no analyzers checkout at ${repo}; the try page will have none`); +} + +fs.writeFileSync(path.join(out, "index.json"), JSON.stringify({ analyzers }, null, "\t") + "\n"); +console.log(`copy-analyzers: ${analyzers.length} analyzers, ` + + `${analyzers.reduce((n, a) => n + a.files.length, 0)} files, ` + + `${(analyzers.reduce((n, a) => n + a.bytes, 0) / 1024 / 1024).toFixed(2)} MB -> public/try/analyzers/`); diff --git a/studio/scripts/copy-samples.mjs b/studio/scripts/copy-samples.mjs index d40ac0b..1140f60 100644 --- a/studio/scripts/copy-samples.mjs +++ b/studio/scripts/copy-samples.mjs @@ -15,11 +15,15 @@ // // Output: public/analyzers/<slug>/... and public/analyzers/index.json. Only // spec/, kb/ and input/ travel, never output/, tmp/ or an engine run's *_log/. +// What travels and how it is copied is scripts/analyzer-files.mjs, shared with +// copy-analyzers.mjs. import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { copyAnalyzer, slug } from "./analyzer-files.mjs"; + const studio = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const out = path.join(studio, "public", "analyzers"); const templates = process.env.ANALYZER_TEMPLATES @@ -29,38 +33,8 @@ const templates = process.env.ANALYZER_TEMPLATES const PICKED_TEMPLATES = [ "Telephone Numbers", "Email Addresses", "Date and Times", "Knowledge Base", "Bare Minimum", ]; -const TRAVELS = ["spec", "kb", "input"]; -const MAX_FILE_BYTES = 1_000_000; - -export const slug = (name) => name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, ""); -function filesUnder(dir, rel = "") { - const found = []; - for (const entry of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) { - const r = rel ? `${rel}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - if (entry.name.endsWith("_log") || entry.name === "output" || entry.name === "tmp") continue; - found.push(...filesUnder(dir, r)); - } else if (fs.statSync(path.join(dir, r)).size <= MAX_FILE_BYTES) { - found.push(r); - } - } - return found; -} - -function copyAnalyzer(from, title, origin) { - if (!fs.existsSync(path.join(from, "spec", "analyzer.seq"))) return null; - const name = slug(title); - const files = TRAVELS.filter((d) => fs.existsSync(path.join(from, d))) - .flatMap((d) => filesUnder(from, d)) - .sort(); - for (const f of files) { - const dest = path.join(out, name, f); - fs.mkdirSync(path.dirname(dest), { recursive: true }); - fs.copyFileSync(path.join(from, f), dest); - } - return { name, title, origin, files }; -} +export { slug }; fs.rmSync(out, { recursive: true, force: true }); fs.mkdirSync(out, { recursive: true }); @@ -69,13 +43,13 @@ const analyzers = []; const samples = path.join(studio, "samples"); for (const entry of fs.readdirSync(samples, { withFileTypes: true })) { if (entry.isDirectory()) { - const a = copyAnalyzer(path.join(samples, entry.name), entry.name, "sample"); + const a = copyAnalyzer(path.join(samples, entry.name), entry.name, "sample", out); if (a) analyzers.push(a); } } if (fs.existsSync(templates)) { for (const title of PICKED_TEMPLATES) { - const a = copyAnalyzer(path.join(templates, title), title, "template"); + const a = copyAnalyzer(path.join(templates, title), title, "template", out); if (a) analyzers.push(a); } } else { diff --git a/studio/src/styles.css b/studio/src/styles.css index 8c7cde9..c8c67bf 100644 --- a/studio/src/styles.css +++ b/studio/src/styles.css @@ -1,53 +1,7 @@ -:root { - --bg: #f7f7f5; - --panel: #ffffff; - --ink: #1d1f21; - --muted: #6b6f76; - --line: #dcdcd8; - --on: #e3ecfb; - --on-ink: #1f4fa3; - --accent: #1f6feb; - --accent-ink: #ffffff; - --ok: #1a7f37; - --error: #b3261e; - --warning: #8a5a00; - color-scheme: light; -} -@media (prefers-color-scheme: dark) { - :root:not([data-theme="light"]) { - --bg: #17181a; --panel: #1e1f22; --ink: #e4e4e4; --muted: #9aa0a6; --line: #34363a; - --on: #243553; --on-ink: #a9c7ff; --accent: #2f81f7; --accent-ink: #ffffff; --ok: #56d364; - --error: #f28b82; --warning: #fdd663; - color-scheme: dark; - } -} -:root[data-theme="dark"] { - --bg: #17181a; --panel: #1e1f22; --ink: #e4e4e4; --muted: #9aa0a6; --line: #34363a; - --on: #243553; --on-ink: #a9c7ff; --accent: #2f81f7; --accent-ink: #ffffff; --ok: #56d364; - --error: #f28b82; --warning: #fdd663; - color-scheme: dark; -} +@import "./theme.css"; -* { box-sizing: border-box; } -html, body { margin: 0; height: 100%; } -body { background: var(--bg); color: var(--ink); font: 14px/1.4 system-ui, -apple-system, "Segoe UI", sans-serif; } -.mono { font-family: ui-monospace, "Cascadia Code", Consolas, monospace; } -.muted { color: var(--muted); } -.grow { flex: 1; } -[hidden] { display: none !important; } - -.app { display: flex; flex-direction: column; height: 100vh; } -.bar { display: flex; align-items: center; gap: 10px; padding: 8px 14px; background: var(--panel); - border-bottom: 1px solid var(--line); flex-wrap: wrap; } -.brand { font-size: 15px; } -.pick { display: inline-flex; align-items: center; gap: 6px; } /* A GitHub analyzer's name carries its repository; capped, so the header stays on one line. */ #analyzer { max-width: min(280px, 40vw); text-overflow: ellipsis; } -select, button { font: inherit; color: var(--ink); background: var(--panel); border: 1px solid var(--line); - border-radius: 6px; padding: 4px 8px; } -button { cursor: pointer; } -button:disabled { cursor: default; opacity: .55; } -button.run { background: var(--accent); border-color: var(--accent); color: var(--accent-ink); font-weight: 600; padding: 4px 14px; } .layout { flex: 1; display: grid; grid-template-columns: minmax(200px, 280px) 1fr; min-height: 0; } @media (max-width: 720px) { .layout { grid-template-columns: 1fr; grid-template-rows: 30vh 1fr; } } diff --git a/studio/src/theme.css b/studio/src/theme.css new file mode 100644 index 0000000..10c250e --- /dev/null +++ b/studio/src/theme.css @@ -0,0 +1,59 @@ +/* The palette, the resets and the plain controls every studio page shares. + * + * Both pages import this: src/styles.css (the editor) and src/try/try.css (the try + * page), so a colour is changed once. Light by default, dark with the system unless + * <html data-theme="light">, and either way forced by <html data-theme>. + * + * @visualtext/analyzer-views/style.css follows the same data-theme, so the lists and + * the code colouring match without being told twice. + */ + +:root { + --bg: #f7f7f5; + --panel: #ffffff; + --ink: #1d1f21; + --muted: #6b6f76; + --line: #dcdcd8; + --on: #e3ecfb; + --on-ink: #1f4fa3; + --accent: #1f6feb; + --accent-ink: #ffffff; + --ok: #1a7f37; + --error: #b3261e; + --warning: #8a5a00; + color-scheme: light; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --bg: #17181a; --panel: #1e1f22; --ink: #e4e4e4; --muted: #9aa0a6; --line: #34363a; + --on: #243553; --on-ink: #a9c7ff; --accent: #2f81f7; --accent-ink: #ffffff; --ok: #56d364; + --error: #f28b82; --warning: #fdd663; + color-scheme: dark; + } +} +:root[data-theme="dark"] { + --bg: #17181a; --panel: #1e1f22; --ink: #e4e4e4; --muted: #9aa0a6; --line: #34363a; + --on: #243553; --on-ink: #a9c7ff; --accent: #2f81f7; --accent-ink: #ffffff; --ok: #56d364; + --error: #f28b82; --warning: #fdd663; + color-scheme: dark; +} + +* { box-sizing: border-box; } +html, body { margin: 0; height: 100%; } +body { background: var(--bg); color: var(--ink); font: 14px/1.4 system-ui, -apple-system, "Segoe UI", sans-serif; } +.mono { font-family: ui-monospace, "Cascadia Code", Consolas, monospace; } +.muted { color: var(--muted); } +.grow { flex: 1; } +[hidden] { display: none !important; } + +.app { display: flex; flex-direction: column; height: 100vh; } +.bar { display: flex; align-items: center; gap: 10px; padding: 8px 14px; background: var(--panel); + border-bottom: 1px solid var(--line); flex-wrap: wrap; } +.brand { font-size: 15px; } +.pick { display: inline-flex; align-items: center; gap: 6px; } + +select, button { font: inherit; color: var(--ink); background: var(--panel); border: 1px solid var(--line); + border-radius: 6px; padding: 4px 8px; } +button { cursor: pointer; } +button:disabled { cursor: default; opacity: .55; } +button.run { background: var(--accent); border-color: var(--accent); color: var(--accent-ink); font-weight: 600; padding: 4px 14px; } diff --git a/studio/src/try/main.ts b/studio/src/try/main.ts new file mode 100644 index 0000000..95711ea --- /dev/null +++ b/studio/src/try/main.ts @@ -0,0 +1,321 @@ +// "Try an analyzer": pick one of VisualText's analyzers, run it on some text, and see +// what it did -- the passes, the knowledge base, what the run wrote, the parse trees, +// and for a -DEV run, each pass's own tree and what its rules matched. +// +// The analyzers are baked in at build time by scripts/copy-analyzers.mjs, from a pinned +// commit of VisualText/analyzers. The visitor edits the text, never the NLP++, so the +// only code the engine runs is ours: see deploy/INSTALL.md on why that matters. +// +// The lists are @visualtext/analyzer-views, the same elements the NLP++ extension's +// views are built from, so a pass is numbered and a knowledge base is filtered here +// exactly as they are in VS Code. + +import "@visualtext/analyzer-views/style.css"; +import "@visualtext/analyzer-views"; +import type { NlpCode, NlpKnowledgeBase, NlpLog, NlpOutput, NlpSequence, NlpTrees, NlpValues, OpenDetail } + from "@visualtext/analyzer-views"; +import { ruleMatches } from "@visualtext/analyzer-views/rules"; + +import { fetchTree, outputFiles, runAnalyzer, serverHealth, type RunResult } from "../run/api"; +import { passTrees, resultTone, valuesOutput, viewTrees } from "./result"; +import "./try.css"; + +// The page is served at <site>/try/, so its analyzers sit beside it and the run server +// is one level up, shared with the studio. Both are relative: the built site works from +// any folder or host (vite.config.ts sets base "./"). +const ANALYZERS = "analyzers"; +const API = "../api"; + +// What scripts/copy-analyzers.mjs wrote. Not AnalyzerEntry from ../analyzers: that one's +// `source` means "fetch it from GitHub through the studio server", and these are static. +interface TryEntry { + name: string; + title: string; + shows: string; + files: string[]; + bytes: number; + input: string | null; + pinned: { repo: string; folder: string; commit: string | null; release: string | null }; +} + +const byId = <T extends HTMLElement>(id: string) => document.getElementById(id) as T; + +const els = { + analyzer: byId<HTMLSelectElement>("analyzer"), + run: byId<HTMLButtonElement>("run"), + debug: byId<HTMLInputElement>("debug"), + status: byId<HTMLElement>("status"), + theme: byId<HTMLButtonElement>("theme"), + shows: byId<HTMLElement>("shows"), + pinned: byId<HTMLElement>("pinned"), + sequence: byId<NlpSequence>("sequence"), + kb: byId<NlpKnowledgeBase>("kb"), + path: byId<HTMLElement>("path"), + pathNote: byId<HTMLElement>("path-note"), + code: byId<NlpCode>("code"), + text: byId<HTMLTextAreaElement>("text"), + resetText: byId<HTMLButtonElement>("reset-text"), + message: byId<HTMLElement>("result-message"), + values: byId<NlpValues>("values"), + output: byId<NlpOutput>("output"), + trees: byId<NlpTrees>("trees"), + log: byId<NlpLog>("log"), +}; + +// What is open now. `ranText` is kept apart from the box on purpose: the matches of a +// pass are marked over the text that actually ran, not over what has been typed since. +const state = { + entry: null as TryEntry | null, + files: new Map<string, string>(), // the analyzer's own files, by path + result: null as RunResult | null, + ranText: "", + busy: false, +}; + +// ---- showing one file ------------------------------------------------------------- + +function show(path: string, text: string, note = "", language?: string): void { + els.path.textContent = path; + els.pathNote.textContent = note; + if (language) els.code.setAttribute("language", language); + else els.code.removeAttribute("language"); + els.code.path = language ? null : path; + els.code.text = text; + els.code.line = null; +} + +function showAnalyzerFile(path: string, line?: number): void { + const text = state.files.get(path); + if (text === undefined) return; + show(path, text); + els.sequence.selected = path; + els.kb.selected = path; + if (line) els.code.line = line; +} + +function showRunFile(name: string): void { + const text = state.result?.output?.[name]; + if (text === undefined) return; + show(`output/${name}`, text, "written by this run"); + els.sequence.selected = null; + els.kb.selected = null; +} + +// A tree is not sent with the result -- they get large -- so it is fetched when opened. +async function withTree(name: string, then: (tree: string) => void): Promise<void> { + const run = state.result?.trees?.run; + if (!run) return; + show(name, "Fetching…", ""); + try { + then(await fetchTree(run, name, API)); + } catch (error) { + show(name, "", "", undefined); + els.message.textContent = `${name}: ${error instanceof Error ? error.message : String(error)}`; + els.message.className = "result-message bad"; + } +} + +// ---- loading an analyzer ---------------------------------------------------------- + +async function loadIndex(): Promise<TryEntry[]> { + const res = await fetch(`${ANALYZERS}/index.json`, { cache: "no-store" }); + if (!res.ok) throw new Error(`analyzers/index.json: HTTP ${res.status} -- run npm run analyzers`); + return ((await res.json()) as { analyzers: TryEntry[] }).analyzers; +} + +async function loadFiles(entry: TryEntry): Promise<Map<string, string>> { + const texts = await Promise.all(entry.files.map(async (path) => { + const url = `${ANALYZERS}/${entry.name}/${path.split("/").map(encodeURIComponent).join("/")}`; + const res = await fetch(url); + return [path, res.ok ? await res.text() : ""] as const; + })); + return new Map(texts); +} + +function clearResults(): void { + state.result = null; + state.ranText = ""; + els.message.textContent = ""; + els.message.className = "result-message"; + els.values.output = null; + els.output.files = []; + els.trees.trees = []; + els.trees.skipped = []; + els.sequence.trees = []; + els.log.problems = []; + els.log.lines = []; +} + +async function openAnalyzer(entry: TryEntry): Promise<void> { + state.entry = entry; + state.files = await loadFiles(entry); + clearResults(); + + const paths = [...state.files.keys()]; + const encoder = new TextEncoder(); + els.sequence.files = paths; + els.sequence.sequence = state.files.get("spec/analyzer.seq") ?? ""; + // Bytes, not characters: a dictionary of accented words is longer than its length. + els.kb.files = paths.map((path) => ({ path, bytes: encoder.encode(state.files.get(path) ?? "").length })); + + els.shows.textContent = entry.shows; + // Say which copy of the analyzer this is: a release tag when it came from the + // bundle, a commit when it came from a checkout. + const { repo, folder, commit, release } = entry.pinned; + const at = release ?? (commit ? commit.slice(0, 8) : null); + els.pinned.textContent = at ? `${repo}/${folder} at ${at}` : `${repo}/${folder}`; + + els.text.value = entry.input ? state.files.get(entry.input) ?? "" : ""; + els.resetText.disabled = !entry.input; + showAnalyzerFile("spec/analyzer.seq"); +} + +// ---- running ---------------------------------------------------------------------- + +function showResult(result: RunResult): void { + state.result = result; + + const ms = typeof result.ms === "number" ? ` (${result.ms} ms)` : ""; + els.message.textContent = result.status === "ok" ? `${result.message}${ms}` : result.message; + const tone = resultTone(result.status); + els.message.className = tone ? `result-message ${tone}` : "result-message"; + + els.output.files = outputFiles(result); + const values = valuesOutput(result); + els.values.output = values.output; + // An output.json the analyzer wrote but no one can parse is worth saying out loud. + if (values.error) { + els.message.textContent = `${els.message.textContent} ${values.error}`.trim(); + els.message.className = "result-message warn"; + } + + const trees = viewTrees(result); + els.trees.trees = trees; + els.trees.skipped = result.trees?.skipped ?? []; + els.sequence.trees = passTrees(trees); + + els.log.problems = result.problems ?? []; + els.log.lines = result.log ?? []; +} + +async function run(): Promise<void> { + if (state.busy || !state.entry) return; + state.busy = true; + els.run.disabled = true; + els.run.classList.add("running"); + els.status.textContent = "Running…"; + els.status.className = "muted grow"; + + const text = els.text.value; + const files = Object.fromEntries(state.files); + try { + const result = await runAnalyzer(files, text, { develop: els.debug.checked }, API); + state.ranText = text; // what the matches will be marked over + showResult(result); + els.status.textContent = state.entry.title; + } finally { + state.busy = false; + els.run.disabled = false; + els.run.classList.remove("running"); + } +} + +// ---- wiring ----------------------------------------------------------------------- + +function wire(): void { + // A pass or a dictionary. + for (const list of [els.sequence, els.kb]) { + list.addEventListener("nlp-open", (e) => { + showAnalyzerFile((e as CustomEvent<OpenDetail>).detail.path); + }); + } + + // A pass's own tree, and the text with what its rules matched -- the two icons a + // -DEV run puts on every pass, as the extension's sequence view does. + els.sequence.addEventListener("nlp-open-tree", (e) => { + const { path } = (e as CustomEvent<OpenDetail>).detail; + void withTree(path, (tree) => show(path, tree, "parse tree after this pass", "tree")); + }); + els.sequence.addEventListener("nlp-open-matches", (e) => { + const { path, pass } = (e as CustomEvent<OpenDetail>).detail; + void withTree(path, (tree) => { + // Marked over the text that ran, not over the box, which may have been edited. + const marked = ruleMatches(tree, state.ranText); + show(`pass ${pass ?? "?"} matches`, marked, "what this pass's rules matched", "txxt"); + }); + }); + + // What the run wrote, and the trees it kept. + els.output.addEventListener("nlp-open", (e) => { + showRunFile((e as CustomEvent<OpenDetail>).detail.path); + }); + els.trees.addEventListener("nlp-open", (e) => { + const { path } = (e as CustomEvent<OpenDetail>).detail; + void withTree(path, (tree) => show(path, tree, "", "tree")); + }); + + // A problem opens its pass at its line. + els.log.addEventListener("nlp-open", (e) => { + const { path, line } = (e as CustomEvent<OpenDetail>).detail; + if (path) showAnalyzerFile(path, line); + }); + + els.analyzer.addEventListener("change", () => { + const entry = entries.find((a) => a.name === els.analyzer.value); + if (entry) void openAnalyzer(entry); + }); + els.run.addEventListener("click", () => void run()); + els.resetText.addEventListener("click", () => { + if (state.entry?.input) els.text.value = state.files.get(state.entry.input) ?? ""; + }); + document.addEventListener("keydown", (e) => { + if (e.key === "F5" && !e.ctrlKey && !e.metaKey) { e.preventDefault(); void run(); } + }); + els.theme.addEventListener("click", () => { + const dark = matchMedia("(prefers-color-scheme: dark)").matches; + const now = document.documentElement.dataset.theme ?? (dark ? "dark" : "light"); + document.documentElement.dataset.theme = now === "dark" ? "light" : "dark"; + }); +} + +// ---- start ------------------------------------------------------------------------ + +let entries: TryEntry[] = []; + +async function start(): Promise<void> { + wire(); + try { + entries = await loadIndex(); + } catch (error) { + els.status.textContent = error instanceof Error ? error.message : String(error); + els.status.className = "grow bad"; + return; + } + if (entries.length === 0) { + els.status.textContent = "No analyzers were built into this page."; + els.status.className = "grow bad"; + return; + } + + els.analyzer.replaceChildren(...entries.map((a) => { + const option = document.createElement("option"); + option.value = a.name; + option.textContent = a.title; + return option; + })); + await openAnalyzer(entries[0]); + + // Say plainly when there is no run server: the page still shows every analyzer, it + // just cannot run one, and a page that says nothing here is the bug we keep refixing. + const health = await serverHealth(API); + if (health) { + els.run.disabled = false; + els.status.textContent = `${entries[0].title} · engine ${health.engine ?? "?"}`; + } else { + els.status.textContent = "No run server answered, so nothing can be run here. " + + "The analyzers below can still be read."; + els.status.className = "grow bad"; + } +} + +void start(); diff --git a/studio/src/try/result.ts b/studio/src/try/result.ts new file mode 100644 index 0000000..322a438 --- /dev/null +++ b/studio/src/try/result.ts @@ -0,0 +1,51 @@ +// Turning a run's result into what the try page's views take. +// +// PURE: no DOM, no fetch. The two translations here are easy to get quietly wrong -- +// a tree list that loses its sizes, or a pass offered a tree it has no business with -- +// and neither shows up as an error, only as a page that looks slightly off. So they +// live here with tests (test/try.test.ts) rather than inline in main.ts. + +import { readOutput, type TreeFile as ViewTree } from "@visualtext/analyzer-views/rules"; + +import type { RunResult, RunStatus } from "../run/api"; + +// The run server calls a tree's size `size`; the views call it `bytes`. One of the two +// names has to be translated, and this is the only place that does it. +export function viewTrees(result: Pick<RunResult, "trees"> | null | undefined): ViewTree[] { + return (result?.trees?.files ?? []).map((tree) => ({ + name: tree.name, + pass: tree.pass, + passName: tree.passName, + bytes: tree.size, + })); +} + +// The trees that give a pass its two icons. final.tree was written after the last pass +// but belongs to none, so handing it to <nlp-sequence> would put the icons on whichever +// pass happened to be numbered null. +export function passTrees(trees: readonly ViewTree[]): ViewTree[] { + return trees.filter((tree) => tree.pass !== null); +} + +// How loudly to say what happened. A run that ran and found problems is the analyzer's +// news, not the page's; anything else is the page failing to do what was asked. +export function resultTone(status: RunStatus): "" | "warn" | "bad" { + if (status === "ok") return ""; + return status === "failed" ? "warn" : "bad"; +} + +// What <nlp-values> lists, when the analyzer wrote an output.json at all. Most of +// VisualText's analyzers write .txt and .kbb files instead, and then there is nothing to +// list and the element hides itself. +// +// readOutput() answers { output } or { error }, while the element wants the parsed value +// itself -- handing it the wrapper would list a field called "output.greetings" instead +// of "greetings". An output.json that is not JSON is the analyzer saying something worth +// repeating, so it comes back as a sentence rather than being dropped. +export function valuesOutput(result: Pick<RunResult, "output"> | null | undefined): + { output: unknown; error: string | null } { + const text = result?.output?.["output.json"]; + if (text === undefined) return { output: null, error: null }; + const read = readOutput(text); + return "error" in read ? { output: null, error: read.error } : { output: read.output, error: null }; +} diff --git a/studio/src/try/try.css b/studio/src/try/try.css new file mode 100644 index 0000000..c2b59a7 --- /dev/null +++ b/studio/src/try/try.css @@ -0,0 +1,82 @@ +@import "../theme.css"; + +/* The try page: three columns that fill the window, each scrolling on its own so the + * page itself never does. Under 900px they stack, and the window scrolls instead. + * + * The lists are @visualtext/analyzer-views; everything here only places and colours + * them. Their own style.css draws the rows and the icons. + */ + +.layout3 { + flex: 1; + display: grid; + grid-template-columns: minmax(230px, 300px) minmax(0, 1.2fr) minmax(320px, 1fr); + min-height: 0; +} + +.col { display: flex; flex-direction: column; min-height: 0; min-width: 0; background: var(--panel); } +.col + .col { border-left: 1px solid var(--line); } + +.col-head { + display: flex; align-items: center; gap: 8px; + padding: 6px 10px; border-bottom: 1px solid var(--line); + font-size: 12.5px; color: var(--muted); + white-space: nowrap; overflow: hidden; +} +.col-head .mono { overflow: hidden; text-overflow: ellipsis; } +.col-head button { padding: 2px 8px; font-size: 12.5px; } + +/* The lists, in this page's colours (analyzer-views reads these). */ +.lists { + overflow: auto; + padding: 10px; + --nlp-muted: var(--muted); --nlp-hover: var(--on); --nlp-selected: var(--on); + --nlp-selected-ink: var(--on-ink); --nlp-changed: var(--warning); --nlp-error: var(--error); + color: var(--ink); +} +.shows { margin: 0 0 10px; font-size: 12.5px; } +.pinned { margin: 12px 0 0; font-size: 11.5px; } +.pinned a { color: inherit; } + +/* The middle column holds one file, however long: it scrolls, the page does not. */ +.viewer { overflow: hidden; } +.code { flex: 1; min-height: 0; overflow: auto; padding: 8px 10px; } + +/* The text to run, over what the run did. */ +.right { overflow: hidden; } +.text { + flex: 0 0 34%; + min-height: 120px; + resize: vertical; + border: 0; border-bottom: 1px solid var(--line); + padding: 10px; + background: var(--panel); color: var(--ink); + font-size: 13px; line-height: 1.45; +} +.text:focus { outline: 2px solid var(--accent); outline-offset: -2px; } + +.results { + flex: 1; min-height: 0; overflow: auto; + padding: 10px; + --nlp-muted: var(--muted); --nlp-hover: var(--on); --nlp-selected: var(--on); + --nlp-selected-ink: var(--on-ink); --nlp-error: var(--error); + color: var(--ink); +} +/* Every outcome of a run is a sentence, including "no run server". It is always shown. */ +.result-message { margin: 0 0 10px; font-size: 12.5px; } +.result-message:empty { display: none; } +.result-message.bad { color: var(--error); } +.result-message.warn { color: var(--warning); } + +.check { display: inline-flex; align-items: center; gap: 5px; font-size: 13px; } +#status.bad { color: var(--error); } +#run.running { opacity: .7; } + +@media (max-width: 900px) { + .app { height: auto; min-height: 100vh; } + .layout3 { grid-template-columns: 1fr; } + .col + .col { border-left: 0; border-top: 1px solid var(--line); } + .lists, .results { overflow: visible; } + .viewer, .right { overflow: visible; } + .code { max-height: 60vh; } +} diff --git a/studio/test/try.test.ts b/studio/test/try.test.ts new file mode 100644 index 0000000..8f26aa2 --- /dev/null +++ b/studio/test/try.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; + +import type { RunResult } from "../src/run/api"; +import { passTrees, resultTone, valuesOutput, viewTrees } from "../src/try/result"; + +// A debug run of an analyzer with a switched-off pass: the engine keeps final.tree and +// a tree after each pass that actually ran. Pass 3 is "/nlp anaphoraSearch", which keeps +// its number and writes nothing -- as VisualText/analyzers' business analyzer does. +const RESULT = { + status: "ok", + message: "Ran.", + trees: { + run: "r1", + files: [ + { name: "final.tree", pass: null, passName: null, file: null, size: 4200 }, + { name: "ana001.tree", pass: 1, passName: "tokenize", file: null, size: 120 }, + { name: "ana002.tree", pass: 2, passName: "words", file: "spec/words.nlp", size: 340 }, + { name: "ana004.tree", pass: 4, passName: "events", file: "spec/events.nlp", size: 900 }, + ], + skipped: ["ana005.tree"], + }, +} as unknown as RunResult; + +describe("viewTrees", () => { + it("carries the server's size across as the views' bytes", () => { + expect(viewTrees(RESULT)).toEqual([ + { name: "final.tree", pass: null, passName: null, bytes: 4200 }, + { name: "ana001.tree", pass: 1, passName: "tokenize", bytes: 120 }, + { name: "ana002.tree", pass: 2, passName: "words", bytes: 340 }, + { name: "ana004.tree", pass: 4, passName: "events", bytes: 900 }, + ]); + }); + + it("is empty for a run that kept no trees, rather than throwing", () => { + expect(viewTrees({ status: "ok", message: "Ran." } as RunResult)).toEqual([]); + expect(viewTrees(null)).toEqual([]); + }); +}); + +describe("passTrees", () => { + it("leaves final.tree out, so it cannot land on a pass", () => { + expect(passTrees(viewTrees(RESULT)).map((t) => t.name)) + .toEqual(["ana001.tree", "ana002.tree", "ana004.tree"]); + }); + + it("keeps the engine's own numbering, gaps and all", () => { + // 3 is missing because that pass is switched off. Renumbering 4 to 3 here would + // label every tree and problem after it with the wrong pass. + expect(passTrees(viewTrees(RESULT)).map((t) => t.pass)).toEqual([1, 2, 4]); + }); +}); + +describe("resultTone", () => { + it("says nothing about a run that worked", () => { + expect(resultTone("ok")).toBe(""); + }); + + it("treats an analyzer that reported problems as the analyzer's news", () => { + expect(resultTone("failed")).toBe("warn"); + }); + + it("treats everything else as the page failing to do what was asked", () => { + for (const status of ["timeout", "crashed", "rejected", "busy", "invalid", + "unauthorized", "unavailable"] as const) { + expect(resultTone(status)).toBe("bad"); + } + }); +}); + +describe("valuesOutput", () => { + it("unwraps readOutput, so a field is not listed as output.greetings", () => { + const result = { output: { "output.json": '{"greetings": 3}' } } as unknown as RunResult; + expect(valuesOutput(result)).toEqual({ output: { greetings: 3 }, error: null }); + }); + + it("has nothing to list when the analyzer wrote no output.json", () => { + // Most of VisualText's analyzers write .txt and .kbb files instead. + const result = { output: { "anaphora.txt": "..." } } as unknown as RunResult; + expect(valuesOutput(result)).toEqual({ output: null, error: null }); + expect(valuesOutput(null)).toEqual({ output: null, error: null }); + }); + + it("says so when output.json is not JSON, rather than dropping it", () => { + const result = { output: { "output.json": "{oops" } } as unknown as RunResult; + const read = valuesOutput(result); + expect(read.output).toBeNull(); + expect(read.error).toMatch(/not valid JSON/); + }); +}); diff --git a/studio/try/index.html b/studio/try/index.html new file mode 100644 index 0000000..e4f35b8 --- /dev/null +++ b/studio/try/index.html @@ -0,0 +1,65 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <title>Try an analyzer · VisualText + + + +
+
+ Try an analyzer + + + + + + Starting… + +
+ +
+ + + + +
+
+   + +
+ +
+ + +
+
+ Text + +
+ +
+

+ + + + +
+
+
+
+ + + diff --git a/studio/vite.config.ts b/studio/vite.config.ts index d9ebf08..8c22d77 100644 --- a/studio/vite.config.ts +++ b/studio/vite.config.ts @@ -13,6 +13,14 @@ export default defineConfig({ target: "es2022", // Monaco's editor core is one large chunk by nature (~4 MB, ~1 MB gzipped). chunkSizeWarningLimit: 5000, + // Two pages: the studio (Monaco, editing, GitHub) and try/, which reads and runs + // the analyzers baked in by scripts/copy-analyzers.mjs and loads no editor at all. + rollupOptions: { + input: { + studio: new URL("index.html", import.meta.url).pathname, + try: new URL("try/index.html", import.meta.url).pathname, + }, + }, }, worker: { format: "es" }, // ../analyzer-views is linked, not copied, and has its own node_modules; one copy of From 789d84244eae1693e013e985674ee4740bfb6370 Mon Sep 17 00:00:00 2001 From: David de Hilster Date: Thu, 17 Sep 2026 17:43:12 -0700 Subject: [PATCH 2/2] Leave the analyzers' rendered HTML behind The NLPFix analyzers carry a syntax-highlighted rendering of every pass and dictionary beside it -- spec/dates.nlp.html next to spec/dates.nlp -- made for a web page. Nothing here reads them: the sequence comes from analyzer.seq and the knowledge base lists only .dict and .kbb, so they were never listed, never opened and never read by the engine. They were still fetched by every visitor and sent up again in the body of every run. They were 46% of the try page's analyzers by size. Without them the catalog is 188 files and 0.78 MB, down from 259 and 1.45 MB; Entities halves and Dates and Times and Formatting lose two thirds each. All eight still run to the same output, the same problems and the same trees. The rule goes in analyzer-files.mjs, with the rest of what an analyzer is made of, rather than in the try page's script alone: a rendering of a pass is not part of an analyzer whichever page is showing it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/TRY-PAGE.md | 18 +++++++++++------- studio/scripts/analyzer-files.mjs | 9 ++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/TRY-PAGE.md b/docs/TRY-PAGE.md index b4dcc49..43bbc84 100644 --- a/docs/TRY-PAGE.md +++ b/docs/TRY-PAGE.md @@ -70,9 +70,13 @@ The source is either of two things, and the distinction matters: Only `spec/`, `kb/` and `input/` travel — never `output/`, `tmp/` or an engine run's `*_log/`, which are a previous run's leavings and would otherwise be read back as though -this page's run had written them. Files over 1 MB are skipped; see -`studio/scripts/analyzer-files.mjs`, which both copy scripts share so they cannot drift on -what an analyzer is. +this page's run had written them. Files over 1 MB are skipped, and so is any `.html`: the +NLPFix analyzers carry a syntax-highlighted rendering of every pass and dictionary beside +it (`spec/dates.nlp.html` next to `spec/dates.nlp`), made for a web page and read by +nothing here — the sequence comes from `analyzer.seq` and the knowledge base lists only +`.dict` and `.kbb`. They were 46% of the catalog by size, downloaded by every visitor and +sent up again with every run. See `studio/scripts/analyzer-files.mjs`, which both copy +scripts share so they cannot drift on what an analyzer is. `studio/public/try/` is generated and git-ignored. It is not committed: it is build output, and a second copy of another repository besides. @@ -80,15 +84,15 @@ and a second copy of another repository besides. ## Which analyzers, and why Eight of the twenty in the bundle, each showing something the others do not. Together they -come to 259 files and 1.45 MB — small enough to be fetched into a tab. +come to 188 files and 0.78 MB — small enough to be fetched into a tab. | On the page | From | Files | Size | Passes | Shows | |---|---|---|---|---|---| | **Corporate** | `corporate` | 43 | 132K | 30 | Companies, money and events, with pronouns resolved back to what they refer to. The showcase analyzer. | | **Regions** | `nlp-tutorials/tutorial-07` | 10 | 38K | 5 | Where NLP++ code, functions and rules live in a pass — and what a code-only pass matches. | -| **Dates and Times** | `nlpfix-analyzers/date-time` | 48 | 273K | 17 | Dates and times in many formats, where a regular expression needs rewriting for each. Four `.dict` files, and the only one that writes an `output.json`. | -| **Formatting** | `nlpfix-analyzers/formatting` | 66 | 225K | 26 | Recovers headings, lists and tables from text that lost its formatting. | -| **Entities** | `nlpfix-analyzers/nlp` | 45 | 661K | 17 | Gathers what a text says about each person or thing into one record. Writes eight files. | +| **Dates and Times** | `nlpfix-analyzers/date-time` | 26 | 81K | 17 | Dates and times in many formats, where a regular expression needs rewriting for each. Four `.dict` files, and the only one that writes an `output.json`. | +| **Formatting** | `nlpfix-analyzers/formatting` | 36 | 76K | 26 | Recovers headings, lists and tables from text that lost its formatting. | +| **Entities** | `nlpfix-analyzers/nlp` | 26 | 314K | 17 | Gathers what a text says about each person or thing into one record. Writes eight files. | | **Variables** | `nlp-tutorials/tutorial-02` | 17 | 49K | 12 | The five NLP++ variables — N, S, X, G and L — on a short résumé. | | **Pronouns** | `nlp-tutorials/tutorial-08` | 19 | 51K | 14 | Builds a knowledge base from the text as it reads, then resolves pronouns with it. | | **Ambiguity** | `nlp-tutorials/tutorial-15` | 11 | 53K | 4 | Choosing between readings of the same words, with a `.dict` and a `.kbb` file. | diff --git a/studio/scripts/analyzer-files.mjs b/studio/scripts/analyzer-files.mjs index 15eb4a7..50177e6 100644 --- a/studio/scripts/analyzer-files.mjs +++ b/studio/scripts/analyzer-files.mjs @@ -14,6 +14,13 @@ import path from "node:path"; // read back as if the page's own run had written it. export const TRAVELS = ["spec", "kb", "input"]; +// Some analyzers carry a syntax-highlighted rendering of each pass and dictionary +// beside it -- spec/dates.nlp.html next to spec/dates.nlp -- made for a web page, +// not for the engine. Nothing reads them: the sequence comes from analyzer.seq and +// the knowledge base lists only .dict and .kbb. They were 46% of the try page's +// analyzers by size, downloaded by every visitor and sent up again with every run. +const RENDERED = /\.html?$/i; + // The run server refuses a request over 8 MB of files (nlp_run.MAX_FILES_BYTES), // so anything larger could be listed but never run. export const MAX_FILE_BYTES = 1_000_000; @@ -30,7 +37,7 @@ export function filesUnder(dir, rel = "") { if (entry.isDirectory()) { if (entry.name.endsWith("_log") || entry.name === "output" || entry.name === "tmp") continue; found.push(...filesUnder(dir, r)); - } else if (fs.statSync(path.join(dir, r)).size <= MAX_FILE_BYTES) { + } else if (!RENDERED.test(entry.name) && fs.statSync(path.join(dir, r)).size <= MAX_FILE_BYTES) { found.push(r); } }