From 28e29395deb28898d664b6f370245a3f4ae5e005 Mon Sep 17 00:00:00 2001 From: Piotr Mankowski Date: Mon, 24 Aug 2026 23:15:45 -0700 Subject: [PATCH 1/4] =?UTF-8?q?feat(demo):=20the=20full=20scenario=20runs?= =?UTF-8?q?=20as=20one=20spec=20=E2=80=94=20e2e=20test=20or=20video=20capt?= =?UTF-8?q?ure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain-language laboratory question becomes checked SQL, is refined in conversation, both results are saved as governed Datasets, a table and a grouped-bar Widget are built over them, a Dashboard collects both, Publish to Superset writes the native bundle, the pinned importer brings it in (the spec runs it, so the Bundle ready -> Imported flip is covered end to end), and the finished dashboard renders in Superset. One spec, two Playwright projects: deterministic asserts with no pacing; demo-video is the same steps with dwells, recorded at 1280x720 for the published cut. The demo never diverges from the test, so the recording stays evidence that the product works. scripts/superset-demo-fixture.mjs remains for SQL Lab work (HIV connection + permalink); the demo itself no longer needs Superset UI driving beyond viewing the imported dashboard. --- catalyst-ui/e2e/full-scenario-demo.plan.json | 161 ++++++++++++ catalyst-ui/e2e/full-scenario-demo.spec.ts | 259 +++++++++++++++++++ catalyst-ui/e2e/support/superset-import.ts | 69 +++++ docs/full-scenario-demo.md | 81 ++++++ scripts/superset-demo-fixture.mjs | 186 +++++++++++++ 5 files changed, 756 insertions(+) create mode 100644 catalyst-ui/e2e/full-scenario-demo.plan.json create mode 100644 catalyst-ui/e2e/full-scenario-demo.spec.ts create mode 100644 catalyst-ui/e2e/support/superset-import.ts create mode 100644 docs/full-scenario-demo.md create mode 100644 scripts/superset-demo-fixture.mjs diff --git a/catalyst-ui/e2e/full-scenario-demo.plan.json b/catalyst-ui/e2e/full-scenario-demo.plan.json new file mode 100644 index 00000000..58e4612e --- /dev/null +++ b/catalyst-ui/e2e/full-scenario-demo.plan.json @@ -0,0 +1,161 @@ +{ + "width": 1280, + "height": 720, + "fps": 25, + "font": "Helvetica Neue", + "footer": "openclinai.org", + "segments": [ + { + "type": "card", + "duration": 4.5, + "kicker": "CATALYST", + "heading": "From a question to a dashboard", + "lines": [ + "Two English sentences become governed SQL,", + "and the results become a live Superset dashboard." + ] + }, + { + "type": "clip", + "from": "source-selected", + "to": "question-typed", + "kind": "read", + "caption": "The Catalyst workbench, pointed at an OpenELIS laboratory database." + }, + { + "type": "card", + "duration": 3.5, + "kicker": "STEP 1", + "heading": "Ask in plain language", + "lines": ["\"Show viral load results since 2026-01-01…\""] + }, + { + "type": "clip", + "from": "question-typed", + "to": "generate-clicked", + "kind": "read", + "caption": "No SQL is written by hand — the question is the input." + }, + { + "type": "clip", + "from": "generate-clicked", + "to": "sql-ready-1", + "kind": "wait", + "target_seconds": 8.0, + "caption": "A writer model drafts the SQL; a reviewer checks it against the approved catalog." + }, + { + "type": "clip", + "from": "sql-ready-1", + "to": "dataset-1", + "kind": "read", + "caption": "The draft is SQL you can read and edit — then it runs against the real database." + }, + { + "type": "clip", + "from": "dataset-1", + "to": "dataset-saved-1", + "kind": "read", + "caption": "The result is saved as a governed Dataset, with its query and provenance." + }, + { + "type": "card", + "duration": 3.5, + "kicker": "STEP 2", + "heading": "Refine in conversation", + "lines": ["The follow-up edits the exact query that just ran."] + }, + { + "type": "clip", + "from": "followup-typed", + "to": "generate-clicked-2", + "kind": "read", + "caption": "\"Now count all results by test name instead…\"" + }, + { + "type": "clip", + "from": "generate-clicked-2", + "to": "sql-ready-2", + "kind": "wait", + "target_seconds": 7.0, + "caption": "The models revise the current query rather than starting over." + }, + { + "type": "clip", + "from": "sql-ready-2", + "to": "dataset-saved-2", + "kind": "read", + "caption": "The aggregate runs and is saved as a second Dataset." + }, + { + "type": "card", + "duration": 3.5, + "kicker": "STEP 3", + "heading": "Build the widgets", + "lines": ["A table and a grouped bar, each bound to a saved Dataset."] + }, + { + "type": "clip", + "from": "widget-table", + "to": "widget-bar", + "kind": "read", + "caption": "Widget types are constrained by what the Dataset's columns support." + }, + { + "type": "card", + "duration": 3.5, + "kicker": "STEP 4", + "heading": "Publish the dashboard", + "lines": ["Catalyst writes a native Superset bundle, deterministically."] + }, + { + "type": "clip", + "from": "widget-bar", + "to": "bundle-ready", + "kind": "read", + "caption": "Both widgets on one dashboard — published as a Superset bundle." + }, + { + "type": "clip", + "from": "bundle-ready", + "to": "imported-visible", + "kind": "wait", + "target_seconds": 6.0, + "caption": "A pinned importer brings the bundle into Superset and records a receipt." + }, + { + "type": "clip", + "from": "imported-visible", + "to": "superset-open", + "kind": "wait", + "target_seconds": 4.0, + "caption": "Imported — Catalyst links straight to the live dashboard." + }, + { + "type": "clip", + "from": "superset-open", + "to": "dashboard-rendered", + "kind": "wait", + "target_seconds": 5.0, + "caption": "Superset renders the published dashboard." + }, + { + "type": "clip", + "from": "dashboard-rendered", + "to": "end", + "kind": "read", + "caption": "The finished product: a table and a graph, from one conversation." + }, + { + "type": "card", + "duration": 5.5, + "kicker": "WHAT THIS SHOWED", + "heading": "A conversation became a dashboard", + "lines": [ + "Small open models drafted and reviewed every query.", + "Nothing ran that the catalog did not allow.", + "openclinai.org" + ] + } + ] +} diff --git a/catalyst-ui/e2e/full-scenario-demo.spec.ts b/catalyst-ui/e2e/full-scenario-demo.spec.ts new file mode 100644 index 00000000..b6080a78 --- /dev/null +++ b/catalyst-ui/e2e/full-scenario-demo.spec.ts @@ -0,0 +1,259 @@ +import { expect, test } from "@playwright/test"; +import { DemoMilestones } from "./support/demo-milestones"; +import { openComposer } from "./support/open-composer"; +import { runSupersetImport } from "./support/superset-import"; + +/* + * The full scenario, end to end, through the product's own dashboard path: + * a plain-language laboratory question becomes checked SQL in the Catalyst + * workbench, is refined in conversation, both results are saved as governed + * Datasets, a table Widget and a grouped-bar Widget are built over them, a + * Dashboard collects both and is published as a native Superset bundle, the + * pinned importer brings it in, and the finished dashboard renders in + * Superset — a table and a graph, from two English sentences. + * + * (An earlier draft of this demo hand-carried the SQL into Superset SQL Lab + * and rebuilt the charts there by hand; see the demo issue log for the 20+ + * Superset quirks that cost. This is the path the product actually ships.) + * + * ONE spec, two modes — the same run either way; the Playwright project + * picks it: + * + * e2e (assertions, no video, no dwells): + * PLAYWRIGHT_LIVE=true PLAYWRIGHT_BASE_URL=http://127.0.0.1:13000 \ + * CATALYST_STACK_DIR= \ + * npx playwright test e2e/full-scenario-demo.spec.ts --project=deterministic + * + * video (same steps, paced for camera): + * …same env… npx playwright test e2e/full-scenario-demo.spec.ts --project=demo-video + * + * Recording is not a different script with different steps — a demo that + * diverges from the test stops being evidence that the product works. The + * only difference is `dwell()`, a no-op outside the video project. + */ + +test.setTimeout(1_800_000); + +const DETAIL_DATASET = "Viral load results since Jan 2026"; +const VOLUME_DATASET = "Result volumes by test"; +const TABLE_WIDGET = "Recent viral load results"; +const BAR_WIDGET = "Volumes by test"; +const DASHBOARD = "Laboratory results overview"; + +test("plain-language question to a published Superset dashboard", async ({ + page, +}, info) => { + test.skip( + process.env.PLAYWRIGHT_LIVE !== "true", + "Live-stack scenario; set PLAYWRIGHT_LIVE=true with PLAYWRIGHT_BASE_URL.", + ); + + const filming = info.project.name === "demo-video"; + const timing = new DemoMilestones("full-scenario-demo"); + /** Hold the frame so a viewer can read; nothing at all when testing. */ + const dwell = async (ms: number) => { + if (filming) await page.waitForTimeout(ms); + }; + /** Type visibly on camera, instantly when testing. */ + const type = async ( + locator: ReturnType, + text: string, + ) => { + if (filming) await locator.pressSequentially(text, { delay: 28 }); + else await locator.fill(text); + }; + + /** Save the current turn's dataset draft under a real name. + * + * The cell's "Save to datasets" opens the dataset review panel; only the + * current turn offers it, so the locator is unique by construction. */ + const saveDataset = async (name: string) => { + await page.getByRole("button", { name: "Save to datasets" }).click(); + const nameBox = page.getByPlaceholder(/Dataset from Query v/); + await expect(nameBox).toBeVisible(); + await nameBox.click(); + await type(nameBox, name); + await page.getByRole("button", { name: "Save Dataset" }).click(); + // Saving swaps the draft chrome for the saved entity; wait for the busy + // label to clear before moving on. + await expect(page.getByRole("button", { name: "Saving…" })).toHaveCount(0); + await dwell(1_500); + const close = page.getByRole("button", { name: "Close review panel" }); + if (await close.count()) await close.click(); + }; + + /** Build one widget over a saved dataset. */ + const saveWidget = async ( + name: string, + dataset: string, + visualization: string, + ) => { + await page.getByRole("button", { name: "New Widget" }).click(); + await type(page.getByRole("textbox", { name: "Widget name" }), name); + await page + .getByRole("combobox", { name: "Reads Dataset" }) + .selectOption({ label: dataset }); + await page + .getByRole("combobox", { name: "Visualization" }) + .selectOption({ label: visualization }); + await dwell(2_000); + await page.getByRole("button", { name: "Save Widget" }).click(); + await expect(page.getByRole("heading", { name })).toBeVisible(); + await dwell(1_500); + }; + + // ---- Act 1: the question ------------------------------------------------ + await page.goto("/?dataSource=openelis"); + await expect(page.getByText("Catalyst", { exact: true })).toBeVisible(); + await expect( + page.getByText("OpenELIS Laboratory", { exact: true }).first(), + ).toBeVisible(); + timing.mark("source-selected"); + await dwell(2_500); + + await expect(page.getByLabel("Model profile")).toBeEnabled(); + // The reviewed profile: a 12B writer drafts, a 14B reviewer checks — the + // same lineup the published validation runs use. + await page + .getByLabel("Model profile") + .selectOption("catalyst-query-gemma-4-12b-qwen2.5-14b-checked"); + await type( + page.getByLabel("Question"), + "Show viral load results since 2026-01-01 with patient, value, and observed date", + ); + timing.mark("question-typed"); + await dwell(1_200); + await page.getByRole("button", { name: "Generate query" }).click(); + timing.mark("generate-clicked"); + + await expect( + page.getByRole("heading", { name: /^Refine \[1\]$/ }), + ).toBeVisible({ timeout: 600_000 }); + // Pinned deliberately: only the curated lab fact's column vocabulary + // actually executes against the analytics database. + await expect(page.getByRole("textbox", { name: "SQL query" })).toContainText( + "lab_result_fact_v1", + ); + timing.mark("sql-ready-1"); + await dwell(6_000); + + await page.getByRole("button", { name: "Run query" }).click(); + await expect(page.locator(".query-turn__dataset").first()).toBeVisible({ + timeout: 120_000, + }); + timing.mark("dataset-1"); + await dwell(5_000); + await saveDataset(DETAIL_DATASET); + timing.mark("dataset-saved-1"); + + // ---- Act 2: refine in conversation --------------------------------------- + await openComposer(page); + await type( + page.getByRole("textbox", { name: "Follow-up instruction" }), + "Now count all results by test name instead, with the highest counts first", + ); + timing.mark("followup-typed"); + await dwell(1_200); + await page.getByRole("button", { name: "Generate next query" }).click(); + timing.mark("generate-clicked-2"); + + await expect( + page.getByRole("heading", { name: /^Refine \[2\]$/ }), + ).toBeVisible({ timeout: 600_000 }); + await expect(page.getByRole("textbox", { name: "SQL query" })).toContainText( + "lab_result_fact_v1", + ); + timing.mark("sql-ready-2"); + await dwell(6_000); + + await page.getByRole("button", { name: "Run query" }).click(); + await expect(page.locator(".query-turn__dataset").last()).toBeVisible({ + timeout: 120_000, + }); + timing.mark("dataset-2"); + await dwell(5_000); + await saveDataset(VOLUME_DATASET); + timing.mark("dataset-saved-2"); + + // ---- Act 3: two widgets over the governed datasets ---------------------- + await page.getByRole("button", { name: "Widgets" }).click(); + await dwell(1_500); + await saveWidget(TABLE_WIDGET, DETAIL_DATASET, "Table"); + timing.mark("widget-table"); + await saveWidget(BAR_WIDGET, VOLUME_DATASET, "Grouped bar"); + timing.mark("widget-bar"); + + // ---- Act 4: the dashboard ------------------------------------------------- + await page.getByRole("button", { name: "Dashboards" }).click(); + await dwell(1_200); + await page.getByRole("button", { name: "New Dashboard" }).click(); + await type(page.getByRole("textbox", { name: "Dashboard name" }), DASHBOARD); + await page.getByRole("checkbox", { name: TABLE_WIDGET }).check(); + await page.getByRole("checkbox", { name: BAR_WIDGET }).check(); + await dwell(1_500); + await page.getByRole("button", { name: "Save Dashboard" }).click(); + + const card = page.locator("article").filter({ hasText: DASHBOARD }); + await expect(card).toBeVisible({ timeout: 60_000 }); + // Saving may already mint the bundle; publish explicitly when it hasn't. + if (!(await card.getByText("Superset bundle ready").count())) { + await card.getByRole("button", { name: "Publish to Superset" }).click(); + } + await expect(card.getByText("Superset bundle ready")).toBeVisible({ + timeout: 60_000, + }); + timing.mark("bundle-ready"); + await dwell(4_000); + + // ---- Act 5: the seam — the pinned importer ------------------------------ + // The MVP has no Superset REST publication; a pinned CLI imports the + // bundle and records a receipt, which is what flips the card to Imported. + timing.mark("import-started"); + runSupersetImport(); + timing.mark("imported"); + // The card reads the receipt on navigation. + await page.getByRole("button", { name: "Workbench" }).click(); + await page.getByRole("button", { name: "Dashboards" }).click(); + await expect(card.getByText("Imported", { exact: true })).toBeVisible({ + timeout: 60_000, + }); + timing.mark("imported-visible"); + await dwell(4_000); + + // ---- Act 6: the finished dashboard in Superset -------------------------- + const openLink = card.getByRole("link", { name: "Open Superset" }); + await expect(openLink).toBeVisible(); + const href = await openLink.getAttribute("href"); + if (!href) throw new Error("Open Superset link has no href"); + const supersetBase = + process.env.PLAYWRIGHT_SUPERSET_URL ?? "http://127.0.0.1:18088"; + const dashboardUrl = new URL(new URL(href).pathname, supersetBase).toString(); + + // Sign in to Superset in the same page so the capture stays one video. + await page.goto(`${supersetBase}/login/`); + await page + .locator("#username") + .fill(process.env.SUPERSET_ADMIN_USERNAME ?? "admin"); + await page + .locator("#password") + .fill(process.env.SUPERSET_ADMIN_PASSWORD ?? "admin"); + await page.getByRole("button", { name: /sign in/i }).click(); + await page.waitForLoadState("networkidle"); + await page.goto(dashboardUrl); + timing.mark("superset-open"); + + await expect( + page.getByText(DASHBOARD, { exact: false }).first(), + ).toBeVisible({ timeout: 120_000 }); + // The table widget shows real rows; the bar chart renders on canvas. + await expect(page.getByText("Viral Load").first()).toBeVisible({ + timeout: 120_000, + }); + await expect(page.locator("canvas").first()).toBeVisible({ + timeout: 120_000, + }); + timing.mark("dashboard-rendered"); + await dwell(9_000); + timing.mark("end"); + timing.save(); +}); diff --git a/catalyst-ui/e2e/support/superset-import.ts b/catalyst-ui/e2e/support/superset-import.ts new file mode 100644 index 00000000..249b4ab0 --- /dev/null +++ b/catalyst-ui/e2e/support/superset-import.ts @@ -0,0 +1,69 @@ +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; + +/* + * Run the pinned Superset importer against the live stack — the seam between + * "Superset bundle ready" and "Imported". + * + * The MVP deliberately has no Superset REST publication: Catalyst writes a + * native bundle to the outbox and an operator runs the pinned CLI. This + * helper is that operator, so the e2e run exercises the seam the product + * actually has, and the recording shows the status flip honestly. + * + * Environment it must match (learned the hard way, see the demo issue log): + * - COMPOSE_PROJECT_NAME must be the RUNNING stack's project, or compose + * creates a second network and the importer can't see Superset. + * - SUPERSET_PORT must be the published port (18088 on the isolated stack) — + * the importer stamps it into the receipt's public URL, which becomes the + * product's own "Open Superset" link. Left defaulted, that link 404s. + * - The stack checkout must be the one whose runtime/ the gateway mounts; + * a bundle published by the gateway is invisible to an importer run from + * a different checkout of the same repo. + */ +export const runSupersetImport = (): string => { + const stackDir = resolve( + process.env.CATALYST_STACK_DIR ?? resolve(__dirname, "..", "..", ".."), + ); + const projectName = + process.env.CATALYST_STACK_PROJECT ?? "catalyst-mvp-isolated"; + const supersetPort = process.env.CATALYST_SUPERSET_PORT ?? "18088"; + const overrideFile = process.env.CATALYST_STACK_OVERRIDE ?? ""; + const revision = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: stackDir, + encoding: "utf-8", + }).trim(); + + const composeArgs = [ + "compose", + "--env-file", + ".env", + "-f", + "docker-compose.mvp.yml", + ...(overrideFile ? ["-f", overrideFile] : []), + "--profile", + "superset-import", + "run", + "--rm", + "--no-deps", + "superset-importer", + "import", + ]; + const output = execFileSync("docker", composeArgs, { + cwd: stackDir, + encoding: "utf-8", + env: { + ...process.env, + COMPOSE_PROJECT_NAME: projectName, + SUPERSET_PORT: supersetPort, + CATALYST_IMPORTER_REVISION: revision, + }, + stdio: ["ignore", "pipe", "pipe"], + timeout: 300_000, + }); + const lastLine = output.trim().split("\n").at(-1) ?? ""; + const receipt = JSON.parse(lastLine) as { status: string; dashboardUrl?: string }; + if (receipt.status !== "imported") { + throw new Error(`superset import did not succeed: ${lastLine}`); + } + return receipt.dashboardUrl ?? ""; +}; diff --git a/docs/full-scenario-demo.md b/docs/full-scenario-demo.md new file mode 100644 index 00000000..884e92cb --- /dev/null +++ b/docs/full-scenario-demo.md @@ -0,0 +1,81 @@ +# The full-scenario demo: one spec, two modes + +`catalyst-ui/e2e/full-scenario-demo.spec.ts` walks the product's whole claim +through the product's own path: a plain-language laboratory question becomes +checked SQL in the workbench, is refined in conversation, both results are +saved as governed Datasets, a table Widget and a grouped-bar Widget are built +over them, a Dashboard collects both, `Publish to Superset` writes the native +bundle, the pinned importer brings it in, and the finished dashboard renders +in Superset. + +It runs two ways, and they are **the same steps**: + +| | project | video | dwells | what it is for | +|---|---|---|---|---| +| **e2e** | `deterministic` | off | none | does the whole path still work end to end | +| **video** | `demo-video` | 1280×720 | yes | raw footage for a published cut | + +The only difference is `dwell()`, which holds the frame long enough to read +something and is a no-op outside the video project. A demo that diverges from +the test stops being evidence that the product works. + +## Prerequisites + +The live stack, including Superset (which does **not** come back on its own +after a host restart): + +```sh +docker start catalyst-mvp-isolated-superset-metadata-db catalyst-mvp-isolated-superset +``` + +Two invariants that cost real debugging time when violated (see the demo +issue log for the full stories): + +- **One checkout.** The gateway's outbox/receipts mounts, and the checkout + the importer runs from, must be the same tree. A gateway recreated from a + different worktree silently reads an empty outbox and never shows + `Imported`. +- **The Superset port.** The importer stamps the public URL into its receipt; + that becomes the product's own "Open Superset" link. Run it with + `SUPERSET_PORT` matching the published port (18088 on the isolated stack) + or the link 404s. + +## Running it + +```sh +cd catalyst-ui + +# as a test +PLAYWRIGHT_LIVE=true PLAYWRIGHT_BASE_URL=http://127.0.0.1:13000 \ + CATALYST_STACK_DIR=/targets/catalyst \ + CATALYST_STACK_OVERRIDE=../../compose/catalyst-mvp-isolated.override.yml \ + npx playwright test e2e/full-scenario-demo.spec.ts --project=deterministic + +# as a recording +…same env… npx playwright test e2e/full-scenario-demo.spec.ts --project=demo-video +``` + +The spec runs the pinned importer itself (`e2e/support/superset-import.ts`), +so the "Superset bundle ready → Imported" flip happens on camera and the e2e +mode genuinely covers the seam. + +The recording lands at `test-results/*/video.webm` and is **wiped by the next +run** — copy it out immediately. Milestones land in +`demo-milestones/full-scenario-demo.json`; the published cut's timeline is +authored from them (`scripts/author_timeline.py` in the harness repo, plan in +`e2e/full-scenario-demo.plan.json`) and rendered by +`scripts/render_demo_video.py` — see `specs/demo-video-recording-guide.md` +there. + +## Environment + +| variable | default | meaning | +|---|---|---| +| `PLAYWRIGHT_LIVE` | — | must be `true`; otherwise the spec skips | +| `PLAYWRIGHT_BASE_URL` | `http://127.0.0.1:4173` | the Catalyst UI | +| `PLAYWRIGHT_SUPERSET_URL` | `http://127.0.0.1:18088` | Superset, for the final act | +| `CATALYST_STACK_DIR` | repo root | the RUNNING stack's checkout (targets/catalyst) | +| `CATALYST_STACK_OVERRIDE` | — | compose override file, e.g. the isolated stack's | +| `CATALYST_STACK_PROJECT` | `catalyst-mvp-isolated` | compose project of the running stack | +| `CATALYST_SUPERSET_PORT` | `18088` | published Superset port, stamped into receipts | +| `SUPERSET_ADMIN_USERNAME` / `_PASSWORD` | `admin` / `admin` | Superset sign-in | diff --git a/scripts/superset-demo-fixture.mjs b/scripts/superset-demo-fixture.mjs new file mode 100644 index 00000000..2ba0d54d --- /dev/null +++ b/scripts/superset-demo-fixture.mjs @@ -0,0 +1,186 @@ +#!/usr/bin/env node +/* + * Prepare Superset for the full-scenario demo, and print the permalink key + * the spec needs. + * + * Two things the shipped stack does not provide: + * + * 1. `superset-init.sh` provisions only the OpenELIS analytics connection, so + * the HIV database the workbench queries (catalyst_analytics_hiv) is not + * reachable from Superset at all. This adds it, read-only. + * 2. Selecting a database and schema in SQL Lab is a three-interaction + * popover. A SQL Lab permalink encodes both plus the tab name in one URL, + * which is steadier to drive and reads better on camera. + * + * Idempotent: re-running reuses the existing connection and mints a fresh + * permalink. Usage: + * node scripts/superset-demo-fixture.mjs # prints the key + * SUPERSET_URL=... SUPERSET_ADMIN_PASSWORD=... node scripts/... + */ + +const SUPERSET_URL = process.env.SUPERSET_URL ?? "http://127.0.0.1:18088"; +const USER = process.env.SUPERSET_ADMIN_USERNAME ?? "admin"; +const PASSWORD = process.env.SUPERSET_ADMIN_PASSWORD ?? "admin"; +const DB_NAME = "Catalyst OpenMRS HIV analytics"; +const ANALYTICS_URI = + process.env.CATALYST_SUPERSET_HIV_URI ?? + "postgresql+psycopg2://catalyst_readonly:demo-readonly-change-me@analytics-db:5432/catalyst_analytics_hiv"; + +const login = async () => { + const res = await fetch(`${SUPERSET_URL}/api/v1/security/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + username: USER, + password: PASSWORD, + provider: "db", + refresh: false, + }), + }); + if (!res.ok) throw new Error(`superset login failed: ${res.status}`); + return (await res.json()).access_token; +}; + +/* Superset's API is CSRF-protected even for token auth: the token comes from + * /security/csrf_token/ and must travel with that endpoint's session cookie. */ +const csrf = async (token) => { + const res = await fetch(`${SUPERSET_URL}/api/v1/security/csrf_token/`, { + headers: { Authorization: `Bearer ${token}` }, + }); + const cookie = res.headers.get("set-cookie") ?? ""; + return { csrfToken: (await res.json()).result, cookie: cookie.split(";")[0] }; +}; + +/* Sign in through the HTML login form, the way a browser does, for the + * legacy endpoints that predate the JWT API and only accept a session + * cookie. Returns that cookie plus the CSRF token minted alongside it. */ +const formLogin = async () => { + const page = await fetch(`${SUPERSET_URL}/login/`); + const jar = new Map(); + const collect = (res) => { + for (const raw of res.headers.getSetCookie?.() ?? []) { + const [pair] = raw.split(";"); + const [name, ...rest] = pair.split("="); + jar.set(name.trim(), rest.join("=")); + } + }; + collect(page); + const html = await page.text(); + const token = /name="csrf_token"[^>]*value="([^"]+)"/.exec(html)?.[1] ?? ""; + const cookieHeader = () => + [...jar].map(([name, value]) => `${name}=${value}`).join("; "); + const submit = await fetch(`${SUPERSET_URL}/login/`, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + Cookie: cookieHeader(), + Referer: `${SUPERSET_URL}/login/`, + }, + body: new URLSearchParams({ + username: USER, + password: PASSWORD, + csrf_token: token, + }), + redirect: "manual", + }); + collect(submit); + return { cookie: cookieHeader(), csrfToken: token }; +}; + +const api = async (path, { token, csrfToken, cookie, method = "GET", body }) => { + const res = await fetch(`${SUPERSET_URL}${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-CSRFToken": csrfToken, + Referer: SUPERSET_URL, + ...(cookie ? { Cookie: cookie } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + const text = await res.text(); + if (!res.ok) throw new Error(`${method} ${path} -> ${res.status}: ${text}`); + return text ? JSON.parse(text) : {}; +}; + +const token = await login(); +const { csrfToken, cookie } = await csrf(token); +const auth = { token, csrfToken, cookie }; + +const existing = await api( + `/api/v1/database/?q=(filters:!((col:database_name,opr:eq,value:'${encodeURIComponent(DB_NAME)}')))`, + auth, +); +let dbId = existing.result?.[0]?.id; +if (!dbId) { + const created = await api("/api/v1/database/", { + ...auth, + method: "POST", + body: { + database_name: DB_NAME, + sqlalchemy_uri: ANALYTICS_URI, + expose_in_sqllab: true, + allow_ctas: false, + allow_cvas: false, + allow_dml: false, + }, + }); + dbId = created.id; + console.error(`created database connection ${dbId} (${DB_NAME})`); +} else { + console.error(`reusing database connection ${dbId} (${DB_NAME})`); +} + +/* Each permalink visit opens another SQL Lab tab, and they persist per user. + * After a few takes the tab strip is a row of identical "HIV program + * snapshot" tabs — clutter in the product, and clutter on camera. + * + * Trim them, but never to zero: with no tabs and no active tab, SQL Lab's + * bootstrap never resolves and the page sits on its loading spinner forever + * (a permalink does not rescue it). One tab is kept as the anchor. */ +try { + // The bootstrap payload lists them; /tabstateview/ deletes one. That + // endpoint predates the JWT API and only accepts a Flask session cookie, + // so this signs in the way the browser does. + const bootstrap = await api("/api/v1/sqllab/", auth); + const all = bootstrap.result?.tab_state_ids ?? []; + const tabs = all.slice(0, -1); // keep the last one + if (tabs.length) { + const session = await formLogin(); + for (const tab of tabs) { + const res = await fetch(`${SUPERSET_URL}/tabstateview/${tab.id}`, { + method: "DELETE", + headers: { + Cookie: session.cookie, + "X-CSRFToken": session.csrfToken, + Referer: SUPERSET_URL, + }, + }); + if (!res.ok) throw new Error(`DELETE tab ${tab.id} -> ${res.status}`); + } + console.error(`cleared ${tabs.length} stale SQL Lab tab(s)`); + } + if (!all.length) { + console.error( + "warn: this user has NO SQL Lab tabs — SQL Lab will hang on its " + + "loading spinner. Open /sqllab/ and click 'Add a new tab' once.", + ); + } +} catch (error) { + console.error(`warn: could not clear SQL Lab tabs: ${error.message}`); +} + +const permalink = await api("/api/v1/sqllab/permalink", { + ...auth, + method: "POST", + body: { + dbId, + name: "HIV program snapshot", + schema: "analytics", + sql: "", + }, +}); + +console.error(`permalink: ${permalink.url}`); +process.stdout.write(`${permalink.key}\n`); From f035f625c5550b040162bb5bc6df871d46638749 Mon Sep 17 00:00:00 2001 From: Piotr Mankowski Date: Mon, 24 Aug 2026 23:42:11 -0700 Subject: [PATCH 2/4] fix(demo): the spec survives contact with the product MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight findings from running it against the live stack, each recorded in the demo issue log: - the dataset tile's button is TurnNotebook's 'Save to datasets', not the panel component's aria-label, and the review dialog only closes through its own Close button — the workspace's control sits behind the backdrop - the tucked composer is only restored by the floating 'back to [n] · ask' pill; the toggle is CSS-hidden and the top-right jump control only scrolls - on this catalog the writer projects COUNT(*) without the alias its own structured output promises, and the advisory validation rejects it — so the demo now pins standing guidance ('Alias aggregate columns explicitly…') before refining, which is both the fix and a governance beat worth showing - the Dashboards library never refetches import receipts on navigation; a reload is what flips Bundle ready to Imported --- catalyst-ui/e2e/full-scenario-demo.plan.json | 23 ++++++-- catalyst-ui/e2e/full-scenario-demo.spec.ts | 57 +++++++++++++++++--- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/catalyst-ui/e2e/full-scenario-demo.plan.json b/catalyst-ui/e2e/full-scenario-demo.plan.json index 58e4612e..9aae9c0c 100644 --- a/catalyst-ui/e2e/full-scenario-demo.plan.json +++ b/catalyst-ui/e2e/full-scenario-demo.plan.json @@ -27,7 +27,9 @@ "duration": 3.5, "kicker": "STEP 1", "heading": "Ask in plain language", - "lines": ["\"Show viral load results since 2026-01-01…\""] + "lines": [ + "\"Show viral load results since 2026-01-01…\"" + ] }, { "type": "clip", @@ -58,12 +60,21 @@ "kind": "read", "caption": "The result is saved as a governed Dataset, with its query and provenance." }, + { + "type": "clip", + "from": "dataset-saved-1", + "to": "guidance-pinned", + "kind": "read", + "caption": "Standing guidance is pinned to the session — every later turn must honor it." + }, { "type": "card", "duration": 3.5, "kicker": "STEP 2", "heading": "Refine in conversation", - "lines": ["The follow-up edits the exact query that just ran."] + "lines": [ + "The follow-up edits the exact query that just ran." + ] }, { "type": "clip", @@ -92,7 +103,9 @@ "duration": 3.5, "kicker": "STEP 3", "heading": "Build the widgets", - "lines": ["A table and a grouped bar, each bound to a saved Dataset."] + "lines": [ + "A table and a grouped bar, each bound to a saved Dataset." + ] }, { "type": "clip", @@ -106,7 +119,9 @@ "duration": 3.5, "kicker": "STEP 4", "heading": "Publish the dashboard", - "lines": ["Catalyst writes a native Superset bundle, deterministically."] + "lines": [ + "Catalyst writes a native Superset bundle, deterministically." + ] }, { "type": "clip", diff --git a/catalyst-ui/e2e/full-scenario-demo.spec.ts b/catalyst-ui/e2e/full-scenario-demo.spec.ts index b6080a78..e25a3d1d 100644 --- a/catalyst-ui/e2e/full-scenario-demo.spec.ts +++ b/catalyst-ui/e2e/full-scenario-demo.spec.ts @@ -1,6 +1,5 @@ import { expect, test } from "@playwright/test"; import { DemoMilestones } from "./support/demo-milestones"; -import { openComposer } from "./support/open-composer"; import { runSupersetImport } from "./support/superset-import"; /* @@ -78,8 +77,15 @@ test("plain-language question to a published Superset dashboard", async ({ // label to clear before moving on. await expect(page.getByRole("button", { name: "Saving…" })).toHaveCount(0); await dwell(1_500); - const close = page.getByRole("button", { name: "Close review panel" }); - if (await close.count()) await close.click(); + // Close the review DIALOG through its own button: the workspace's + // "Close review panel" control sits behind the dialog's backdrop, so a + // click on it never lands while the panel is open. + await page + .getByRole("dialog") + .getByRole("button", { name: "Close" }) + .first() + .click(); + await expect(page.getByRole("dialog")).toHaveCount(0); }; /** Build one widget over a saved dataset. */ @@ -146,11 +152,46 @@ test("plain-language question to a published Superset dashboard", async ({ await saveDataset(DETAIL_DATASET); timing.mark("dataset-saved-1"); + /** Reach the follow-up composer the way a person does. + * + * After the dataset-review detour the composer is tucked: its restore + * toggle is CSS-hidden and the top-right jump control only scrolls. The + * affordance that actually restores it is the floating "↓ back to [n] · + * ask" pill, whose click sets the composer mode directly. + */ + const ensureComposerOpen = async () => { + const composer = page.locator("#refine-openelis"); + if ((await composer.getAttribute("data-mode")) === "full") return; + const jumpPill = page.locator(".turn-composer__jump"); + const toggle = page.locator("#refine-openelis-toggle"); + if (await jumpPill.isVisible()) await jumpPill.click(); + else if (await toggle.isVisible()) await toggle.click(); + await expect(composer).toHaveAttribute("data-mode", "full"); + }; + + // ---- Act 1½: pin standing guidance --------------------------------------- + // On this catalog the writer reliably projects COUNT(*) without the alias + // its own structured output promises, and the advisory validation rejects + // the mismatch. Guidance pinning is the product's answer: a standing + // instruction every later turn must honor — and a beat worth showing. + const pinBox = page.getByPlaceholder(/Pin guidance/); + await pinBox.click(); + await type( + pinBox, + "Alias aggregate columns explicitly, e.g. COUNT(*) AS count.", + ); + await page.getByRole("button", { name: "Pin", exact: true }).click(); + await expect( + page.getByText("Alias aggregate columns explicitly", { exact: false }).first(), + ).toBeVisible(); + timing.mark("guidance-pinned"); + await dwell(2_500); + // ---- Act 2: refine in conversation --------------------------------------- - await openComposer(page); + await ensureComposerOpen(); await type( page.getByRole("textbox", { name: "Follow-up instruction" }), - "Now count all results by test name instead, with the highest counts first", + "Now count the results by test name instead, highest count first", ); timing.mark("followup-typed"); await dwell(1_200); @@ -211,8 +252,10 @@ test("plain-language question to a published Superset dashboard", async ({ timing.mark("import-started"); runSupersetImport(); timing.mark("imported"); - // The card reads the receipt on navigation. - await page.getByRole("button", { name: "Workbench" }).click(); + // The library only refetches receipts on a fresh load — tab navigation + // keeps the stale publication state, so the flip never shows without it. + await page.reload(); + await expect(page.getByText("Catalyst", { exact: true })).toBeVisible(); await page.getByRole("button", { name: "Dashboards" }).click(); await expect(card.getByText("Imported", { exact: true })).toBeVisible({ timeout: 60_000, From d6dd9a49a312553a9171c86b3ea7af98d03f9499 Mon Sep 17 00:00:00 2001 From: Piotr Mankowski Date: Tue, 25 Aug 2026 00:10:04 -0700 Subject: [PATCH 3/4] fix(ui): the guidance composer leaves the workbench surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin bar sat unstyled at the top of the thread — truncated placeholder, orphaned button, and nothing on the page saying what it was for. The owner's read on seeing it: ugly, malformed, no use case visible. Session guidance stays an API-level capability (the harness pins through the gateway and asserts it lands; the demo spec now pins the same way), and the composer can return when it has a designed home. The demo plan also says much less: one-word step cards, captions only where the footage cannot speak for itself. --- catalyst-ui/e2e/full-scenario-demo.plan.json | 96 ++++++------------- catalyst-ui/e2e/full-scenario-demo.spec.ts | 40 +++++--- .../features/query/QueryWorkspace.test.tsx | 69 ------------- .../src/features/query/QueryWorkspace.tsx | 37 +------ 4 files changed, 57 insertions(+), 185 deletions(-) diff --git a/catalyst-ui/e2e/full-scenario-demo.plan.json b/catalyst-ui/e2e/full-scenario-demo.plan.json index 9aae9c0c..7684bd99 100644 --- a/catalyst-ui/e2e/full-scenario-demo.plan.json +++ b/catalyst-ui/e2e/full-scenario-demo.plan.json @@ -7,170 +7,130 @@ "segments": [ { "type": "card", - "duration": 4.5, + "duration": 3.5, "kicker": "CATALYST", - "heading": "From a question to a dashboard", - "lines": [ - "Two English sentences become governed SQL,", - "and the results become a live Superset dashboard." - ] + "heading": "From a question to a dashboard" }, { "type": "clip", "from": "source-selected", "to": "question-typed", - "kind": "read", - "caption": "The Catalyst workbench, pointed at an OpenELIS laboratory database." - }, - { - "type": "card", - "duration": 3.5, - "kicker": "STEP 1", - "heading": "Ask in plain language", - "lines": [ - "\"Show viral load results since 2026-01-01…\"" - ] + "kind": "read" }, { "type": "clip", "from": "question-typed", "to": "generate-clicked", "kind": "read", - "caption": "No SQL is written by hand — the question is the input." + "caption": "No SQL written by hand." }, { "type": "clip", "from": "generate-clicked", "to": "sql-ready-1", "kind": "wait", - "target_seconds": 8.0, - "caption": "A writer model drafts the SQL; a reviewer checks it against the approved catalog." + "target_seconds": 12.0, + "caption": "A writer model drafts; a reviewer checks it against the catalog." }, { "type": "clip", "from": "sql-ready-1", "to": "dataset-1", "kind": "read", - "caption": "The draft is SQL you can read and edit — then it runs against the real database." + "caption": "Readable SQL — run against the real database." }, { "type": "clip", "from": "dataset-1", "to": "dataset-saved-1", "kind": "read", - "caption": "The result is saved as a governed Dataset, with its query and provenance." - }, - { - "type": "clip", - "from": "dataset-saved-1", - "to": "guidance-pinned", - "kind": "read", - "caption": "Standing guidance is pinned to the session — every later turn must honor it." + "caption": "Saved as a governed Dataset." }, { "type": "card", - "duration": 3.5, + "duration": 3.0, "kicker": "STEP 2", - "heading": "Refine in conversation", - "lines": [ - "The follow-up edits the exact query that just ran." - ] + "heading": "Refine" }, { "type": "clip", "from": "followup-typed", "to": "generate-clicked-2", - "kind": "read", - "caption": "\"Now count all results by test name instead…\"" + "kind": "read" }, { "type": "clip", "from": "generate-clicked-2", "to": "sql-ready-2", "kind": "wait", - "target_seconds": 7.0, - "caption": "The models revise the current query rather than starting over." + "target_seconds": 11.0, + "caption": "The models revise the current query." }, { "type": "clip", "from": "sql-ready-2", "to": "dataset-saved-2", - "kind": "read", - "caption": "The aggregate runs and is saved as a second Dataset." + "kind": "read" }, { "type": "card", - "duration": 3.5, + "duration": 3.0, "kicker": "STEP 3", - "heading": "Build the widgets", - "lines": [ - "A table and a grouped bar, each bound to a saved Dataset." - ] + "heading": "Widgets" }, { "type": "clip", "from": "widget-table", "to": "widget-bar", "kind": "read", - "caption": "Widget types are constrained by what the Dataset's columns support." + "caption": "A table and a bar chart, bound to the saved Datasets." }, { "type": "card", - "duration": 3.5, + "duration": 3.0, "kicker": "STEP 4", - "heading": "Publish the dashboard", - "lines": [ - "Catalyst writes a native Superset bundle, deterministically." - ] + "heading": "Publish" }, { "type": "clip", "from": "widget-bar", "to": "bundle-ready", - "kind": "read", - "caption": "Both widgets on one dashboard — published as a Superset bundle." + "kind": "read" }, { "type": "clip", "from": "bundle-ready", "to": "imported-visible", "kind": "wait", - "target_seconds": 6.0, - "caption": "A pinned importer brings the bundle into Superset and records a receipt." + "target_seconds": 10.0, + "caption": "A pinned importer brings the bundle into Superset." }, { "type": "clip", "from": "imported-visible", "to": "superset-open", "kind": "wait", - "target_seconds": 4.0, - "caption": "Imported — Catalyst links straight to the live dashboard." + "target_seconds": 6.0 }, { "type": "clip", "from": "superset-open", "to": "dashboard-rendered", "kind": "wait", - "target_seconds": 5.0, - "caption": "Superset renders the published dashboard." + "target_seconds": 8.0 }, { "type": "clip", "from": "dashboard-rendered", "to": "end", "kind": "read", - "caption": "The finished product: a table and a graph, from one conversation." + "caption": "One conversation. A live dashboard." }, { "type": "card", - "duration": 5.5, - "kicker": "WHAT THIS SHOWED", - "heading": "A conversation became a dashboard", - "lines": [ - "Small open models drafted and reviewed every query.", - "Nothing ran that the catalog did not allow.", - "openclinai.org" - ] + "duration": 4.5, + "kicker": "OPENCLINAI.ORG", + "heading": "A conversation became a dashboard" } ] } diff --git a/catalyst-ui/e2e/full-scenario-demo.spec.ts b/catalyst-ui/e2e/full-scenario-demo.spec.ts index e25a3d1d..cad01879 100644 --- a/catalyst-ui/e2e/full-scenario-demo.spec.ts +++ b/catalyst-ui/e2e/full-scenario-demo.spec.ts @@ -169,23 +169,33 @@ test("plain-language question to a published Superset dashboard", async ({ await expect(composer).toHaveAttribute("data-mode", "full"); }; - // ---- Act 1½: pin standing guidance --------------------------------------- - // On this catalog the writer reliably projects COUNT(*) without the alias - // its own structured output promises, and the advisory validation rejects - // the mismatch. Guidance pinning is the product's answer: a standing - // instruction every later turn must honor — and a beat worth showing. - const pinBox = page.getByPlaceholder(/Pin guidance/); - await pinBox.click(); - await type( - pinBox, - "Alias aggregate columns explicitly, e.g. COUNT(*) AS count.", + // ---- Act 1½: standing guidance (through the API) ------------------------- + // On this catalog the writer projects COUNT(*) without the alias its own + // structured output promises, and the advisory validation rejects the + // mismatch. Guidance pinning is the product's lever for that; its composer + // bar was removed from the workbench surface, so the pin goes through the + // gateway — the same way the harness pins guidance in validation runs. + const gatewayUrl = + process.env.CATALYST_GATEWAY_URL ?? "http://127.0.0.1:18000"; + const sessionList = await ( + await page.request.get(`${gatewayUrl}/v1/catalyst/workbench/sessions`) + ).json(); + const sessionId = sessionList.sessions?.[0]?.sessionId; + if (!sessionId) throw new Error("no workbench session to pin guidance on"); + const pinned = await page.request.post( + `${gatewayUrl}/v1/catalyst/workbench/sessions/${sessionId}/guidance`, + { + data: { + contractVersion: "catalyst.workbench.guidance.request.v1", + text: "Alias aggregate columns explicitly, e.g. COUNT(*) AS count.", + source: "human", + }, + }, ); - await page.getByRole("button", { name: "Pin", exact: true }).click(); - await expect( - page.getByText("Alias aggregate columns explicitly", { exact: false }).first(), - ).toBeVisible(); + if (!pinned.ok()) { + throw new Error(`guidance pin failed: ${pinned.status()}`); + } timing.mark("guidance-pinned"); - await dwell(2_500); // ---- Act 2: refine in conversation --------------------------------------- await ensureComposerOpen(); diff --git a/catalyst-ui/src/features/query/QueryWorkspace.test.tsx b/catalyst-ui/src/features/query/QueryWorkspace.test.tsx index 8fbf1682..5a5ba6e4 100644 --- a/catalyst-ui/src/features/query/QueryWorkspace.test.tsx +++ b/catalyst-ui/src/features/query/QueryWorkspace.test.tsx @@ -1099,72 +1099,3 @@ describe("Dashboard Builder Ask shell", () => { }); - it("pins session guidance beside the composer and it survives reloads", async () => { - const user = userEvent.setup(); - const client = api(); - const entry = { - entryId: "g-1", - text: "Exclude do_not_perform requests.", - source: "human", - active: true, - }; - client.pinWorkbenchGuidance = vi - .fn() - .mockResolvedValue({ ...session, guidance: [entry] }); - client.unpinWorkbenchGuidance = vi - .fn() - .mockResolvedValue({ ...session, guidance: [] }); - window.localStorage.setItem( - "catalyst.workbench.activeSessionId", - session.sessionId, - ); - render(); - - await user.type( - await screen.findByRole("textbox", { name: "Pin session guidance" }), - "Exclude do_not_perform requests.", - ); - await user.click(screen.getByRole("button", { name: "Pin" })); - - // The pinned instruction is visible, verbatim, with a way to unpin it. - expect( - await screen.findByText("Exclude do_not_perform requests."), - ).toBeVisible(); - expect(client.pinWorkbenchGuidance).toHaveBeenCalledWith( - session.sessionId, - "Exclude do_not_perform requests.", - ); - - await user.click( - screen.getByRole("button", { name: /Unpin: Exclude do_not_perform/ }), - ); - await waitFor(() => - expect( - screen.queryByText("Exclude do_not_perform requests."), - ).not.toBeInTheDocument(), - ); - }); - - it("shows guidance already pinned on a restored session", async () => { - const client = api(); - client.getWorkbenchSession = vi.fn().mockResolvedValue({ - ...session, - guidance: [ - { - entryId: "g-9", - text: "Always exclude cancelled orders.", - source: "human", - active: true, - }, - ], - }); - window.localStorage.setItem( - "catalyst.workbench.activeSessionId", - session.sessionId, - ); - render(); - - expect( - await screen.findByText("Always exclude cancelled orders."), - ).toBeVisible(); - }); diff --git a/catalyst-ui/src/features/query/QueryWorkspace.tsx b/catalyst-ui/src/features/query/QueryWorkspace.tsx index 5ceb6cf6..5e65729c 100644 --- a/catalyst-ui/src/features/query/QueryWorkspace.tsx +++ b/catalyst-ui/src/features/query/QueryWorkspace.tsx @@ -21,7 +21,6 @@ import { type NotebookGrounding, type NotebookTurn, } from "./components/TurnNotebook"; -import { SessionGuidance } from "./components/SessionGuidance"; import { WorkbenchPanel } from "./components/WorkbenchPanel"; import { WorkbenchRail } from "./components/WorkbenchRail"; import { @@ -1032,32 +1031,6 @@ export const QueryWorkspace = ({ } }; - const pinGuidance = async (text: string) => { - if (!workbenchSession || !api.pinWorkbenchGuidance) return; - try { - const updated = await api.pinWorkbenchGuidance( - workbenchSession.sessionId, - text, - ); - setWorkbenchSession(updated); - } catch (error) { - setWorkbenchError(messageFromError(error)); - } - }; - - const unpinGuidance = async (entryId: string) => { - if (!workbenchSession || !api.unpinWorkbenchGuidance) return; - try { - const updated = await api.unpinWorkbenchGuidance( - workbenchSession.sessionId, - entryId, - ); - setWorkbenchSession(updated); - } catch (error) { - setWorkbenchError(messageFromError(error)); - } - }; - const generateNextWorkbenchQuery = async () => { if ( followupBusy || @@ -1672,12 +1645,10 @@ export const QueryWorkspace = ({ {notebookShowing && workbenchSession && workbenchTimeline && ( <> - + {/* Session guidance stays an API-level capability (the harness pins + through the gateway and asserts it lands); the composer bar that + used to sit here was unstyled, unexplained, and confused every + reader of the page, so the surface shipped without it. */} Date: Wed, 26 Aug 2026 00:30:10 -0700 Subject: [PATCH 4/4] test(catalyst): make full scenario evidence honest --- catalyst-ui/e2e/full-scenario-demo.spec.ts | 180 +++++++++++++----- catalyst-ui/e2e/phase1-journeys.spec.ts | 29 ++- catalyst-ui/e2e/support/superset-import.ts | 94 ++++----- .../features/query/QueryWorkspace.test.tsx | 3 - docs/full-scenario-demo.md | 44 ++--- scripts/superset-demo-fixture.mjs | 75 -------- tests/analytics/test_full_scenario_demo.py | 72 +++++++ 7 files changed, 284 insertions(+), 213 deletions(-) create mode 100644 tests/analytics/test_full_scenario_demo.py diff --git a/catalyst-ui/e2e/full-scenario-demo.spec.ts b/catalyst-ui/e2e/full-scenario-demo.spec.ts index cad01879..f6e7a22a 100644 --- a/catalyst-ui/e2e/full-scenario-demo.spec.ts +++ b/catalyst-ui/e2e/full-scenario-demo.spec.ts @@ -1,3 +1,6 @@ +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; import { expect, test } from "@playwright/test"; import { DemoMilestones } from "./support/demo-milestones"; import { runSupersetImport } from "./support/superset-import"; @@ -20,7 +23,7 @@ import { runSupersetImport } from "./support/superset-import"; * * e2e (assertions, no video, no dwells): * PLAYWRIGHT_LIVE=true PLAYWRIGHT_BASE_URL=http://127.0.0.1:13000 \ - * CATALYST_STACK_DIR= \ + * CATALYST_HARNESS_DIR= \ * npx playwright test e2e/full-scenario-demo.spec.ts --project=deterministic * * video (same steps, paced for camera): @@ -33,11 +36,45 @@ import { runSupersetImport } from "./support/superset-import"; test.setTimeout(1_800_000); -const DETAIL_DATASET = "Viral load results since Jan 2026"; -const VOLUME_DATASET = "Result volumes by test"; -const TABLE_WIDGET = "Recent viral load results"; -const BAR_WIDGET = "Volumes by test"; -const DASHBOARD = "Laboratory results overview"; +const DETAIL_DATASET_BASE = "Viral load results since Jan 2026"; +const VOLUME_DATASET_BASE = "Result volumes by test"; +const TABLE_WIDGET_BASE = "Recent viral load results"; +const BAR_WIDGET_BASE = "Volumes by test"; +const DASHBOARD_BASE = "Laboratory results overview"; + +type CohortFixture = { + expected: { + testTypes: number; + viralLoadResults: number; + }; + terminology: { mappings: Array<{ test: string }> }; +}; + +const cohortFixture = JSON.parse( + readFileSync( + resolve( + import.meta.dirname, + "..", + "..", + "analytics", + "openelis", + "catalyst-cohort-v1.json", + ), + "utf-8", + ), +) as CohortFixture; +const EXPECTED_TOP_GROUP = cohortFixture.terminology.mappings.find( + ({ test: testName }) => testName === "Viral Load", +)?.test; +const EXPECTED_TOP_GROUP_COUNT = cohortFixture.expected.viralLoadResults; +const EXPECTED_GROUPS = cohortFixture.expected.testTypes; +if ( + !EXPECTED_TOP_GROUP || + !Number.isInteger(EXPECTED_TOP_GROUP_COUNT) || + !Number.isInteger(EXPECTED_GROUPS) +) { + throw new Error("Catalyst cohort fixture does not define grouped result totals"); +} test("plain-language question to a published Superset dashboard", async ({ page, @@ -49,6 +86,17 @@ test("plain-language question to a published Superset dashboard", async ({ const filming = info.project.name === "demo-video"; const timing = new DemoMilestones("full-scenario-demo"); + // Dashboard Builder state is intentionally retained. A fresh artifact name + // keeps this run separate from every prior take without resetting or + // reseeding the application databases. Callers may supply a meaningful run + // ID for a published take; the default is unique for every invocation. + const runId = process.env.CATALYST_DEMO_RUN_ID?.trim() || randomUUID(); + const runName = (base: string) => `${base} · ${runId}`; + const detailDataset = runName(DETAIL_DATASET_BASE); + const volumeDataset = runName(VOLUME_DATASET_BASE); + const tableWidget = runName(TABLE_WIDGET_BASE); + const barWidget = runName(BAR_WIDGET_BASE); + const dashboard = runName(DASHBOARD_BASE); /** Hold the frame so a viewer can read; nothing at all when testing. */ const dwell = async (ms: number) => { if (filming) await page.waitForTimeout(ms); @@ -144,12 +192,24 @@ test("plain-language question to a published Superset dashboard", async ({ await dwell(6_000); await page.getByRole("button", { name: "Run query" }).click(); - await expect(page.locator(".query-turn__dataset").first()).toBeVisible({ + const detailResult = page.locator(".query-turn__dataset").first(); + await expect(detailResult).toBeVisible({ timeout: 120_000, }); + await expect(detailResult.getByRole("columnheader")).toHaveCount(3); + for (const column of ["patient_id", "result_value", "observed_at"]) { + await expect( + detailResult.getByRole("columnheader", { name: column, exact: true }), + ).toBeVisible(); + } + await expect( + detailResult.getByText("Showing 1–10 of 100 returned rows", { + exact: true, + }), + ).toBeVisible(); timing.mark("dataset-1"); await dwell(5_000); - await saveDataset(DETAIL_DATASET); + await saveDataset(detailDataset); timing.mark("dataset-saved-1"); /** Reach the follow-up composer the way a person does. @@ -169,39 +229,11 @@ test("plain-language question to a published Superset dashboard", async ({ await expect(composer).toHaveAttribute("data-mode", "full"); }; - // ---- Act 1½: standing guidance (through the API) ------------------------- - // On this catalog the writer projects COUNT(*) without the alias its own - // structured output promises, and the advisory validation rejects the - // mismatch. Guidance pinning is the product's lever for that; its composer - // bar was removed from the workbench surface, so the pin goes through the - // gateway — the same way the harness pins guidance in validation runs. - const gatewayUrl = - process.env.CATALYST_GATEWAY_URL ?? "http://127.0.0.1:18000"; - const sessionList = await ( - await page.request.get(`${gatewayUrl}/v1/catalyst/workbench/sessions`) - ).json(); - const sessionId = sessionList.sessions?.[0]?.sessionId; - if (!sessionId) throw new Error("no workbench session to pin guidance on"); - const pinned = await page.request.post( - `${gatewayUrl}/v1/catalyst/workbench/sessions/${sessionId}/guidance`, - { - data: { - contractVersion: "catalyst.workbench.guidance.request.v1", - text: "Alias aggregate columns explicitly, e.g. COUNT(*) AS count.", - source: "human", - }, - }, - ); - if (!pinned.ok()) { - throw new Error(`guidance pin failed: ${pinned.status()}`); - } - timing.mark("guidance-pinned"); - // ---- Act 2: refine in conversation --------------------------------------- await ensureComposerOpen(); await type( page.getByRole("textbox", { name: "Follow-up instruction" }), - "Now count the results by test name instead, highest count first", + "Now replace the detail rows with counts across the full dataset by test name. Call the count column result_count and put the highest count first", ); timing.mark("followup-typed"); await dwell(1_200); @@ -218,37 +250,85 @@ test("plain-language question to a published Superset dashboard", async ({ await dwell(6_000); await page.getByRole("button", { name: "Run query" }).click(); - await expect(page.locator(".query-turn__dataset").last()).toBeVisible({ + const volumeResult = page.locator(".query-turn__dataset").last(); + await expect(volumeResult).toBeVisible({ timeout: 120_000, }); + await expect(volumeResult.getByRole("columnheader")).toHaveCount(2); + for (const column of ["test_name", "result_count"]) { + await expect( + volumeResult.getByRole("columnheader", { name: column, exact: true }), + ).toBeVisible(); + } + const groupedRows = volumeResult.locator("tbody tr"); + await expect(groupedRows).toHaveCount(EXPECTED_GROUPS); + const topGroupRow = groupedRows.first(); + await expect(topGroupRow.getByRole("cell")).toHaveCount(2); + await expect( + topGroupRow.getByRole("cell", { + name: EXPECTED_TOP_GROUP, + exact: true, + }), + ).toBeVisible(); + await expect( + topGroupRow.getByRole("cell", { + name: String(EXPECTED_TOP_GROUP_COUNT), + exact: true, + }), + ).toBeVisible(); timing.mark("dataset-2"); await dwell(5_000); - await saveDataset(VOLUME_DATASET); + await saveDataset(volumeDataset); timing.mark("dataset-saved-2"); // ---- Act 3: two widgets over the governed datasets ---------------------- await page.getByRole("button", { name: "Widgets" }).click(); await dwell(1_500); - await saveWidget(TABLE_WIDGET, DETAIL_DATASET, "Table"); + await saveWidget(tableWidget, detailDataset, "Table"); timing.mark("widget-table"); - await saveWidget(BAR_WIDGET, VOLUME_DATASET, "Grouped bar"); + await saveWidget(barWidget, volumeDataset, "Grouped bar"); timing.mark("widget-bar"); // ---- Act 4: the dashboard ------------------------------------------------- await page.getByRole("button", { name: "Dashboards" }).click(); await dwell(1_200); await page.getByRole("button", { name: "New Dashboard" }).click(); - await type(page.getByRole("textbox", { name: "Dashboard name" }), DASHBOARD); - await page.getByRole("checkbox", { name: TABLE_WIDGET }).check(); - await page.getByRole("checkbox", { name: BAR_WIDGET }).check(); + await type(page.getByRole("textbox", { name: "Dashboard name" }), dashboard); + await page.getByRole("checkbox", { name: tableWidget, exact: true }).check(); + await page.getByRole("checkbox", { name: barWidget, exact: true }).check(); await dwell(1_500); + const savedDashboardResponse = page.waitForResponse( + (response) => + response.request().method() === "POST" && + new URL(response.url()).pathname === + "/v1/catalyst/dashboard-builder/dashboards", + ); await page.getByRole("button", { name: "Save Dashboard" }).click(); + const savedDashboard = (await (await savedDashboardResponse).json()) as { + versionId?: unknown; + }; + if (typeof savedDashboard.versionId !== "string") { + throw new Error("saved Dashboard response did not include its version ID"); + } - const card = page.locator("article").filter({ hasText: DASHBOARD }); + const card = page.locator("article").filter({ hasText: dashboard }); await expect(card).toBeVisible({ timeout: 60_000 }); - // Saving may already mint the bundle; publish explicitly when it hasn't. - if (!(await card.getByText("Superset bundle ready").count())) { - await card.getByRole("button", { name: "Publish to Superset" }).click(); + const publishedDashboardResponse = page.waitForResponse( + (response) => + response.request().method() === "POST" && + new URL(response.url()).pathname === + `/v1/catalyst/dashboard-builder/dashboards/${encodeURIComponent(savedDashboard.versionId as string)}/publish`, + ); + await card.getByRole("button", { name: "Publish to Superset" }).click(); + const publication = (await (await publishedDashboardResponse).json()) as { + pointer?: { bundle?: { sha256?: unknown } }; + }; + const bundleDigest = publication.pointer?.bundle?.sha256; + if ( + typeof bundleDigest !== "string" || + !/^[a-f0-9]{64}$/.test(bundleDigest) + ) { + throw new Error("published Dashboard response did not include a bundle digest"); } await expect(card.getByText("Superset bundle ready")).toBeVisible({ timeout: 60_000, @@ -260,7 +340,7 @@ test("plain-language question to a published Superset dashboard", async ({ // The MVP has no Superset REST publication; a pinned CLI imports the // bundle and records a receipt, which is what flips the card to Imported. timing.mark("import-started"); - runSupersetImport(); + runSupersetImport(bundleDigest); timing.mark("imported"); // The library only refetches receipts on a fresh load — tab navigation // keeps the stale publication state, so the flip never shows without it. @@ -296,7 +376,7 @@ test("plain-language question to a published Superset dashboard", async ({ timing.mark("superset-open"); await expect( - page.getByText(DASHBOARD, { exact: false }).first(), + page.getByText(dashboard, { exact: false }).first(), ).toBeVisible({ timeout: 120_000 }); // The table widget shows real rows; the bar chart renders on canvas. await expect(page.getByText("Viral Load").first()).toBeVisible({ diff --git a/catalyst-ui/e2e/phase1-journeys.spec.ts b/catalyst-ui/e2e/phase1-journeys.spec.ts index df974e2d..1282a21a 100644 --- a/catalyst-ui/e2e/phase1-journeys.spec.ts +++ b/catalyst-ui/e2e/phase1-journeys.spec.ts @@ -1,10 +1,10 @@ import { expect, test, type Page } from "@playwright/test"; -// The three Phase 1 deployed-proof journeys (roadmap G6), run against a live -// stack with the selected team. Deployment and user-flow checks, not model -// scores. Run with: +// Three real-product journeys, run against a live stack with the profile named +// for this run. These verify user-visible behavior, not model scores or a team +// selection. Run with: // PLAYWRIGHT_LIVE=true PLAYWRIGHT_BASE_URL=http://127.0.0.1:13000 \ -// PHASE1_PROFILE= \ +// PHASE1_PROFILE= \ // npx playwright test e2e/phase1-journeys.spec.ts test.setTimeout(1_200_000); @@ -74,30 +74,27 @@ test("journey 2: ambiguous ask -> clarification -> frozen answer -> ready; refre await expect(page.getByText(/SELECT/i).first()).toBeVisible(); }); -test("journey 3: pinned guidance survives reload and is honored; addresses are unsupported", async ({ +test("journey 3: conversation instructions survive reload; addresses are unsupported", async ({ page, }) => { live(); await openHivSession(page); - await ask(page, "Count medication requests by medication name."); + await ask( + page, + "Count medication requests by medication name, excluding do_not_perform requests.", + ); await expect(page.getByText(/SELECT/i).first()).toBeVisible({ timeout: 300_000, }); - // Pin standing guidance, reload, and see it still standing. - await page - .getByRole("textbox", { name: "Pin session guidance" }) - .fill("Exclude do_not_perform requests."); - await page.getByRole("button", { name: "Pin" }).click(); - await expect( - page.getByText("Exclude do_not_perform requests."), - ).toBeVisible(); + // Reload, then continue the same visible conversation. The opening user's + // exclusion remains part of the session history without a hidden control. await page.reload(); await expect( - page.getByText("Exclude do_not_perform requests."), + page.getByText(/excluding do_not_perform requests/i), ).toBeVisible(); - // The later regroup must honor the pin without it being repeated. + // The later regroup must honor that earlier instruction without repeating it. await page .getByRole("textbox", { name: "Follow-up instruction" }) .fill("Regroup that by patient gender as well as medication name."); diff --git a/catalyst-ui/e2e/support/superset-import.ts b/catalyst-ui/e2e/support/superset-import.ts index 249b4ab0..36604829 100644 --- a/catalyst-ui/e2e/support/superset-import.ts +++ b/catalyst-ui/e2e/support/superset-import.ts @@ -1,69 +1,69 @@ import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import { resolve } from "node:path"; /* * Run the pinned Superset importer against the live stack — the seam between - * "Superset bundle ready" and "Imported". + * "Superset bundle ready" and "Imported" — through the Clinical AI + * Validation Harness operator wrapper. The wrapper owns the isolated Compose + * assembly, ports, sibling Hub context, and dependency startup/waits. * * The MVP deliberately has no Superset REST publication: Catalyst writes a * native bundle to the outbox and an operator runs the pinned CLI. This * helper is that operator, so the e2e run exercises the seam the product * actually has, and the recording shows the status flip honestly. * - * Environment it must match (learned the hard way, see the demo issue log): - * - COMPOSE_PROJECT_NAME must be the RUNNING stack's project, or compose - * creates a second network and the importer can't see Superset. - * - SUPERSET_PORT must be the published port (18088 on the isolated stack) — - * the importer stamps it into the receipt's public URL, which becomes the - * product's own "Open Superset" link. Left defaulted, that link 404s. - * - The stack checkout must be the one whose runtime/ the gateway mounts; - * a bundle published by the gateway is invisible to an importer run from - * a different checkout of the same repo. + * CATALYST_HARNESS_DIR must identify the checkout that owns the running + * isolated stack. Its wrapper verifies the exact pinned Catalyst checkout, + * so the importer and Gateway necessarily share one outbox and receipt tree. */ -export const runSupersetImport = (): string => { - const stackDir = resolve( - process.env.CATALYST_STACK_DIR ?? resolve(__dirname, "..", "..", ".."), +export const runSupersetImport = (expectedBundleDigest: string): string => { + if (!/^[a-f0-9]{64}$/.test(expectedBundleDigest)) { + throw new Error("expected Superset bundle digest is not a SHA-256 value"); + } + const configuredHarnessDir = process.env.CATALYST_HARNESS_DIR?.trim(); + if (!configuredHarnessDir) { + throw new Error("CATALYST_HARNESS_DIR is required for the isolated import"); + } + const harnessDir = resolve(configuredHarnessDir); + const currentPointerPath = resolve( + harnessDir, + "targets", + "catalyst", + "runtime", + "superset", + "outbox", + "current.json", ); - const projectName = - process.env.CATALYST_STACK_PROJECT ?? "catalyst-mvp-isolated"; - const supersetPort = process.env.CATALYST_SUPERSET_PORT ?? "18088"; - const overrideFile = process.env.CATALYST_STACK_OVERRIDE ?? ""; - const revision = execFileSync("git", ["rev-parse", "HEAD"], { - cwd: stackDir, - encoding: "utf-8", - }).trim(); - - const composeArgs = [ - "compose", - "--env-file", - ".env", - "-f", - "docker-compose.mvp.yml", - ...(overrideFile ? ["-f", overrideFile] : []), - "--profile", - "superset-import", - "run", - "--rm", - "--no-deps", - "superset-importer", - "import", - ]; - const output = execFileSync("docker", composeArgs, { - cwd: stackDir, + const currentPointer = JSON.parse( + readFileSync(currentPointerPath, "utf-8"), + ) as { bundle?: { sha256?: unknown } }; + if (currentPointer.bundle?.sha256 !== expectedBundleDigest) { + throw new Error( + `Superset outbox points to ${String(currentPointer.bundle?.sha256 ?? "no bundle")}; expected ${expectedBundleDigest}`, + ); + } + const wrapper = resolve(harnessDir, "scripts", "catalyst-mvp.sh"); + const output = execFileSync(wrapper, ["superset-import"], { + cwd: harnessDir, encoding: "utf-8", - env: { - ...process.env, - COMPOSE_PROJECT_NAME: projectName, - SUPERSET_PORT: supersetPort, - CATALYST_IMPORTER_REVISION: revision, - }, + env: process.env, stdio: ["ignore", "pipe", "pipe"], timeout: 300_000, }); const lastLine = output.trim().split("\n").at(-1) ?? ""; - const receipt = JSON.parse(lastLine) as { status: string; dashboardUrl?: string }; - if (receipt.status !== "imported") { + const receipt = JSON.parse(lastLine) as { + status: string; + bundleDigest?: string; + dashboardUrl?: string; + }; + if (!["imported", "already_imported"].includes(receipt.status)) { throw new Error(`superset import did not succeed: ${lastLine}`); } + if (receipt.bundleDigest !== expectedBundleDigest) { + throw new Error( + `superset imported ${receipt.bundleDigest ?? "no bundle"}; expected ${expectedBundleDigest}`, + ); + } return receipt.dashboardUrl ?? ""; }; diff --git a/catalyst-ui/src/features/query/QueryWorkspace.test.tsx b/catalyst-ui/src/features/query/QueryWorkspace.test.tsx index 5a5ba6e4..d768dcb0 100644 --- a/catalyst-ui/src/features/query/QueryWorkspace.test.tsx +++ b/catalyst-ui/src/features/query/QueryWorkspace.test.tsx @@ -206,7 +206,6 @@ const api = (): CatalystApi => ({ ), ), }); - /** * The session control owns both the session list and the data source, so * reaching either means opening it the way a user does. @@ -1097,5 +1096,3 @@ describe("Dashboard Builder Ask shell", () => { }); }); }); - - diff --git a/docs/full-scenario-demo.md b/docs/full-scenario-demo.md index 884e92cb..e3632c23 100644 --- a/docs/full-scenario-demo.md +++ b/docs/full-scenario-demo.md @@ -21,24 +21,25 @@ the test stops being evidence that the product works. ## Prerequisites -The live stack, including Superset (which does **not** come back on its own -after a host restart): +Start and check the isolated stack through the Clinical AI Validation Harness +operator wrapper. `up` retains the existing databases; do not use `boot`, +`seed`, or `reset` between takes: ```sh -docker start catalyst-mvp-isolated-superset-metadata-db catalyst-mvp-isolated-superset +cd +./scripts/catalyst-mvp.sh up +./scripts/catalyst-mvp.sh health ``` -Two invariants that cost real debugging time when violated (see the demo -issue log for the full stories): +The scenario gives every Dataset, Widget, and Dashboard a run-specific name, +so reruns are safe against the retained Dashboard Builder state. It does not +reset or reseed the stack. Omit `CATALYST_DEMO_RUN_ID` for the unique default; +if you set it for a named recording, use a fresh value for every take. -- **One checkout.** The gateway's outbox/receipts mounts, and the checkout - the importer runs from, must be the same tree. A gateway recreated from a - different worktree silently reads an empty outbox and never shows - `Imported`. -- **The Superset port.** The importer stamps the public URL into its receipt; - that becomes the product's own "Open Superset" link. Run it with - `SUPERSET_PORT` matching the published port (18088 on the isolated stack) - or the link 404s. +The same Harness checkout must own both the running Gateway and this run. Its +wrapper verifies the pinned Catalyst checkout and supplies the isolated +override, project name, ports, and sibling Hub context. This is why the demo +accepts the Harness root rather than reconstructing those settings itself. ## Running it @@ -47,17 +48,18 @@ cd catalyst-ui # as a test PLAYWRIGHT_LIVE=true PLAYWRIGHT_BASE_URL=http://127.0.0.1:13000 \ - CATALYST_STACK_DIR=/targets/catalyst \ - CATALYST_STACK_OVERRIDE=../../compose/catalyst-mvp-isolated.override.yml \ + CATALYST_HARNESS_DIR= \ npx playwright test e2e/full-scenario-demo.spec.ts --project=deterministic # as a recording …same env… npx playwright test e2e/full-scenario-demo.spec.ts --project=demo-video ``` -The spec runs the pinned importer itself (`e2e/support/superset-import.ts`), -so the "Superset bundle ready → Imported" flip happens on camera and the e2e -mode genuinely covers the seam. +The spec runs the pinned importer itself (`e2e/support/superset-import.ts`) +through the Harness's supported `scripts/catalyst-mvp.sh superset-import` +wrapper. That wrapper starts and waits for the required Superset services, so +the "Superset bundle ready → Imported" flip happens on camera and the e2e mode +genuinely covers the seam. The recording lands at `test-results/*/video.webm` and is **wiped by the next run** — copy it out immediately. Milestones land in @@ -74,8 +76,6 @@ there. | `PLAYWRIGHT_LIVE` | — | must be `true`; otherwise the spec skips | | `PLAYWRIGHT_BASE_URL` | `http://127.0.0.1:4173` | the Catalyst UI | | `PLAYWRIGHT_SUPERSET_URL` | `http://127.0.0.1:18088` | Superset, for the final act | -| `CATALYST_STACK_DIR` | repo root | the RUNNING stack's checkout (targets/catalyst) | -| `CATALYST_STACK_OVERRIDE` | — | compose override file, e.g. the isolated stack's | -| `CATALYST_STACK_PROJECT` | `catalyst-mvp-isolated` | compose project of the running stack | -| `CATALYST_SUPERSET_PORT` | `18088` | published Superset port, stamped into receipts | +| `CATALYST_HARNESS_DIR` | — | required root of the Harness checkout that owns the running isolated stack | +| `CATALYST_DEMO_RUN_ID` | a random UUID | unique suffix for this run's retained builder artifacts | | `SUPERSET_ADMIN_USERNAME` / `_PASSWORD` | `admin` / `admin` | Superset sign-in | diff --git a/scripts/superset-demo-fixture.mjs b/scripts/superset-demo-fixture.mjs index 2ba0d54d..565baee7 100644 --- a/scripts/superset-demo-fixture.mjs +++ b/scripts/superset-demo-fixture.mjs @@ -51,42 +51,6 @@ const csrf = async (token) => { return { csrfToken: (await res.json()).result, cookie: cookie.split(";")[0] }; }; -/* Sign in through the HTML login form, the way a browser does, for the - * legacy endpoints that predate the JWT API and only accept a session - * cookie. Returns that cookie plus the CSRF token minted alongside it. */ -const formLogin = async () => { - const page = await fetch(`${SUPERSET_URL}/login/`); - const jar = new Map(); - const collect = (res) => { - for (const raw of res.headers.getSetCookie?.() ?? []) { - const [pair] = raw.split(";"); - const [name, ...rest] = pair.split("="); - jar.set(name.trim(), rest.join("=")); - } - }; - collect(page); - const html = await page.text(); - const token = /name="csrf_token"[^>]*value="([^"]+)"/.exec(html)?.[1] ?? ""; - const cookieHeader = () => - [...jar].map(([name, value]) => `${name}=${value}`).join("; "); - const submit = await fetch(`${SUPERSET_URL}/login/`, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - Cookie: cookieHeader(), - Referer: `${SUPERSET_URL}/login/`, - }, - body: new URLSearchParams({ - username: USER, - password: PASSWORD, - csrf_token: token, - }), - redirect: "manual", - }); - collect(submit); - return { cookie: cookieHeader(), csrfToken: token }; -}; - const api = async (path, { token, csrfToken, cookie, method = "GET", body }) => { const res = await fetch(`${SUPERSET_URL}${path}`, { method, @@ -132,45 +96,6 @@ if (!dbId) { console.error(`reusing database connection ${dbId} (${DB_NAME})`); } -/* Each permalink visit opens another SQL Lab tab, and they persist per user. - * After a few takes the tab strip is a row of identical "HIV program - * snapshot" tabs — clutter in the product, and clutter on camera. - * - * Trim them, but never to zero: with no tabs and no active tab, SQL Lab's - * bootstrap never resolves and the page sits on its loading spinner forever - * (a permalink does not rescue it). One tab is kept as the anchor. */ -try { - // The bootstrap payload lists them; /tabstateview/ deletes one. That - // endpoint predates the JWT API and only accepts a Flask session cookie, - // so this signs in the way the browser does. - const bootstrap = await api("/api/v1/sqllab/", auth); - const all = bootstrap.result?.tab_state_ids ?? []; - const tabs = all.slice(0, -1); // keep the last one - if (tabs.length) { - const session = await formLogin(); - for (const tab of tabs) { - const res = await fetch(`${SUPERSET_URL}/tabstateview/${tab.id}`, { - method: "DELETE", - headers: { - Cookie: session.cookie, - "X-CSRFToken": session.csrfToken, - Referer: SUPERSET_URL, - }, - }); - if (!res.ok) throw new Error(`DELETE tab ${tab.id} -> ${res.status}`); - } - console.error(`cleared ${tabs.length} stale SQL Lab tab(s)`); - } - if (!all.length) { - console.error( - "warn: this user has NO SQL Lab tabs — SQL Lab will hang on its " + - "loading spinner. Open /sqllab/ and click 'Add a new tab' once.", - ); - } -} catch (error) { - console.error(`warn: could not clear SQL Lab tabs: ${error.message}`); -} - const permalink = await api("/api/v1/sqllab/permalink", { ...auth, method: "POST", diff --git a/tests/analytics/test_full_scenario_demo.py b/tests/analytics/test_full_scenario_demo.py new file mode 100644 index 00000000..54d2429b --- /dev/null +++ b/tests/analytics/test_full_scenario_demo.py @@ -0,0 +1,72 @@ +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +class FullScenarioDemoContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.spec = ( + ROOT / "catalyst-ui/e2e/full-scenario-demo.spec.ts" + ).read_text() + cls.import_helper = ( + ROOT / "catalyst-ui/e2e/support/superset-import.ts" + ).read_text() + cls.superset_fixture = ( + ROOT / "scripts/superset-demo-fixture.mjs" + ).read_text() + cls.runbook = (ROOT / "docs/full-scenario-demo.md").read_text() + cls.phase1_journeys = ( + ROOT / "catalyst-ui/e2e/phase1-journeys.spec.ts" + ).read_text() + cls.cohort = json.loads( + (ROOT / "analytics/openelis/catalyst-cohort-v1.json").read_text() + ) + + def test_flagship_has_only_visible_user_instructions_and_unique_artifacts(self): + self.assertNotIn("/guidance", self.spec) + self.assertNotIn("page.request", self.spec) + self.assertNotIn("Pin session guidance", self.phase1_journeys) + self.assertNotIn("selected team", self.phase1_journeys) + self.assertIn("excluding do_not_perform requests", self.phase1_journeys) + self.assertIn("CATALYST_DEMO_RUN_ID", self.spec) + self.assertIn("randomUUID()", self.spec) + + def test_result_assertions_use_the_versioned_cohort_contract(self): + self.assertEqual(self.cohort["expected"]["testTypes"], 9) + self.assertEqual(self.cohort["expected"]["viralLoadResults"], 384) + self.assertIn("viralLoadResults", self.spec) + self.assertIn("expected.testTypes", self.spec) + for column in ( + "patient_id", + "result_value", + "observed_at", + "test_name", + "result_count", + ): + self.assertIn(f'"{column}"', self.spec) + + def test_superset_operations_stay_inside_supported_wrappers(self): + self.assertIn('"catalyst-mvp.sh"', self.import_helper) + self.assertIn( + 'execFileSync(wrapper, ["superset-import"]', self.import_helper + ) + self.assertNotIn('execFileSync("docker"', self.import_helper) + self.assertIn('"already_imported"', self.import_helper) + self.assertIn( + "receipt.bundleDigest !== expectedBundleDigest", self.import_helper + ) + self.assertIn("currentPointer.bundle?.sha256", self.import_helper) + self.assertIn("./scripts/catalyst-mvp.sh up", self.runbook) + self.assertNotIn("docker start", self.runbook) + + def test_sql_lab_fixture_does_not_delete_retained_user_tabs(self): + self.assertNotIn("tabstateview", self.superset_fixture) + self.assertNotIn('method: "DELETE"', self.superset_fixture) + + +if __name__ == "__main__": + unittest.main()