From b09f2dc9d62dc8f5084bc2d67ac312bfe23e04f6 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 19:49:10 +0000 Subject: [PATCH 01/12] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/link-assistant/web-capture/issues/141 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..a4c88ad --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-06-25T19:49:09.783Z for PR creation at branch issue-141-f649a07590ac for issue https://github.com/link-assistant/web-capture/issues/141 \ No newline at end of file From 65d7dcc0d51e0e2173fa37b83c97c93a377f6867 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 20:23:09 +0000 Subject: [PATCH 02/12] feat: add shared dialog capture --- .gitkeep | 1 - README.md | 42 +- .../chatgpt-share-6a3825b9.demo_memory.lino | 29 + .../raw-data/chatgpt-share-6a3825b9.html | 11 + .../google-ai-mode-VG0HhpnAXrBkC0QgP.html | 20 + docs/formalai-contract.md | 95 +- js/.changeset/shared-dialog-capture.md | 5 + js/README.md | 16 + js/bin/web-capture.js | 70 +- js/src/index.js | 12 + js/src/shared-dialog-cli.js | 25 + js/src/shared-dialog.js | 950 +++++++++++++++ js/tests/unit/cli.test.js | 46 +- js/tests/unit/shared-dialog.test.js | 173 +++ rust/CHANGELOG.md | 7 + rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- rust/README.md | 6 + rust/src/lib.rs | 1 + rust/src/main.rs | 108 +- rust/src/shared_dialog.rs | 1050 +++++++++++++++++ rust/tests/unit/mod.rs | 1 + rust/tests/unit/shared_dialog.rs | 109 ++ 23 files changed, 2726 insertions(+), 55 deletions(-) delete mode 100644 .gitkeep create mode 100644 docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.demo_memory.lino create mode 100644 docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html create mode 100644 docs/case-studies/issue-141/raw-data/google-ai-mode-VG0HhpnAXrBkC0QgP.html create mode 100644 js/.changeset/shared-dialog-capture.md create mode 100644 js/src/shared-dialog-cli.js create mode 100644 js/src/shared-dialog.js create mode 100644 js/tests/unit/shared-dialog.test.js create mode 100644 rust/src/shared_dialog.rs create mode 100644 rust/tests/unit/shared_dialog.rs diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index a4c88ad..0000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-06-25T19:49:09.783Z for PR creation at branch issue-141-f649a07590ac for issue https://github.com/link-assistant/web-capture/issues/141 \ No newline at end of file diff --git a/README.md b/README.md index eff59fe..5ec0bca 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A CLI and microservice to fetch URLs and render them as: - **ZIP archive**: Markdown/HTML + locally downloaded images - **PDF**: Print-quality document - **DOCX**: Word document +- **Shared AI dialog transcript**: ChatGPT share captures with structured diagnostics for unsupported providers ## Language Implementations @@ -60,6 +61,7 @@ Both implementations expose the same API: | `GET /fetch?url=` | Proxy fetch content | | `GET /stream?url=` | Stream content | | `GET /search?q=` | Capture structured search-provider results | +| `GET /shared-dialog?url=` | Capture shared AI dialog transcripts as JSON, Markdown, text, or Links Notation | FormalAI integration should use the stable HTTP/CLI contract documented in [`docs/formalai-contract.md`](docs/formalai-contract.md). @@ -88,6 +90,9 @@ web-capture https://example.com --format png -o screenshot.png # Create a ZIP archive web-capture https://example.com --archive +# Capture a shared AI dialog transcript as structured JSON +web-capture shared-dialog https://chatgpt.com/share/SHARE_ID + # Start as API server web-capture --serve @@ -97,24 +102,24 @@ web-capture --serve --port 8080 ## CLI Options -| Option | Short | Description | Default | -| ------------------------ | ----- | ----------------------------------------------------------------------------------------------------- | --------------------- | -| `--serve` | `-s` | Start as HTTP API server | - | -| `--port` | `-p` | Port to listen on | 3000 | -| `--format` | `-f` | Output format: `markdown`/`md`, `html`, `txt`/`text`, `image`/`png`, `jpeg`, `pdf`, `docx`, `archive` | `markdown` | -| `--output` | `-o` | Output file path. Use `-o -` for stdout | auto-derived from URL | -| `--data-dir` | | Base directory for auto-derived output paths | `./data/web-capture` | -| `--engine` | `-e` | Browser engine (JS only): `puppeteer`, `playwright` | `puppeteer` | -| `--embed-images` | | Keep images inline as base64 data URIs (self-contained file) | `false` | -| `--no-extract-images` | | Alias for `--embed-images` | `false` | -| `--extract-images[=DIR]` | | Extract images to `DIR/images/` (or next to the output) and download remote images | - | -| `--keep-original-links` | | Keep remote image URLs as direct links (the default markdown behavior) | `false` | -| `--images-dir` | | Subdirectory name for extracted images | `images` | -| `--archive` | | Create archive: `zip` (default), `7z`, `tar.gz`, `tar` | - | -| `--extract-latex` | | Extract LaTeX formulas | `true` | -| `--extract-metadata` | | Extract article metadata | `true` | -| `--post-process` | | Apply post-processing | `true` | -| `--detect-code-language` | | Detect code block languages | `true` | +| Option | Short | Description | Default | +| ------------------------ | ----- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| `--serve` | `-s` | Start as HTTP API server | - | +| `--port` | `-p` | Port to listen on | 3000 | +| `--format` | `-f` | Output format: `markdown`/`md`, `html`, `txt`/`text`, `image`/`png`, `jpeg`, `pdf`, `docx`, `archive`, `json`, `meta-language`, `demo-memory` | `markdown` | +| `--output` | `-o` | Output file path. Use `-o -` for stdout | auto-derived from URL | +| `--data-dir` | | Base directory for auto-derived output paths | `./data/web-capture` | +| `--engine` | `-e` | Browser engine (JS only): `puppeteer`, `playwright` | `puppeteer` | +| `--embed-images` | | Keep images inline as base64 data URIs (self-contained file) | `false` | +| `--no-extract-images` | | Alias for `--embed-images` | `false` | +| `--extract-images[=DIR]` | | Extract images to `DIR/images/` (or next to the output) and download remote images | - | +| `--keep-original-links` | | Keep remote image URLs as direct links (the default markdown behavior) | `false` | +| `--images-dir` | | Subdirectory name for extracted images | `images` | +| `--archive` | | Create archive: `zip` (default), `7z`, `tar.gz`, `tar` | - | +| `--extract-latex` | | Extract LaTeX formulas | `true` | +| `--extract-metadata` | | Extract article metadata | `true` | +| `--post-process` | | Apply post-processing | `true` | +| `--detect-code-language` | | Detect code block languages | `true` | ## Image Handling @@ -168,6 +173,7 @@ Both implementations expose the same API: | `GET /docx?url=` | DOCX with embedded images | | `GET /fetch?url=` | Proxy fetch content | | `GET /stream?url=` | Stream content | +| `GET /shared-dialog?url=` | Shared AI dialog transcript capture with structured diagnostics | ## Docker diff --git a/docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.demo_memory.lino b/docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.demo_memory.lino new file mode 100644 index 0000000..d1cef50 --- /dev/null +++ b/docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.demo_memory.lino @@ -0,0 +1,29 @@ +demo_memory + event "0c9f0151-b5a1-402f-afc3-6bd34a0d01d2" + role "user" + content "box@87ffc301f5eb:~$ sleep 30m && hive-cleanup -f\n\nmake a loop of that (infinite), answer with only single line" + demoLabel "issue-552-chatgpt-share" + conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + conversationTitle "Infinite loop script" + evidence "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + event "fc908946-280f-4e2b-adcc-f27a97538685" + role "assistant" + content "```bash\nwhile true; do sleep 30m && hive-cleanup -f; done\n```" + demoLabel "issue-552-chatgpt-share" + conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + conversationTitle "Infinite loop script" + evidence "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + event "a9f6893d-b2bf-43aa-a55c-9cea559f5190" + role "user" + content "Can we use `screen -R auto-cleanup` to execute that line inside, also making it all using single line?" + demoLabel "issue-552-chatgpt-share" + conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + conversationTitle "Infinite loop script" + evidence "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + event "a5fb0b5c-6831-4ccf-863e-bb0770e65d96" + role "assistant" + content "```bash\nscreen -dmS auto-cleanup bash -c 'while true; do sleep 30m && hive-cleanup -f; done'\n```" + demoLabel "issue-552-chatgpt-share" + conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24" + conversationTitle "Infinite loop script" + evidence "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24" diff --git a/docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html b/docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html new file mode 100644 index 0000000..049390e --- /dev/null +++ b/docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html @@ -0,0 +1,11 @@ +ChatGPT - Infinite loop script
Skip to content

Chat history

\ No newline at end of file diff --git a/docs/case-studies/issue-141/raw-data/google-ai-mode-VG0HhpnAXrBkC0QgP.html b/docs/case-studies/issue-141/raw-data/google-ai-mode-VG0HhpnAXrBkC0QgP.html new file mode 100644 index 0000000..606786f --- /dev/null +++ b/docs/case-studies/issue-141/raw-data/google-ai-mode-VG0HhpnAXrBkC0QgP.html @@ -0,0 +1,20 @@ +Google Search \ No newline at end of file diff --git a/docs/formalai-contract.md b/docs/formalai-contract.md index 63a8464..59f256d 100644 --- a/docs/formalai-contract.md +++ b/docs/formalai-contract.md @@ -13,6 +13,7 @@ The contract covers the shared endpoints requested in issue 135: - `GET /archive` - `GET /stream` - `GET /search` +- `GET /shared-dialog` The JavaScript package is published as `@link-assistant/web-capture`. The Rust crate is published as `web-capture`, but the Rust crate currently declares @@ -33,6 +34,9 @@ The structured response exceptions are: - `/search`, which returns normalized JSON by default. - `/markdown?converter=kreuzberg&format=json`, which returns the structured converter output. +- `/shared-dialog`, which returns normalized transcript JSON by default and + structured unsupported-provider diagnostics when transcript data is blocked + or unavailable. For reproducible FormalAI integration: @@ -47,16 +51,17 @@ For reproducible FormalAI integration: ## HTTP Endpoints -| Endpoint | Required query | Stable success shape | -| ----------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/fetch` | `url` | Proxies upstream status, content type, selected headers, and response bytes. | -| `/stream` | `url` | Streams or proxies upstream status, content type, selected headers, and response bytes. | -| `/html` | `url` | `200`, `Content-Type: text/html; charset=utf-8`, rendered or fetched HTML with relative URLs normalized to absolute URLs where supported. | -| `/txt` | `url` | `200`, `Content-Type: text/plain; charset=utf-8`, `Content-Disposition` attachment, plain text body. | -| `/markdown` | `url` | `200`, `Content-Type: text/markdown`, Markdown body. | -| `/image` | `url` | `200`, `Content-Type: image/png` by default or `image/jpeg` when requested, binary image bytes. | -| `/archive` | `url` | `200`, `Content-Type: application/zip`, ZIP bytes. Default archive contains `document.md` and `document.html`; local assets use relative folders such as `images/`. | -| `/search` | `q` or `query` | `200`, `Content-Type: application/json` by default, normalized search JSON. | +| Endpoint | Required query | Stable success shape | +| ---------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/fetch` | `url` | Proxies upstream status, content type, selected headers, and response bytes. | +| `/stream` | `url` | Streams or proxies upstream status, content type, selected headers, and response bytes. | +| `/html` | `url` | `200`, `Content-Type: text/html; charset=utf-8`, rendered or fetched HTML with relative URLs normalized to absolute URLs where supported. | +| `/txt` | `url` | `200`, `Content-Type: text/plain; charset=utf-8`, `Content-Disposition` attachment, plain text body. | +| `/markdown` | `url` | `200`, `Content-Type: text/markdown`, Markdown body. | +| `/image` | `url` | `200`, `Content-Type: image/png` by default or `image/jpeg` when requested, binary image bytes. | +| `/archive` | `url` | `200`, `Content-Type: application/zip`, ZIP bytes. Default archive contains `document.md` and `document.html`; local assets use relative folders such as `images/`. | +| `/search` | `q` or `query` | `200`, `Content-Type: application/json` by default, normalized search JSON. | +| `/shared-dialog` | `url` | `200`, `Content-Type: application/json` by default, normalized shared-dialog transcript JSON or an unsupported diagnostic. | Common HTTP parameters: @@ -134,6 +139,73 @@ Provider catalog: Provider IDs are a strict allow-list. Unknown providers return `400` over HTTP or a non-zero CLI exit. +## Shared Dialog Contract + +`GET /shared-dialog?url=&format=json|meta-language|demo-memory|markdown|txt` + +CLI equivalent: + +```bash +web-capture shared-dialog "" +web-capture shared-dialog "" --format demo-memory -o - +``` + +JSON response: + +```json +{ + "provider": "chatgpt", + "sourceUrl": "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24", + "captureMethod": "static_http", + "capturedAt": "2026-06-25T00:00:00.000Z", + "status": "ok", + "conversationId": "6a3825b9-8de4-83ee-9c24-52fd1eb38d24", + "title": "Infinite loop script", + "turns": [ + { + "id": "0c9f0151-b5a1-402f-afc3-6bd34a0d01d2", + "role": "user", + "content": "make a loop of that", + "visibility": "visible", + "sourceEvidence": [ + { + "kind": "chatgpt_linear_conversation", + "sourceUrl": "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24", + "captureMethod": "static_http", + "pointer": "linear_conversation[0].message" + } + ] + } + ], + "diagnostics": { + "status": "ok", + "httpStatus": 200, + "warnings": [] + } +} +``` + +Unsupported providers keep the same top-level fields and return: + +```json +{ + "provider": "google_ai_mode", + "sourceUrl": "https://share.google/aimode/VG0HhpnAXrBkC0QgP", + "captureMethod": "browser", + "status": "unsupported", + "turns": [], + "diagnostics": { + "status": "unsupported", + "unsupportedReason": "provider_challenge_interstitial", + "message": "Google AI Mode capture returned a Google Search JavaScript/interstitial page instead of transcript data." + } +} +``` + +Supported output formats are `json`, `meta-language`, `demo-memory`, +`markdown`/`md`, `txt`/`text`, and `html`. ChatGPT share pages are decoded from +embedded `linear_conversation` data and hidden turns are omitted from `turns`. + ## web-search Relationship `web-capture` currently owns the five-provider `/search` catalog above. No @@ -172,6 +244,7 @@ Stable CLI behavior: | `image` / `png` | `web-capture --format png --output screenshot.png` | PNG file bytes at the output path. | | `jpeg` | JavaScript implementation | JPEG file bytes at the output path. | | `search` | `web-capture search "" --provider wikipedia` | Normalized search JSON on stdout by default. | +| `shared-dialog` | `web-capture shared-dialog "" --format json -o -` | Normalized shared-dialog JSON on stdout. | CLI failures use a non-zero exit code and a human-readable stderr message. FormalAI should normalize the CLI diagnostic with: @@ -190,7 +263,9 @@ The contract is covered by smoke tests in: - `js/tests/integration/formalai-contract.test.js` - `js/tests/unit/cli.test.js` +- `js/tests/unit/shared-dialog.test.js` - `rust/tests/integration/formalai_contract.rs` +- `rust/tests/unit/shared_dialog.rs` These tests assert the stable HTTP content types, binary signatures, ZIP contents, search JSON diagnostics, provider allow-list, and CLI output shapes diff --git a/js/.changeset/shared-dialog-capture.md b/js/.changeset/shared-dialog-capture.md new file mode 100644 index 0000000..485a3e6 --- /dev/null +++ b/js/.changeset/shared-dialog-capture.md @@ -0,0 +1,5 @@ +--- +'@link-assistant/web-capture': patch +--- + +Add shared AI dialog capture for ChatGPT share links with structured transcript output and unsupported-provider diagnostics. diff --git a/js/README.md b/js/README.md index 89d2e19..17eafb5 100644 --- a/js/README.md +++ b/js/README.md @@ -15,6 +15,7 @@ A CLI and microservice to fetch URLs and render them as: - **ZIP archive**: Markdown + locally downloaded images - **PDF**: Print-quality document with embedded images - **DOCX**: Word document with embedded images +- **Shared AI dialog transcript**: ChatGPT share captures with structured diagnostics for unsupported providers ## Installation @@ -99,6 +100,10 @@ web-capture https://example.com --no-extract-latex --no-post-process -o page.md # Structured search-provider capture (JSON by default) web-capture search "formal methods" --provider wikipedia +# Shared AI dialog capture (JSON by default) +web-capture shared-dialog https://chatgpt.com/share/SHARE_ID +web-capture shared-dialog https://chatgpt.com/share/SHARE_ID --format demo-memory -o - + # Start as API server web-capture --serve @@ -321,6 +326,17 @@ Example response (`format=json`): } ``` +### Shared Dialog Endpoint + +``` +GET /shared-dialog?url=&format=json|meta-language|demo-memory|markdown|txt +``` + +Captures replayable shared AI dialog transcripts. ChatGPT share links are +decoded from their embedded `linear_conversation` data. Providers that cannot +currently expose transcript data, such as Google AI Mode interstitial pages, +return a structured unsupported diagnostic. + ## CLI Reference ### Server Mode diff --git a/js/bin/web-capture.js b/js/bin/web-capture.js index 5ef23c0..362a003 100755 --- a/js/bin/web-capture.js +++ b/js/bin/web-capture.js @@ -1,8 +1,4 @@ #!/usr/bin/env node -// CLI entry point for web-capture -// Supports two modes: -// 1. Server mode: web-capture --serve [--port 3000] -// 2. Capture mode: web-capture [options] import fs from 'fs'; import path from 'path'; @@ -27,7 +23,11 @@ function makeVerboseLog(enabled) { }); } -// Create configuration using lino-arguments pattern +const originalConsoleLog = console.log; +console.log = (...args) => + String(args[0]).includes(' variables from ') + ? console.error(...args) + : originalConsoleLog(...args); const config = makeConfig({ yargs: ({ yargs, getenv }) => yargs @@ -49,6 +49,15 @@ const config = makeConfig({ description: 'Search query', }) ) + .command( + 'shared-dialog ', + 'Capture replayable shared AI dialog transcripts', + (yargs) => + yargs.positional('sharedDialogUrl', { + type: 'string', + description: 'Shared AI dialog URL', + }) + ) .option('provider', { type: 'string', description: @@ -78,7 +87,7 @@ const config = makeConfig({ alias: 'f', type: 'string', description: - 'Output format: markdown, md, html, txt, text, image, png, jpeg, pdf, docx, archive', + 'Output format: markdown, md, html, txt, text, image, png, jpeg, pdf, docx, archive, json, meta-language, demo-memory', default: 'markdown', }) .option('theme', { @@ -289,7 +298,8 @@ const config = makeConfig({ 'Capture private Google Doc with API token' ) .epilogue( - 'API Endpoints (in server mode):\n GET /html?url=&engine= Get rendered HTML\n GET /markdown?url= Get Markdown conversion\n GET /txt?url= Get text content\n GET /image?url=&format=png|jpeg&theme=light|dark Screenshot\n GET /archive?url=&localImages=true&documentFormat=markdown|html ZIP archive\n GET /pdf?url=&theme=light|dark PDF with embedded images\n GET /docx?url= DOCX with embedded images\n GET /fetch?url= Proxy fetch\n GET /stream?url= Streaming proxy' + 'API Endpoints (in server mode):\n GET /html?url=&engine= Get rendered HTML\n GET /markdown?url= Get Markdown conversion\n GET /txt?url= Get text content\n GET /image?url=&format=png|jpeg&theme=light|dark Screenshot\n GET /archive?url=&localImages=true&documentFormat=markdown|html ZIP archive\n GET /pdf?url=&theme=light|dark PDF with embedded images\n GET /docx?url= DOCX with embedded images\n GET /fetch?url= Proxy fetch\n GET /stream?url= Streaming proxy' + + '\n GET /shared-dialog?url=&format=json|meta-language|demo-memory|markdown|txt Shared AI dialog capture' ) .strict(), lenv: { @@ -297,6 +307,7 @@ const config = makeConfig({ path: '.lenv', }, }); +console.log = originalConsoleLog; function getUrlArgument() { if (typeof config.url === 'string' && config.url.length > 0) { @@ -331,6 +342,7 @@ async function startServer(port) { console.log(` GET /docx?url= - DOCX with embedded images`); console.log(` GET /fetch?url= - Proxy fetch content`); console.log(` GET /stream?url= - Stream content`); + console.log(` GET /shared-dialog?url= - Shared AI dialog capture`); console.log(''); console.log('Press Ctrl+C to stop the server'); resolve(server); @@ -1370,14 +1382,8 @@ async function captureUrl(url, options) { async function runSearch(query) { const { search, formatSearchAsMarkdown } = await import('../src/search.js'); - // `--format` defaults to `markdown` for capture mode, but search results are - // primarily structured JSON. Only honor the format flag when the user passed - // it explicitly; otherwise default search output to JSON. - const formatPassedExplicitly = process.argv.some( - (arg) => arg === '-f' || arg === '--format' || arg.startsWith('--format=') - ); const format = ( - formatPassedExplicitly ? config.format : 'json' + formatPassedExplicitly() ? config.format : 'json' ).toLowerCase(); const result = await search({ query, @@ -1391,13 +1397,40 @@ async function runSearch(query) { } } +async function runSharedDialog(sharedDialogUrl) { + const { runSharedDialogCli } = await import('../src/shared-dialog-cli.js'); + await runSharedDialogCli(sharedDialogUrl, { + capture: config.capture, + engine: config.engine, + format: formatPassedExplicitly() ? config.format : 'json', + output: config.output, + }); +} + +function formatPassedExplicitly() { + return process.argv.some( + (arg) => arg === '-f' || arg === '--format' || arg.startsWith('--format=') + ); +} + async function main() { const url = getUrlArgument(); - // The `query` key is only present when the `search ` command matched. const isSearch = Object.prototype.hasOwnProperty.call(config, 'query'); + const isSharedDialog = Object.prototype.hasOwnProperty.call( + config, + 'sharedDialogUrl' + ); - if (isSearch) { - // Search mode + if (isSharedDialog) { + const sharedDialogUrl = + typeof config.sharedDialogUrl === 'string' ? config.sharedDialogUrl : ''; + if (!sharedDialogUrl.trim()) { + console.error('Error: Missing shared-dialog URL'); + console.error('Usage: web-capture shared-dialog [--format ]'); + process.exit(1); + } + await runSharedDialog(sharedDialogUrl); + } else if (isSearch) { const query = typeof config.query === 'string' ? config.query : ''; if (!query.trim()) { console.error('Error: Missing search query'); @@ -1406,7 +1439,6 @@ async function main() { } await runSearch(query); } else if (config.serve) { - // Server mode await startServer(config.port); } else if (url) { // --no-extract-images is an alias for --embed-images @@ -1429,7 +1461,6 @@ async function main() { config.archiveFormat = archiveFormat === 'gz' ? 'tar.gz' : archiveFormat; config.format = 'archive'; } - // Capture mode await captureUrl(url, { format: config.format, output: config.output, @@ -1457,7 +1488,6 @@ async function main() { capture: config.capture, }); } else { - // No arguments - show error console.error('Error: Missing URL or --serve flag'); console.error('Run with --help for usage information'); process.exit(1); diff --git a/js/src/index.js b/js/src/index.js index e79b5f3..52e72a7 100644 --- a/js/src/index.js +++ b/js/src/index.js @@ -13,6 +13,7 @@ import { animationHandler } from './animation.js'; import { figuresHandler } from './figures.js'; import { themedImageHandler } from './themed-image.js'; import { searchHandler } from './search.js'; +import { sharedDialogHandler } from './shared-dialog.js'; const app = express(); const port = process.env.PORT || 3000; @@ -31,6 +32,7 @@ app.get('/animation', animationHandler); app.get('/figures', figuresHandler); app.get('/themed-image', themedImageHandler); app.get('/search', searchHandler); +app.get('/shared-dialog', sharedDialogHandler); // Start the server if this is the main module const isMainModule = @@ -78,3 +80,13 @@ export { formatSearchAsMarkdown, SEARCH_PROVIDERS, } from './search.js'; +export { + captureSharedDialog, + formatSharedDialogAsDemoMemory, + formatSharedDialogAsMarkdown, + formatSharedDialogAsMetaLanguage, + formatSharedDialogAsText, + formatSharedDialogResult, + parseSharedDialog, + sharedDialogHandler, +} from './shared-dialog.js'; diff --git a/js/src/shared-dialog-cli.js b/js/src/shared-dialog-cli.js new file mode 100644 index 0000000..43a966e --- /dev/null +++ b/js/src/shared-dialog-cli.js @@ -0,0 +1,25 @@ +import fs from 'fs'; +import path from 'path'; + +import { + captureSharedDialog, + formatSharedDialogResult, +} from './shared-dialog.js'; + +export async function runSharedDialogCli(sharedDialogUrl, options = {}) { + const format = (options.format || 'json').toLowerCase(); + const capture = await captureSharedDialog({ + url: sharedDialogUrl, + capture: options.capture, + engine: options.engine, + }); + const body = formatSharedDialogResult(capture, format); + + if (options.output && options.output !== '-') { + fs.mkdirSync(path.dirname(options.output), { recursive: true }); + fs.writeFileSync(options.output, body, 'utf-8'); + console.error(`Shared dialog (${format}) saved to: ${options.output}`); + } else { + process.stdout.write(body); + } +} diff --git a/js/src/shared-dialog.js b/js/src/shared-dialog.js new file mode 100644 index 0000000..4fbdf6a --- /dev/null +++ b/js/src/shared-dialog.js @@ -0,0 +1,950 @@ +/** + * Shared AI dialog capture and normalization (issue #141). + * + * The primary supported source today is ChatGPT shared-page HTML. ChatGPT + * embeds a React Router stream containing a devalue table; this module decodes + * that table, finds `linear_conversation`, and keeps visible user/assistant + * messages. Providers that do not expose transcript data return a structured + * unsupported diagnostic instead of guessed content. + */ + +import fetch from 'node-fetch'; +import he from 'he'; +import { URL } from 'node:url'; + +import { createBrowser as defaultCreateBrowser } from './browser.js'; + +const USER_AGENT = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; + +const ENQUEUE_MARKER = 'window.__reactRouterContext.streamController.enqueue('; + +export const SHARED_DIALOG_FORMATS = [ + 'json', + 'meta-language', + 'demo-memory', + 'markdown', + 'md', + 'txt', + 'text', + 'html', +]; + +function normalizeSourceUrl(url) { + if (!url) { + throw new Error('Missing shared-dialog URL'); + } + if (url.startsWith('http://') || url.startsWith('https://')) { + return url; + } + return `https://${url}`; +} + +function providerFromUrl(sourceUrl) { + try { + const parsed = new URL(sourceUrl); + const host = parsed.hostname.toLowerCase(); + if (host === 'chatgpt.com' || host.endsWith('.chatgpt.com')) { + return 'chatgpt'; + } + if (host === 'share.google' && parsed.pathname.startsWith('/aimode/')) { + return 'google_ai_mode'; + } + } catch { + // Fall through to body-based detection. + } + return null; +} + +function detectProvider(input, sourceUrl) { + const fromUrl = providerFromUrl(sourceUrl); + if (fromUrl) { + return fromUrl; + } + if (input.includes(ENQUEUE_MARKER) || input.includes('linear_conversation')) { + return 'chatgpt'; + } + if (looksLikeGoogleAiModeInterstitial(input)) { + return 'google_ai_mode'; + } + return 'unknown'; +} + +function okCapture({ + provider, + sourceUrl, + captureMethod, + capturedAt, + httpStatus, + title, + conversationId, + turns, + warnings = [], +}) { + return { + provider, + sourceUrl, + captureMethod, + capturedAt, + status: 'ok', + conversationId, + title, + turns, + diagnostics: { + status: 'ok', + httpStatus, + warnings, + }, + }; +} + +function unsupportedCapture({ + provider, + sourceUrl, + captureMethod, + capturedAt, + httpStatus, + unsupportedReason, + message, + warnings = [], + error, +}) { + return { + provider, + sourceUrl, + captureMethod, + capturedAt, + status: 'unsupported', + conversationId: null, + title: null, + turns: [], + diagnostics: { + status: 'unsupported', + httpStatus, + unsupportedReason, + message, + warnings, + ...(error ? { error } : {}), + }, + }; +} + +function parseUnsupported(input, provider) { + if (looksLikeGoogleAiModeInterstitial(input)) { + return { + reason: 'provider_challenge_interstitial', + message: + 'Google AI Mode capture returned a Google Search JavaScript/interstitial page instead of transcript data.', + }; + } + if (/log in|sign in|login required|authentication required/i.test(input)) { + return { + reason: 'login_required', + message: + 'The shared dialog page requires login before transcript data is visible.', + }; + } + if (/deleted|expired|not found|unavailable/i.test(input)) { + return { + reason: 'deleted_or_expired_share', + message: + 'The shared dialog appears to be deleted, expired, or unavailable.', + }; + } + if (provider === 'unknown') { + return { + reason: 'unsupported_provider_format', + message: + 'The URL or captured DOM is not a supported shared-dialog provider format.', + }; + } + return { + reason: 'no_transcript_in_captured_dom', + message: 'The captured DOM did not contain replayable transcript data.', + }; +} + +/** + * Parse a captured shared-dialog DOM/transcript into the normalized contract. + * + * @param {string} input - Captured HTML or compact transcript + * @param {Object} options - Parse metadata + * @returns {Object} Normalized shared-dialog capture + */ +export function parseSharedDialog(input, options = {}) { + const sourceUrl = options.sourceUrl || ''; + const captureMethod = options.captureMethod || 'static_http'; + const capturedAt = options.capturedAt; + const httpStatus = options.httpStatus; + const provider = options.provider || detectProvider(input || '', sourceUrl); + const warnings = [...(options.warnings || [])]; + + if (provider === 'chatgpt') { + try { + return parseChatGptShareHtml(input || '', { + sourceUrl, + captureMethod, + capturedAt, + httpStatus, + warnings, + }); + } catch (err) { + const diagnostic = parseUnsupported(input || '', provider); + return unsupportedCapture({ + provider, + sourceUrl, + captureMethod, + capturedAt, + httpStatus, + unsupportedReason: + diagnostic.reason === 'no_transcript_in_captured_dom' + ? 'no_transcript_in_captured_dom' + : diagnostic.reason, + message: + diagnostic.reason === 'no_transcript_in_captured_dom' + ? `ChatGPT share capture did not contain a parseable linear_conversation transcript: ${err.message}` + : diagnostic.message, + warnings, + }); + } + } + + const markdown = parseMarkdownTranscript(input || '', { + provider, + sourceUrl, + captureMethod, + capturedAt, + httpStatus, + warnings, + }); + if (markdown.status === 'ok') { + return markdown; + } + + const diagnostic = parseUnsupported(input || '', provider); + return unsupportedCapture({ + provider, + sourceUrl, + captureMethod, + capturedAt, + httpStatus, + unsupportedReason: diagnostic.reason, + message: diagnostic.message, + warnings, + }); +} + +function parseChatGptShareHtml(input, metadata) { + const table = extractChatGptDevalueTable(input); + const resolved = resolveDevalueRoot(table); + const data = findObjectWithArrayKey(resolved, 'linear_conversation'); + if (!data) { + throw new Error('linear_conversation data was not found'); + } + const linear = Array.isArray(data.linear_conversation) + ? data.linear_conversation + : null; + if (!linear) { + throw new Error('linear_conversation field was not an array'); + } + + const turns = []; + linear.forEach((item, index) => { + const turn = chatGptTurnFromItem(item, index, metadata); + if (turn) { + turns.push(turn); + } + }); + if (turns.length === 0) { + throw new Error('no visible user or assistant turns were found'); + } + + const title = + stringField(data, 'title') || + titleFromHtml(input) || + metadata.conversationTitle || + null; + const conversationId = + stringField(data, 'conversation_id') || + chatGptShareId(metadata.sourceUrl) || + null; + + return okCapture({ + provider: 'chatgpt', + sourceUrl: metadata.sourceUrl, + captureMethod: metadata.captureMethod, + capturedAt: metadata.capturedAt, + httpStatus: metadata.httpStatus, + title, + conversationId, + turns, + warnings: metadata.warnings, + }); +} + +function chatGptTurnFromItem(item, index, metadata) { + const message = item?.message || item; + const role = message?.author?.role; + if (role !== 'user' && role !== 'assistant') { + return null; + } + if (messageIsHidden(message)) { + return null; + } + const content = messageContentText(message?.content).trim(); + if (!content) { + return null; + } + const id = typeof message?.id === 'string' ? message.id : ''; + return { + id: id || `chatgpt-turn-${index + 1}`, + role, + content, + visibility: 'visible', + sourceEvidence: [ + { + kind: 'chatgpt_linear_conversation', + sourceUrl: metadata.sourceUrl, + captureMethod: metadata.captureMethod, + pointer: `linear_conversation[${index}].message`, + }, + ], + }; +} + +function messageIsHidden(message) { + return Boolean(message?.metadata?.is_visually_hidden_from_conversation); +} + +function messageContentText(content) { + if (!content) { + return ''; + } + if (typeof content === 'string') { + return content; + } + if (Array.isArray(content.parts)) { + const parts = []; + for (const part of content.parts) { + if (typeof part === 'string' && part) { + parts.push(part); + } else if (typeof part?.text === 'string' && part.text) { + parts.push(part.text); + } + } + return parts.join('\n\n'); + } + return typeof content.text === 'string' ? content.text : ''; +} + +function extractChatGptDevalueTable(input) { + for (const chunk of extractReactRouterStreamChunks(input)) { + const trimmed = chunk.trimStart(); + if (!trimmed.startsWith('[')) { + continue; + } + const value = JSON.parse(trimmed); + if (Array.isArray(value)) { + return value; + } + } + throw new Error('ChatGPT share capture did not contain a JSON devalue table'); +} + +function extractReactRouterStreamChunks(input) { + const chunks = []; + let cursor = 0; + while (cursor < input.length) { + const relative = input.indexOf(ENQUEUE_MARKER, cursor); + if (relative === -1) { + break; + } + let literalStart = relative + ENQUEUE_MARKER.length; + while (/[\s]/.test(input[literalStart] || '')) { + literalStart += 1; + } + if (input[literalStart] !== '"') { + cursor = literalStart + 1; + continue; + } + const { literal, length } = extractJsonStringLiteral( + input.slice(literalStart) + ); + chunks.push(JSON.parse(literal)); + cursor = literalStart + length; + } + if (chunks.length === 0) { + throw new Error( + 'ChatGPT share capture did not contain React Router chunks' + ); + } + return chunks; +} + +function extractJsonStringLiteral(input) { + if (input[0] !== '"') { + throw new Error('expected a JSON string literal'); + } + let index = 1; + while (index < input.length) { + if (input[index] === '\\') { + index += 2; + } else if (input[index] === '"') { + return { + literal: input.slice(0, index + 1), + length: index + 1, + }; + } else { + index += 1; + } + } + throw new Error('unterminated JSON string literal'); +} + +function resolveDevalueRoot(table) { + if (!table.length) { + throw new Error('ChatGPT devalue table was empty'); + } + return resolveTableIndex(table, 0, []); +} + +function resolveTableIndex(table, index, stack) { + if (index >= table.length || stack.includes(index)) { + return null; + } + stack.push(index); + const value = resolveTableValue(table, table[index], stack); + stack.pop(); + return value; +} + +function resolveTableValue(table, value, stack) { + if (Array.isArray(value)) { + return value.map((item) => resolveEncodedReference(table, item, stack)); + } + if (value && typeof value === 'object') { + const resolved = {}; + for (const [encodedKey, encodedValue] of Object.entries(value)) { + const key = resolveObjectKey(table, encodedKey, stack); + resolved[key] = resolveEncodedReference(table, encodedValue, stack); + } + return resolved; + } + return value; +} + +function resolveEncodedReference(table, value, stack) { + if (Number.isInteger(value)) { + if (value < 0) { + return null; + } + return resolveTableIndex(table, value, stack); + } + return resolveTableValue(table, value, stack); +} + +function resolveObjectKey(table, encodedKey, stack) { + if (encodedKey.startsWith('_')) { + const index = Number.parseInt(encodedKey.slice(1), 10); + if (Number.isInteger(index)) { + const key = resolveTableIndex(table, index, stack); + if (typeof key === 'string') { + return key; + } + } + } + return encodedKey; +} + +function findObjectWithArrayKey(value, key) { + if (Array.isArray(value)) { + for (const item of value) { + const found = findObjectWithArrayKey(item, key); + if (found) { + return found; + } + } + return null; + } + if (value && typeof value === 'object') { + if (Array.isArray(value[key])) { + return value; + } + for (const child of Object.values(value)) { + const found = findObjectWithArrayKey(child, key); + if (found) { + return found; + } + } + } + return null; +} + +function parseMarkdownTranscript(input, metadata) { + const turns = []; + let currentRole = null; + let currentLines = []; + + for (const line of input.split(/\r?\n/)) { + const prefixed = markdownTurnPrefix(line); + if (prefixed) { + pushMarkdownTurn(turns, currentRole, currentLines, metadata); + currentRole = prefixed.role; + currentLines = [prefixed.rest.trimStart()]; + } else if (currentRole) { + currentLines.push(line); + } + } + pushMarkdownTurn(turns, currentRole, currentLines, metadata); + + if (turns.length === 0) { + return unsupportedCapture({ + provider: metadata.provider, + sourceUrl: metadata.sourceUrl, + captureMethod: metadata.captureMethod, + capturedAt: metadata.capturedAt, + httpStatus: metadata.httpStatus, + unsupportedReason: 'no_transcript_in_captured_dom', + message: 'No compact Markdown transcript turns were found.', + warnings: metadata.warnings, + }); + } + + return okCapture({ + provider: metadata.provider, + sourceUrl: metadata.sourceUrl, + captureMethod: metadata.captureMethod, + capturedAt: metadata.capturedAt, + httpStatus: metadata.httpStatus, + title: null, + conversationId: null, + turns, + warnings: metadata.warnings, + }); +} + +function markdownTurnPrefix(line) { + const trimmed = line.trimStart(); + for (const [prefix, role] of [ + ['U:', 'user'], + ['User:', 'user'], + ['A:', 'assistant'], + ['Assistant:', 'assistant'], + ]) { + if ( + trimmed.slice(0, prefix.length).toLowerCase() === prefix.toLowerCase() + ) { + return { role, rest: trimmed.slice(prefix.length) }; + } + } + return null; +} + +function pushMarkdownTurn(turns, role, lines, metadata) { + if (!role) { + return; + } + const content = trimmedContentLines(lines); + if (!content) { + return; + } + turns.push({ + id: `markdown-turn-${turns.length + 1}`, + role, + content, + visibility: 'visible', + sourceEvidence: [ + { + kind: 'markdown_transcript', + sourceUrl: metadata.sourceUrl, + captureMethod: metadata.captureMethod, + pointer: `turns[${turns.length}]`, + }, + ], + }); +} + +function trimmedContentLines(lines) { + const start = lines.findIndex((line) => line.trim()); + if (start === -1) { + return ''; + } + let end = lines.length - 1; + while (end > start && !lines[end].trim()) { + end -= 1; + } + return lines + .slice(start, end + 1) + .join('\n') + .trim(); +} + +function stringField(object, key) { + return typeof object?.[key] === 'string' ? object[key] : null; +} + +function titleFromHtml(input) { + const match = input.match(/([\s\S]*?)<\/title>/i); + if (!match) { + return null; + } + const title = he.decode(match[1]).trim(); + return title.replace(/^ChatGPT\s*-\s*/i, '') || null; +} + +function chatGptShareId(sourceUrl) { + if (!sourceUrl) { + return null; + } + try { + const parsed = new URL(sourceUrl); + const parts = parsed.pathname.split('/').filter(Boolean); + const shareIndex = parts.indexOf('share'); + if (shareIndex !== -1 && parts[shareIndex + 1]) { + return parts[shareIndex + 1]; + } + } catch { + return null; + } + return null; +} + +function looksLikeGoogleAiModeInterstitial(input) { + return ( + input.includes('share.google/aimode') || + (input.includes('Google Search') && + input.includes("If you're having trouble accessing Google Search")) || + (input.includes('/search?q=') && input.includes('enablejs')) || + (input.includes('/httpservice/retry/enablejs') && + input.includes('Google Search')) + ); +} + +/** + * Render a URL through the repository browser abstraction and return DOM HTML. + * + * @param {string} url - Absolute URL + * @param {Object} options - Browser options + * @returns {Promise<string>} Rendered DOM + */ +export async function renderSharedDialogWithBrowser(url, options = {}) { + const browser = await (options.createBrowser || defaultCreateBrowser)( + options.engine || 'puppeteer' + ); + try { + const page = await browser.newPage(); + await page.setExtraHTTPHeaders({ + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Charset': 'utf-8', + }); + await page.setUserAgent(USER_AGENT); + await page.setViewport({ width: 1280, height: 800 }); + await page.goto(url, { waitUntil: 'networkidle0', timeout: 30000 }); + await page.waitForTimeout?.(5000); + return await page.content(); + } finally { + await browser.close(); + } +} + +/** + * Fetch and normalize a shared dialog URL. Browser mode first tries static HTTP + * because some providers expose all transcript data there, then falls back to + * rendered DOM when the static capture is unsupported. + * + * @param {Object} options - Capture options + * @returns {Promise<Object>} Normalized shared-dialog capture + */ +export async function captureSharedDialog({ + url, + capture = 'browser', + fetchImpl = fetch, + renderHtml = renderSharedDialogWithBrowser, + engine = 'puppeteer', + now = () => new Date().toISOString(), +} = {}) { + const sourceUrl = normalizeSourceUrl(url); + new URL(sourceUrl); + const capturedAt = now(); + let staticHtml = ''; + let httpStatus = 0; + let warnings = []; + + try { + const response = await fetchImpl(sourceUrl, { + headers: { + 'User-Agent': USER_AGENT, + Accept: 'text/html,application/xhtml+xml', + 'Accept-Language': 'en-US,en;q=0.9', + }, + }); + httpStatus = response.status || 0; + staticHtml = await response.text(); + } catch (err) { + warnings = [`static_http_failed: ${err.message}`]; + } + + const staticCapture = parseSharedDialog(staticHtml, { + sourceUrl, + captureMethod: 'static_http', + capturedAt, + httpStatus, + warnings, + }); + + if (staticCapture.status === 'ok' || capture.toLowerCase() === 'api') { + return staticCapture; + } + if (capture.toLowerCase() !== 'browser') { + return withWarning( + staticCapture, + `unsupported_capture_method: ${capture}; use browser or api` + ); + } + + try { + const renderedHtml = await renderHtml(sourceUrl, { engine }); + return parseSharedDialog(renderedHtml, { + sourceUrl, + captureMethod: 'browser', + capturedAt, + warnings: + staticCapture.status === 'unsupported' + ? [ + ...staticCapture.diagnostics.warnings, + `static_http_unsupported: ${staticCapture.diagnostics.unsupportedReason}`, + ] + : staticCapture.diagnostics.warnings, + }); + } catch (err) { + return withWarning(staticCapture, `browser_render_failed: ${err.message}`); + } +} + +function withWarning(capture, warning) { + return { + ...capture, + diagnostics: { + ...capture.diagnostics, + warnings: [...(capture.diagnostics.warnings || []), warning], + }, + }; +} + +function escapeLino(value) { + return String(value ?? '') + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\n/g, '\\n') + .replace(/\r/g, '\\r') + .replace(/\t/g, '\\t'); +} + +function pushField(lines, indent, name, value) { + if (value === undefined || value === null || value === '') { + return; + } + lines.push(`${indent}${name} "${escapeLino(value)}"`); +} + +export function formatSharedDialogAsMetaLanguage(capture) { + const lines = ['shared_dialog_capture']; + pushField(lines, ' ', 'provider', capture.provider); + pushField(lines, ' ', 'sourceUrl', capture.sourceUrl); + pushField(lines, ' ', 'captureMethod', capture.captureMethod); + pushField(lines, ' ', 'capturedAt', capture.capturedAt); + pushField(lines, ' ', 'status', capture.status); + pushField(lines, ' ', 'conversationId', capture.conversationId); + pushField(lines, ' ', 'title', capture.title); + lines.push(' diagnostics'); + pushField(lines, ' ', 'status', capture.diagnostics.status); + pushField(lines, ' ', 'httpStatus', capture.diagnostics.httpStatus); + pushField( + lines, + ' ', + 'unsupportedReason', + capture.diagnostics.unsupportedReason + ); + pushField(lines, ' ', 'message', capture.diagnostics.message); + for (const warning of capture.diagnostics.warnings || []) { + pushField(lines, ' ', 'warning', warning); + } + for (const turn of capture.turns || []) { + lines.push(` turn "${escapeLino(turn.id)}"`); + pushField(lines, ' ', 'role', turn.role); + pushField(lines, ' ', 'visibility', turn.visibility); + pushField(lines, ' ', 'content', turn.content); + for (const evidence of turn.sourceEvidence || []) { + lines.push(' evidence'); + pushField(lines, ' ', 'kind', evidence.kind); + pushField(lines, ' ', 'sourceUrl', evidence.sourceUrl); + pushField(lines, ' ', 'captureMethod', evidence.captureMethod); + pushField(lines, ' ', 'pointer', evidence.pointer); + } + } + return `${lines.join('\n')}\n`; +} + +export function formatSharedDialogAsDemoMemory( + capture, + { demoLabel = 'web-capture-shared-dialog' } = {} +) { + const lines = ['demo_memory']; + if (capture.status !== 'ok') { + lines.push(' diagnostic "shared-dialog-unsupported"'); + pushField(lines, ' ', 'provider', capture.provider); + pushField(lines, ' ', 'sourceUrl', capture.sourceUrl); + pushField(lines, ' ', 'captureMethod', capture.captureMethod); + pushField( + lines, + ' ', + 'unsupportedReason', + capture.diagnostics.unsupportedReason + ); + pushField(lines, ' ', 'message', capture.diagnostics.message); + return `${lines.join('\n')}\n`; + } + + for (const turn of capture.turns) { + lines.push(` event "${escapeLino(turn.id)}"`); + pushField(lines, ' ', 'role', turn.role); + pushField(lines, ' ', 'content', turn.content); + pushField(lines, ' ', 'demoLabel', demoLabel); + pushField(lines, ' ', 'conversationId', capture.conversationId); + pushField(lines, ' ', 'conversationTitle', capture.title); + for (const evidence of turn.sourceEvidence || []) { + pushField(lines, ' ', 'evidence', evidence.sourceUrl); + } + } + return `${lines.join('\n')}\n`; +} + +export function formatSharedDialogAsMarkdown(capture) { + const lines = []; + lines.push(`# ${capture.title || 'Shared Dialog Capture'}`); + lines.push(''); + lines.push(`- Provider: \`${capture.provider}\``); + lines.push(`- Source: ${capture.sourceUrl || ''}`); + lines.push(`- Capture method: \`${capture.captureMethod}\``); + lines.push(`- Status: \`${capture.status}\``); + if (capture.conversationId) { + lines.push(`- Conversation ID: \`${capture.conversationId}\``); + } + if (capture.status !== 'ok') { + lines.push( + `- Unsupported reason: \`${capture.diagnostics.unsupportedReason}\`` + ); + if (capture.diagnostics.message) { + lines.push(''); + lines.push(capture.diagnostics.message); + } + return `${lines.join('\n')}\n`; + } + lines.push(''); + for (const turn of capture.turns) { + lines.push(`**${turn.role === 'user' ? 'User' : 'Assistant'}**`); + lines.push(''); + lines.push(turn.content); + lines.push(''); + } + return `${lines.join('\n').trimEnd()}\n`; +} + +export function formatSharedDialogAsText(capture) { + if (capture.status !== 'ok') { + return [ + `Provider: ${capture.provider}`, + `Source: ${capture.sourceUrl || ''}`, + `Capture method: ${capture.captureMethod}`, + `Status: ${capture.status}`, + `Unsupported reason: ${capture.diagnostics.unsupportedReason}`, + capture.diagnostics.message || '', + '', + ].join('\n'); + } + const lines = []; + for (const turn of capture.turns) { + lines.push(`${turn.role === 'user' ? 'User' : 'Assistant'}:`); + lines.push(turn.content); + lines.push(''); + } + return lines.join('\n'); +} + +export function formatSharedDialogAsHtml(capture) { + const markdown = formatSharedDialogAsMarkdown(capture); + return `<!doctype html> +<html> +<head><meta charset="utf-8"><title>${he.escape(capture.title || 'Shared Dialog Capture')} +
${he.escape(markdown)}
+ +`; +} + +export function formatSharedDialogResult(capture, format = 'json') { + const normalized = format.toLowerCase(); + if (normalized === 'meta-language' || normalized === 'meta_language') { + return formatSharedDialogAsMetaLanguage(capture); + } + if (normalized === 'demo-memory' || normalized === 'demo_memory') { + return formatSharedDialogAsDemoMemory(capture); + } + if (normalized === 'markdown' || normalized === 'md') { + return formatSharedDialogAsMarkdown(capture); + } + if (normalized === 'txt' || normalized === 'text') { + return formatSharedDialogAsText(capture); + } + if (normalized === 'html') { + return formatSharedDialogAsHtml(capture); + } + return `${JSON.stringify(capture, null, 2)}\n`; +} + +export function sharedDialogOutputExtension(format = 'json') { + const normalized = format.toLowerCase(); + if (normalized === 'meta-language' || normalized === 'meta_language') { + return 'lino'; + } + if (normalized === 'demo-memory' || normalized === 'demo_memory') { + return 'lino'; + } + if (normalized === 'markdown' || normalized === 'md') { + return 'md'; + } + if (normalized === 'txt' || normalized === 'text') { + return 'txt'; + } + if (normalized === 'html') { + return 'html'; + } + return 'json'; +} + +export async function sharedDialogHandler(req, res) { + const url = req.query.url; + if (!url) { + return res.status(400).send('Missing `url` parameter'); + } + const format = (req.query.format || 'json').toLowerCase(); + try { + const capture = await captureSharedDialog({ + url, + capture: req.query.capture || 'browser', + engine: req.query.engine || req.query.browser || 'puppeteer', + }); + const body = formatSharedDialogResult(capture, format); + if (format === 'json') { + res.type('application/json').send(body); + } else if (format === 'markdown' || format === 'md') { + res.type('text/markdown').send(body); + } else if (format === 'html') { + res.type('text/html').send(body); + } else { + res.type('text/plain').send(body); + } + } catch (err) { + res.status(500).send(`Error capturing shared dialog: ${err.message}`); + } +} diff --git a/js/tests/unit/cli.test.js b/js/tests/unit/cli.test.js index a18d2cc..61fc562 100644 --- a/js/tests/unit/cli.test.js +++ b/js/tests/unit/cli.test.js @@ -43,7 +43,10 @@ function startFixtureServer({ } = {}) { return new Promise((resolve, reject) => { const server = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': contentType }); + res.writeHead(200, { + 'Content-Type': contentType, + 'Content-Length': Buffer.byteLength(body), + }); res.end(body); }); @@ -351,5 +354,46 @@ describe('CLI', () => { }); expect(body.diagnostics.sourceUrl).toContain('en.wikipedia.org'); }, 20000); + + test('emits shared-dialog demo_memory from the shared-dialog subcommand', async () => { + const fixturePath = resolve( + __dirname, + '../../../docs/case-studies/issue-141/raw-data/chatgpt-share-6a3825b9.html' + ); + const body = readFileSync(fixturePath, 'utf-8'); + const { server, url } = await startFixtureServer({ body }); + const shareUrl = url.replace( + '/article', + '/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24' + ); + + try { + const result = await runCli( + [ + 'shared-dialog', + shareUrl, + '--format', + 'demo-memory', + '--output', + '-', + ], + { + cwd: resolve(__dirname, '../..'), + } + ); + + expect(result.code).toBe(0); + expect(result.stdout.startsWith('demo_memory')).toBe(true); + expect(result.stdout).toContain('demo_memory'); + expect(result.stdout.match(/\n {2}event "/g)).toHaveLength(4); + expect(result.stdout).toContain('role "user"'); + expect(result.stdout).toContain('role "assistant"'); + expect(result.stdout).toContain( + 'conversationTitle "Infinite loop script"' + ); + } finally { + await stopFixtureServer(server); + } + }, 20000); }); }); diff --git a/js/tests/unit/shared-dialog.test.js b/js/tests/unit/shared-dialog.test.js new file mode 100644 index 0000000..96663f6 --- /dev/null +++ b/js/tests/unit/shared-dialog.test.js @@ -0,0 +1,173 @@ +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { jest } from '@jest/globals'; + +import { + captureSharedDialog, + formatSharedDialogAsDemoMemory, + formatSharedDialogAsMarkdown, + formatSharedDialogAsMetaLanguage, + parseSharedDialog, +} from '../../src/shared-dialog.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const repoRoot = resolve(__dirname, '../../..'); + +const CHATGPT_SHARE_URL = + 'https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24'; +const GOOGLE_AI_MODE_URL = 'https://share.google/aimode/VG0HhpnAXrBkC0QgP'; + +function readCaseStudyFixture(name) { + return readFileSync( + resolve(repoRoot, 'docs/case-studies/issue-141/raw-data', name), + 'utf-8' + ); +} + +describe('shared-dialog module (#141)', () => { + it('extracts the four visible ChatGPT shared-dialog turns', () => { + const capture = parseSharedDialog( + readCaseStudyFixture('chatgpt-share-6a3825b9.html'), + { + sourceUrl: CHATGPT_SHARE_URL, + captureMethod: 'static_http', + } + ); + + expect(capture).toMatchObject({ + provider: 'chatgpt', + sourceUrl: CHATGPT_SHARE_URL, + captureMethod: 'static_http', + status: 'ok', + conversationId: '6a3825b9-8de4-83ee-9c24-52fd1eb38d24', + title: 'Infinite loop script', + }); + expect(capture.turns).toHaveLength(4); + expect(capture.turns.map((turn) => turn.role)).toEqual([ + 'user', + 'assistant', + 'user', + 'assistant', + ]); + expect(capture.turns[0]).toMatchObject({ + visibility: 'visible', + sourceEvidence: [ + { + sourceUrl: CHATGPT_SHARE_URL, + kind: 'chatgpt_linear_conversation', + }, + ], + }); + expect(capture.turns[0].content).toContain('make a loop of that'); + expect(capture.turns[1].content).toContain( + 'while true; do sleep 30m && hive-cleanup -f; done' + ); + expect(capture.turns[3].content).toContain( + 'screen -dmS auto-cleanup bash -c' + ); + }); + + it('formats ChatGPT captures as demo_memory', () => { + const capture = parseSharedDialog( + readCaseStudyFixture('chatgpt-share-6a3825b9.html'), + { + sourceUrl: CHATGPT_SHARE_URL, + captureMethod: 'static_http', + } + ); + + const memory = formatSharedDialogAsDemoMemory(capture, { + demoLabel: 'issue-552-chatgpt-share', + }); + + expect(memory).toContain('demo_memory'); + expect(memory.match(/\n {2}event "/g)).toHaveLength(4); + expect(memory).toContain('role "user"'); + expect(memory).toContain('role "assistant"'); + expect(memory).toContain( + 'conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24"' + ); + expect(memory).toContain('conversationTitle "Infinite loop script"'); + expect(memory).toContain(`evidence "${CHATGPT_SHARE_URL}"`); + expect(memory).toContain( + "screen -dmS auto-cleanup bash -c 'while true; do sleep 30m && hive-cleanup -f; done'" + ); + }); + + it('formats captures as shared-dialog meta-language and Markdown', () => { + const capture = parseSharedDialog( + readCaseStudyFixture('chatgpt-share-6a3825b9.html'), + { + sourceUrl: CHATGPT_SHARE_URL, + captureMethod: 'static_http', + } + ); + + const metaLanguage = formatSharedDialogAsMetaLanguage(capture); + expect(metaLanguage).toContain('shared_dialog_capture'); + expect(metaLanguage).toContain('provider "chatgpt"'); + expect(metaLanguage).toContain( + 'turn "0c9f0151-b5a1-402f-afc3-6bd34a0d01d2"' + ); + + const markdown = formatSharedDialogAsMarkdown(capture); + expect(markdown).toContain('# Infinite loop script'); + expect(markdown).toContain('**User**'); + expect(markdown).toContain('**Assistant**'); + }); + + it('returns a structured Google AI Mode interstitial diagnostic', () => { + const capture = parseSharedDialog( + readCaseStudyFixture('google-ai-mode-VG0HhpnAXrBkC0QgP.html'), + { + sourceUrl: GOOGLE_AI_MODE_URL, + captureMethod: 'static_http', + } + ); + + expect(capture).toMatchObject({ + provider: 'google_ai_mode', + sourceUrl: GOOGLE_AI_MODE_URL, + captureMethod: 'static_http', + status: 'unsupported', + turns: [], + diagnostics: { + unsupportedReason: 'provider_challenge_interstitial', + }, + }); + }); + + it('uses browser-rendered DOM when static Google AI Mode capture is blocked', async () => { + const googleHtml = readCaseStudyFixture( + 'google-ai-mode-VG0HhpnAXrBkC0QgP.html' + ); + const renderHtml = jest.fn(async () => googleHtml); + const capture = await captureSharedDialog({ + url: GOOGLE_AI_MODE_URL, + capture: 'browser', + fetchImpl: async () => ({ + ok: true, + status: 200, + text: async () => googleHtml, + }), + renderHtml, + now: () => '2026-06-25T00:00:00.000Z', + }); + + expect(renderHtml).toHaveBeenCalledWith(GOOGLE_AI_MODE_URL, { + engine: 'puppeteer', + }); + expect(capture).toMatchObject({ + provider: 'google_ai_mode', + captureMethod: 'browser', + status: 'unsupported', + capturedAt: '2026-06-25T00:00:00.000Z', + diagnostics: { + unsupportedReason: 'provider_challenge_interstitial', + }, + }); + }); +}); diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index dbac7b5..ca83c11 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.3.32 + +### Patch Changes + +- Add shared AI dialog capture for ChatGPT share links with structured + transcript output and unsupported-provider diagnostics. + ## 0.3.31 ### Patch Changes diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 48159c4..b9e1ca7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -3593,7 +3593,7 @@ dependencies = [ [[package]] name = "web-capture" -version = "0.3.31" +version = "0.3.32" dependencies = [ "anyhow", "async-tungstenite", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 5b88ba1..a001d22 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "web-capture" -version = "0.3.31" +version = "0.3.32" edition = "2021" description = "CLI and microservice to render web pages as HTML, Markdown, or PNG" license = "Unlicense" diff --git a/rust/README.md b/rust/README.md index fa71d0e..ecc8a94 100644 --- a/rust/README.md +++ b/rust/README.md @@ -13,6 +13,7 @@ A CLI and microservice to fetch URLs and render them as: - **HTML**: Rendered page content - **Plain text**: Raw text downloads for paste-like URLs such as xpaste.pro - **PNG screenshot**: Full page capture +- **Shared AI dialog transcript**: ChatGPT share captures with structured diagnostics for unsupported providers This is the Rust implementation of web-capture, providing the same API as the JavaScript version. @@ -68,6 +69,10 @@ web-capture https://example.com --embed-images -o page.md # Structured search-provider capture (JSON by default) web-capture search "formal methods" --provider wikipedia +# Shared AI dialog capture (JSON by default) +web-capture shared-dialog https://chatgpt.com/share/SHARE_ID +web-capture shared-dialog https://chatgpt.com/share/SHARE_ID --format demo-memory -o - + # Start as API server web-capture --serve @@ -90,6 +95,7 @@ FormalAI integration should use the stable HTTP/CLI contract documented in - **PNG screenshot**: `GET /image?url=` - **Archive**: `GET /archive?url=` (ZIP containing `document.md` and `document.html`) - **Search**: `GET /search?q=&provider=&format=json|markdown` +- **Shared dialog**: `GET /shared-dialog?url=&format=json|meta-language|demo-memory|markdown|txt` For xpaste.pro paste URLs, `/markdown` captures the visual paste page in visible order and appends the raw paste text as `xpaste-pro-.txt` when the final diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 652049c..a198438 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -49,6 +49,7 @@ pub mod markdown; pub mod metadata; pub mod postprocess; pub mod search; +pub mod shared_dialog; pub mod stackoverflow; pub mod themed_image; pub mod verify; diff --git a/rust/src/main.rs b/rust/src/main.rs index fc899ac..7c66b4e 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -50,11 +50,11 @@ use web_capture::{ #[allow(clippy::struct_excessive_bools)] struct Args { /// URL to capture (required in capture mode). Use the literal `search` - /// here to enter structured search mode: `web-capture search `. + /// or `shared-dialog` to enter those structured capture modes. #[arg(index = 1)] url: Option, - /// Search query (used only in `web-capture search ` mode) + /// Search query or shared dialog URL (used only in structured subcommands) #[arg(index = 2)] query: Option, @@ -74,7 +74,7 @@ struct Args { #[arg(short, long, default_value = "3000", env = "PORT")] port: u16, - /// Output format: markdown/md, html, txt/text, image/png + /// Output format: markdown/md, html, txt/text, image/png, json, meta-language, demo-memory #[arg(short, long, default_value = "markdown")] format: String, @@ -226,6 +226,16 @@ struct SearchQuery { format: Option, } +/// Query parameters for the /shared-dialog endpoint +#[derive(Debug, Deserialize)] +struct SharedDialogQuery { + url: String, + #[serde(default)] + format: Option, + #[serde(default)] + capture: Option, +} + /// Current UTC time as an RFC 3339 timestamp (e.g. `2026-05-30T12:34:56Z`). /// /// Implemented without a calendar dependency via Howard Hinnant's @@ -400,6 +410,9 @@ async fn main() -> anyhow::Result<()> { } else if args.url.as_deref() == Some("search") { // Structured search mode: `web-capture search ` run_search(&args).await?; + } else if args.url.as_deref() == Some("shared-dialog") { + // Shared AI dialog mode: `web-capture shared-dialog ` + run_shared_dialog(&args).await?; } else if let Some(ref url) = args.url { // Capture mode capture_url(url, &effective_format, args.output.as_ref(), &args).await?; @@ -426,6 +439,7 @@ async fn start_server(port: u16) -> anyhow::Result<()> { .route("/figures", get(figures_handler)) .route("/themed-image", get(themed_image_handler)) .route("/search", get(search_handler)) + .route("/shared-dialog", get(shared_dialog_handler)) .layer(TraceLayer::new_for_http()); let addr = SocketAddr::from(([0, 0, 0, 0], port)); @@ -444,6 +458,7 @@ async fn start_server(port: u16) -> anyhow::Result<()> { info!(" GET /figures?url= - Extract figure images"); info!(" GET /themed-image?url= - Dual-theme screenshots"); info!(" GET /search?q= - Structured search-provider capture"); + info!(" GET /shared-dialog?url= - Shared AI dialog capture"); info!(""); info!("Press Ctrl+C to stop the server"); @@ -1079,6 +1094,39 @@ async fn search_handler(Query(params): Query) -> Response { } } +/// Shared AI dialog capture endpoint handler. +async fn shared_dialog_handler(Query(params): Query) -> Response { + let url = match normalize_url(¶ms.url) { + Ok(url) => url, + Err(e) => return (StatusCode::BAD_REQUEST, e).into_response(), + }; + let format = params.format.unwrap_or_else(|| "json".to_string()); + let capture = params.capture.unwrap_or_else(|| "browser".to_string()); + + let result = match web_capture::shared_dialog::capture_shared_dialog( + &url, + &capture, + Some(now_rfc3339()), + ) + .await + { + Ok(result) => result, + Err(e) => return (StatusCode::BAD_REQUEST, e).into_response(), + }; + + let body = match web_capture::shared_dialog::format_shared_dialog_result(&result, &format) { + Ok(body) => body, + Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), + }; + + ( + StatusCode::OK, + [("Content-Type", shared_dialog_content_type(&format))], + body, + ) + .into_response() +} + /// Run structured search from the CLI (`web-capture search `). async fn run_search(args: &Args) -> anyhow::Result<()> { let Some(query) = args.query.as_deref() else { @@ -1109,6 +1157,60 @@ async fn run_search(args: &Args) -> anyhow::Result<()> { Ok(()) } +/// Run shared AI dialog capture from the CLI (`web-capture shared-dialog `). +async fn run_shared_dialog(args: &Args) -> anyhow::Result<()> { + let Some(url) = args.query.as_deref() else { + eprintln!("Error: Missing shared dialog URL. Usage: web-capture shared-dialog "); + std::process::exit(1); + }; + let absolute_url = normalize_url(url).map_err(|e| anyhow::anyhow!(e))?; + + // Shared dialog output defaults to JSON unless --format/-f was passed explicitly. + let format_explicit = std::env::args().any(|arg| { + arg == "-f" || arg == "--format" || arg.starts_with("--format=") || arg.starts_with("-f=") + }); + let format = if format_explicit { + args.format.clone() + } else { + "json".to_string() + }; + + let result = web_capture::shared_dialog::capture_shared_dialog( + &absolute_url, + &args.capture, + Some(now_rfc3339()), + ) + .await + .map_err(|e| anyhow::anyhow!(e))?; + let body = web_capture::shared_dialog::format_shared_dialog_result(&result, &format) + .map_err(|e| anyhow::anyhow!(e))?; + + if let Some(path) = effective_output_path( + &absolute_url, + web_capture::shared_dialog::shared_dialog_output_extension(&format), + args.output.as_ref(), + &args.data_dir, + ) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).ok(); + } + fs::write(&path, &body).await?; + eprintln!("Shared dialog ({format}) saved to: {}", path.display()); + } else { + print!("{body}"); + } + Ok(()) +} + +fn shared_dialog_content_type(format: &str) -> &'static str { + match format.to_ascii_lowercase().as_str() { + "json" => "application/json; charset=utf-8", + "markdown" | "md" => "text/markdown; charset=utf-8", + "html" => "text/html; charset=utf-8", + _ => "text/plain; charset=utf-8", + } +} + /// Capture a URL and save/output the result #[allow(clippy::too_many_lines)] async fn capture_url( diff --git a/rust/src/shared_dialog.rs b/rust/src/shared_dialog.rs new file mode 100644 index 0000000..ee52aef --- /dev/null +++ b/rust/src/shared_dialog.rs @@ -0,0 +1,1050 @@ +//! Shared AI dialog capture and normalization. +//! +//! `ChatGPT` shared pages embed a React Router stream with a devalue table that +//! contains `linear_conversation`. This module decodes that table into a +//! provider-neutral capture result and reports unsupported provider states as +//! structured diagnostics. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use url::Url; + +const ENQUEUE_MARKER: &str = "window.__reactRouterContext.streamController.enqueue("; +const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SharedDialogParseOptions { + pub source_url: Option, + pub capture_method: String, + pub captured_at: Option, + pub http_status: Option, + pub provider: Option, + pub warnings: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SharedDialogCapture { + pub provider: String, + pub source_url: String, + pub capture_method: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub captured_at: Option, + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + pub turns: Vec, + pub diagnostics: SharedDialogDiagnostics, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SharedDialogTurn { + pub id: String, + pub role: String, + pub content: String, + pub visibility: String, + pub source_evidence: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SharedDialogEvidence { + pub kind: String, + pub source_url: String, + pub capture_method: String, + pub pointer: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SharedDialogDiagnostics { + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub http_status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub unsupported_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub warnings: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct UnsupportedDiagnostic { + reason: &'static str, + message: &'static str, +} + +/// Parse captured HTML or a compact transcript into a normalized shared-dialog +/// result. Unsupported inputs return a structured diagnostic instead of an +/// error. +#[must_use] +pub fn parse_shared_dialog(input: &str, options: &SharedDialogParseOptions) -> SharedDialogCapture { + let source_url = options.source_url.clone().unwrap_or_default(); + let capture_method = if options.capture_method.is_empty() { + "static_http".to_string() + } else { + options.capture_method.clone() + }; + let provider = options + .provider + .clone() + .unwrap_or_else(|| detect_provider(input, &source_url)); + + if provider == "chatgpt" { + return match parse_chatgpt_share_html(input, options, &source_url, &capture_method) { + Ok(capture) => capture, + Err(error) => { + let diagnostic = parse_unsupported(input, &provider); + unsupported_capture(UnsupportedCaptureInput { + provider, + source_url, + capture_method, + captured_at: options.captured_at.clone(), + http_status: options.http_status, + unsupported_reason: diagnostic.reason, + message: if diagnostic.reason == "no_transcript_in_captured_dom" { + format!( + "ChatGPT share capture did not contain a parseable linear_conversation transcript: {error}" + ) + } else { + diagnostic.message.to_string() + }, + warnings: options.warnings.clone(), + error: None, + }) + } + }; + } + + let markdown = + parse_markdown_transcript(input, options, &provider, &source_url, &capture_method); + if markdown.status == "ok" { + return markdown; + } + + let diagnostic = parse_unsupported(input, &provider); + unsupported_capture(UnsupportedCaptureInput { + provider, + source_url, + capture_method, + captured_at: options.captured_at.clone(), + http_status: options.http_status, + unsupported_reason: diagnostic.reason, + message: diagnostic.message.to_string(), + warnings: options.warnings.clone(), + error: None, + }) +} + +/// Fetch and normalize a shared dialog URL. +/// +/// Browser mode first tries static HTTP, then falls back to rendered DOM when +/// the static capture is unsupported. +pub async fn capture_shared_dialog( + url: &str, + capture: &str, + captured_at: Option, +) -> Result { + let source_url = normalize_source_url(url)?; + let mut http_status = None; + let mut warnings = Vec::new(); + let mut static_html = String::new(); + + match reqwest::Client::builder().user_agent(USER_AGENT).build() { + Ok(client) => match client + .get(&source_url) + .header("Accept", "text/html,application/xhtml+xml") + .header("Accept-Language", "en-US,en;q=0.9") + .send() + .await + { + Ok(response) => { + http_status = Some(response.status().as_u16()); + match response.text().await { + Ok(body) => static_html = body, + Err(error) => warnings.push(format!("static_http_read_failed: {error}")), + } + } + Err(error) => warnings.push(format!("static_http_failed: {error}")), + }, + Err(error) => warnings.push(format!("static_http_client_failed: {error}")), + } + + let static_capture = parse_shared_dialog( + &static_html, + &SharedDialogParseOptions { + source_url: Some(source_url.clone()), + capture_method: "static_http".to_string(), + captured_at: captured_at.clone(), + http_status, + warnings, + ..SharedDialogParseOptions::default() + }, + ); + + if static_capture.status == "ok" || capture.eq_ignore_ascii_case("api") { + return Ok(static_capture); + } + if !capture.eq_ignore_ascii_case("browser") { + return Ok(with_warning( + static_capture, + format!("unsupported_capture_method: {capture}; use browser or api"), + )); + } + + match crate::render_html(&source_url).await { + Ok(rendered) => { + let mut render_warnings = static_capture.diagnostics.warnings.clone(); + if let Some(reason) = static_capture.diagnostics.unsupported_reason.as_ref() { + render_warnings.push(format!("static_http_unsupported: {reason}")); + } + Ok(parse_shared_dialog( + &rendered, + &SharedDialogParseOptions { + source_url: Some(source_url), + capture_method: "browser".to_string(), + captured_at, + warnings: render_warnings, + ..SharedDialogParseOptions::default() + }, + )) + } + Err(error) => Ok(with_warning( + static_capture, + format!("browser_render_failed: {error}"), + )), + } +} + +fn parse_chatgpt_share_html( + input: &str, + options: &SharedDialogParseOptions, + source_url: &str, + capture_method: &str, +) -> Result { + let table = extract_chatgpt_devalue_table(input)?; + let resolved = resolve_devalue_root(&table)?; + let data = find_object_with_array_key(&resolved, "linear_conversation") + .ok_or_else(|| "linear_conversation data was not found".to_string())?; + let linear = data + .get("linear_conversation") + .and_then(Value::as_array) + .ok_or_else(|| "linear_conversation field was not an array".to_string())?; + + let mut turns = Vec::new(); + for (index, item) in linear.iter().enumerate() { + if let Some(turn) = chatgpt_turn_from_item(item, index, source_url, capture_method) { + turns.push(turn); + } + } + if turns.is_empty() { + return Err("no visible user or assistant turns were found".to_string()); + } + + let title = string_field(data, "title").or_else(|| title_from_html(input)); + let conversation_id = + string_field(data, "conversation_id").or_else(|| chatgpt_share_id(source_url)); + + Ok(ok_capture(OkCaptureInput { + provider: "chatgpt".to_string(), + source_url: source_url.to_string(), + capture_method: capture_method.to_string(), + captured_at: options.captured_at.clone(), + http_status: options.http_status, + title, + conversation_id, + turns, + warnings: options.warnings.clone(), + })) +} + +fn chatgpt_turn_from_item( + item: &Value, + index: usize, + source_url: &str, + capture_method: &str, +) -> Option { + let message = item.get("message").unwrap_or(item); + let role = message + .get("author") + .and_then(|author| author.get("role")) + .and_then(Value::as_str)?; + if role != "user" && role != "assistant" { + return None; + } + if message_is_hidden(message) { + return None; + } + let content = message + .get("content") + .map(message_content_text) + .unwrap_or_default() + .trim() + .to_string(); + if content.is_empty() { + return None; + } + let id = message + .get("id") + .and_then(Value::as_str) + .map_or_else(|| format!("chatgpt-turn-{}", index + 1), ToOwned::to_owned); + Some(SharedDialogTurn { + id, + role: role.to_string(), + content, + visibility: "visible".to_string(), + source_evidence: vec![SharedDialogEvidence { + kind: "chatgpt_linear_conversation".to_string(), + source_url: source_url.to_string(), + capture_method: capture_method.to_string(), + pointer: format!("linear_conversation[{index}].message"), + }], + }) +} + +fn message_is_hidden(message: &Value) -> bool { + message + .get("metadata") + .and_then(|metadata| metadata.get("is_visually_hidden_from_conversation")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn message_content_text(content: &Value) -> String { + if let Some(text) = content.as_str() { + return text.to_string(); + } + if let Some(parts) = content.get("parts").and_then(Value::as_array) { + let mut texts = Vec::new(); + for part in parts { + if let Some(text) = part.as_str() { + if !text.is_empty() { + texts.push(text.to_string()); + } + } else if let Some(text) = part.get("text").and_then(Value::as_str) { + if !text.is_empty() { + texts.push(text.to_string()); + } + } + } + return texts.join("\n\n"); + } + content + .get("text") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() +} + +fn extract_chatgpt_devalue_table(input: &str) -> Result, String> { + for chunk in extract_react_router_stream_chunks(input)? { + let trimmed = chunk.trim_start(); + if !trimmed.starts_with('[') { + continue; + } + let value = serde_json::from_str::(trimmed) + .map_err(|error| format!("failed to parse React Router payload JSON: {error}"))?; + if let Value::Array(table) = value { + return Ok(table); + } + } + Err("ChatGPT share capture did not contain a JSON devalue table".to_string()) +} + +fn extract_react_router_stream_chunks(input: &str) -> Result, String> { + let mut chunks = Vec::new(); + let mut cursor = 0; + let bytes = input.as_bytes(); + while let Some(relative) = input[cursor..].find(ENQUEUE_MARKER) { + let mut literal_start = cursor + relative + ENQUEUE_MARKER.len(); + while matches!(bytes.get(literal_start), Some(b' ' | b'\n' | b'\r' | b'\t')) { + literal_start += 1; + } + if bytes.get(literal_start) != Some(&b'"') { + cursor = literal_start.saturating_add(1); + continue; + } + let (literal, literal_len) = extract_json_string_literal(&input[literal_start..])?; + let chunk = serde_json::from_str::(literal) + .map_err(|error| format!("failed to decode React Router stream string: {error}"))?; + chunks.push(chunk); + cursor = literal_start + literal_len; + } + if chunks.is_empty() { + return Err("ChatGPT share capture did not contain React Router chunks".to_string()); + } + Ok(chunks) +} + +fn extract_json_string_literal(input: &str) -> Result<(&str, usize), String> { + let bytes = input.as_bytes(); + if bytes.first() != Some(&b'"') { + return Err("expected a JSON string literal".to_string()); + } + let mut index = 1; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => return Ok((&input[..=index], index + 1)), + _ => index += 1, + } + } + Err("unterminated JSON string literal".to_string()) +} + +fn resolve_devalue_root(table: &[Value]) -> Result { + if table.is_empty() { + return Err("ChatGPT devalue table was empty".to_string()); + } + Ok(resolve_table_index(table, 0, &mut Vec::new())) +} + +fn resolve_table_index(table: &[Value], index: usize, stack: &mut Vec) -> Value { + if index >= table.len() || stack.contains(&index) { + return Value::Null; + } + stack.push(index); + let value = resolve_table_value(table, &table[index], stack); + stack.pop(); + value +} + +fn resolve_table_value(table: &[Value], value: &Value, stack: &mut Vec) -> Value { + match value { + Value::Array(items) => Value::Array( + items + .iter() + .map(|item| resolve_encoded_reference(table, item, stack)) + .collect(), + ), + Value::Object(map) => { + let mut resolved = Map::new(); + for (encoded_key, encoded_value) in map { + let key = resolve_object_key(table, encoded_key, stack); + let value = resolve_encoded_reference(table, encoded_value, stack); + resolved.insert(key, value); + } + Value::Object(resolved) + } + other => other.clone(), + } +} + +fn resolve_encoded_reference(table: &[Value], value: &Value, stack: &mut Vec) -> Value { + if let Some(index) = value.as_i64() { + if index < 0 { + return Value::Null; + } + if let Ok(index) = usize::try_from(index) { + return resolve_table_index(table, index, stack); + } + return Value::Null; + } + resolve_table_value(table, value, stack) +} + +fn resolve_object_key(table: &[Value], encoded_key: &str, stack: &mut Vec) -> String { + if let Some(raw_index) = encoded_key.strip_prefix('_') { + if let Ok(index) = raw_index.parse::() { + if let Value::String(key) = resolve_table_index(table, index, stack) { + return key; + } + } + } + encoded_key.to_string() +} + +fn find_object_with_array_key<'a>(value: &'a Value, key: &str) -> Option<&'a Map> { + match value { + Value::Object(map) => { + if map.get(key).and_then(Value::as_array).is_some() { + return Some(map); + } + map.values() + .find_map(|child| find_object_with_array_key(child, key)) + } + Value::Array(items) => items + .iter() + .find_map(|child| find_object_with_array_key(child, key)), + _ => None, + } +} + +fn parse_markdown_transcript( + input: &str, + options: &SharedDialogParseOptions, + provider: &str, + source_url: &str, + capture_method: &str, +) -> SharedDialogCapture { + let mut turns = Vec::new(); + let mut current_role: Option<&'static str> = None; + let mut current_lines = Vec::new(); + + for line in input.lines() { + if let Some((role, rest)) = markdown_turn_prefix(line) { + push_markdown_turn( + &mut turns, + current_role.take(), + ¤t_lines, + source_url, + capture_method, + ); + current_role = Some(role); + current_lines.clear(); + current_lines.push(rest.trim_start().to_string()); + } else if current_role.is_some() { + current_lines.push(line.to_string()); + } + } + push_markdown_turn( + &mut turns, + current_role.take(), + ¤t_lines, + source_url, + capture_method, + ); + + if turns.is_empty() { + return unsupported_capture(UnsupportedCaptureInput { + provider: provider.to_string(), + source_url: source_url.to_string(), + capture_method: capture_method.to_string(), + captured_at: options.captured_at.clone(), + http_status: options.http_status, + unsupported_reason: "no_transcript_in_captured_dom", + message: "No compact Markdown transcript turns were found.".to_string(), + warnings: options.warnings.clone(), + error: None, + }); + } + + ok_capture(OkCaptureInput { + provider: provider.to_string(), + source_url: source_url.to_string(), + capture_method: capture_method.to_string(), + captured_at: options.captured_at.clone(), + http_status: options.http_status, + title: None, + conversation_id: None, + turns, + warnings: options.warnings.clone(), + }) +} + +fn markdown_turn_prefix(line: &str) -> Option<(&'static str, &str)> { + let trimmed = line.trim_start(); + for (prefix, role) in [ + ("U:", "user"), + ("User:", "user"), + ("A:", "assistant"), + ("Assistant:", "assistant"), + ] { + if trimmed.len() >= prefix.len() && trimmed[..prefix.len()].eq_ignore_ascii_case(prefix) { + return Some((role, &trimmed[prefix.len()..])); + } + } + None +} + +fn push_markdown_turn( + turns: &mut Vec, + role: Option<&'static str>, + lines: &[String], + source_url: &str, + capture_method: &str, +) { + let Some(role) = role else { + return; + }; + let content = trimmed_content_lines(lines); + if content.is_empty() { + return; + } + let index = turns.len(); + turns.push(SharedDialogTurn { + id: format!("markdown-turn-{}", index + 1), + role: role.to_string(), + content, + visibility: "visible".to_string(), + source_evidence: vec![SharedDialogEvidence { + kind: "markdown_transcript".to_string(), + source_url: source_url.to_string(), + capture_method: capture_method.to_string(), + pointer: format!("turns[{index}]"), + }], + }); +} + +fn trimmed_content_lines(lines: &[String]) -> String { + let Some(start) = lines.iter().position(|line| !line.trim().is_empty()) else { + return String::new(); + }; + let end = lines + .iter() + .rposition(|line| !line.trim().is_empty()) + .unwrap_or(start); + lines[start..=end].join("\n").trim().to_string() +} + +fn detect_provider(input: &str, source_url: &str) -> String { + if let Some(provider) = provider_from_url(source_url) { + return provider; + } + if input.contains(ENQUEUE_MARKER) || input.contains("linear_conversation") { + return "chatgpt".to_string(); + } + if looks_like_google_ai_mode_interstitial(input) { + return "google_ai_mode".to_string(); + } + "unknown".to_string() +} + +fn provider_from_url(source_url: &str) -> Option { + let parsed = Url::parse(source_url).ok()?; + let host = parsed.host_str()?.to_ascii_lowercase(); + if host == "chatgpt.com" || host.ends_with(".chatgpt.com") { + return Some("chatgpt".to_string()); + } + if host == "share.google" && parsed.path().starts_with("/aimode/") { + return Some("google_ai_mode".to_string()); + } + None +} + +fn parse_unsupported(input: &str, provider: &str) -> UnsupportedDiagnostic { + if looks_like_google_ai_mode_interstitial(input) { + return UnsupportedDiagnostic { + reason: "provider_challenge_interstitial", + message: "Google AI Mode capture returned a Google Search JavaScript/interstitial page instead of transcript data.", + }; + } + let lower = input.to_ascii_lowercase(); + if lower.contains("log in") + || lower.contains("sign in") + || lower.contains("login required") + || lower.contains("authentication required") + { + return UnsupportedDiagnostic { + reason: "login_required", + message: "The shared dialog page requires login before transcript data is visible.", + }; + } + if lower.contains("deleted") + || lower.contains("expired") + || lower.contains("not found") + || lower.contains("unavailable") + { + return UnsupportedDiagnostic { + reason: "deleted_or_expired_share", + message: "The shared dialog appears to be deleted, expired, or unavailable.", + }; + } + if provider == "unknown" { + return UnsupportedDiagnostic { + reason: "unsupported_provider_format", + message: "The URL or captured DOM is not a supported shared-dialog provider format.", + }; + } + UnsupportedDiagnostic { + reason: "no_transcript_in_captured_dom", + message: "The captured DOM did not contain replayable transcript data.", + } +} + +fn looks_like_google_ai_mode_interstitial(input: &str) -> bool { + input.contains("share.google/aimode") + || (input.contains("Google Search") + && input.contains("If you're having trouble accessing Google Search")) + || (input.contains("/search?q=") && input.contains("enablejs")) + || (input.contains("/httpservice/retry/enablejs") && input.contains("Google Search")) +} + +fn title_from_html(input: &str) -> Option { + let start = input.find("")? + "<title>".len(); + let end = input[start..].find("")? + start; + let title = html_escape::decode_html_entities(&input[start..end]) + .trim() + .to_string(); + let stripped = title + .strip_prefix("ChatGPT - ") + .unwrap_or(&title) + .trim() + .to_string(); + if stripped.is_empty() { + None + } else { + Some(stripped) + } +} + +fn string_field(map: &Map, key: &str) -> Option { + map.get(key).and_then(Value::as_str).map(ToOwned::to_owned) +} + +fn chatgpt_share_id(source_url: &str) -> Option { + let parsed = Url::parse(source_url).ok()?; + let parts: Vec<_> = parsed + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + let index = parts.iter().position(|part| *part == "share")?; + parts.get(index + 1).map(|value| (*value).to_string()) +} + +struct OkCaptureInput { + provider: String, + source_url: String, + capture_method: String, + captured_at: Option, + http_status: Option, + title: Option, + conversation_id: Option, + turns: Vec, + warnings: Vec, +} + +fn ok_capture(input: OkCaptureInput) -> SharedDialogCapture { + SharedDialogCapture { + provider: input.provider, + source_url: input.source_url, + capture_method: input.capture_method, + captured_at: input.captured_at, + status: "ok".to_string(), + conversation_id: input.conversation_id, + title: input.title, + turns: input.turns, + diagnostics: SharedDialogDiagnostics { + status: "ok".to_string(), + http_status: input.http_status, + unsupported_reason: None, + message: None, + warnings: input.warnings, + error: None, + }, + } +} + +struct UnsupportedCaptureInput { + provider: String, + source_url: String, + capture_method: String, + captured_at: Option, + http_status: Option, + unsupported_reason: &'static str, + message: String, + warnings: Vec, + error: Option, +} + +fn unsupported_capture(input: UnsupportedCaptureInput) -> SharedDialogCapture { + SharedDialogCapture { + provider: input.provider, + source_url: input.source_url, + capture_method: input.capture_method, + captured_at: input.captured_at, + status: "unsupported".to_string(), + conversation_id: None, + title: None, + turns: Vec::new(), + diagnostics: SharedDialogDiagnostics { + status: "unsupported".to_string(), + http_status: input.http_status, + unsupported_reason: Some(input.unsupported_reason.to_string()), + message: Some(input.message), + warnings: input.warnings, + error: input.error, + }, + } +} + +fn with_warning(mut capture: SharedDialogCapture, warning: String) -> SharedDialogCapture { + capture.diagnostics.warnings.push(warning); + capture +} + +fn normalize_source_url(url: &str) -> Result { + let source_url = if url.starts_with("http://") || url.starts_with("https://") { + url.to_string() + } else { + format!("https://{url}") + }; + Url::parse(&source_url) + .map_err(|error| format!("Invalid shared-dialog URL \"{url}\": {error}"))?; + Ok(source_url) +} + +fn escape_lino(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + .replace('\r', "\\r") + .replace('\t', "\\t") +} + +fn push_field(lines: &mut Vec, indent: &str, name: &str, value: Option<&str>) { + if let Some(value) = value { + if !value.is_empty() { + lines.push(format!("{indent}{name} \"{}\"", escape_lino(value))); + } + } +} + +/// Format a capture as repository-local Links Notation style meta-language. +#[must_use] +pub fn format_shared_dialog_as_meta_language(capture: &SharedDialogCapture) -> String { + let mut lines = vec!["shared_dialog_capture".to_string()]; + push_field(&mut lines, " ", "provider", Some(&capture.provider)); + push_field(&mut lines, " ", "sourceUrl", Some(&capture.source_url)); + push_field( + &mut lines, + " ", + "captureMethod", + Some(&capture.capture_method), + ); + push_field( + &mut lines, + " ", + "capturedAt", + capture.captured_at.as_deref(), + ); + push_field(&mut lines, " ", "status", Some(&capture.status)); + push_field( + &mut lines, + " ", + "conversationId", + capture.conversation_id.as_deref(), + ); + push_field(&mut lines, " ", "title", capture.title.as_deref()); + lines.push(" diagnostics".to_string()); + push_field( + &mut lines, + " ", + "status", + Some(&capture.diagnostics.status), + ); + if let Some(status) = capture.diagnostics.http_status { + lines.push(format!(" httpStatus \"{status}\"")); + } + push_field( + &mut lines, + " ", + "unsupportedReason", + capture.diagnostics.unsupported_reason.as_deref(), + ); + push_field( + &mut lines, + " ", + "message", + capture.diagnostics.message.as_deref(), + ); + for warning in &capture.diagnostics.warnings { + push_field(&mut lines, " ", "warning", Some(warning)); + } + for turn in &capture.turns { + lines.push(format!(" turn \"{}\"", escape_lino(&turn.id))); + push_field(&mut lines, " ", "role", Some(&turn.role)); + push_field(&mut lines, " ", "visibility", Some(&turn.visibility)); + push_field(&mut lines, " ", "content", Some(&turn.content)); + for evidence in &turn.source_evidence { + lines.push(" evidence".to_string()); + push_field(&mut lines, " ", "kind", Some(&evidence.kind)); + push_field( + &mut lines, + " ", + "sourceUrl", + Some(&evidence.source_url), + ); + push_field( + &mut lines, + " ", + "captureMethod", + Some(&evidence.capture_method), + ); + push_field(&mut lines, " ", "pointer", Some(&evidence.pointer)); + } + } + format!("{}\n", lines.join("\n")) +} + +/// Format a capture as formal-ai compatible `demo_memory` Links Notation. +#[must_use] +pub fn format_shared_dialog_as_demo_memory( + capture: &SharedDialogCapture, + demo_label: Option<&str>, +) -> String { + let mut lines = vec!["demo_memory".to_string()]; + if capture.status != "ok" { + lines.push(" diagnostic \"shared-dialog-unsupported\"".to_string()); + push_field(&mut lines, " ", "provider", Some(&capture.provider)); + push_field(&mut lines, " ", "sourceUrl", Some(&capture.source_url)); + push_field( + &mut lines, + " ", + "captureMethod", + Some(&capture.capture_method), + ); + push_field( + &mut lines, + " ", + "unsupportedReason", + capture.diagnostics.unsupported_reason.as_deref(), + ); + push_field( + &mut lines, + " ", + "message", + capture.diagnostics.message.as_deref(), + ); + return format!("{}\n", lines.join("\n")); + } + + let label = demo_label.unwrap_or("web-capture-shared-dialog"); + for turn in &capture.turns { + lines.push(format!(" event \"{}\"", escape_lino(&turn.id))); + push_field(&mut lines, " ", "role", Some(&turn.role)); + push_field(&mut lines, " ", "content", Some(&turn.content)); + push_field(&mut lines, " ", "demoLabel", Some(label)); + push_field( + &mut lines, + " ", + "conversationId", + capture.conversation_id.as_deref(), + ); + push_field( + &mut lines, + " ", + "conversationTitle", + capture.title.as_deref(), + ); + for evidence in &turn.source_evidence { + push_field(&mut lines, " ", "evidence", Some(&evidence.source_url)); + } + } + format!("{}\n", lines.join("\n")) +} + +/// Format a capture as Markdown. +#[must_use] +pub fn format_shared_dialog_as_markdown(capture: &SharedDialogCapture) -> String { + let mut lines = Vec::new(); + lines.push(format!( + "# {}", + capture.title.as_deref().unwrap_or("Shared Dialog Capture") + )); + lines.push(String::new()); + lines.push(format!("- Provider: `{}`", capture.provider)); + lines.push(format!("- Source: {}", capture.source_url)); + lines.push(format!("- Capture method: `{}`", capture.capture_method)); + lines.push(format!("- Status: `{}`", capture.status)); + if let Some(conversation_id) = capture.conversation_id.as_ref() { + lines.push(format!("- Conversation ID: `{conversation_id}`")); + } + if capture.status != "ok" { + if let Some(reason) = capture.diagnostics.unsupported_reason.as_ref() { + lines.push(format!("- Unsupported reason: `{reason}`")); + } + if let Some(message) = capture.diagnostics.message.as_ref() { + lines.push(String::new()); + lines.push(message.clone()); + } + return format!("{}\n", lines.join("\n")); + } + lines.push(String::new()); + for turn in &capture.turns { + let label = if turn.role == "user" { + "User" + } else { + "Assistant" + }; + lines.push(format!("**{label}**")); + lines.push(String::new()); + lines.push(turn.content.clone()); + lines.push(String::new()); + } + format!("{}\n", lines.join("\n").trim_end()) +} + +/// Format a capture as plain text. +#[must_use] +pub fn format_shared_dialog_as_text(capture: &SharedDialogCapture) -> String { + if capture.status != "ok" { + return format!( + "Provider: {}\nSource: {}\nCapture method: {}\nStatus: {}\nUnsupported reason: {}\n{}\n", + capture.provider, + capture.source_url, + capture.capture_method, + capture.status, + capture + .diagnostics + .unsupported_reason + .as_deref() + .unwrap_or_default(), + capture.diagnostics.message.as_deref().unwrap_or_default() + ); + } + let mut lines = Vec::new(); + for turn in &capture.turns { + let label = if turn.role == "user" { + "User" + } else { + "Assistant" + }; + lines.push(format!("{label}:")); + lines.push(turn.content.clone()); + lines.push(String::new()); + } + lines.join("\n") +} + +/// Format a capture as simple HTML containing the Markdown representation. +#[must_use] +pub fn format_shared_dialog_as_html(capture: &SharedDialogCapture) -> String { + let title = + html_escape::encode_text(capture.title.as_deref().unwrap_or("Shared Dialog Capture")); + let markdown = format_shared_dialog_as_markdown(capture); + let body = html_escape::encode_text(&markdown); + format!( + "\n\n{title}\n
{body}
\n\n" + ) +} + +/// Format a capture according to a CLI/API format name. +/// +/// # Errors +/// +/// Returns a serialization error for JSON output if serde cannot serialize the +/// capture. +pub fn format_shared_dialog_result( + capture: &SharedDialogCapture, + format: &str, +) -> Result { + Ok(match format.to_ascii_lowercase().as_str() { + "meta-language" | "meta_language" => format_shared_dialog_as_meta_language(capture), + "demo-memory" | "demo_memory" => format_shared_dialog_as_demo_memory(capture, None), + "markdown" | "md" => format_shared_dialog_as_markdown(capture), + "txt" | "text" => format_shared_dialog_as_text(capture), + "html" => format_shared_dialog_as_html(capture), + _ => format!("{}\n", serde_json::to_string_pretty(capture)?), + }) +} + +/// Return the conventional extension for a shared-dialog output format. +#[must_use] +pub fn shared_dialog_output_extension(format: &str) -> &'static str { + match format.to_ascii_lowercase().as_str() { + "meta-language" | "meta_language" | "demo-memory" | "demo_memory" => "lino", + "markdown" | "md" => "md", + "txt" | "text" => "txt", + "html" => "html", + _ => "json", + } +} diff --git a/rust/tests/unit/mod.rs b/rust/tests/unit/mod.rs index d82405a..ebd3d15 100644 --- a/rust/tests/unit/mod.rs +++ b/rust/tests/unit/mod.rs @@ -7,4 +7,5 @@ mod localize_images; mod markdown; mod metadata; mod postprocess; +mod shared_dialog; mod verify; diff --git a/rust/tests/unit/shared_dialog.rs b/rust/tests/unit/shared_dialog.rs new file mode 100644 index 0000000..0b862c4 --- /dev/null +++ b/rust/tests/unit/shared_dialog.rs @@ -0,0 +1,109 @@ +use std::fs; +use std::path::Path; + +use web_capture::shared_dialog::{ + format_shared_dialog_as_demo_memory, format_shared_dialog_as_markdown, + format_shared_dialog_as_meta_language, parse_shared_dialog, SharedDialogParseOptions, +}; + +const CHATGPT_SHARE_URL: &str = "https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24"; +const GOOGLE_AI_MODE_URL: &str = "https://share.google/aimode/VG0HhpnAXrBkC0QgP"; + +#[test] +fn chatgpt_share_html_extracts_visible_dialog_turns() { + let html = read_case_study_fixture("chatgpt-share-6a3825b9.html"); + let capture = parse_shared_dialog( + &html, + &SharedDialogParseOptions { + source_url: Some(CHATGPT_SHARE_URL.to_string()), + capture_method: "static_http".to_string(), + ..SharedDialogParseOptions::default() + }, + ); + + assert_eq!(capture.provider, "chatgpt"); + assert_eq!(capture.status, "ok"); + assert_eq!(capture.capture_method, "static_http"); + assert_eq!(capture.title.as_deref(), Some("Infinite loop script")); + assert_eq!( + capture.conversation_id.as_deref(), + Some("6a3825b9-8de4-83ee-9c24-52fd1eb38d24") + ); + assert_eq!(capture.turns.len(), 4); + assert_eq!(capture.turns[0].role, "user"); + assert_eq!(capture.turns[1].role, "assistant"); + assert_eq!(capture.turns[2].role, "user"); + assert_eq!(capture.turns[3].role, "assistant"); + assert_eq!(capture.turns[0].visibility, "visible"); + assert_eq!( + capture.turns[0].source_evidence[0].source_url, + CHATGPT_SHARE_URL + ); + assert!(capture.turns[0].content.contains("make a loop of that")); + assert!(capture.turns[1] + .content + .contains("while true; do sleep 30m && hive-cleanup -f; done")); + assert!(capture.turns[3] + .content + .contains("screen -dmS auto-cleanup bash -c")); +} + +#[test] +fn chatgpt_share_formats_demo_memory_meta_language_and_markdown() { + let html = read_case_study_fixture("chatgpt-share-6a3825b9.html"); + let capture = parse_shared_dialog( + &html, + &SharedDialogParseOptions { + source_url: Some(CHATGPT_SHARE_URL.to_string()), + capture_method: "static_http".to_string(), + ..SharedDialogParseOptions::default() + }, + ); + + let memory = format_shared_dialog_as_demo_memory(&capture, Some("issue-552-chatgpt-share")); + assert!(memory.contains("demo_memory")); + assert_eq!(memory.matches("\n event \"").count(), 4); + assert!(memory.contains("conversationTitle \"Infinite loop script\"")); + assert!(memory + .contains("evidence \"https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24\"")); + + let meta_language = format_shared_dialog_as_meta_language(&capture); + assert!(meta_language.contains("shared_dialog_capture")); + assert!(meta_language.contains("provider \"chatgpt\"")); + assert!(meta_language.contains("turn \"0c9f0151-b5a1-402f-afc3-6bd34a0d01d2\"")); + + let markdown = format_shared_dialog_as_markdown(&capture); + assert!(markdown.contains("# Infinite loop script")); + assert!(markdown.contains("**User**")); + assert!(markdown.contains("**Assistant**")); +} + +#[test] +fn google_ai_mode_interstitial_returns_structured_diagnostic() { + let html = read_case_study_fixture("google-ai-mode-VG0HhpnAXrBkC0QgP.html"); + let capture = parse_shared_dialog( + &html, + &SharedDialogParseOptions { + source_url: Some(GOOGLE_AI_MODE_URL.to_string()), + capture_method: "static_http".to_string(), + ..SharedDialogParseOptions::default() + }, + ); + + assert_eq!(capture.provider, "google_ai_mode"); + assert_eq!(capture.status, "unsupported"); + assert!(capture.turns.is_empty()); + assert_eq!( + capture.diagnostics.unsupported_reason.as_deref(), + Some("provider_challenge_interstitial") + ); +} + +fn read_case_study_fixture(name: &str) -> String { + fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../docs/case-studies/issue-141/raw-data") + .join(name), + ) + .expect("case-study fixture should be readable") +} From fb5f5dea647a34ccfbef68727328b30185e59d8f Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 20:33:58 +0000 Subject: [PATCH 03/12] test(js): harden process e2e loopback URLs --- js/tests/e2e/process.test.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/js/tests/e2e/process.test.js b/js/tests/e2e/process.test.js index c1f9ebf..6178e0a 100644 --- a/js/tests/e2e/process.test.js +++ b/js/tests/e2e/process.test.js @@ -23,11 +23,13 @@ beforeAll(async () => { res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); res.end(MOCK_HTML); }); - await new Promise((resolve) => mockServer.listen(mockPort, resolve)); - mockUrl = `http://localhost:${mockPort}`; + await new Promise((resolve) => + mockServer.listen(mockPort, '127.0.0.1', resolve) + ); + mockUrl = `http://127.0.0.1:${mockPort}`; const port = await getPort(); - baseUrl = `http://localhost:${port}`; + baseUrl = `http://127.0.0.1:${port}`; serverProcess = spawn( 'node', From a3f7f2b73ce5d379455a715cac38a4d4d9acea9c Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 20:39:31 +0000 Subject: [PATCH 04/12] test(js): isolate process e2e fixture server --- js/tests/e2e/process.test.js | 108 ++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/js/tests/e2e/process.test.js b/js/tests/e2e/process.test.js index 6178e0a..332da1e 100644 --- a/js/tests/e2e/process.test.js +++ b/js/tests/e2e/process.test.js @@ -1,13 +1,13 @@ import { spawn } from 'child_process'; -import http from 'http'; import fetch from 'node-fetch'; import getPort from 'get-port'; import path from 'path'; const WAIT_FOR_READY = 5000; // ms let serverProcess; +let serverOutput = ''; let baseUrl; -let mockServer; +let mockProcess; let mockUrl; const MOCK_HTML = ` @@ -17,15 +17,9 @@ const MOCK_HTML = ` `; beforeAll(async () => { - // Start a local mock server to avoid depending on external network const mockPort = await getPort(); - mockServer = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); - res.end(MOCK_HTML); - }); - await new Promise((resolve) => - mockServer.listen(mockPort, '127.0.0.1', resolve) - ); + mockProcess = startMockServer(mockPort); + await waitForProcessOutput(mockProcess, 'mock fixture server listening'); mockUrl = `http://127.0.0.1:${mockPort}`; const port = await getPort(); @@ -39,6 +33,9 @@ beforeAll(async () => { stdio: ['ignore', 'pipe', 'pipe'], } ); + collectProcessOutput(serverProcess, (chunk) => { + serverOutput += chunk; + }); // Wait for the server to be ready (simple delay or poll) await new Promise((resolve, reject) => { @@ -81,8 +78,8 @@ afterAll(() => { if (serverProcess) { serverProcess.kill(); } - if (mockServer) { - mockServer.close(); + if (mockProcess) { + mockProcess.kill(); } }); @@ -91,8 +88,7 @@ describe('E2E: Web Capture Microservice', () => { const res = await fetch( `${baseUrl}/html?url=${encodeURIComponent(mockUrl)}` ); - expect(res.status).toBe(200); - const text = await res.text(); + const text = await expectTextResponse(res); expect(text).toMatch(/ { const res = await fetch( `${baseUrl}/markdown?url=${encodeURIComponent(mockUrl)}` ); - expect(res.status).toBe(200); - const text = await res.text(); + const text = await expectTextResponse(res); expect(text).toMatch(/example/i); }); @@ -109,7 +104,7 @@ describe('E2E: Web Capture Microservice', () => { const res = await fetch( `${baseUrl}/image?url=${encodeURIComponent(mockUrl)}` ); - expect(res.status).toBe(200); + await expectStatusOk(res); expect(res.headers.get('content-type')).toMatch(/^image\/png/); const buf = Buffer.from(await res.arrayBuffer()); // PNG signature check @@ -124,9 +119,7 @@ describe('E2E: Web Capture Microservice', () => { const res = await fetch( `${baseUrl}/stream?url=${encodeURIComponent(mockUrl)}` ); - expect(res.status).toBe(200); - // Get the response as text - const text = await res.text(); + const text = await expectTextResponse(res); expect(text).toMatch(/ { const res = await fetch( `${baseUrl}/fetch?url=${encodeURIComponent(mockUrl)}` ); - expect(res.status).toBe(200); - const text = await res.text(); + const text = await expectTextResponse(res); expect(text).toMatch(/ { + res.writeHead(200, { + 'Content-Type': 'text/html; charset=utf-8', + 'Content-Length': Buffer.byteLength(html), + }); + res.end(html); + }); + server.listen(${port}, '127.0.0.1', () => { + console.log('mock fixture server listening'); + }); + process.on('SIGTERM', () => server.close(() => process.exit(0))); + process.on('SIGINT', () => server.close(() => process.exit(0))); + `; + return spawn('node', ['-e', script], { + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function collectProcessOutput(child, onChunk) { + child.stdout.on('data', (data) => onChunk(data.toString())); + child.stderr.on('data', (data) => onChunk(data.toString())); +} + +async function waitForProcessOutput(child, expected) { + let output = ''; + collectProcessOutput(child, (chunk) => { + output += chunk; + }); + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject( + new Error(`Timed out waiting for "${expected}". Output:\n${output}`) + ); + }, WAIT_FOR_READY); + child.stdout.on('data', (data) => { + if (data.toString().includes(expected)) { + clearTimeout(timeout); + resolve(); + } + }); + child.on('exit', (code, signal) => { + clearTimeout(timeout); + reject(new Error(`Process exited with code ${code}, signal ${signal}`)); + }); + }); +} + +async function expectTextResponse(res) { + const text = await res.text(); + expectResponseStatus(res.status, text); + return text; +} + +async function expectStatusOk(res) { + if (res.status === 200) { + return; + } + expectResponseStatus(res.status, await res.text()); +} + +function expectResponseStatus(status, body) { + if (status !== 200) { + throw new Error( + `Expected HTTP 200, received ${status}\nBody:\n${body}\nServer output:\n${serverOutput}` + ); + } +} From 40f5e095874147700a2777e5cb6e7a5ca5f7c2dc Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 20:45:41 +0000 Subject: [PATCH 05/12] fix(js): harden proxy response headers --- js/src/fetch.js | 17 +---- js/src/proxy-headers.js | 60 +++++++++++++++++ js/src/stream.js | 19 ++---- js/tests/e2e/process.test.js | 9 ++- js/tests/unit/proxy-headers.test.js | 100 ++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 31 deletions(-) create mode 100644 js/src/proxy-headers.js create mode 100644 js/tests/unit/proxy-headers.test.js diff --git a/js/src/fetch.js b/js/src/fetch.js index ebce729..f7fcd9c 100644 --- a/js/src/fetch.js +++ b/js/src/fetch.js @@ -1,5 +1,6 @@ import fetch from 'node-fetch'; import { convertGoogleDriveUrl } from './lib.js'; +import { copyProxyResponseHeaders } from './proxy-headers.js'; export async function fetchHandler(req, res) { const url = req.query.url; @@ -10,21 +11,7 @@ export async function fetchHandler(req, res) { const response = await fetch(convertGoogleDriveUrl(url)); // Copy status and headers res.status(response.status); - - // Set default content type if not present - const contentType = response.headers.get('content-type') || 'text/plain'; - res.setHeader('Content-Type', contentType); - - // Copy other headers - for (const [key, value] of response.headers.entries()) { - if ( - key.toLowerCase() !== 'transfer-encoding' && - key.toLowerCase() !== 'content-encoding' && - key.toLowerCase() !== 'content-length' - ) { - res.setHeader(key, value); - } - } + copyProxyResponseHeaders(response.headers, res); // Get the response body as buffer and send it const buffer = await response.buffer(); diff --git a/js/src/proxy-headers.js b/js/src/proxy-headers.js new file mode 100644 index 0000000..a199484 --- /dev/null +++ b/js/src/proxy-headers.js @@ -0,0 +1,60 @@ +const HOP_BY_HOP_HEADERS = new Set([ + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', +]); + +const TRANSFORMED_BODY_HEADERS = new Set(['content-encoding']); + +export function getProxyHeaderEntries(sourceHeaders, options = {}) { + const { preserveContentLength = false } = options; + const connectionTokens = getConnectionTokens(sourceHeaders); + const isEncoded = Boolean(sourceHeaders.get('content-encoding')); + const entries = []; + + for (const [key, value] of sourceHeaders.entries()) { + const lowerKey = key.toLowerCase(); + if ( + HOP_BY_HOP_HEADERS.has(lowerKey) || + TRANSFORMED_BODY_HEADERS.has(lowerKey) || + connectionTokens.has(lowerKey) + ) { + continue; + } + if ( + lowerKey === 'content-length' && + (!preserveContentLength || isEncoded) + ) { + continue; + } + entries.push([key, value]); + } + + return entries; +} + +export function copyProxyResponseHeaders(sourceHeaders, res, options = {}) { + const contentType = sourceHeaders.get('content-type') || 'text/plain'; + res.setHeader('Content-Type', contentType); + + for (const [key, value] of getProxyHeaderEntries(sourceHeaders, options)) { + if (key.toLowerCase() !== 'content-type') { + res.setHeader(key, value); + } + } +} + +function getConnectionTokens(sourceHeaders) { + const connection = sourceHeaders.get('connection') || ''; + return new Set( + connection + .split(',') + .map((token) => token.trim().toLowerCase()) + .filter(Boolean) + ); +} diff --git a/js/src/stream.js b/js/src/stream.js index a589baa..6ff7748 100644 --- a/js/src/stream.js +++ b/js/src/stream.js @@ -1,6 +1,7 @@ import fetch from 'node-fetch'; import { pipeline } from 'stream'; import { convertGoogleDriveUrl } from './lib.js'; +import { copyProxyResponseHeaders } from './proxy-headers.js'; export async function streamHandler(req, res) { const url = req.query.url; @@ -11,21 +12,9 @@ export async function streamHandler(req, res) { const response = await fetch(convertGoogleDriveUrl(url)); // Copy status and headers res.status(response.status); - - // Set default content type if not present - const contentType = response.headers.get('content-type') || 'text/plain'; - res.setHeader('Content-Type', contentType); - - // Copy other headers - for (const [key, value] of response.headers.entries()) { - if ( - key.toLowerCase() !== 'transfer-encoding' && - key.toLowerCase() !== 'content-encoding' && - key.toLowerCase() !== 'content-length' - ) { - res.setHeader(key, value); - } - } + copyProxyResponseHeaders(response.headers, res, { + preserveContentLength: true, + }); // Stream the response body if (response.body) { diff --git a/js/tests/e2e/process.test.js b/js/tests/e2e/process.test.js index 332da1e..78d722a 100644 --- a/js/tests/e2e/process.test.js +++ b/js/tests/e2e/process.test.js @@ -186,7 +186,14 @@ async function waitForProcessOutput(child, expected) { } async function expectTextResponse(res) { - const text = await res.text(); + let text; + try { + text = await res.text(); + } catch (error) { + throw new Error( + `${error.message}\nServer output:\n${serverOutput || '(empty)'}` + ); + } expectResponseStatus(res.status, text); return text; } diff --git a/js/tests/unit/proxy-headers.test.js b/js/tests/unit/proxy-headers.test.js new file mode 100644 index 0000000..e1f096f --- /dev/null +++ b/js/tests/unit/proxy-headers.test.js @@ -0,0 +1,100 @@ +import { + copyProxyResponseHeaders, + getProxyHeaderEntries, +} from '../../src/proxy-headers.js'; + +describe('proxy header helpers', () => { + it('removes hop-by-hop headers and headers named by Connection', () => { + const entries = getProxyHeaderEntries( + headers({ + 'Content-Type': 'text/html', + Connection: 'keep-alive, X-Internal-Hop', + 'Keep-Alive': 'timeout=5', + TE: 'trailers', + Trailer: 'Expires', + Upgrade: 'websocket', + 'Proxy-Authenticate': 'Basic', + 'Proxy-Authorization': 'Basic token', + 'Transfer-Encoding': 'chunked', + 'X-Internal-Hop': 'remove me', + 'X-End-To-End': 'keep me', + }) + ); + + expect(Object.fromEntries(entries)).toEqual({ + 'content-type': 'text/html', + 'x-end-to-end': 'keep me', + }); + }); + + it('preserves content length only for unencoded streaming responses', () => { + expect( + Object.fromEntries( + getProxyHeaderEntries( + headers({ + 'Content-Type': 'text/plain', + 'Content-Length': '12', + }), + { preserveContentLength: true } + ) + ) + ).toEqual({ + 'content-type': 'text/plain', + 'content-length': '12', + }); + + expect( + Object.fromEntries( + getProxyHeaderEntries( + headers({ + 'Content-Type': 'text/plain', + 'Content-Length': '12', + 'Content-Encoding': 'gzip', + }), + { preserveContentLength: true } + ) + ) + ).toEqual({ + 'content-type': 'text/plain', + }); + }); + + it('sets a default content type while copying safe headers', () => { + const res = responseRecorder(); + + copyProxyResponseHeaders( + headers({ + ETag: '"abc"', + }), + res + ); + + expect(res.headers).toEqual({ + 'content-type': 'text/plain', + etag: '"abc"', + }); + }); +}); + +function headers(values) { + const map = new Map( + Object.entries(values).map(([key, value]) => [key.toLowerCase(), value]) + ); + return { + get(key) { + return map.get(key.toLowerCase()) || null; + }, + entries() { + return map.entries(); + }, + }; +} + +function responseRecorder() { + return { + headers: {}, + setHeader(key, value) { + this.headers[key.toLowerCase()] = value; + }, + }; +} From b51fa9216ed0fc46ae622acfa18982490e9681e3 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 21:04:17 +0000 Subject: [PATCH 06/12] fix(js): request identity encoding for html fetches --- js/src/lib.js | 13 +++++++++++-- js/tests/unit/fetch-html.test.js | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 js/tests/unit/fetch-html.test.js diff --git a/js/src/lib.js b/js/src/lib.js index 6a48779..508de65 100644 --- a/js/src/lib.js +++ b/js/src/lib.js @@ -15,6 +15,9 @@ import { retry } from './retry.js'; const STACKPRINTER_RETRIES = 3; const STACKPRINTER_RETRY_BASE_DELAY_MS = 1000; const GOOGLE_DRIVE_FILE_ID_RE = /^[A-Za-z0-9_-]+$/; +const HTML_FETCH_HEADERS = { + 'accept-encoding': 'identity', +}; export async function fetchHtml(url) { if (!url) { @@ -30,7 +33,7 @@ export async function fetchHtml(url) { return await fetchStackPrinterHtml(stackPrinterUrl); } - const response = await fetch(url); + const response = await fetchHtmlResponse(url); return response.text(); } @@ -194,7 +197,7 @@ function imageExtensionForContentType(contentType) { async function fetchStackPrinterHtml(url) { return await retry( async () => { - const response = await fetch(url); + const response = await fetchHtmlResponse(url); if (!response.ok) { throw new Error(`StackPrinter returned HTTP ${response.status}`); } @@ -212,6 +215,12 @@ async function fetchStackPrinterHtml(url) { ); } +function fetchHtmlResponse(url) { + return fetch(url, { + headers: HTML_FETCH_HEADERS, + }); +} + function isStackPrinterTransientError(html) { return /Ooooops|Please try again later/i.test(html); } diff --git a/js/tests/unit/fetch-html.test.js b/js/tests/unit/fetch-html.test.js new file mode 100644 index 0000000..75e6461 --- /dev/null +++ b/js/tests/unit/fetch-html.test.js @@ -0,0 +1,24 @@ +import nock from 'nock'; +import { fetchHtml } from '../../src/lib.js'; + +afterEach(() => { + nock.cleanAll(); +}); + +describe('fetchHtml', () => { + it('requests identity encoding for document HTML fetches', async () => { + nock('https://example.com', { + reqheaders: { + 'accept-encoding': 'identity', + }, + }) + .get('/article') + .reply(200, 'ok', { + 'content-type': 'text/html; charset=utf-8', + }); + + await expect(fetchHtml('https://example.com/article')).resolves.toContain( + 'ok' + ); + }); +}); From e35556bf6c3c201b2d906b80aed0e7906b78e58b Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 21:20:29 +0000 Subject: [PATCH 07/12] fix(js): request identity encoding for gdocs exports --- js/src/gdocs.js | 12 ++++++---- js/tests/unit/gdocs-export-headers.test.js | 28 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 js/tests/unit/gdocs-export-headers.test.js diff --git a/js/src/gdocs.js b/js/src/gdocs.js index 8c3da45..017b673 100644 --- a/js/src/gdocs.js +++ b/js/src/gdocs.js @@ -42,6 +42,12 @@ const GDOCS_EXPORT_BASE = 'https://docs.google.com/document/d'; const GDOCS_API_BASE = 'https://docs.googleapis.com/v1/documents'; const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; +const PUBLIC_EXPORT_HEADERS = { + 'User-Agent': DEFAULT_USER_AGENT, + 'Accept-Charset': 'utf-8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'identity', +}; /** * Supported Google Docs export formats. @@ -165,11 +171,7 @@ export async function fetchGoogleDoc(url, options = {}) { hasApiToken: Boolean(apiToken), })); - const headers = { - 'User-Agent': DEFAULT_USER_AGENT, - 'Accept-Charset': 'utf-8', - 'Accept-Language': 'en-US,en;q=0.9', - }; + const headers = { ...PUBLIC_EXPORT_HEADERS }; if (apiToken) { headers.Authorization = `Bearer ${apiToken}`; diff --git a/js/tests/unit/gdocs-export-headers.test.js b/js/tests/unit/gdocs-export-headers.test.js new file mode 100644 index 0000000..3d173e4 --- /dev/null +++ b/js/tests/unit/gdocs-export-headers.test.js @@ -0,0 +1,28 @@ +import nock from 'nock'; +import { fetchGoogleDocAsMarkdown } from '../../src/gdocs.js'; + +afterEach(() => { + nock.cleanAll(); +}); + +describe('Google Docs public export headers', () => { + it('requests identity encoding for public export fetches', async () => { + nock('https://docs.google.com', { + reqheaders: { + 'accept-encoding': 'identity', + }, + }) + .get('/document/d/export-doc/export') + .query({ format: 'html' }) + .reply(200, '

Export Doc

', { + 'content-type': 'text/html; charset=utf-8', + }); + + const result = await fetchGoogleDocAsMarkdown( + 'https://docs.google.com/document/d/export-doc/edit' + ); + + expect(result.markdown).toContain('Export Doc'); + expect(nock.isDone()).toBe(true); + }); +}); From 8b4e041045c4503a26c7efa13cba0eba3cba6453 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 21:37:47 +0000 Subject: [PATCH 08/12] fix(js): request identity encoding for github api --- js/src/github.js | 1 + js/tests/unit/github.test.js | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/js/src/github.js b/js/src/github.js index 261db52..8c4d3da 100644 --- a/js/src/github.js +++ b/js/src/github.js @@ -244,6 +244,7 @@ async function fetchGithubText(url, options = {}) { function githubHeaders(accept) { const headers = { Accept: accept, + 'Accept-Encoding': 'identity', 'User-Agent': GITHUB_USER_AGENT, 'X-GitHub-Api-Version': '2022-11-28', }; diff --git a/js/tests/unit/github.test.js b/js/tests/unit/github.test.js index f71f93a..c9f8dd4 100644 --- a/js/tests/unit/github.test.js +++ b/js/tests/unit/github.test.js @@ -1,4 +1,6 @@ +import nock from 'nock'; import { + fetchGithubRepositorySnapshot, formatGithubRepositoryMarkdown, formatGithubRepositoryText, getGithubRepositoryTextFilename, @@ -42,6 +44,10 @@ const SNAPSHOT = { }, }; +afterEach(() => { + nock.cleanAll(); +}); + describe('GitHub repository URLs', () => { it('detects plain GitHub repository pages', () => { expect( @@ -70,6 +76,59 @@ describe('GitHub repository URLs', () => { }); }); +describe('GitHub repository snapshot fetching', () => { + it('requests identity encoding from GitHub JSON and raw README fetches', async () => { + const owner = 'octocat'; + const repo = 'Hello-World'; + const readme = '# Hello World\n\nRaw README content.'; + + const githubApi = nock('https://api.github.com', { + reqheaders: { + 'accept-encoding': 'identity', + }, + }) + .get(`/repos/${owner}/${repo}`) + .reply(200, { + full_name: `${owner}/${repo}`, + html_url: `https://github.com/${owner}/${repo}`, + default_branch: 'master', + }) + .get(`/repos/${owner}/${repo}/readme`) + .query({ ref: 'master' }) + .reply(200, { + name: 'README.md', + path: 'README.md', + download_url: `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`, + }) + .get(`/repos/${owner}/${repo}/contents`) + .query({ ref: 'master' }) + .reply(200, [ + { + name: 'README.md', + path: 'README.md', + type: 'file', + size: readme.length, + }, + ]); + + const rawGithub = nock('https://raw.githubusercontent.com', { + reqheaders: { + 'accept-encoding': 'identity', + }, + }) + .get(`/${owner}/${repo}/master/README.md`) + .reply(200, readme); + + const snapshot = await fetchGithubRepositorySnapshot( + `https://github.com/${owner}/${repo}` + ); + + expect(snapshot.readme.content).toBe(readme); + expect(githubApi.isDone()).toBe(true); + expect(rawGithub.isDone()).toBe(true); + }); +}); + describe('GitHub repository snapshot formatting', () => { it('formats a compact markdown snapshot with metadata, file tree, and README', () => { const markdown = formatGithubRepositoryMarkdown(SNAPSHOT); From 20046e48ad639f1cbc9b9bc8bd6451f74ac9db4a Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 22:04:10 +0000 Subject: [PATCH 09/12] fix(js): harden stream proxy response handling --- js/docker-compose.yml | 2 + js/src/stream.js | 28 ++++++------ js/tests/e2e/docker.test.js | 5 +-- js/tests/unit/stream.test.js | 83 ++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 js/tests/unit/stream.test.js diff --git a/js/docker-compose.yml b/js/docker-compose.yml index ef73e34..f89c4a8 100644 --- a/js/docker-compose.yml +++ b/js/docker-compose.yml @@ -5,5 +5,7 @@ services: dockerfile: Dockerfile ports: - '3000:3000' + extra_hosts: + - 'host.docker.internal:host-gateway' environment: - PORT=3000 diff --git a/js/src/stream.js b/js/src/stream.js index 6ff7748..96ae4e1 100644 --- a/js/src/stream.js +++ b/js/src/stream.js @@ -1,32 +1,36 @@ import fetch from 'node-fetch'; -import { pipeline } from 'stream'; import { convertGoogleDriveUrl } from './lib.js'; import { copyProxyResponseHeaders } from './proxy-headers.js'; +const STREAM_FETCH_HEADERS = { + 'accept-encoding': 'identity', +}; + export async function streamHandler(req, res) { const url = req.query.url; if (!url) { return res.status(400).send('Missing `url` parameter'); } try { - const response = await fetch(convertGoogleDriveUrl(url)); + const response = await fetch(convertGoogleDriveUrl(url), { + headers: STREAM_FETCH_HEADERS, + }); // Copy status and headers res.status(response.status); - copyProxyResponseHeaders(response.headers, res, { - preserveContentLength: true, - }); + copyProxyResponseHeaders(response.headers, res); // Stream the response body if (response.body) { - pipeline(response.body, res, (err) => { - if (err) { - console.error('Pipeline error in /stream:', err); - if (!res.headersSent) { - res.status(500); - res.end('Error proxying content'); - } + response.body.on('error', (err) => { + console.error('Upstream stream error in /stream:', err); + if (!res.headersSent) { + res.status(500); + res.end('Error proxying content'); + } else if (!res.writableEnded) { + res.end(); } }); + response.body.pipe(res); } else { res.end(); } diff --git a/js/tests/e2e/docker.test.js b/js/tests/e2e/docker.test.js index 0be54ba..05de3d4 100644 --- a/js/tests/e2e/docker.test.js +++ b/js/tests/e2e/docker.test.js @@ -40,9 +40,8 @@ beforeAll(async () => { await new Promise((resolve) => mockServer.listen(mockPort, '0.0.0.0', resolve) ); - // Use host.docker.internal or the host IP to make it reachable from Docker - // On Linux (CI), Docker containers can reach the host via 172.17.0.1 - mockUrl = `http://172.17.0.1:${mockPort}`; + // docker-compose.yml maps this hostname to the host gateway on Linux CI. + mockUrl = `http://host.docker.internal:${mockPort}`; console.log('Checking if Docker service is already running...'); const alreadyRunning = await isServiceRunning(); diff --git a/js/tests/unit/stream.test.js b/js/tests/unit/stream.test.js new file mode 100644 index 0000000..5dea75d --- /dev/null +++ b/js/tests/unit/stream.test.js @@ -0,0 +1,83 @@ +import express from 'express'; +import http from 'http'; +import request from 'supertest'; +import { streamHandler } from '../../src/stream.js'; + +describe('streamHandler', () => { + it('streams upstream content with proxy-safe framing', async () => { + const body = 'Example Domain'; + let upstreamHeaders; + const { server, url } = await startUpstreamServer((req, res) => { + upstreamHeaders = req.headers; + res.writeHead(200, { + 'Content-Type': 'text/html; charset=utf-8', + 'Content-Length': Buffer.byteLength(body), + }); + res.end(body); + }); + + try { + const res = await request(testApp()) + .get('/stream') + .query({ url }) + .expect(200); + + expect(upstreamHeaders['accept-encoding']).toBe('identity'); + expect(res.headers['content-type']).toBe('text/html; charset=utf-8'); + expect(res.headers['content-length']).toBeUndefined(); + expect(res.text).toContain('Example Domain'); + } finally { + await closeServer(server); + } + }); + + it('ends the downstream response when upstream closes after body bytes', async () => { + const body = 'Example Domain'; + const { server, url } = await startUpstreamServer((req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/html; charset=utf-8', + }); + res.write(body); + res.socket.end(); + }); + + try { + const res = await request(testApp()) + .get('/stream') + .query({ url }) + .expect(200); + + expect(res.text).toContain('Example Domain'); + } finally { + await closeServer(server); + } + }); +}); + +function testApp() { + const app = express(); + app.get('/stream', streamHandler); + return app; +} + +async function startUpstreamServer(handler) { + const server = http.createServer(handler); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + return { + server, + url: `http://127.0.0.1:${port}`, + }; +} + +async function closeServer(server) { + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); +} From 4e9f1351b64c4ecd4d673bc362844a119cffacde Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 22:10:30 +0000 Subject: [PATCH 10/12] fix(js): handle upstream stream closes --- js/src/stream.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/js/src/stream.js b/js/src/stream.js index 96ae4e1..bfd3a6a 100644 --- a/js/src/stream.js +++ b/js/src/stream.js @@ -21,14 +21,28 @@ export async function streamHandler(req, res) { // Stream the response body if (response.body) { - response.body.on('error', (err) => { - console.error('Upstream stream error in /stream:', err); + let upstreamEnded = false; + let upstreamFailureHandled = false; + const finishAfterUpstreamFailure = (err) => { + if (upstreamEnded || upstreamFailureHandled) { + return; + } + upstreamFailureHandled = true; + console.error('Upstream stream failed in /stream:', err); if (!res.headersSent) { res.status(500); res.end('Error proxying content'); } else if (!res.writableEnded) { res.end(); } + }; + + response.body.once('end', () => { + upstreamEnded = true; + }); + response.body.once('error', finishAfterUpstreamFailure); + response.body.once('close', () => { + finishAfterUpstreamFailure(new Error('Upstream closed before end')); }); response.body.pipe(res); } else { From d8cc305f24fcfcb3f7a55e73c9d23219db129975 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 22:15:57 +0000 Subject: [PATCH 11/12] fix(js): stream proxy responses with explicit writes --- js/src/stream.js | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/js/src/stream.js b/js/src/stream.js index bfd3a6a..8ede1ac 100644 --- a/js/src/stream.js +++ b/js/src/stream.js @@ -21,30 +21,7 @@ export async function streamHandler(req, res) { // Stream the response body if (response.body) { - let upstreamEnded = false; - let upstreamFailureHandled = false; - const finishAfterUpstreamFailure = (err) => { - if (upstreamEnded || upstreamFailureHandled) { - return; - } - upstreamFailureHandled = true; - console.error('Upstream stream failed in /stream:', err); - if (!res.headersSent) { - res.status(500); - res.end('Error proxying content'); - } else if (!res.writableEnded) { - res.end(); - } - }; - - response.body.once('end', () => { - upstreamEnded = true; - }); - response.body.once('error', finishAfterUpstreamFailure); - response.body.once('close', () => { - finishAfterUpstreamFailure(new Error('Upstream closed before end')); - }); - response.body.pipe(res); + await writeStreamResponse(response.body, res); } else { res.end(); } @@ -56,3 +33,24 @@ export async function streamHandler(req, res) { } } } + +async function writeStreamResponse(body, res) { + try { + for await (const chunk of body) { + if (!res.write(chunk)) { + await new Promise((resolve) => res.once('drain', resolve)); + } + } + if (!res.writableEnded) { + res.end(); + } + } catch (err) { + console.error('Upstream stream failed in /stream:', err); + if (!res.headersSent) { + res.status(500); + res.end('Error proxying content'); + } else if (!res.writableEnded) { + res.end(); + } + } +} From 78225bca251233ecc139f9ce1427ebdb092ae24a Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 25 Jun 2026 22:28:20 +0000 Subject: [PATCH 12/12] fix(js): preserve stream content length --- js/src/stream.js | 4 +++- js/tests/unit/stream.test.js | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/js/src/stream.js b/js/src/stream.js index 8ede1ac..6530c68 100644 --- a/js/src/stream.js +++ b/js/src/stream.js @@ -17,7 +17,9 @@ export async function streamHandler(req, res) { }); // Copy status and headers res.status(response.status); - copyProxyResponseHeaders(response.headers, res); + copyProxyResponseHeaders(response.headers, res, { + preserveContentLength: true, + }); // Stream the response body if (response.body) { diff --git a/js/tests/unit/stream.test.js b/js/tests/unit/stream.test.js index 5dea75d..f4bc9d5 100644 --- a/js/tests/unit/stream.test.js +++ b/js/tests/unit/stream.test.js @@ -24,7 +24,9 @@ describe('streamHandler', () => { expect(upstreamHeaders['accept-encoding']).toBe('identity'); expect(res.headers['content-type']).toBe('text/html; charset=utf-8'); - expect(res.headers['content-length']).toBeUndefined(); + expect(res.headers['content-length']).toBe( + String(Buffer.byteLength(body)) + ); expect(res.text).toContain('Example Domain'); } finally { await closeServer(server);