From 963fabecc4e44bef28112de8fcdf158f6714d96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=E1=BA=BF=20Long?= <73347418+the-long-ride@users.noreply.github.com> Date: Sun, 30 Aug 2026 07:03:04 +0700 Subject: [PATCH 1/6] feat: implement workspace insights and wiki links --- chromium-xtension/src/chrome-host-insights.ts | 271 +++ chromium-xtension/src/chrome-host-search.ts | 6 + chromium-xtension/src/insights-host-router.ts | 62 + ...026-08-27-workspace-insights-wiki-links.md | 1456 +++++++++++++++++ docs/workspace-insights-design.md | 983 +++++++++++ electron/core/ipc-handlers.js | 23 +- electron/core/main-bootstrap.js | 26 +- electron/core/runtime-insights-external.js | 322 ++++ electron/core/runtime-insights.js | 366 +++++ package.json | 2 +- pnpm-lock.yaml | 54 +- tauri/src/app_state.rs | 5 +- tauri/src/dispatcher/commands.rs | 8 +- tauri/src/insights/external.rs | 342 ++++ tauri/src/insights/external_host.rs | 140 ++ tauri/src/insights/mod.rs | 246 +++ tauri/src/insights/scan.rs | 322 ++++ tauri/src/lib.rs | 8 +- tests/manifest/coverage-manifest.test.ts | 2 + .../workspace-insights-coverage-manifest.ts | 43 + .../chromium/chrome-host-insights.test.ts | 74 + tests/unit/electron/insights-external.test.ts | 88 + tests/unit/electron/runtime-insights.test.ts | 98 ++ .../insights-gallery-links.test.tsx | 62 + .../insights-graph-related-reports.test.tsx | 73 + .../insights-lint-duplicates.test.tsx | 41 + .../insights-settings-integration.test.tsx | 141 ++ .../ui/components/scope-view-modal.test.tsx | 15 +- .../ui/components/sidebar-render.test.tsx | 4 + .../workspace-insights-panel.test.tsx | 73 + .../ui/contexts/insights-translations.test.ts | 39 + .../unit/ui/insights/analyze-document.test.ts | 65 + tests/unit/ui/insights/cache.test.ts | 92 ++ tests/unit/ui/insights/contracts.test.ts | 54 + tests/unit/ui/insights/duplicates.test.ts | 63 + tests/unit/ui/insights/graph.test.ts | 31 + tests/unit/ui/insights/host-protocol.test.ts | 196 +++ tests/unit/ui/insights/index.test.ts | 77 + tests/unit/ui/insights/lint.test.ts | 87 + tests/unit/ui/insights/patterns.test.ts | 34 + tests/unit/ui/insights/relationships.test.ts | 66 + .../ui/insights/settings-portability.test.ts | 92 ++ tests/unit/ui/insights/settings-store.test.ts | 62 + .../insights/use-workspace-insights.test.tsx | 273 ++++ tests/unit/ui/insights/worker-client.test.ts | 59 + .../ui/insights/workspace-identity.test.ts | 40 + tests/unit/ui/markdown/anchors.test.ts | 44 + tests/unit/ui/markdown/frontmatter.test.ts | 54 + tests/unit/ui/markdown/references.test.ts | 83 + tests/unit/ui/markdown/transclusion.test.ts | 94 ++ tests/unit/ui/markdown/wiki-links.test.ts | 139 ++ tests/unit/ui/markdown/wiki-renderer.test.ts | 26 + tests/unit/ui/navigation-wiki-links.test.tsx | 69 + tests/unit/vscode/insights-external.test.ts | 63 + tests/unit/vscode/panel-insights.test.ts | 59 + ui/package.json | 3 +- ui/src/components/Insights/DuplicatesView.tsx | 97 ++ ui/src/components/Insights/GalleryView.tsx | 120 ++ ui/src/components/Insights/GraphView.tsx | 134 ++ .../components/Insights/InsightsSettings.tsx | 287 ++++ ui/src/components/Insights/LinksView.tsx | 143 ++ ui/src/components/Insights/LintView.tsx | 84 + ui/src/components/Insights/RelatedView.tsx | 106 ++ .../Insights/WorkspaceInsightsEntry.tsx | 111 ++ .../Insights/WorkspaceInsightsPanel.tsx | 186 +++ ui/src/components/Settings/SettingsModal.tsx | 7 +- .../components/Sidebar/SidebarTabsHeader.tsx | 2 + ui/src/constants/storage.ts | 3 + ui/src/contexts/NavigationContext.tsx | 47 +- ui/src/contexts/auditedUiTranslations.ts | 290 +++- ui/src/contexts/auditedUiTranslationsBase.ts | 113 ++ ui/src/contexts/insightsTranslations.ts | 100 ++ ui/src/dom/globalHandlers.ts | 29 + ui/src/insights/analyzeDocument.ts | 276 ++++ ui/src/insights/cache.ts | 191 +++ ui/src/insights/config.ts | 283 ++++ ui/src/insights/contracts.ts | 116 ++ ui/src/insights/duplicates.ts | 220 +++ ui/src/insights/graph.ts | 182 +++ ui/src/insights/index.ts | 314 ++++ ui/src/insights/insights.worker.ts | 65 + ui/src/insights/lint.ts | 318 ++++ ui/src/insights/patterns.ts | 158 ++ ui/src/insights/relationships.ts | 151 ++ ui/src/insights/reports.ts | 149 ++ ui/src/insights/settingsStore.ts | 107 ++ ui/src/insights/useWorkspaceInsights.ts | 420 +++++ ui/src/insights/workerClient.ts | 122 ++ ui/src/insights/workerProtocol.ts | 25 + ui/src/insights/workspaceIdentity.ts | 15 + ui/src/insights/workspaceInsightsSession.ts | 288 ++++ ui/src/markdown/anchors.ts | 65 + ui/src/markdown/frontmatter.ts | 313 ++++ ui/src/markdown/inline.ts | 42 +- ui/src/markdown/references.ts | 247 +++ ui/src/markdown/renderer.ts | 12 +- ui/src/markdown/sourceMapping.ts | 55 +- ui/src/markdown/transclusion.ts | 119 ++ ui/src/markdown/wikiLinks.ts | 323 ++++ ui/src/platform/bridge.ts | 230 ++- ui/src/settings/settingsImportExport.ts | 331 +--- ui/src/settings/settingsImportExportBase.ts | 312 ++++ .../global/global-workspace-insights.css | 198 +++ ui/src/types/hostMessages.ts | 37 + ui/src/types/webviewMessages.ts | 28 + vscode/src/core/panelInsights.ts | 253 +++ vscode/src/core/panelInsightsExternal.ts | 349 ++++ vscode/src/fonts/panelFontBridge.ts | 48 +- website-app/src/web-file-utility-router.ts | 7 +- website-app/src/web-insights-host.ts | 3 + 110 files changed, 15049 insertions(+), 502 deletions(-) create mode 100644 chromium-xtension/src/chrome-host-insights.ts create mode 100644 chromium-xtension/src/insights-host-router.ts create mode 100644 docs/superpowers/plans/2026-08-27-workspace-insights-wiki-links.md create mode 100644 docs/workspace-insights-design.md create mode 100644 electron/core/runtime-insights-external.js create mode 100644 electron/core/runtime-insights.js create mode 100644 tauri/src/insights/external.rs create mode 100644 tauri/src/insights/external_host.rs create mode 100644 tauri/src/insights/mod.rs create mode 100644 tauri/src/insights/scan.rs create mode 100644 tests/manifest/workspace-insights-coverage-manifest.ts create mode 100644 tests/unit/chromium/chrome-host-insights.test.ts create mode 100644 tests/unit/electron/insights-external.test.ts create mode 100644 tests/unit/electron/runtime-insights.test.ts create mode 100644 tests/unit/ui/components/insights-gallery-links.test.tsx create mode 100644 tests/unit/ui/components/insights-graph-related-reports.test.tsx create mode 100644 tests/unit/ui/components/insights-lint-duplicates.test.tsx create mode 100644 tests/unit/ui/components/insights-settings-integration.test.tsx create mode 100644 tests/unit/ui/components/workspace-insights-panel.test.tsx create mode 100644 tests/unit/ui/contexts/insights-translations.test.ts create mode 100644 tests/unit/ui/insights/analyze-document.test.ts create mode 100644 tests/unit/ui/insights/cache.test.ts create mode 100644 tests/unit/ui/insights/contracts.test.ts create mode 100644 tests/unit/ui/insights/duplicates.test.ts create mode 100644 tests/unit/ui/insights/graph.test.ts create mode 100644 tests/unit/ui/insights/host-protocol.test.ts create mode 100644 tests/unit/ui/insights/index.test.ts create mode 100644 tests/unit/ui/insights/lint.test.ts create mode 100644 tests/unit/ui/insights/patterns.test.ts create mode 100644 tests/unit/ui/insights/relationships.test.ts create mode 100644 tests/unit/ui/insights/settings-portability.test.ts create mode 100644 tests/unit/ui/insights/settings-store.test.ts create mode 100644 tests/unit/ui/insights/use-workspace-insights.test.tsx create mode 100644 tests/unit/ui/insights/worker-client.test.ts create mode 100644 tests/unit/ui/insights/workspace-identity.test.ts create mode 100644 tests/unit/ui/markdown/anchors.test.ts create mode 100644 tests/unit/ui/markdown/frontmatter.test.ts create mode 100644 tests/unit/ui/markdown/references.test.ts create mode 100644 tests/unit/ui/markdown/transclusion.test.ts create mode 100644 tests/unit/ui/markdown/wiki-links.test.ts create mode 100644 tests/unit/ui/markdown/wiki-renderer.test.ts create mode 100644 tests/unit/ui/navigation-wiki-links.test.tsx create mode 100644 tests/unit/vscode/insights-external.test.ts create mode 100644 tests/unit/vscode/panel-insights.test.ts create mode 100644 ui/src/components/Insights/DuplicatesView.tsx create mode 100644 ui/src/components/Insights/GalleryView.tsx create mode 100644 ui/src/components/Insights/GraphView.tsx create mode 100644 ui/src/components/Insights/InsightsSettings.tsx create mode 100644 ui/src/components/Insights/LinksView.tsx create mode 100644 ui/src/components/Insights/LintView.tsx create mode 100644 ui/src/components/Insights/RelatedView.tsx create mode 100644 ui/src/components/Insights/WorkspaceInsightsEntry.tsx create mode 100644 ui/src/components/Insights/WorkspaceInsightsPanel.tsx create mode 100644 ui/src/contexts/auditedUiTranslationsBase.ts create mode 100644 ui/src/contexts/insightsTranslations.ts create mode 100644 ui/src/insights/analyzeDocument.ts create mode 100644 ui/src/insights/cache.ts create mode 100644 ui/src/insights/config.ts create mode 100644 ui/src/insights/contracts.ts create mode 100644 ui/src/insights/duplicates.ts create mode 100644 ui/src/insights/graph.ts create mode 100644 ui/src/insights/index.ts create mode 100644 ui/src/insights/insights.worker.ts create mode 100644 ui/src/insights/lint.ts create mode 100644 ui/src/insights/patterns.ts create mode 100644 ui/src/insights/relationships.ts create mode 100644 ui/src/insights/reports.ts create mode 100644 ui/src/insights/settingsStore.ts create mode 100644 ui/src/insights/useWorkspaceInsights.ts create mode 100644 ui/src/insights/workerClient.ts create mode 100644 ui/src/insights/workerProtocol.ts create mode 100644 ui/src/insights/workspaceIdentity.ts create mode 100644 ui/src/insights/workspaceInsightsSession.ts create mode 100644 ui/src/markdown/anchors.ts create mode 100644 ui/src/markdown/frontmatter.ts create mode 100644 ui/src/markdown/references.ts create mode 100644 ui/src/markdown/transclusion.ts create mode 100644 ui/src/markdown/wikiLinks.ts create mode 100644 ui/src/settings/settingsImportExportBase.ts create mode 100644 ui/src/styles/global/global-workspace-insights.css create mode 100644 vscode/src/core/panelInsights.ts create mode 100644 vscode/src/core/panelInsightsExternal.ts create mode 100644 website-app/src/web-insights-host.ts diff --git a/chromium-xtension/src/chrome-host-insights.ts b/chromium-xtension/src/chrome-host-insights.ts new file mode 100644 index 00000000..d162b234 --- /dev/null +++ b/chromium-xtension/src/chrome-host-insights.ts @@ -0,0 +1,271 @@ +const DEFAULT_SOFT_LIMIT_BYTES = 10 * 1024 * 1024; +const DEFAULT_HARD_LIMIT_BYTES = 64 * 1024 * 1024; +const SCAN_BATCH_SIZE = 200; +const POLL_INTERVAL_MS = 2500; +const HARD_EXCLUDED = new Set(['.git', '.hg', '.svn']); +const DEFAULT_EXCLUDED = new Set(['node_modules', '.next', 'dist', 'build', 'coverage', '.cache']); + +const MIME_TYPES: Record = { + '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', + '.webp': 'image/webp', '.svg': 'image/svg+xml', '.avif': 'image/avif', + '.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime', + '.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg', '.m4a': 'audio/mp4', + '.md': 'text/markdown', '.mdx': 'text/mdx', '.pdf': 'application/pdf', +}; + +interface ChromeInsightsHostDeps { + getActiveHandle: () => FileSystemDirectoryHandle | null; + send: (message: any) => void; + setInterval?: typeof window.setInterval; + clearInterval?: typeof window.clearInterval; + pollIntervalMs?: number; +} + +function normalizePath(value: string): string | null { + const parts: string[] = []; + for (const part of String(value || '').replace(/\\/g, '/').split('/')) { + if (!part || part === '.') continue; + if (part === '..') { + if (!parts.length) return null; + parts.pop(); + continue; + } + parts.push(part); + } + return parts.join('/'); +} + +function resolveReference(documentPath: string, resourcePath: string): string | null { + const raw = String(resourcePath || '').split(/[?#]/, 1)[0]; + if (!raw || /^(?:https?:|data:|blob:|javascript:|file:\/\/)/i.test(raw)) return null; + if (raw.startsWith('/')) return normalizePath(raw.slice(1)); + const base = String(documentPath || '').replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + return normalizePath(`${base}/${raw}`); +} + +function extension(path: string): string { + const file = path.split('/').pop() || ''; + const index = file.lastIndexOf('.'); + return index > 0 ? file.slice(index).toLowerCase() : ''; +} + +function isExcluded(relativePath: string, userPatterns: readonly string[] = []): boolean { + const segments = relativePath.split('/').filter(Boolean); + if (segments.some(segment => HARD_EXCLUDED.has(segment))) return true; + let result = segments.some(segment => DEFAULT_EXCLUDED.has(segment)); + for (const raw of userPatterns) { + const value = String(raw || '').trim(); + if (!value || value.startsWith('#')) continue; + const negated = value.startsWith('!'); + const needle = (negated ? value.slice(1) : value).replace(/^\//, '').replace(/\/$/, ''); + if (relativePath === needle || relativePath.startsWith(`${needle}/`) || (!needle.includes('/') && segments.includes(needle))) { + result = !negated; + } + } + return result; +} + +async function getFileHandle(root: FileSystemDirectoryHandle, relativePath: string): Promise { + const normalized = normalizePath(relativePath); + if (!normalized) return null; + const parts = normalized.split('/'); + let directory = root; + try { + for (let index = 0; index < parts.length - 1; index += 1) { + directory = await directory.getDirectoryHandle(parts[index]); + } + return await directory.getFileHandle(parts.at(-1)!); + } catch { + return null; + } +} + +async function getAnyHandle(root: FileSystemDirectoryHandle, relativePath: string): Promise { + const normalized = normalizePath(relativePath); + if (!normalized) return null; + const parts = normalized.split('/'); + let directory = root; + try { + for (let index = 0; index < parts.length - 1; index += 1) directory = await directory.getDirectoryHandle(parts[index]); + const name = parts.at(-1)!; + try { return await directory.getFileHandle(name); } + catch { return await directory.getDirectoryHandle(name); } + } catch { + return null; + } +} + +async function sha256(text: string): Promise { + try { + if (!globalThis.crypto?.subtle) return undefined; + const digest = await globalThis.crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)); + return Array.from(new Uint8Array(digest), value => value.toString(16).padStart(2, '0')).join(''); + } catch { + return undefined; + } +} + +export function createChromeInsightsHost(deps: ChromeInsightsHostDeps) { + const cancelled = new Set(); + const setIntervalImpl = deps.setInterval ?? window.setInterval.bind(window); + const clearIntervalImpl = deps.clearInterval ?? window.clearInterval.bind(window); + let pollTimer: number | null = null; + let pollGeneration = 0; + let previousPoll = new Map(); + + async function collectEntries(root: FileSystemDirectoryHandle, userPatterns: readonly string[] = [], requestId?: string) { + const results: any[] = []; + let excludedEntries = 0; + let skippedEntries = 0; + const walk = async (directory: FileSystemDirectoryHandle, prefix = ''): Promise => { + try { + for await (const [name, handle] of (directory as any).entries()) { + if (requestId && cancelled.has(requestId)) break; + const relativePath = prefix ? `${prefix}/${name}` : name; + if (isExcluded(relativePath, userPatterns)) { excludedEntries += 1; continue; } + if (handle.kind === 'directory') { + await walk(handle as FileSystemDirectoryHandle, relativePath); + continue; + } + if (handle.kind !== 'file') continue; + try { + const file = await (handle as FileSystemFileHandle).getFile(); + results.push({ + relativePath, + canonicalRelativePath: relativePath, + kind: 'file', + sizeBytes: file.size, + mtimeMs: file.lastModified || 0, + extension: extension(relativePath) || undefined, + isSymlink: false, + }); + } catch { skippedEntries += 1; } + } + } catch { skippedEntries += 1; } + }; + await walk(root); + return { entries: results, excludedEntries, skippedEntries }; + } + + async function probeWorkspaceResource(message: any): Promise { + const root = deps.getActiveHandle(); + const requestId = String(message.requestId || ''); + if (!root) { deps.send({ command: 'workspaceResourceProbeResult', requestId, status: 'missing' }); return; } + const relativePath = resolveReference(String(message.documentPath || ''), String(message.resourcePath || '')); + if (!relativePath) { deps.send({ command: 'workspaceResourceProbeResult', requestId, status: 'outside-workspace' }); return; } + const handle = await getAnyHandle(root, relativePath); + if (!handle) { deps.send({ command: 'workspaceResourceProbeResult', requestId, status: 'missing' }); return; } + if (handle.kind === 'directory') { + deps.send({ command: 'workspaceResourceProbeResult', requestId, status: 'exists', relativePath, kind: 'directory' }); + return; + } + try { + const file = await (handle as FileSystemFileHandle).getFile(); + deps.send({ + command: 'workspaceResourceProbeResult', requestId, status: 'exists', relativePath, kind: 'file', + sizeBytes: file.size, mimeType: file.type || MIME_TYPES[extension(relativePath)] || 'application/octet-stream', + }); + } catch { + deps.send({ command: 'workspaceResourceProbeResult', requestId, status: 'unreadable' }); + } + } + + async function readInsightsDocumentSource(message: any): Promise { + const root = deps.getActiveHandle(); + const requestId = String(message.requestId || ''); + const relativePath = normalizePath(String(message.relativePath || '')) || ''; + const respond = (payload: any) => deps.send({ command: 'insightsDocumentSourceResult', requestId, relativePath, ...payload }); + if (!root || !relativePath) { respond({ status: 'missing' }); return; } + if (!/\.mdx?$/i.test(relativePath)) { respond({ status: 'unsupported' }); return; } + const handle = await getFileHandle(root, relativePath); + if (!handle) { respond({ status: 'missing' }); return; } + try { + const file = await handle.getFile(); + const soft = Math.max(1, Number(message.softLimitBytes) || DEFAULT_SOFT_LIMIT_BYTES); + const hard = Math.min(Math.max(1, Number(message.hardLimitBytes) || DEFAULT_HARD_LIMIT_BYTES), DEFAULT_HARD_LIMIT_BYTES); + if (file.size > hard) { respond({ status: 'too-large', sizeBytes: file.size, mtimeMs: file.lastModified || 0, hardLimit: true }); return; } + if (file.size > soft) { respond({ status: 'too-large', sizeBytes: file.size, mtimeMs: file.lastModified || 0, hardLimit: false }); return; } + const source = await file.text(); + respond({ status: 'ok', source, sizeBytes: file.size, mtimeMs: file.lastModified || 0, contentHash: await sha256(source) }); + } catch { + respond({ status: 'unreadable' }); + } + } + + async function scanInsightsWorkspace(message: any): Promise { + const root = deps.getActiveHandle(); + const requestId = String(message.requestId || ''); + cancelled.delete(requestId); + if (!root) { + deps.send({ command: 'insightsScanComplete', requestId, totalEntries: 0, excludedEntries: 0, skippedEntries: 0, truncated: false }); + return; + } + const { entries, excludedEntries, skippedEntries } = await collectEntries(root, Array.isArray(message.userPatterns) ? message.userPatterns : [], requestId); + for (let index = 0; index < entries.length; index += SCAN_BATCH_SIZE) { + if (cancelled.has(requestId)) break; + const batch = entries.slice(index, index + SCAN_BATCH_SIZE); + deps.send({ command: 'insightsScanBatch', requestId, entries: batch, scannedEntries: Math.min(index + batch.length, entries.length), excludedEntries }); + } + const wasCancelled = cancelled.delete(requestId); + deps.send({ + command: 'insightsScanComplete', requestId, totalEntries: entries.length, excludedEntries, skippedEntries, + truncated: false, ...(wasCancelled ? { cancelled: true } : {}), + }); + } + + function cancelInsightsScan(message: any): void { + const requestId = String(message.requestId || ''); + if (requestId) cancelled.add(requestId); + } + + function stopPolling(): void { + pollGeneration += 1; + if (pollTimer !== null) clearIntervalImpl(pollTimer); + pollTimer = null; + previousPoll = new Map(); + } + + async function pollOnce(message: any, generation: number): Promise { + const root = deps.getActiveHandle(); + if (!root || generation !== pollGeneration) return; + const { entries } = await collectEntries(root); + if (generation !== pollGeneration) return; + const next = new Map(entries.map(entry => [entry.relativePath, entry])); + const deltas: any[] = []; + for (const [relativePath, entry] of next) { + const previous = previousPoll.get(relativePath); + if (!previous || previous.sizeBytes !== entry.sizeBytes || previous.mtimeMs !== entry.mtimeMs) { + deltas.push({ kind: previous ? 'update' : 'add', entry }); + } + } + for (const relativePath of previousPoll.keys()) if (!next.has(relativePath)) deltas.push({ kind: 'delete', relativePath }); + previousPoll = next; + if (deltas.length) deps.send({ command: 'insightsFsDelta', requestId: message.requestId, workspaceOperationId: message.workspaceOperationId, deltas }); + } + + function setInsightsWatchState(message: any): void { + stopPolling(); + deps.send({ + command: 'insightsRuntimeCapabilities', requestId: message.requestId, + capabilities: { fileChanges: 'polling', externalLinkChecking: false, documentPreviewReuse: true }, + }); + if (message.active !== true || message.visible !== true || !deps.getActiveHandle()) return; + const generation = pollGeneration; + void pollOnce(message, generation); + pollTimer = setIntervalImpl(() => { void pollOnce(message, generation); }, deps.pollIntervalMs ?? POLL_INTERVAL_MS); + } + + function dispose(): void { stopPolling(); cancelled.clear(); } + + return { + capabilities: { fileChanges: 'polling' as const, externalLinkChecking: false, documentPreviewReuse: true }, + probeWorkspaceResource, + readInsightsDocumentSource, + scanInsightsWorkspace, + cancelInsightsScan, + setInsightsWatchState, + dispose, + }; +} + +export { DEFAULT_SOFT_LIMIT_BYTES, DEFAULT_HARD_LIMIT_BYTES, SCAN_BATCH_SIZE, POLL_INTERVAL_MS, resolveReference }; diff --git a/chromium-xtension/src/chrome-host-search.ts b/chromium-xtension/src/chrome-host-search.ts index da66fa82..800949d2 100644 --- a/chromium-xtension/src/chrome-host-search.ts +++ b/chromium-xtension/src/chrome-host-search.ts @@ -2,6 +2,7 @@ import type { FolderNode, MdFile } from '../../ui/src/types'; import type { BrowserSearchIndex } from './search-index'; import { handleChromeExportHostCommand } from './chrome-host-export'; import { filterSearchIndexTabs, isValidExternalUrl, normalizeSearchQuery, resolveWorkspaceTextResourcePath } from './chrome-host-utils'; +import { handleBrowserInsightsHostCommand } from './insights-host-router'; import { resolveWorkspaceSearchItems } from './workspace-search-items'; interface ChromeHostSearchContext { @@ -15,6 +16,11 @@ interface ChromeHostSearchContext { } export async function handleChromeHostUtilityCommand(message: any, context: ChromeHostSearchContext): Promise { + if (await handleBrowserInsightsHostCommand(message, { + activeHandle: context.activeHandle, + send: context.send, + })) return true; + if (await handleChromeExportHostCommand(message, { activeHandle: context.activeHandle, send: context.send, diff --git a/chromium-xtension/src/insights-host-router.ts b/chromium-xtension/src/insights-host-router.ts new file mode 100644 index 00000000..d1ddbf41 --- /dev/null +++ b/chromium-xtension/src/insights-host-router.ts @@ -0,0 +1,62 @@ +import { createChromeInsightsHost } from './chrome-host-insights'; + +interface InsightsHostRouterContext { + activeHandle: FileSystemDirectoryHandle | null; + send: (message: any) => void; +} + +let currentHandle: FileSystemDirectoryHandle | null = null; +let currentSend: (message: any) => void = () => {}; + +const host = createChromeInsightsHost({ + getActiveHandle: () => currentHandle, + send: (message) => currentSend(message), +}); + +export async function handleBrowserInsightsHostCommand( + message: any, + context: InsightsHostRouterContext, +): Promise { + currentHandle = context.activeHandle; + currentSend = context.send; + + switch (message.command) { + case 'scanInsightsWorkspace': + await host.scanInsightsWorkspace(message); + return true; + case 'cancelInsightsScan': + host.cancelInsightsScan(message); + return true; + case 'readInsightsDocumentSource': + await host.readInsightsDocumentSource(message); + return true; + case 'probeWorkspaceResource': + await host.probeWorkspaceResource(message); + return true; + case 'setInsightsWatchState': + host.setInsightsWatchState(message); + return true; + case 'checkExternalLinks': + for (const url of Array.isArray(message.urls) ? message.urls : []) { + context.send({ + command: 'externalLinkCheckResult', + requestId: message.requestId, + url, + status: 'unsupported', + reason: 'Browser runtime cannot provide DNS/IP-pinned HTTP status checks.', + }); + } + context.send({ command: 'externalLinkCheckComplete', requestId: message.requestId, cancelled: false }); + return true; + case 'cancelExternalLinkChecks': + context.send({ command: 'externalLinkCheckComplete', requestId: message.requestId, cancelled: true }); + return true; + default: + return false; + } +} + +export function disposeBrowserInsightsHost(): void { + host.dispose(); + currentHandle = null; +} diff --git a/docs/superpowers/plans/2026-08-27-workspace-insights-wiki-links.md b/docs/superpowers/plans/2026-08-27-workspace-insights-wiki-links.md new file mode 100644 index 00000000..6cca7024 --- /dev/null +++ b/docs/superpowers/plans/2026-08-27-workspace-insights-wiki-links.md @@ -0,0 +1,1456 @@ +# Workspace Insights and Wiki Links Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add first-class Wiki Links/transclusion and a scalable, offline-first Workspace Insights subsystem with Gallery, Links, Lint, Duplicates, Graph, and Related views across Markdown Explorer runtimes. + +**Architecture:** Shared TypeScript Markdown modules own syntax, resolution, extraction, linting, indexing, duplicates, backlinks/graph, and relationship scoring. Runtime hosts own recursive workspace enumeration, bounded source reads, metadata-only probes, filesystem change signals, document-preview reuse, and secure external HTTP checking; a dedicated Web Worker maintains the live index, while React coordinates lazy lifecycle, persistence, settings, progress, and presentation. + +**Tech Stack:** React 19, TypeScript 5.8, Vite 8, Vitest 4, existing Markdown Explorer parser/renderer/bridge infrastructure, Web Worker APIs, IndexedDB/localStorage, Node host APIs for Electron/VS Code, Chromium File System Access APIs, Rust/Tauri, Mermaid, typed translation catalogs. + +**Spec:** `docs/workspace-insights-design.md` + +## Global Constraints + +- External HTTP(S) checking defaults **OFF** and requires an enabled setting plus explicit user action. +- Opening a workspace, opening Insights, local Refresh, local transclusion, duplicate analysis, graph generation, and relationship scoring perform **zero external HTTP requests**. +- Use the existing parser/source-mapping semantics for Markdown structure; do not create a regex-only competing structural parser. +- Heading anchors must use the renderer's existing `slugify()` behavior plus duplicate suffixes `base`, `base-1`, `base-2`, ... +- Dedicated Insights scans must not inherit the legacy workspace/search file-count cap; any safety ceiling must report `truncated: true`. +- Default Markdown/MDX soft analysis limit is **10 MiB**; per-file/pattern overrides may exceed it; absolute non-overridable source ceiling is **64 MiB**. +- `probeWorkspaceResource` is metadata/stat only and must never read/base64 binary contents or delegate to `readWorkspaceExportResource`. +- Symlinks are followed only when the canonical target remains inside the active workspace. +- Wiki Link matching is Unicode-normalized and case-insensitive; ambiguity is explicit and never silently guessed. +- Wiki transclusion recursion depth is **5** with independent cycle detection. +- Remote media/content is never auto-loaded. +- Near-duplicate default threshold is **90%** and candidate generation must avoid a full O(n²) workspace pass. +- Focused graph default visible-node cap is **100**, with deterministic radial layout and synchronized accessible list. +- Persistent derived cache default global cap is **500 MiB**, contains no full Markdown/MDX bodies, and uses hybrid LRU eviction. +- External checker concurrency is **4 global / 2 per origin**, timeout defaults to **10 s** configurable **3–30 s**, redirect cap is **5**, and transient failures get at most **one retry**. +- External checker sends no application cookies, Authorization headers, or session credentials and must validate/pin destination IPs against private-network policy on every redirect hop. +- Diagnostics are read-only in v1; no source rewrite, auto-fix, or rename rewrite. +- All user-visible Insights/Wiki-Link strings must be added to every supported typed locale. + +--- + +## File Structure + +The implementation should converge on these responsibility boundaries. If an existing file already owns the named responsibility, extend it instead of duplicating it. + +### Shared Markdown / Wiki semantics + +- `ui/src/markdown/frontmatter.ts` — YAML frontmatter parsing, typed title/aliases/tags extraction, duplicate-key/malformed diagnostics. +- `ui/src/markdown/sourceMapping.ts` — frontmatter/body source offset mapping; delegate parsing to `frontmatter.ts`. +- `ui/src/markdown/anchors.ts` — renderer-consistent anchor index generation and static HTML anchor extraction. +- `ui/src/markdown/wikiLinks.ts` — Wiki Link token parsing/escaping and pure resolution against a document catalog. +- `ui/src/markdown/references.ts` — Markdown/HTML/MDX link/media/tag extraction using parser ranges. +- `ui/src/markdown/inline.ts` — Wiki Link/link/embed HTML emission. +- `ui/src/markdown/renderer.ts` — shared anchor allocator and transclusion render context. +- `ui/src/markdown/types.ts` — transclusion/render-source identity types as needed. + +### Insights domain / worker + +- `ui/src/insights/contracts.ts` — shared host, worker, finding, status, and report types. +- `ui/src/insights/config.ts` — built-in defaults and normalization. +- `ui/src/insights/patterns.ts` — gitignore-style rule compilation/precedence. +- `ui/src/insights/analyzeDocument.ts` — one-document parser/extractor/lint pipeline. +- `ui/src/insights/lint.ts` — configurable lint rules and suppression matching. +- `ui/src/insights/duplicates.ts` — exact, section/window, and near-duplicate candidate logic. +- `ui/src/insights/relationships.ts` — inverted candidate generation and deterministic scoring. +- `ui/src/insights/graph.ts` — focused graph projection and deterministic radial coordinates. +- `ui/src/insights/index.ts` — incremental `WorkspaceInsightsIndex`. +- `ui/src/insights/workerProtocol.ts` — batched worker messages. +- `ui/src/insights/insights.worker.ts` — dedicated worker entrypoint. +- `ui/src/insights/workerClient.ts` — worker lifecycle plus cooperative degraded fallback. +- `ui/src/insights/cache.ts` — IndexedDB derived-cache schema/versioning/eviction. +- `ui/src/insights/workspaceIdentity.ts` — app-local workspace ID/path-history recognition. +- `ui/src/insights/useWorkspaceInsights.ts` — active-workspace session orchestration. +- `ui/src/insights/reports.ts` — Markdown/JSON snapshots. + +### UI + +- `ui/src/components/Insights/WorkspaceInsightsPanel.tsx` — resizable shell, view switching, progress/completeness. +- `ui/src/components/Insights/GalleryView.tsx` +- `ui/src/components/Insights/LinksView.tsx` +- `ui/src/components/Insights/LintView.tsx` +- `ui/src/components/Insights/DuplicatesView.tsx` +- `ui/src/components/Insights/GraphView.tsx` +- `ui/src/components/Insights/RelatedView.tsx` +- `ui/src/components/Insights/InsightsSettings.tsx` +- `ui/src/components/Sidebar/SidebarTabsHeader.tsx` / `Sidebar.tsx` — discovery entry only. +- `ui/src/App.tsx` / `ui/src/AppView.tsx` — panel state and main-area composition. +- `ui/src/hooks/useResize.ts` — reuse existing resize mechanics; do not invent a second pointer-resize system. +- `ui/src/styles/global/global-workspace-insights.css` — panel/views/graph styling through existing theme tokens. + +### Shared bridge / settings + +- `ui/src/types/webviewMessages.ts` +- `ui/src/types/hostMessages.ts` +- `ui/src/platform/bridge.ts` +- `ui/src/settings/settingsImportExport.ts` +- `ui/src/types/settings.ts` +- `ui/src/constants/storage.ts` +- `ui/src/contexts/translations.ts`, `translationsData.ts`, `translationTypes.ts`, `auditedUiTranslations.ts`, `auditedUiTranslationTypes.ts`. + +### Runtime hosts + +- Electron: extend `electron/core/runtime-workspace-resources.js`, `runtime-workspace-handlers.js`, `runtime-command-handlers.js`, `ipc-handlers.js`; add focused `electron/core/runtime-insights.js` and `electron/core/runtime-insights-external.js`. +- VS Code: extend `vscode/src/types.ts`, `vscode/src/core/panel.ts`; add focused `vscode/src/core/panelInsights.ts`. +- Chromium: add `chromium-xtension/src/chrome-host-insights.ts`; modify `chromium-xtension/src/chrome-host.ts` and reuse `chrome-host-utils.ts`/`file-access.ts`/`incremental-workspace-scan.ts` where applicable. +- Website app: add `website-app/src/web-insights-host.ts`; modify `website-app/src/web-file-utility-router.ts`. +- Tauri: extend dispatcher routing; add `tauri/src/insights/mod.rs`, `scan.rs`, `external.rs`. + +--- + +### Task 1: Define Insights contracts and configuration normalization + +**Files:** +- Create: `ui/src/insights/contracts.ts` +- Create: `ui/src/insights/config.ts` +- Create: `ui/src/insights/patterns.ts` +- Modify: `ui/src/types/webviewMessages.ts` +- Modify: `ui/src/types/hostMessages.ts` +- Test: `tests/unit/ui/insights/contracts.test.ts` +- Test: `tests/unit/ui/insights/patterns.test.ts` + +**Interfaces:** +- Produces: `InsightsWorkspaceEntry`, `InsightsScanRequest`, `InsightsScanBatch`, `InsightsScanComplete`, `InsightsSourceResult`, `WorkspaceResourceProbeResult`, `InsightsFsDelta`, `InsightsRuntimeCapabilities`, `ExternalLinkCheckRequest`, `ExternalLinkCheckResult`, `InsightsSettings`, `InsightsWorkspaceOverrides`, `normalizeInsightsSettings()`, `createInsightsPathMatcher()`. + +- [ ] **Step 1: Write failing contract/default tests** + +```ts +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_INSIGHTS_SETTINGS, + normalizeInsightsSettings, +} from '../../../../ui/src/insights/config'; + +it('uses approved safety and UX defaults', () => { + expect(DEFAULT_INSIGHTS_SETTINGS.externalLinks.enabled).toBe(false); + expect(DEFAULT_INSIGHTS_SETTINGS.externalLinks.timeoutMs).toBe(10_000); + expect(DEFAULT_INSIGHTS_SETTINGS.sourceSoftLimitBytes).toBe(10 * 1024 * 1024); + expect(DEFAULT_INSIGHTS_SETTINGS.sourceHardLimitBytes).toBe(64 * 1024 * 1024); + expect(DEFAULT_INSIGHTS_SETTINGS.nearDuplicateThreshold).toBe(0.90); + expect(DEFAULT_INSIGHTS_SETTINGS.graphNodeCap).toBe(100); + expect(DEFAULT_INSIGHTS_SETTINGS.cacheCapBytes).toBe(500 * 1024 * 1024); +}); + +it('clamps external timeout to 3-30 seconds', () => { + expect(normalizeInsightsSettings({ externalLinks: { timeoutMs: 100 } }).externalLinks.timeoutMs).toBe(3_000); + expect(normalizeInsightsSettings({ externalLinks: { timeoutMs: 90_000 } }).externalLinks.timeoutMs).toBe(30_000); +}); +``` + +- [ ] **Step 2: Run tests and verify RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/contracts.test.ts tests/unit/ui/insights/patterns.test.ts +``` + +Expected: FAIL because the Insights modules/types do not exist. + +- [ ] **Step 3: Implement contracts/defaults and gitignore-style precedence** + +```ts +export interface InsightsWorkspaceEntry { + readonly relativePath: string; + readonly canonicalRelativePath: string; + readonly kind: 'file' | 'directory'; + readonly sizeBytes: number; + readonly mtimeMs: number; + readonly extension?: string; + readonly isSymlink?: boolean; +} + +export type WorkspaceResourceProbeStatus = + | 'exists' | 'missing' | 'outside-workspace' | 'unreadable' | 'unsupported'; + +export interface WorkspaceResourceProbeResult { + readonly status: WorkspaceResourceProbeStatus; + readonly relativePath?: string; + readonly kind?: 'file' | 'directory'; + readonly sizeBytes?: number; + readonly mimeType?: string; +} + +export const DEFAULT_INSIGHTS_SETTINGS = { + externalLinks: { enabled: false, timeoutMs: 10_000 }, + sourceSoftLimitBytes: 10 * 1024 * 1024, + sourceHardLimitBytes: 64 * 1024 * 1024, + nearDuplicateThreshold: 0.90, + graphNodeCap: 100, + cacheCapBytes: 500 * 1024 * 1024, +} as const; +``` + +Use one path matcher for scan/watch/poll/refresh/oversized decisions. Hard exclusions win permanently; built-in defaults and `.gitignore` are then overridden by user rules in user-defined last-match-wins order. + +- [ ] **Step 4: Run focused tests and typecheck** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/contracts.test.ts tests/unit/ui/insights/patterns.test.ts +pnpm --filter ./ui exec tsc -b --pretty false +``` + +Expected: PASS and TypeScript exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights ui/src/types/webviewMessages.ts ui/src/types/hostMessages.ts tests/unit/ui/insights +git commit -m "feat: define workspace insights contracts" +``` + +--- + +### Task 2: Replace simplistic frontmatter parsing with YAML metadata parsing + +**Files:** +- Create: `ui/src/markdown/frontmatter.ts` +- Modify: `ui/src/markdown/sourceMapping.ts` +- Modify: `ui/package.json` +- Modify: `pnpm-lock.yaml` +- Modify: `ui/src/types/content.ts` +- Test: `tests/unit/ui/markdown/frontmatter.test.ts` +- Test: existing parser/source-mapping tests under `tests/unit/ui/markdown/` + +**Interfaces:** +- Produces: `parseFrontmatterDocument(source): ParsedFrontmatterDocument`. +- `ParsedFrontmatterDocument` contains `body`, `sourceSegments`, `flatFrontmatter`, `metadata: { title?: string; aliases: string[]; tags: string[] }`, and diagnostics for malformed YAML, duplicate keys, invalid Insights metadata. +- Existing renderer consumers continue receiving a compatible flat frontmatter representation. + +- [ ] **Step 1: Add failing YAML behavior tests** + +```ts +it('parses title, aliases and tags without losing source mapping', () => { + const parsed = parseFrontmatterDocument(`--- +title: Setup +aliases: + - Install + - Setup Guide +tags: [api, docs] +--- +# Body +`); + expect(parsed.metadata).toEqual({ + title: 'Setup', + aliases: ['Install', 'Setup Guide'], + tags: ['api', 'docs'], + }); + expect(parsed.body).toBe('# Body\n'); +}); + +it('ignores only a duplicated key and preserves unrelated metadata', () => { + const parsed = parseFrontmatterDocument(`--- +title: One +title: Two +tags: [docs] +--- +Body +`); + expect(parsed.metadata.title).toBeUndefined(); + expect(parsed.metadata.tags).toEqual(['docs']); + expect(parsed.diagnostics.map(d => d.ruleId)).toContain('frontmatter/duplicate-key'); +}); +``` + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/frontmatter.test.ts +``` + +Expected: FAIL because `parseFrontmatterDocument` does not exist and current parsing only supports scalar `key: value`. + +- [ ] **Step 3: Add the YAML dependency and implement AST-based duplicate handling** + +```bash +pnpm --filter ./ui add yaml +``` + +Implement with `yaml` document parsing configured so duplicate map items can be inspected rather than silently applying first/last-wins. Build a duplicate-key set, omit duplicated keys from typed metadata, and keep unrelated valid fields. Preserve `scanFrontmatterPreamble()` behavior and source segments so parser ranges stay correct. + +- [ ] **Step 4: Run frontmatter/parser regression tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/frontmatter.test.ts tests/unit/ui/markdown +pnpm --filter ./ui exec tsc -b --pretty false +``` + +Expected: PASS; malformed YAML yields diagnostics while body parsing remains available. + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/markdown/frontmatter.ts ui/src/markdown/sourceMapping.ts ui/src/types/content.ts ui/package.json pnpm-lock.yaml tests/unit/ui/markdown +git commit -m "feat: parse yaml frontmatter metadata" +``` + +--- + +### Task 3: Centralize anchor generation and static-anchor extraction + +**Files:** +- Create: `ui/src/markdown/anchors.ts` +- Modify: `ui/src/markdown/renderer.ts` +- Modify: `ui/src/markdown/utils.ts` only if a shared normalization helper is required +- Test: `tests/unit/ui/markdown/anchors.test.ts` +- Test: existing renderer tests + +**Interfaces:** +- Produces: `createHeadingIdAllocator()`, `buildDocumentAnchorIndex(tokens, staticHtmlSource?)`. +- Renderer and Insights both consume the same allocator. + +- [ ] **Step 1: Write failing duplicate/Setext/static-anchor tests** + +```ts +it('matches renderer duplicate suffixes', () => { + const next = createHeadingIdAllocator(); + expect(next('API Usage')).toBe('api-usage'); + expect(next('API Usage')).toBe('api-usage-1'); + expect(next('API Usage')).toBe('api-usage-2'); +}); + +it('includes literal HTML id and legacy anchor name', () => { + const anchors = extractStaticAnchors('
'); + expect(anchors).toEqual(new Set(['details', 'legacy'])); +}); +``` + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/anchors.test.ts +``` + +- [ ] **Step 3: Implement allocator and refactor renderer** + +```ts +export function createHeadingIdAllocator() { + const counts = new Map(); + return (text: string): string => { + const base = slugify(text); + const index = counts.get(base) ?? 0; + counts.set(base, index + 1); + return index === 0 ? base : `${base}-${index}`; + }; +} +``` + +Replace `HtmlRenderer.headingIdCounts`/`nextHeadingId()` internals with this shared allocator. Extract only literal `id="..."`, `id='...'`, and ``; dynamic MDX attributes are not anchors. + +- [ ] **Step 4: Run renderer/anchor tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/anchors.test.ts tests/unit/ui/markdown +``` + +Expected: PASS with unchanged renderer IDs. + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/markdown/anchors.ts ui/src/markdown/renderer.ts ui/src/markdown/utils.ts tests/unit/ui/markdown +git commit -m "refactor: share markdown anchor generation" +``` + +--- + +### Task 4: Implement Wiki Link syntax, reference extraction, and pure resolution + +**Files:** +- Create: `ui/src/markdown/wikiLinks.ts` +- Create: `ui/src/markdown/references.ts` +- Test: `tests/unit/ui/markdown/wiki-links.test.ts` +- Test: `tests/unit/ui/markdown/references.test.ts` + +**Interfaces:** +- Produces: + +```ts +export interface WikiLinkToken { + readonly kind: 'link' | 'embed'; + readonly raw: string; + readonly target: string; + readonly fragment?: string; + readonly label?: string; + readonly sourceStart: number; + readonly sourceEnd: number; +} + +export type WikiResolution = + | { status: 'resolved'; documentPath: string; canonicalPath: string; fragment?: string; caseMismatch: boolean } + | { status: 'ambiguous'; candidates: readonly string[] } + | { status: 'missing' | 'outside-workspace' | 'invalid-anchor' }; + +export function parseWikiLink(raw: string, offset?: number): WikiLinkToken | WikiParseFailure; +export function resolveWikiLink(token: WikiLinkToken, context: WikiResolverContext): WikiResolution; +``` + +- `extractDocumentReferences()` emits standard links, Wiki links/embeds, static HTML/MDX refs, dynamic refs, media categories, tags, and source ranges. + +- [ ] **Step 1: Write failing syntax/resolution tests** + +Cover exact approved forms: + +```ts +expect(parseWikiLink('[[Guide#Install|Setup]]')).toMatchObject({ + kind: 'link', target: 'Guide', fragment: 'Install', label: 'Setup', +}); +expect(parseWikiLink('[[a\\#b\\|c]]')).toMatchObject({ target: 'a#b|c' }); +expect(parseWikiLink('![[../media\\image.png]]')).toMatchObject({ + kind: 'embed', target: '../media/image.png', +}); +``` + +Add cases for `.md/.mdx` ambiguity, title/alias candidates, `[[#Heading]]`, directory targets (`README.md/.mdx`, `index.md/.mdx`), case mismatch, malformed syntax, standard Markdown literal-extension semantics, percent-decoded fragments, query stripping, static HTML refs, dynamic MDX refs, and Markdown-aware tags. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/wiki-links.test.ts tests/unit/ui/markdown/references.test.ts +``` + +- [ ] **Step 3: Implement parser/extractor/resolver** + +Use parser/source ranges to exclude fenced code, inline code, link destinations, HTML attributes, and other non-prose tag contexts. Normalize resolver keys with Unicode normalization and locale-independent case folding while preserving canonical display casing. Resolution precedence is explicit/relative path, filename/stem, canonical title, aliases; ambiguous candidates remain ambiguous. + +Normal Markdown `[Guide](Guide)` stays literal. Wiki `[[Guide]]` may try `.md`/`.mdx`. Directory links resolve only when exactly one recognized index document is viable. + +- [ ] **Step 4: Run focused and parser tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/wiki-links.test.ts tests/unit/ui/markdown/references.test.ts tests/unit/ui/markdown +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/markdown/wikiLinks.ts ui/src/markdown/references.ts tests/unit/ui/markdown +git commit -m "feat: add wiki link resolution semantics" +``` + +--- + +### Task 5: Add renderer/navigation support for Wiki Links and transclusion + +**Files:** +- Modify: `ui/src/markdown/inline.ts` +- Modify: `ui/src/markdown/renderer.ts` +- Modify: `ui/src/markdown/types.ts` +- Modify: `ui/src/contexts/NavigationContext.tsx` +- Modify: `ui/src/dom/globalHandlers.ts` +- Create: `ui/src/markdown/transclusion.ts` +- Test: `tests/unit/ui/markdown/wiki-renderer.test.ts` +- Test: `tests/unit/ui/markdown/transclusion.test.ts` +- Test: `tests/unit/ui/navigation-wiki-links.test.tsx` + +**Interfaces:** +- Consumes: `parseWikiLink()`, `resolveWikiLink()`, shared anchor allocator. +- Produces: `TransclusionRenderContext` with `sourceDocumentPath`, `depth`, `ancestorDocumentPaths`, and async `resolve/read/render` callbacks. +- UI navigation exposes one `navigateWikiLink(rawTarget, sourceDocumentPath)` path shared by renderer and Insights. + +- [ ] **Step 1: Write failing render/navigation/transclusion tests** + +```ts +it('renders a wiki link as an internal resolvable action', () => { + const html = renderInline('See [[Setup|Install guide]]'); + expect(html).toContain('data-mdn-wiki-target="Setup"'); + expect(html).toContain('Install guide'); +}); + +it('stops a transclusion cycle', async () => { + const result = await renderTransclusion('A.md', 'B.md', { + depth: 2, + ancestorDocumentPaths: ['A.md', 'B.md'], + }); + expect(result.status).toBe('cycle'); +}); +``` + +Also test depth 5, nested source-relative resolution, interactive links inside embedded Markdown, missing/ambiguous placeholders, and supported non-Markdown preview delegation. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown/wiki-renderer.test.ts tests/unit/ui/markdown/transclusion.test.ts tests/unit/ui/navigation-wiki-links.test.tsx +``` + +- [ ] **Step 3: Implement renderer hooks without starting Insights eagerly** + +Render Wiki syntax into data attributes/actions, not hard-coded guessed paths. `navigateWikiLink()` and `renderTransclusion()` accept an injected `WikiResolverContext`/catalog reader, so this task is independently testable with an in-memory catalog and does not start Insights. Task 15 wires that interface to the lazy active-workspace host/index session. + +Keep embedded source identity in data/source-mapping attributes so bookmarks/navigation refer to the embedded source document. Use an injected existing-preview callback for supported PDF/DOCX/XLSX/PPTX/HTML/RTF embeds. Remote embeds render an explicit unloaded placeholder. + +- [ ] **Step 4: Run markdown/navigation regression suite** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/markdown tests/unit/ui/navigation-wiki-links.test.tsx +pnpm --filter ./ui exec tsc -b --pretty false +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/markdown ui/src/contexts/NavigationContext.tsx ui/src/dom/globalHandlers.ts tests/unit/ui/markdown tests/unit/ui/navigation-wiki-links.test.tsx +git commit -m "feat: render wiki links and transclusions" +``` + +--- + +### Task 6: Add UI-to-host Insights protocol and bridge helpers + +**Files:** +- Modify: `ui/src/types/webviewMessages.ts` +- Modify: `ui/src/types/hostMessages.ts` +- Modify: `ui/src/platform/bridge.ts` +- Test: `tests/unit/ui/platform-bridges.test.ts` +- Test: `tests/unit/ui/insights/host-protocol.test.ts` + +**Interfaces:** +- Produces bridge request helpers: + - `scanInsightsWorkspace(request)` + - `cancelInsightsScan(requestId)` + - `readInsightsDocumentSource(request)` + - `probeWorkspaceResource(request)` + - `setInsightsWatchState(request)` + - `checkExternalLinks(request)` + - `cancelExternalLinkChecks(requestId)` +- Host messages stream batches/results keyed by request ID and workspace operation ID. + +- [ ] **Step 1: Write failing bridge correlation tests** + +```ts +it('correlates streamed scan batches and completion by request id', async () => { + const scan = bridge.scanInsightsWorkspace({ requestId: 'scan-1', workspaceOperationId: 'ws-1', patterns: [] }); + host.emit({ command: 'insightsScanBatch', requestId: 'scan-1', entries: [entry] }); + host.emit({ command: 'insightsScanComplete', requestId: 'scan-1', truncated: false, excludedCount: 0 }); + await expect(scan.done).resolves.toMatchObject({ truncated: false }); +}); +``` + +In the test fixture, define `entry` as an `InsightsWorkspaceEntry` and use the existing bridge fake host/event helper from `platform-bridges.test.ts`. Verify probe results carry metadata only and external results preserve per-URL status. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/platform-bridges.test.ts tests/unit/ui/insights/host-protocol.test.ts +``` + +- [ ] **Step 3: Implement typed message unions and request helpers** + +Keep legacy `readWorkspaceExportResource` unchanged for explicit binary preview/export. Do not reuse it from the probe helper. Ensure cancellation rejects/finishes outstanding iterators without leaking listeners. + +- [ ] **Step 4: Run bridge tests/typecheck** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/platform-bridges.test.ts tests/unit/ui/insights/host-protocol.test.ts +pnpm --filter ./ui exec tsc -b --pretty false +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/types ui/src/platform/bridge.ts tests/unit/ui/platform-bridges.test.ts tests/unit/ui/insights/host-protocol.test.ts +git commit -m "feat: add insights host protocol" +``` + +--- + +### Task 7: Implement workspace scan/source/probe/watch capabilities in all runtimes + +**Files:** +- Electron: create `electron/core/runtime-insights.js`; modify `electron/core/runtime-workspace-resources.js`, `electron/core/runtime-workspace-handlers.js`, `electron/core/runtime-command-handlers.js`, `electron/core/ipc-handlers.js`. +- VS Code: create `vscode/src/core/panelInsights.ts`; modify `vscode/src/core/panel.ts`, `vscode/src/types.ts`. +- Chromium: create `chromium-xtension/src/chrome-host-insights.ts`; modify `chromium-xtension/src/chrome-host.ts` and reuse `chromium-xtension/src/file-access.ts`/`incremental-workspace-scan.ts`. +- Website: create `website-app/src/web-insights-host.ts`; modify `website-app/src/web-file-utility-router.ts`. +- Tauri: create `tauri/src/insights/mod.rs`, `tauri/src/insights/scan.rs`; modify `tauri/src/dispatcher.rs` and `tauri/src/dispatcher/handlers.rs`. +- Test: `tests/unit/electron/runtime-insights.test.ts` +- Test: `tests/unit/vscode/panel-insights.test.ts` +- Test: `tests/unit/chromium/chrome-host-insights.test.ts` +- Test: `tests/node/insights-host-contracts.test.mjs` +- Test: Tauri Rust unit tests in `tauri/src/insights/scan.rs` + +**Interfaces:** +- Consumes host protocol from Task 6. +- Produces recursive metadata scan, bounded Markdown source read, metadata-only probe, watcher capability or visible-only polling capability. + +- [ ] **Step 1: Write host contract tests before implementations** + +Required assertions per runtime: + +```ts +expect(await probe('docs/image.png')).toMatchObject({ + status: 'exists', + kind: 'file', +}); +expect(binaryReadSpy).not.toHaveBeenCalled(); + +const MiB = 1024 * 1024; +expect(await readSource('large.md', { softLimitBytes: 10 * MiB })) + .toMatchObject({ status: 'too-large' }); +expect(await readSource('allowed-large.md', { softLimitBytes: 20 * MiB })) + .toMatchObject({ status: 'ok' }); +expect(await readSource('over-hard-limit.md', { softLimitBytes: 100 * MiB })) + .toMatchObject({ status: 'too-large', hardLimit: true }); +``` + +Create >1000-file fixture metadata for hosts that previously cap normal scanning and assert Insights reports all eligible entries or an explicit truncation status, never silent completion. + +- [ ] **Step 2: Run RED across runtimes** + +```bash +pnpm exec vitest run --project electron tests/unit/electron/runtime-insights.test.ts +pnpm exec vitest run --project vscode tests/unit/vscode/panel-insights.test.ts +pnpm exec vitest run --project chromium tests/unit/chromium/chrome-host-insights.test.ts +node --experimental-strip-types --test tests/node/insights-host-contracts.test.mjs +cargo test --manifest-path tauri/Cargo.toml insights_scan -- --test-threads=1 +``` + +- [ ] **Step 3: Implement canonical scan/read/probe behavior** + +For every host: +1. canonicalize/realpath candidate; +2. reject symlink escapes; +3. apply hard/default/gitignore/user rules consistently; +4. stream metadata batches; +5. enforce 10 MiB soft / 64 MiB hard source policy; +6. stat probes only; +7. expose watcher when reliable. + +Chromium/website runtimes without reliable change observers use metadata polling only while Insights is visible and stop when hidden. Manual Refresh remains available everywhere. + +- [ ] **Step 4: Run runtime contract tests** + +Run all commands from Step 2 plus: + +```bash +pnpm run build:vscode +pnpm run build:chromium +pnpm run build:website-app +cargo test --manifest-path tauri/Cargo.toml -- --test-threads=1 +``` + +Expected: all contract tests/builds pass. + +- [ ] **Step 5: Commit** + +```bash +git add electron vscode chromium-xtension website-app tauri tests/unit/electron tests/unit/vscode tests/unit/chromium tests/node/insights-host-contracts.test.mjs +git commit -m "feat: add cross-runtime insights filesystem hosts" +``` + +--- + +### Task 8: Implement secure host-backed external HTTP checking + +**Files:** +- Create: `electron/core/runtime-insights-external.js` +- Extend: `vscode/src/core/panelInsights.ts` +- Create: `tauri/src/insights/external.rs` +- Modify: `tauri/Cargo.toml` only if the existing HTTP stack cannot pin resolved IPs safely +- Extend: `chromium-xtension/src/chrome-host-insights.ts` and `website-app/src/web-insights-host.ts` to report `unsupported` unless they can satisfy the full status/DNS/IP contract. +- Test: `tests/unit/electron/insights-external.test.ts` +- Test: `tests/unit/vscode/insights-external.test.ts` +- Test: `tests/unit/chromium/chrome-host-insights.test.ts` +- Test: Tauri tests in `tauri/src/insights/external.rs` + +**Interfaces:** +- Produces real `ExternalLinkCheckResult` statuses: `reachable`, `auth-required`, `broken`, `rate-limited`, `server-error`, `unreachable`, `unknown`, `private-confirmation-required`, `unsupported`. + +- [ ] **Step 1: Write failing security/behavior tests** + +Use injected DNS and HTTP transports so tests never depend on public internet: + +```ts +it('does not connect when DNS resolves to private space without approval', async () => { + dns.resolve.mockResolvedValue(['127.0.0.1']); + const result = await checker.check('http://example.test/', session); + expect(result.status).toBe('private-confirmation-required'); + expect(http.request).not.toHaveBeenCalled(); +}); + +it('pins the validated address and revalidates redirects', async () => { + dns.resolve + .mockResolvedValueOnce(['203.0.113.10']) + .mockResolvedValueOnce(['10.0.0.8']); + http.head.mockResolvedValue({ status: 302, location: 'http://private.test/' }); + expect((await checker.check('https://public.test/', session)).status) + .toBe('private-confirmation-required'); +}); +``` + +Add HEAD→GET fallback, no Cookie/Authorization, 404/410, 401/403, 429, 5xx one-retry, Retry-After, timeout, TLS failure, redirect limit 5, HTTPS downgrade flag, 4-global/2-origin concurrency, and abort tests. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project electron tests/unit/electron/insights-external.test.ts +pnpm exec vitest run --project vscode tests/unit/vscode/insights-external.test.ts +cargo test --manifest-path tauri/Cargo.toml insights_external -- --test-threads=1 +``` + +- [ ] **Step 3: Implement request state machine** + +```text +parse URL +→ resolve all addresses +→ classify addresses +→ require origin-scoped private approval when needed +→ select/pin validated IP +→ HEAD +→ if 405/501/known inconclusive HEAD behavior: bounded GET and abort body +→ validate every redirect from scratch +→ classify status +→ optional single transient retry +``` + +Never use browser `no-cors` responses for status. Chromium/website report `unsupported` unless their host context can satisfy real status + DNS/IP policy. + +- [ ] **Step 4: Run checker suites and runtime builds** + +```bash +pnpm exec vitest run --project electron tests/unit/electron/insights-external.test.ts +pnpm exec vitest run --project vscode tests/unit/vscode/insights-external.test.ts +pnpm exec vitest run --project chromium tests/unit/chromium/chrome-host-insights.test.ts +cargo test --manifest-path tauri/Cargo.toml insights_external -- --test-threads=1 +pnpm run build:vscode +``` + +- [ ] **Step 5: Commit** + +```bash +git add electron/core/runtime-insights-external.js vscode/src/core/panelInsights.ts chromium-xtension/src/chrome-host-insights.ts website-app/src/web-insights-host.ts tauri/src/insights/external.rs tauri/Cargo.toml tests +git commit -m "feat: add secure external link checker" +``` + +--- + +### Task 9: Build one-document analysis and configurable linting + +**Files:** +- Create: `ui/src/insights/analyzeDocument.ts` +- Create: `ui/src/insights/lint.ts` +- Test: `tests/unit/ui/insights/analyze-document.test.ts` +- Test: `tests/unit/ui/insights/lint.test.ts` + +**Interfaces:** +- Consumes parser/frontmatter/anchors/references from Tasks 2–4. +- Produces `AnalyzedDocument` with title/aliases/tags, anchors, links/media, sections, terminology/signatures, and lint findings. +- Produces `applyLintSuppressions(findings, suppressions)`. + +- [ ] **Step 1: Write failing analysis/lint tests** + +```ts +it('keeps body findings when frontmatter is malformed', () => { + const result = analyzeDocument({ + path: 'guide.md', + source: '---\ntitle: [bad\n---\n# A\n### C\n', + revision: 'r1', + }); + expect(result.lint.map(f => f.ruleId)).toContain('frontmatter/malformed'); + expect(result.lint.map(f => f.ruleId)).toContain('heading/skipped-level'); +}); +``` + +Cover duplicate heading, malformed table delimiter, table column count, list marker/indentation, trailing whitespace, malformed Wiki syntax, case mismatch, malformed URI, Mermaid failure input, and suppression scopes. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/analyze-document.test.ts tests/unit/ui/insights/lint.test.ts +``` + +- [ ] **Step 3: Implement single-parse analysis** + +Call the existing parser once per revision, then derive source-range-aware references/lint data from tokens plus focused extractors. Do not render transcluded content into duplicate/signature source. Keep readable significant terms live-only and emit hashed/signature forms for persistence. + +- [ ] **Step 4: Run analysis/lint tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/analyze-document.test.ts tests/unit/ui/insights/lint.test.ts tests/unit/ui/markdown +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/analyzeDocument.ts ui/src/insights/lint.ts tests/unit/ui/insights +git commit -m "feat: analyze and lint workspace documents" +``` + +--- + +### Task 10: Implement incremental workspace index, backlinks, and graph projection + +**Files:** +- Create: `ui/src/insights/index.ts` +- Create: `ui/src/insights/graph.ts` +- Test: `tests/unit/ui/insights/index.test.ts` +- Test: `tests/unit/ui/insights/graph.test.ts` + +**Interfaces:** +- Produces `WorkspaceInsightsIndex`: + +```ts +class WorkspaceInsightsIndex { + applyDocument(document: AnalyzedDocument): IndexDeltaResult; + removeDocument(canonicalPath: string): IndexDeltaResult; + renameDocument(change: HighConfidenceRename): IndexDeltaResult; + applyActiveOverlay(document: AnalyzedDocument): IndexDeltaResult; + clearActiveOverlay(canonicalPath: string): IndexDeltaResult; + snapshot(): WorkspaceInsightsSnapshot; +} +``` + +- Produces `buildFocusedGraph(snapshot, { centerPath, nodeCap, includeInferred, showTags, showHeadings })`. + +- [ ] **Step 1: Write failing incremental tests** + +```ts +it('removes deleted documents and turns surviving references into broken links', () => { + const index = new WorkspaceInsightsIndex(); + index.applyDocument(analyzeDocument({ path: 'a.md', source: '[[B]]', revision: '1' })); + index.applyDocument(analyzeDocument({ path: 'b.md', source: '# B', revision: '1' })); + index.removeDocument('b.md'); + expect(index.snapshot().documents.has('b.md')).toBe(false); + expect(index.snapshot().brokenLinks[0].status).toBe('missing'); +}); +``` + +Add active-overlay replacement/removal, link/embed edge distinction, same-document fragment exclusion, high-confidence rename migration, ambiguity, and deterministic radial coordinates for the same graph state. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/index.test.ts tests/unit/ui/insights/graph.test.ts +``` + +- [ ] **Step 3: Implement inverted maps and deterministic graph** + +Maintain document/path/title/alias/tag/heading/link indexes incrementally. Graph projection ranks explicit edges before inferred edges, caps at configurable node count, reports hidden count, and generates stable radial coordinates from sorted node identity rather than force simulation. + +- [ ] **Step 4: Run index/graph tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/index.test.ts tests/unit/ui/insights/graph.test.ts +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/index.ts ui/src/insights/graph.ts tests/unit/ui/insights +git commit -m "feat: maintain insights backlinks and graph" +``` + +--- + +### Task 11: Implement duplicate candidate engines + +**Files:** +- Create: `ui/src/insights/duplicates.ts` +- Test: `tests/unit/ui/insights/duplicates.test.ts` + +**Interfaces:** +- Produces `normalizeExactDuplicateSource()`, `buildSectionFingerprints()`, `buildPassageFingerprints()`, `buildNearDuplicateCandidates()`, `scoreNearDuplicate()`, `findDuplicateGroups()`. + +- [ ] **Step 1: Write failing exact/repeated/near tests** + +```ts +it('treats BOM, CRLF and trailing whitespace as exact-normalized duplicates', () => { + expect(normalizeExactDuplicateSource('\uFEFF# A \r\n')) + .toBe(normalizeExactDuplicateSource('# A\n')); +}); + +it('does not compare every document pair', () => { + const corpus = Array.from({ length: 2_000 }, (_, index) => ({ + path: `doc-${index}.md`, + normalizedTokens: [`topic-${index % 200}`, `unique-${index}`], + })); + const scorer = vi.fn(scoreNearDuplicate); + findDuplicateGroups(corpus, { scorePair: scorer, threshold: 0.90 }); + expect(scorer.mock.calls.length).toBeLessThan(20_000); +}); +``` + +Test >=100 normalized characters / >=20 meaningful-token section threshold, ~120-token sliding windows with ~50% stride, boilerplate suppression, threshold override, and duplicate suppression presentation. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/duplicates.test.ts +``` + +- [ ] **Step 3: Implement hashed buckets/candidate generation** + +Exact: normalized hash groups. Repeated passages: section hash plus bounded rolling/window signatures. Near duplicates: use shingle/signature buckets or inverted significant-term evidence to generate a sparse candidate set; only candidate pairs receive exact similarity scoring. + +- [ ] **Step 4: Run duplicate tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/duplicates.test.ts +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/duplicates.ts tests/unit/ui/insights/duplicates.test.ts +git commit -m "feat: detect duplicate workspace content" +``` + +--- + +### Task 12: Implement relationship candidate generation and explainable scoring + +**Files:** +- Create: `ui/src/insights/relationships.ts` +- Test: `tests/unit/ui/insights/relationships.test.ts` + +**Interfaces:** +- Produces `RELATIONSHIP_PRESETS`, `normalizeRelationshipWeights()`, `buildRelationshipCandidates()`, `scoreRelatedDocument()`, `getRelationshipEvidence()`. + +- [ ] **Step 1: Write failing scoring/preset tests** + +```ts +it('uses approved default weights', () => { + expect(RELATIONSHIP_PRESETS.default).toEqual({ + links: 35, tags: 20, headings: 15, title: 10, terminology: 20, + }); +}); + +it('returns actual evidence without persisting readable terminology', () => { + const result = scoreRelatedDocument( + fixtureDocument({ tags: ['api'], terms: ['refresh token'] }), + fixtureDocument({ tags: ['api'], terms: ['refresh token'] }), + RELATIONSHIP_PRESETS.default, + ); + expect(result.score).toBeGreaterThan(0); + expect(result.evidence.sharedTags).toContain('api'); + expect(result.evidence.sharedTerms).toContain('refresh token'); + expect(result.persisted.terminologySignatures.every( + (value: string) => /^[a-f0-9]{16,}$/i.test(value), + )).toBe(true); +}); +``` + +Define `fixtureDocument()` in the test file as a small factory returning the concrete relationship-input type. Test Link-focused, Tag-focused, Terminology-focused, custom normalization, omission of no-signal candidates, and candidate-count bounds for large sparse corpora. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/relationships.test.ts +``` + +- [ ] **Step 3: Implement inverted candidate indexes and 0–100 scoring** + +Candidates come from links/tags/headings/title/significant-term indexes. Normalize custom weights before scoring. Persist only hashed/signature terminology; readable terms remain live or are reconstructed on demand. + +- [ ] **Step 4: Run relationship tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/relationships.test.ts +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/relationships.ts tests/unit/ui/insights/relationships.test.ts +git commit -m "feat: rank related workspace documents" +``` + +--- + +### Task 13: Add worker protocol, dedicated worker, and degraded fallback + +**Files:** +- Create: `ui/src/insights/workerProtocol.ts` +- Create: `ui/src/insights/insights.worker.ts` +- Create: `ui/src/insights/workerClient.ts` +- Modify: `ui/vite.config.ts` only if worker bundling needs explicit configuration +- Test: `tests/unit/ui/insights/worker-client.test.ts` + +**Interfaces:** +- Worker input: `initialize`, `applySourceBatch`, `applyFsDeltaBatch`, `setActiveOverlay`, `clearActiveOverlay`, `updateConfig`, `requestSnapshot`, `cancel`. +- Worker output: `progress`, `documentResults`, `snapshotDelta`, `complete`, `cancelled`, `error`. +- Produces `createInsightsWorkerClient()` and cooperative `createChunkedInsightsFallback()`. + +- [ ] **Step 1: Write failing batching/lifecycle tests** + +```ts +it('batches source work and emits provisional results before complete', async () => { + const client = createTestWorkerClient(); + await client.applySourceBatch([ + { path: 'a.md', source: '# A', revision: '1' }, + { path: 'b.md', source: '# B', revision: '1' }, + ]); + expect(client.events.some(e => e.type === 'documentResults')).toBe(true); + expect(client.events.at(-1)?.type).toBe('complete'); +}); + +it('reports degraded mode when Worker construction fails', () => { + const client = createInsightsWorkerClient({ createWorker: () => { throw new Error('blocked'); } }); + expect(client.mode).toBe('degraded'); +}); +``` + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/worker-client.test.ts +``` + +- [ ] **Step 3: Implement worker around `WorkspaceInsightsIndex`** + +Batch messages; never emit one message per token/finding. Cancellation stops queued jobs and keeps completed index state reusable. Fallback processes bounded document chunks using `scheduler.yield` when available or `setTimeout(0)`/microtask scheduling, and exposes `mode: 'degraded'`. + +- [ ] **Step 4: Run worker tests/build** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/worker-client.test.ts tests/unit/ui/insights +pnpm run build:ui +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/workerProtocol.ts ui/src/insights/insights.worker.ts ui/src/insights/workerClient.ts ui/vite.config.ts tests/unit/ui/insights +git commit -m "feat: run workspace insights in a worker" +``` + +--- + +### Task 14: Add cache, workspace identity, settings precedence, and settings import/export + +**Files:** +- Create: `ui/src/insights/cache.ts` +- Create: `ui/src/insights/workspaceIdentity.ts` +- Extend: `ui/src/insights/config.ts` +- Modify: `ui/src/settings/settingsImportExport.ts` +- Modify: `ui/src/constants/storage.ts` +- Modify: `ui/src/types/settings.ts` +- Test: `tests/unit/ui/insights/cache.test.ts` +- Test: `tests/unit/ui/insights/workspace-identity.test.ts` +- Test: `tests/unit/ui/settings-import-export.test.ts` + +**Interfaces:** +- Produces `InsightsCacheStore`, `resolveWorkspaceIdentity()`, `resolveInsightsSettings(global, workspace)`, `resetWorkspaceInsightsOverrides()`. +- Cache schema stores derived data only; full source/session network/private approvals are structurally absent. + +- [ ] **Step 1: Write failing persistence/privacy tests** + +```ts +it('never serializes source bodies or external session state', async () => { + await store.putWorkspace(makeCacheFixture({ source: '# secret', externalSession: { url: 'https://x' } })); + const raw = JSON.stringify(await dumpIndexedDb(store)); + expect(raw).not.toContain('# secret'); + expect(raw).not.toContain('externalSession'); +}); + +it('applies workspace override over global over built-in', () => { + const resolved = resolveInsightsSettings( + { nearDuplicateThreshold: 0.88 }, + { nearDuplicateThreshold: 0.93 }, + ); + expect(resolved.nearDuplicateThreshold).toBe(0.93); +}); +``` + +Define `makeCacheFixture()` and `dumpIndexedDb()` in the test file using `fake-indexeddb`. Add schema-component invalidation, normal metadata-first restore, hybrid per-file then whole-workspace LRU, 500 MiB default cap, high-confidence moved-workspace path history, uncertain-new-workspace behavior, and settings import validation. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/cache.test.ts tests/unit/ui/insights/workspace-identity.test.ts tests/unit/ui/settings-import-export.test.ts +``` + +- [ ] **Step 3: Implement persistence/config integration** + +Store schema/component versions. Evict inactive per-file entries first, then inactive workspace caches. Keep settings/suppressions separate from disposable cache. Extend existing settings JSON envelope rather than creating a second settings file format. + +- [ ] **Step 4: Run cache/settings tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/cache.test.ts tests/unit/ui/insights/workspace-identity.test.ts tests/unit/ui/settings-import-export.test.ts +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/cache.ts ui/src/insights/workspaceIdentity.ts ui/src/insights/config.ts ui/src/settings/settingsImportExport.ts ui/src/constants/storage.ts ui/src/types/settings.ts tests/unit/ui +git commit -m "feat: persist workspace insights state" +``` + +--- + +### Task 15: Orchestrate lazy active-workspace Insights sessions and unsaved overlays + +**Files:** +- Create: `ui/src/insights/useWorkspaceInsights.ts` +- Modify: `ui/src/contexts/useAppStateEffects.ts` +- Modify: `ui/src/App.tsx` +- Test: `tests/unit/ui/insights/use-workspace-insights.test.tsx` + +**Interfaces:** +- Produces one `WorkspaceInsightsSession` for the active workspace. +- Session methods: `open()`, `closePanel()`, `refreshLocal()`, `pause()`, `dispose()`, `applyActiveOverlay()`, `clearActiveOverlay()`, `checkExternalLinks()`, `cancelExternalChecks()`. +- Provides the injected `WikiResolverContext`/catalog reader required by Task 5, reusing the live index when available and lazily constructing the minimum document catalog when Wiki navigation occurs before Insights has been opened. + +- [ ] **Step 1: Write failing lifecycle tests** + +```ts +it('does not scan or start a worker until first Insights open', () => { + renderHook(() => useWorkspaceInsights(props)); + expect(host.scanInsightsWorkspace).not.toHaveBeenCalled(); + expect(workerFactory).not.toHaveBeenCalled(); +}); + +it('pauses expensive work on panel close but keeps completed state warm', async () => { + const { result } = renderHook(() => useWorkspaceInsights(props)); + await act(() => result.current.open()); + act(() => result.current.closePanel()); + expect(worker.pauseExpensiveWork).toHaveBeenCalled(); + expect(result.current.snapshot.documents.size).toBeGreaterThan(0); +}); +``` + +Add active workspace switch serialize/teardown, streamed provisional results, complete-with-warnings, manual Refresh hashes every eligible source and performs zero external checks, overlay save/revert/close behavior, watcher/polling start/stop, and cancellation/resume. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/use-workspace-insights.test.tsx +``` + +- [ ] **Step 3: Implement session coordinator** + +Normal reopen validation uses path/size/mtime first and hashes suspicious entries. Manual Refresh re-enumerates and hashes all eligible Markdown/MDX documents but reparses only changed/incompatible content. Keep one live session for active workspace only. + +- [ ] **Step 4: Run lifecycle tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/insights/use-workspace-insights.test.tsx tests/unit/ui/insights/worker-client.test.ts +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/insights/useWorkspaceInsights.ts ui/src/contexts/useAppStateEffects.ts ui/src/App.tsx tests/unit/ui/insights/use-workspace-insights.test.tsx +git commit -m "feat: orchestrate workspace insights sessions" +``` + +--- + +### Task 16: Add sidebar discovery entry and dedicated resizable panel shell + +**Files:** +- Create: `ui/src/components/Insights/WorkspaceInsightsPanel.tsx` +- Create: `ui/src/components/Insights/InsightsSettings.tsx` +- Modify: `ui/src/components/Sidebar/SidebarTabsHeader.tsx` +- Modify: `ui/src/components/Sidebar/Sidebar.tsx` +- Modify: `ui/src/AppView.tsx` +- Modify: `ui/src/App.tsx` +- Modify: `ui/src/hooks/useResize.ts` only to reuse/generalize existing behavior +- Create: `ui/src/styles/global/global-workspace-insights.css` +- Modify: global stylesheet entrypoint that imports the existing `global-*.css` files +- Test: `tests/unit/ui/components/workspace-insights-panel.test.tsx` + +**Interfaces:** +- Panel receives `WorkspaceInsightsSessionViewModel`; view keys are `gallery | links | lint | duplicates | graph | related`. +- Sidebar action only opens/focuses the panel. + +- [ ] **Step 1: Write failing shell/UI tests** + +```tsx +it('opens Insights from the sidebar into the main resizable panel', async () => { + render(); + await user.click(screen.getByRole('button', { name: /workspace insights/i })); + expect(screen.getByRole('region', { name: /workspace insights/i })).toBeVisible(); + expect(screen.getByRole('tab', { name: /gallery/i })).toHaveAttribute('aria-selected', 'true'); +}); +``` + +Define `TestApp` with the same app/provider test helpers already used by UI component tests. Test resize keyboard/pointer behavior, progress/provisional status, warning/truncated state, cancel, Refresh, settings, and panel close. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/components/workspace-insights-panel.test.tsx +``` + +- [ ] **Step 3: Implement shell using existing theme/resize patterns** + +Do not add a narrow fourth content tab containing all results. The sidebar entry opens a wider main-area panel. Use existing CSS variables/theme tokens and `useResize` behavior. Keep view content lazy but session shared. + +- [ ] **Step 4: Run UI/style tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/components/workspace-insights-panel.test.tsx +pnpm run lint:ui-styles +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/components/Insights ui/src/components/Sidebar ui/src/App.tsx ui/src/AppView.tsx ui/src/hooks/useResize.ts ui/src/styles tests/unit/ui/components +git commit -m "feat: add workspace insights panel" +``` + +--- + +### Task 17: Implement Gallery, Links, Lint, and Duplicates views + +**Files:** +- Create: `ui/src/components/Insights/GalleryView.tsx` +- Create: `ui/src/components/Insights/LinksView.tsx` +- Create: `ui/src/components/Insights/LintView.tsx` +- Create: `ui/src/components/Insights/DuplicatesView.tsx` +- Test: `tests/unit/ui/components/insights-gallery-links.test.tsx` +- Test: `tests/unit/ui/components/insights-lint-duplicates.test.tsx` + +**Interfaces:** +- Consumes session snapshot plus source-navigation/probe/external-check/suppression actions. + +- [ ] **Step 1: Write failing view behavior tests** + +Gallery assertions: +- referenced media only; +- categories image/diagram/video/audio/document; +- invalid Mermaid visible with failed status; +- local existence invokes metadata probe; +- remote preview is unloaded by default and requires explicit `Load Preview`. + +Links assertions: +- distinct missing/invalid-anchor/ambiguous/outside/dynamic/unsupported/non-checkable statuses; +- dynamic refs excluded from broken count; +- `file:` outside workspace classified outside-workspace. + +Lint/Duplicate assertions: +- severity/rule filters; +- `Show suppressed`; +- finding/rule/path/workspace suppression actions; +- exact/repeated/near groups; threshold display and reversible duplicate suppression. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/components/insights-gallery-links.test.tsx tests/unit/ui/components/insights-lint-duplicates.test.tsx +``` + +- [ ] **Step 3: Implement views with source navigation and explicit network boundaries** + +Remote Gallery cards must not set actual remote `src` until `Load Preview` is clicked. External-link status in Links remains `unchecked` until the separate check action runs. Use session-level probes/checks rather than direct `fetch()`. + +- [ ] **Step 4: Run view tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/components/insights-gallery-links.test.tsx tests/unit/ui/components/insights-lint-duplicates.test.tsx +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/components/Insights/GalleryView.tsx ui/src/components/Insights/LinksView.tsx ui/src/components/Insights/LintView.tsx ui/src/components/Insights/DuplicatesView.tsx tests/unit/ui/components +git commit -m "feat: add insights media links lint and duplicates views" +``` + +--- + +### Task 18: Implement Graph and Related views, external-check UX, and report export + +**Files:** +- Create: `ui/src/components/Insights/GraphView.tsx` +- Create: `ui/src/components/Insights/RelatedView.tsx` +- Create: `ui/src/insights/reports.ts` +- Extend: `ui/src/components/Insights/LinksView.tsx` +- Extend: `ui/src/components/Insights/WorkspaceInsightsPanel.tsx` +- Test: `tests/unit/ui/components/insights-graph-related.test.tsx` +- Test: `tests/unit/ui/components/insights-external-check.test.tsx` +- Test: `tests/unit/ui/insights/reports.test.ts` + +**Interfaces:** +- Graph consumes `FocusedGraph`. +- External checker UI holds session-only result cache and origin approvals; neither is persisted. +- Reports expose `createInsightsMarkdownReport(snapshot, scope)` and `createInsightsJsonReport(snapshot, scope)`. + +- [ ] **Step 1: Write failing graph/accessibility/network/report tests** + +```tsx +it('synchronizes graph and accessible list selection', async () => { + render(); + await user.click(screen.getByRole('button', { name: /b\.md/i })); + expect(screen.getByRole('treeitem', { name: /b\.md/i })).toHaveAttribute('aria-selected', 'true'); +}); + +it('checks unique URLs only after explicit action', async () => { + render(); + expect(host.checkExternalLinks).not.toHaveBeenCalled(); + await user.click(screen.getByRole('button', { name: /check external links/i })); + expect(host.checkExternalLinks).toHaveBeenCalledTimes(1); +}); +``` + +Define `fixture`, `props`, and `host` with concrete typed fixtures in the test files. Report tests verify full/filter/selection scope, completeness/provisional metadata, external summary-only fields, and absence of DNS/private approvals/credentials/source bodies. + +- [ ] **Step 2: Run RED** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/components/insights-graph-related.test.tsx tests/unit/ui/components/insights-external-check.test.tsx tests/unit/ui/insights/reports.test.ts +``` + +- [ ] **Step 3: Implement graph/related/network/export UX** + +Graph: deterministic SVG plus synchronized keyboard list; no force simulation. Inferred edges toggle defaults off. + +External checks: +- setting must be enabled; +- selected/filtered scope is default when present; +- explicit full-workspace option; +- show unique URL count; +- origin-scoped private confirmation; +- cancellation aborts queued/in-flight; +- session result cache with age and Recheck bypass. + +Export both Markdown and JSON through the existing save-export bridge. + +- [ ] **Step 4: Run focused tests** + +```bash +pnpm exec vitest run --project ui tests/unit/ui/components/insights-graph-related.test.tsx tests/unit/ui/components/insights-external-check.test.tsx tests/unit/ui/insights/reports.test.ts +``` + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/components/Insights ui/src/insights/reports.ts tests/unit/ui/components tests/unit/ui/insights/reports.test.ts +git commit -m "feat: add graph related checks and insights reports" +``` + +--- + +### Task 19: Complete settings UI, localization, accessibility, documentation, and full verification + +**Files:** +- Extend: `ui/src/components/Insights/InsightsSettings.tsx` +- Modify: `ui/src/components/Settings/SettingsModal.tsx` +- Modify: `ui/src/contexts/translations.ts` +- Modify: `ui/src/contexts/translationsData.ts` +- Modify: `ui/src/contexts/translationTypes.ts` +- Modify: `ui/src/contexts/auditedUiTranslations.ts` +- Modify: `ui/src/contexts/auditedUiTranslationTypes.ts` +- Modify: `docs/instructions/05-reference/01-ui-to-host-command-catalog.md` +- Modify: `docs/instructions/03-features/12-settings-preferences-import-export.md` +- Modify: `docs/instructions/05-reference/10-localization-catalog.md` +- Test: `tests/contracts/translations-coverage.test.ts` +- Test: `tests/contracts/ui-style-contract.test.ts` +- Test: `tests/node/localization-settings-doc-sync-contract.test.mjs` + +**Interfaces:** +- Settings UI exposes global defaults, workspace overrides/reset, pattern validation, 10 MiB override patterns, duplicate threshold/suppressions, lint severity/suppressions, relationship presets/weights, graph cap, external timeout/toggle, 500 MiB cache cap/usage/clear. +- All user-facing copy is typed/localized. + +- [ ] **Step 1: Add failing translation/settings/accessibility contract tests** + +```ts +it('has localized Workspace Insights domains in every locale', () => { + for (const locale of supportedLocales) { + expect(AUDITED_UI_TRANSLATIONS[locale].insights.externalLinksDescription).toBeTruthy(); + expect(AUDITED_UI_TRANSLATIONS[locale].rendererUi.wikiTransclusionCycle).toBeTruthy(); + } +}); +``` + +Add keyboard tests for graph/list, panel tabs, suppression controls, private confirmation dialog, and non-color edge/status semantics. + +- [ ] **Step 2: Run RED** + +```bash +pnpm run test:translations +pnpm run lint:ui-styles +node --experimental-strip-types --test tests/node/localization-settings-doc-sync-contract.test.mjs +pnpm exec vitest run --project ui tests/unit/ui/components +``` + +- [ ] **Step 3: Implement all locale/settings/doc updates** + +Use the exact approved external-link setting meaning in every locale: anonymous checks, no cookies/Authorization, private-network confirmation, possible unknown/transient results, and zero external requests while disabled. + +Document new host commands/results and runtime `unsupported` behavior. Document that manual Refresh is local-only and that remote media preview is separately explicit. + +- [ ] **Step 4: Run the full fresh verification matrix** + +Run from repository root: + +```bash +pnpm test:ui +pnpm test:electron +pnpm test:vscode +pnpm test:chromium +pnpm test:contracts +pnpm test:translations +pnpm run lint:ui-styles +pnpm run build:ui +pnpm run build:vscode +pnpm run build:chromium +pnpm run build:website-app +cargo test --manifest-path tauri/Cargo.toml -- --test-threads=1 +``` + +Then verify the branch diff contains no stale prototype files, generated artifacts, repository-local cache/config, direct UI `no-cors` external checking, or probe→binary-read coupling. + +Expected: every command exits 0. If any command cannot run in the available environment, record it explicitly as unverified rather than claiming success. + +- [ ] **Step 5: Commit final integration/docs** + +```bash +git add ui electron vscode chromium-xtension website-app tauri tests docs pnpm-lock.yaml +git commit -m "docs: document workspace insights behavior" +``` + +After the commit, re-read PR #44 review threads/checks and address only technically valid new findings. Do not mark the implementation complete until the fresh verification matrix and final diff review support that claim. + +--- + +## Plan Self-Review / Spec Coverage + +This mapping is part of the implementation checklist; executors should preserve it when tasks are split among workers. + +| Spec requirement | Implementation task(s) | +| --- | --- | +| Dedicated resizable six-view panel, lazy active-workspace lifecycle | 15–18 | +| Dedicated uncapped scan, exclusions, 10/64 MiB source rules, probes, watchers/polling | 1, 6, 7 | +| Greptile binary-probe finding | 1, 6, 7 | +| Greptile opaque-response finding | 6, 8, 18 | +| Shared parser/source mapping and exact renderer anchors | 2–4, 9 | +| YAML title/aliases/tags, malformed/duplicate handling | 2, 9 | +| Wiki syntax, ambiguity, case folding, paths, directory index behavior | 4–5 | +| Depth-5 interactive transclusion and preview-pipeline reuse | 5, 7, 15 | +| Markdown/HTML/MDX refs, dynamic refs, tags, local schemes | 4, 9 | +| Referenced-only Gallery, Mermaid failure, remote no-auto-load | 9, 17 | +| Broken Link status model | 4, 7, 9, 10, 17 | +| Secure explicit external checking/private approvals/session cache | 8, 15, 18 | +| Lint rules/severity/suppressions | 2, 9, 14, 17 | +| Exact/repeated/near duplicates without O(n²) | 11 | +| Backlinks/focused deterministic graph/accessibility | 10, 18 | +| Explainable relationship presets/custom weights | 12, 18 | +| Unsaved active overlay, delete/rename semantics | 10, 15 | +| Derived cache/versioning/hybrid 500 MiB eviction/workspace identity | 14 | +| Settings precedence/reset/import/export | 1, 14, 19 | +| Manual Refresh full hash/no network | 15 | +| Markdown + JSON report scopes/completeness/privacy | 18 | +| Localization and accessibility | 16–19 | +| Cross-runtime/security/build verification | 7, 8, 19 | + +### Type consistency checkpoint + +The plan intentionally uses these canonical names across tasks: + +- `InsightsWorkspaceEntry` +- `WorkspaceResourceProbeResult` +- `ExternalLinkCheckRequest` / `ExternalLinkCheckResult` +- `WikiLinkToken` / `WikiResolution` +- `AnalyzedDocument` +- `WorkspaceInsightsIndex` +- `WorkspaceInsightsSnapshot` +- `InsightsSettings` / `InsightsWorkspaceOverrides` +- `WorkspaceInsightsSession` + +If implementation discovers an existing repository name that should replace one of these, rename it once at the defining task and update all later task references in this plan before continuing. diff --git a/docs/workspace-insights-design.md b/docs/workspace-insights-design.md new file mode 100644 index 00000000..0eaec847 --- /dev/null +++ b/docs/workspace-insights-design.md @@ -0,0 +1,983 @@ +# Workspace Insights and Wiki Links — Architecture Specification + +## Status + +**Approved design; implementation has not started.** + +This specification supersedes the original Workspace Insights design and the obsolete implementation plan that were committed earlier on `feature/workspace-insights`. + +The design was audited against the current Markdown Explorer renderer, parser, host-message model, settings import/export path, runtime constraints, and Greptile feedback on PR #44. In particular: + +- Insights adds a metadata-only workspace resource probe instead of abusing binary export reads for existence checks. +- External link checks are host-backed and return real HTTP/network status. Browser `no-cors` opaque responses are not used for reachability classification. +- Insights uses a dedicated full workspace scan rather than the existing capped file list/search inventory. +- Structural Markdown analysis reuses the renderer parser/source-mapping model and the renderer's actual heading slug logic. +- Wiki Links and embeds are first-class renderer/navigation features, not an Insights-only syntax. + +## Goals + +Deliver six workspace-wide tools from one shared local index: + +1. **Image and Media Gallery** — browse every image, diagram, video, audio, and supported embedded document referenced by Markdown/MDX in the workspace. +2. **Broken-Link Inspector** — report missing local files, invalid anchors, ambiguous Wiki Links, missing local resources, dynamic/uncheckable references, and explicitly checked external-link failures. +3. **Markdown Linter Panel** — report conservative structural/style diagnostics with configurable rule enablement and severity. +4. **Duplicate Content Finder** — find normalized exact duplicates, repeated sections/passages, and near-identical notes without an O(n²) workspace comparison. +5. **Backlinks and Knowledge Graph** — show inbound references and a focused, deterministic graph of explicit links/embeds with optional inferred relationships. +6. **Document Relationships Panel** — rank related documents with explainable, deterministic signals. + +The same work also adds first-class Wiki Link navigation and transclusion because Insights must not invent a link language that the renderer itself does not understand. + +## Constraints and principles + +- Shared behavior lives in the shared React/TypeScript UI and shared Markdown analysis modules. Runtime hosts provide filesystem/network capabilities, not competing analysis implementations. +- Offline-first: opening a workspace, opening Insights, refreshing local analysis, rendering local embeds, and building relationships do not contact external URLs. +- External HTTP(S) checking is disabled by default and requires both an enabled setting and an explicit user action. +- No telemetry, cloud embedding service, cloud similarity service, or remote content analysis is introduced. +- Parse each document once per source revision and derive all six tools from one incremental index. +- No silent caps. Any runtime safety ceiling or incomplete scan is surfaced as a warning/truncated state with counts/reasons. +- One unreadable, oversized, unsupported, or transiently failed file must not prevent the rest of the workspace from completing. +- User-visible Insights strings and settings are localized in every supported locale. +- Diagnostics do not rewrite source in v1. + +## Information architecture + +### Entry point + +Add **Workspace Insights** as a sidebar discovery action. Activating it opens a **dedicated resizable Workspace Insights panel** in the main workspace area rather than forcing six tools into the narrow sidebar. + +The panel contains six views: + +- Gallery +- Links +- Lint +- Duplicates +- Graph +- Related + +The sidebar remains the navigation/discovery surface. The wider Insights panel owns filtering, tables, graph visualization, detailed evidence, settings, progress, cancellation, and exports. + +### Lazy activation + +Insights is lazy: + +- ordinary workspace open does not restore the Insights cache, enumerate the full Insights workspace, or start the Insights worker; +- the first Insights open restores/validates cache and starts scanning; +- switching among Insights views reuses the same live index; +- closing the panel pauses/cancels expensive analysis while retaining completed state and minimal staleness tracking for the active workspace; +- switching workspaces serializes eligible derived cache for the old workspace and tears down its worker/watch state. + +Only the **active workspace** has a live Insights worker/index/watch session. Inactive workspace derived state is cached locally. + +## Architecture + +### Responsibilities + +**Runtime host** + +- recursively enumerate workspace entries for Insights; +- canonicalize paths and enforce workspace/symlink boundaries; +- read Markdown/MDX source with explicit size limits; +- probe local resources using metadata only; +- expose filesystem deltas where reliable; +- provide metadata polling fallback where direct watching is unavailable; +- perform external HTTP(S) checks with real status, DNS/IP validation, timeouts, redirects, and cancellation; +- reuse the existing document conversion/preview pipeline for supported non-Markdown embeds. + +**Dedicated Insights Web Worker** + +- parse Markdown/MDX using the shared parser semantics; +- extract links, Wiki Links, static HTML/MDX references, media, tags, headings, sections, and metadata; +- maintain inverted indexes and incremental derived state; +- lint documents; +- compute duplicate signatures/candidates; +- build backlinks and graph edges; +- score relationships; +- emit batched partial results. + +**React UI** + +- panel navigation and filters; +- progress/provisional/completeness states; +- source navigation; +- settings and suppressions; +- graph/list interaction; +- explicit external-link check controls and private-origin confirmation; +- report export; +- worker lifecycle and host request coordination. + +If Web Workers are unavailable, analysis falls back to cooperative chunked work on the UI runtime. The UI must display a **degraded performance** state; it must not pretend the worker is active. + +### Shared Markdown semantics + +Structural meaning comes from the existing Markdown parser/source mapping used by rendering: headings, Setext/ATX forms, lists, tables, fences, frontmatter boundaries, source ranges, and nested Markdown must not be independently reinterpreted by a regex-only Insights parser. + +A focused secondary extractor is allowed only for semantics not exposed by the parser, such as Wiki Link tokens, standard link destinations, static HTML/MDX `href`/`src` attributes, inline tags, and related metadata. Extraction must respect parser ranges so code/fences and other non-prose contexts are excluded correctly. + +Heading anchor generation calls the same shared `slugify()` behavior used by `HtmlRenderer`, including duplicate suffixes (`base`, `base-1`, `base-2`, ...). Insights must not implement a different GitHub-style approximation. + +## Host capability contracts + +Names below describe the required contract; final message/type names may follow existing repository conventions. + +### 1. Dedicated Insights workspace scan + +`scanInsightsWorkspace` is separate from the existing workspace `fileList`/search scan and has no silent 1000-file-style cap. + +Input includes: + +- workspace operation/request ID; +- user include/exclude rules; +- built-in exclusion profile version; +- cancellation token/request identity. + +The host recursively streams batches of metadata first: + +```ts +interface InsightsWorkspaceEntry { + relativePath: string; + canonicalRelativePath: string; + kind: 'file' | 'directory'; + sizeBytes: number; + mtimeMs: number; + extension?: string; + isSymlink?: boolean; +} +``` + +The scan also reports progress, excluded/skipped counts, and completion metadata. If a runtime must stop at an absolute safety boundary, the response is explicitly `truncated: true` with a reason and count; it is never presented as complete. + +### 2. Insights source read + +`readInsightsDocumentSource` reads a specific eligible Markdown/MDX file after metadata filtering. + +Result categories: + +- `ok` +- `missing` +- `outside-workspace` +- `unreadable` +- `unsupported` +- `too-large` + +The default soft analysis limit is **10 MiB per Markdown/MDX file**. Users can override the soft limit for specific files or gitignore-style path patterns. An absolute **64 MiB per-source safety ceiling** is not overridable in v1. + +The derived persistent cache never stores the full returned source body. + +### 3. Metadata-only workspace resource probe + +Add a generic `probeWorkspaceResource({ documentPath, resourcePath })` capability for any workspace-local referenced file. + +Representative result: + +```ts +interface WorkspaceResourceProbeResult { + status: 'exists' | 'missing' | 'outside-workspace' | 'unreadable' | 'unsupported'; + relativePath?: string; + kind?: 'file' | 'directory'; + sizeBytes?: number; + mimeType?: string; +} +``` + +This operation performs canonical path resolution and metadata/stat work only. It **must not read file bytes, base64-encode content, or call `readWorkspaceExportResource` internally**. + +Binary reads remain a separate, explicit operation used only when the user actually requests a preview/export that needs bytes. + +### 4. Incremental filesystem deltas + +Where a runtime has a reliable filesystem watcher, expose batched Insights deltas for: + +- add +- update +- delete +- rename/move hint when the platform supplies one + +Deleted files are removed from the live index immediately. Surviving inbound references become broken-link findings. + +Rename/move identity is best-effort. High-confidence correlation may use native rename events plus prior path, content fingerprint, size, and mtime. High-confidence renames migrate derived cache entries, duplicate/lint suppressions, and per-file overrides. Low-confidence cases are treated as delete + add rather than attaching state to the wrong file. + +### 5. Browser capability and polling fallback + +If a browser-backed runtime lacks a reliable observer, use lightweight metadata polling only while Insights is active/visible. Polling stops when Insights is inactive. The UI labels this as polling, not real-time watching. + +Every runtime exposes **Refresh** regardless of watcher support. + +### 6. Active unsaved document overlay + +Disk content is the source of truth for all non-active documents. The active Markdown/MDX document may overlay the indexed disk source with the unsaved source already supplied to rendering, together with a monotonic source revision/version. + +The overlay affects the full derived index: + +- outbound links and backlinks; +- graph and embed edges; +- broken links; +- Gallery and lint; +- duplicate signatures; +- relationship indexes/scores. + +Save collapses the overlay into persistent disk state. Revert/close removes it. Only the active document needs unsaved overlay support in v1. + +### 7. Host-backed external HTTP checking + +External checking is a host capability, not `fetch(..., { mode: 'no-cors' })` in the UI. A runtime that cannot obtain reliable status while enforcing the network-safety contract returns `unsupported` instead of guessing. + +The checker accepts a batch/request ID and streams per-URL results so progress/cancellation can be reflected immediately. + +## Workspace scanning and exclusions + +### Exclusion sources + +The effective matcher uses gitignore-style syntax everywhere: initial scan, watcher reconciliation, polling, manual Refresh, and oversized overrides. + +**Hard safety exclusions** cannot be overridden. They are intentionally minimal: + +- VCS internals such as `.git/`; +- Markdown Explorer runtime/cache/internal storage paths; +- runtime-internal paths that must not be traversed for correctness/safety. + +**Built-in default exclusions** are overridable and cover common high-volume generated/dependency directories such as `node_modules/`, `dist/`, `build/`, `.next/`, `coverage/`, common cache directories, and vendor output. + +Then apply repository `.gitignore` rules. + +Finally apply user Insights rules **in user-defined order with gitignore last-match-wins semantics**. User negation/re-include rules may override `.gitignore` and built-in defaults but can never override hard safety exclusions. + +Invalid user patterns are rejected at settings-save/import normalization with a visible validation message rather than being silently ignored. + +The UI reports excluded/skipped counts and, where practical, the effective rule/source that caused a path to be excluded. + +## Index lifecycle and persistence + +### Initial indexing + +On first Insights open: + +1. derive/resolve the app-local workspace identity; +2. restore compatible derived cache if present; +3. enumerate the dedicated Insights workspace metadata stream; +4. compare metadata with cached entries; +5. load eligible changed/uncached Markdown/MDX source in bounded batches; +6. send source/revision batches to the worker; +7. stream partial results into the six views; +8. establish watcher or visible-only polling after scan reconciliation. + +Results are **provisional** while indexing is incomplete. Per-document Gallery/Lint data may appear immediately. Workspace-wide duplicates, graph, backlinks, and relationships update incrementally but remain marked `still indexing` until every eligible file was attempted. + +A completed index may contain warnings. Completion means every eligible file was attempted; unreadable, oversized, unsupported, or failed documents remain visible with a reason and affected views warn that results may be incomplete. + +### Refresh + +Manual **Refresh** is intentionally stronger than ordinary reopen validation: + +- re-enumerate eligible documents; +- hash every eligible Markdown/MDX source; +- reparse only content whose fingerprint changed or whose analyzer/cache component is incompatible; +- reconcile add/delete/rename/exclusion/size-override changes; +- re-probe affected local resources as necessary; +- rebuild only affected derived indexes. + +Refresh performs **zero external HTTP requests**. External checks have their own explicit Check/Recheck actions. + +### Normal reopen validation + +Ordinary cache restore compares canonical path, size, and mtime first. Entries that appear changed/suspicious are fingerprinted; unchanged content is reused. Metadata-only reuse is an optimization, not the source of truth. + +### Persistent cache + +The cache is app-local and never written into the repository/workspace. + +Persistent cache may contain: + +- workspace identity/path history; +- file identity, canonical relative path, size, mtime, content fingerprint; +- headings, tags, link targets, anchors, media metadata; +- duplicate fingerprints/signatures; +- hashed terminology/signatures used for candidate generation; +- graph/index metadata; +- analyzer/cache component versions. + +It must not contain: + +- full Markdown/MDX bodies; +- binary media bodies; +- remote response bodies; +- cookies, Authorization headers, credentials; +- private-origin confirmations; +- external-check session cache; +- readable significant terminology solely for similarity ranking. + +Readable filenames/headings/tags/titles may be cached where required for restored UI. Human-readable relationship terminology is reconstructed from source on demand when an explanation is expanded. + +### Cache versioning + +Store a cache schema version plus component versions for parser/resolver, lint, duplicate signatures, relationships, and other derived models. Invalidate/rebuild incompatible portions while reusing compatible entries. If safe compatibility cannot be determined, discard the affected workspace cache. + +### Cache capacity and eviction + +Default global cache cap: **500 MiB**, user-configurable subject to runtime storage quota. + +Eviction is hybrid: + +1. remove least-recently-used per-file derived entries from inactive workspaces; +2. retain lightweight workspace metadata where useful for validation; +3. if still over cap, evict entire inactive workspace caches by LRU. + +Never evict the active workspace live index mid-session. Cache eviction never deletes Insights user settings/suppressions. + +Expose cache usage and **Clear Insights cache** globally and per workspace. + +### Workspace identity + +Per-workspace settings/cache use an app-local generated identity plus known canonical path history. A bounded fingerprint of stable workspace characteristics is used for best-effort move/rename recognition. High-confidence recognition migrates path history/cache/settings; uncertain matches become a new workspace. No identity file is written into the repository. + +## Wiki Links + +Wiki Links are part of the shared renderer/navigation/resolver contract. + +### Syntax + +Supported links: + +- `[[Note]]` +- `[[Note#Heading]]` +- `[[Note|Label]]` +- `[[Note#Heading|Label]]` +- `[[#Heading]]` for a same-document fragment +- relative path forms such as `[[./Local]]` and `[[../shared/Guide]]` + +Supported embeds: + +- `![[Note]]` +- `![[Note#Heading]]` +- `![[image.png]]` +- other supported local media/document targets. + +Wiki Link escaping uses backslash only: + +- `\#` literal `#` +- `\|` literal `|` +- `\\` literal backslash + +The parser splits at the first unescaped `|` for label and first unescaped `#` for fragment. Percent encoding is not a second Wiki Link escaping system. + +Both `/` and `\` are accepted as Wiki Link path separators and normalized internally to `/`. Original source text is preserved. + +Malformed Wiki Link syntax renders as plain text and produces a warning diagnostic. No graph/backlink/embed edge is created for malformed syntax. + +### Resolution + +Matching uses Unicode normalization plus locale-independent case-insensitive matching while preserving original/canonical casing for display. + +Canonical document title precedence: + +1. YAML frontmatter `title`; +2. first rendered H1; +3. filename stem. + +Resolver keys include: + +- explicit/relative workspace path; +- filename/stem; +- canonical document title; +- YAML frontmatter `aliases`. + +YAML aliases support scalar/list forms normalized to strings. YAML is the only frontmatter format interpreted in v1. + +Path-qualified targets take precedence over bare-name title/alias search. Bare-name resolution gathers viable filename/title/alias candidates. If more than one viable target remains, the result is **ambiguous** and the UI presents candidates; no target is silently guessed. + +Extensionless Wiki Link paths try `.md` and `.mdx`. If both remain viable, resolution is ambiguous. An explicit extension is authoritative. + +Relative paths resolve from the source document directory. Canonical resolution must remain inside the workspace after symlink resolution. + +`[[#Heading]]` resolves within the source document. Inside transcluded content, “source document” means the embedded document, not the parent container. + +Canonical filesystem casing is displayed for resolved paths. A case-only source mismatch remains valid but produces an `info` portability diagnostic. + +Wiki Link source is never rewritten automatically after file rename/move. + +### Directory targets + +A Markdown or Wiki Link that targets a directory is valid only when the directory contains exactly one unambiguous recognized index document among: + +- `README.md` +- `README.mdx` +- `index.md` +- `index.mdx` + +Matching is case-insensitive under the same normalization rules. Multiple viable index candidates produce an ambiguous target. A directory without one is unresolved. + +### Transclusion + +Wiki embeds are full transclusion, not an Insights-only preview. + +For Markdown/MDX embeds: + +- render the embedded document/section using the normal renderer; +- preserve embedded source-document identity and source mapping; +- resolve nested links relative to the embedded source document; +- keep links, headings, tables, callouts, code, Mermaid, media, and supported controls interactive; +- cap recursive transclusion depth at **5**; +- detect cycles independently of depth and render an explicit cycle placeholder; +- render ambiguity/missing/depth/preview failures as non-fatal placeholders. + +Graph/backlink semantics distinguish normal links from embed/transclusion edges. Duplicate detection operates on source documents only; rendered transcluded content is never counted as copied source. + +For supported non-Markdown local documents (for example PDF/DOCX/XLSX/PPTX/HTML/RTF where the application already supports preview/conversion), reuse the existing preview pipeline. Unsupported binaries fall back to attachment cards. + +Remote media/content is never automatically downloaded by transclusion. Remote preview remains a separate explicit user action. + +## Standard Markdown, HTML, and MDX references + +### Normal Markdown links + +Normal Markdown links preserve standard path semantics. `[Guide](Guide)` checks the literal local target `Guide`; it does not implicitly try `Guide.md`/`.mdx`. + +For local links, query strings are excluded from filesystem resolution but preserved as link metadata. Fragments are validated separately. + +Normal URL fragments are percent-decoded before anchor comparison. Invalid percent encoding produces a malformed-link diagnostic rather than a guessed resolution. + +### Anchors + +Valid local anchors are: + +- renderer-generated Markdown heading IDs using the exact shared slug/duplicate algorithm; +- literal static HTML/MDX `id="..."` anchors; +- legacy literal `` anchors. + +Dynamic `id={expression}` is not evaluated and is classified as dynamic/unresolved. + +### Static HTML/MDX references + +Analyze literal link/media attributes such as: + +- `href` +- `src` +- `poster` +- `` + +Dynamic expressions such as `href={buildUrl()}` or `src={assetPath}` are never evaluated. They appear in Links as **dynamic / not statically checkable**, are excluded from broken counts, and preserve source location/expression evidence. + +### Inline tags + +Recognize `#tag` only in Markdown-aware prose/list contexts. Exclude fenced code, inline code, URLs, link destinations, HTML attributes, and anchor fragments. Support nested forms such as `#project/backend` plus YAML frontmatter `tags`. + +Matching is normalized case-insensitively while original spelling is preserved for display. + +## Local link/resource policy + +Symlinks are followed only when their canonical target remains inside the current workspace. Canonical targets are deduplicated and cycles are prevented. Display uses the workspace-relative path users recognize. + +`file:` URLs are validated only when canonicalized inside the current workspace. Outside-workspace targets are classified **outside workspace**, never opened automatically, and are not treated as generic broken files. + +Non-HTTP schemes such as `mailto:`, `tel:`, `data:`, `vscode:`, and custom schemes are visible as **valid/non-checkable** inventory entries and excluded from broken counts. Malformed URI syntax may still produce a lint diagnostic. + +## Gallery + +Gallery is **referenced-media only**. It does not become a general filesystem asset browser. + +Sources include: + +- Markdown image/media references; +- Wiki embeds; +- static HTML/MDX media attributes; +- Mermaid blocks and supported diagram sources; +- supported embedded local documents. + +Categories: + +- `image` — PNG/JPEG/GIF/WebP/SVG by default; +- `diagram` — Mermaid and explicitly supported diagram formats; +- `video`; +- `audio`; +- `document`. + +SVG defaults to image unless a renderer explicitly treats the source as a diagram format. + +Local entries use metadata probes for existence. Binary bytes are loaded only for an explicit preview that requires them. + +Remote entries show URL/metadata/placeholder and are never auto-loaded. **Load Preview** is a separate explicit network action from **Open externally** and is not implicitly enabled by external-link checking. + +Invalid Mermaid remains visible as a failed diagram entry with source range/error summary. Mermaid render/syntax failure produces a lint diagnostic with default severity `warning`. + +## Broken-Link Inspector + +Local statuses include at least: + +- valid +- missing +- invalid anchor +- ambiguous +- outside workspace +- unreadable +- dynamic / unchecked +- unsupported +- non-checkable scheme + +Ambiguous is not collapsed into missing. Dynamic/uncheckable is not counted as broken. + +The view retains original source text, canonical resolved target where available, source file/range, and reason. + +## External HTTP(S) checking + +### User control + +Setting: **Check external links** — default **OFF**. + +Required localized description: + +> Verify HTTP(S) links from your Markdown files when requested. Checks are anonymous and do not send cookies or authorization credentials. Private-network destinations require confirmation. Some sites may block automated requests, require authentication, rate-limit checks, or return transient errors, so results can be reachable, broken, unreachable, or unknown. External links are never contacted when this setting is disabled. + +When disabled: + +- zero external checker requests are issued; +- URLs remain listed; +- status is `unchecked`; +- check actions are hidden/disabled. + +### Scope + +**Check external links** supports: + +- current filtered/selected scope (default when such scope exists); +- explicit **Check entire workspace**. + +The UI shows the number of unique URLs that will be contacted. Duplicate URLs are checked once and shared across references. + +### Anonymous requests + +Never send application/session cookies, Authorization headers, or user credentials. System proxy/network stack may be used. `401`/`403` are valid reachability results indicating authentication/authorization is required. + +### Method + +1. try `HEAD`; +2. if method-related/inconclusive (for example `405`, `501`, or equivalent host-detected HEAD incompatibility), issue a streaming `GET`; +3. inspect status/headers and abort body transfer immediately. + +Do not download response bodies for reachability checking. + +### Status semantics + +- `2xx`, successful redirect chain -> `reachable` +- `401`, `403` -> `reachable/auth-required` +- `404`, `410` -> `broken` +- `429` -> `rate-limited/unknown` +- `5xx` -> `server-error/retryable`, not immediately `broken` +- DNS failure, refused connection, timeout, TLS validation failure -> `unreachable` +- private-not-confirmed, blocked, unsupported -> `unchecked/unknown` + +TLS verification is never disabled. + +### SSRF/private-network policy + +Resolve DNS before connecting and classify all resolved addresses. Private classes include localhost/loopback, link-local, RFC1918/private ranges, and equivalent IPv6 ranges. + +Connect only to an IP that passed policy validation while preserving the original hostname for TLS SNI/Host semantics. Re-resolve and revalidate every redirect target. This prevents DNS rebinding from validating one address and connecting to another. + +A public URL redirecting to a private target requires confirmation. + +Private confirmation is scoped to **scheme + host + port (origin)** for the **current active workspace session only**. A redirect to another private origin requires a new confirmation. Approvals are discarded on workspace deactivation/close and are never persisted/exported. + +### Redirects and transport + +- maximum redirects: **5**; +- redirect-limit exhaustion -> `unknown / redirect limit exceeded`; +- HTTPS -> HTTP downgrade may be followed but adds an **insecure downgrade** warning separate from reachability status; +- TLS certificate failures -> `unreachable`; +- every hop repeats DNS/IP/private validation. + +### Concurrency, timeout, retry + +- global concurrency: **4**; +- per-origin concurrency: **2**; +- default timeout: **10 seconds per attempt**; +- configurable timeout range: **3–30 seconds**, global default with per-workspace override; +- at most **one retry** after the initial attempt; +- retry only transient network failures and retryable `5xx`; +- use exponential backoff; +- honor `Retry-After`; +- do not retry `404`, `410`, `401`, `403`, or explicit cancellation; +- `429` stays rate-limited/unknown and must not trigger aggressive retries. + +### Session cache and cancellation + +Results are cached only for the active workspace session with status, code/category, final URL, and timestamp. The UI shows checked age. Explicit **Recheck** bypasses the session cache. + +Closing Insights, switching workspace, or pressing Cancel aborts both queued and in-flight checks. Canceled URLs revert to their previous session result or `unchecked`; cancellation is never classified as unreachable/broken. + +Manual local **Refresh** never rechecks external URLs. + +## Markdown linter + +The linter is diagnostics-only in v1: finding, severity, explanation, source range, navigation. No auto-format, quick-fix, or source rewrite. + +Each rule supports enable/disable and severity (`info`, `warning`, `error`) where severity is meaningful. Configuration follows global defaults plus per-workspace overrides and reset. + +Conservative initial rules include: + +| Rule | Default | Behavior | +| --- | --- | --- | +| `frontmatter/malformed` | error | YAML frontmatter cannot be parsed; body analysis continues | +| `frontmatter/duplicate-key` | error | duplicated key is ignored rather than first/last-wins | +| `frontmatter/invalid-insights-metadata` | warning | invalid `title`/`aliases`/`tags` shape | +| `heading/duplicate` | warning | repeated normalized heading; renderer suffix behavior still works | +| `heading/skipped-level` | warning | structural heading jump | +| `table/malformed-delimiter` | error | table-like header has invalid delimiter row | +| `table/column-count` | warning | inconsistent table structure | +| `list/inconsistent-marker` | warning | sibling list marker inconsistency | +| `list/inconsistent-indent` | warning | inconsistent nesting indentation | +| `format/trailing-whitespace` | info | avoidable line-end whitespace | +| `wiki/malformed` | warning | malformed Wiki Link/embed rendered as plain text | +| `link/case-mismatch` | info | resolved local path differs only by casing | +| `link/malformed-uri` | warning | invalid URI/percent-encoding syntax | +| `mermaid/render-failure` | warning | Mermaid parse/render failure | + +Malformed YAML does **not** remove the document from Insights. Ignore unusable metadata fields and continue body analysis. + +Duplicate YAML keys are errors. The duplicated field is ignored; unrelated valid frontmatter fields and body analysis continue. + +### Lint suppressions + +App-local suppressions support: + +- one finding; +- one rule for one file; +- one rule for a path/glob pattern; +- one rule for an entire workspace. + +Suppressions are visible, reversible, included in settings export/import, and never inserted into Markdown source. Suppressed findings remain internally detectable and can be shown with **Show suppressed**. + +High-confidence rename detection migrates applicable per-file suppressions. + +## Duplicate Content Finder + +Analysis always uses source documents, never rendered transcluded output. + +### Exact duplicates + +Exact duplicate normalization: + +- strip UTF-8 BOM; +- normalize CRLF/CR to LF; +- remove trailing whitespace per line; +- normalize final newline handling; +- preserve all other Markdown/frontmatter content. + +Hash normalized content and group equal hashes. + +### Repeated sections/passages + +Use two candidate systems: + +1. heading-delimited normalized section fingerprints; +2. bounded sliding-window fingerprints for substantial unheaded/cross-heading passages. + +Ignore trivial/common boilerplate and tiny fragments. Default implementation constants should be conservative: heading/passages need meaningful content (approximately >=100 normalized characters and >=20 meaningful tokens); sliding windows should be bounded (for example ~120 normalized tokens with ~50% stride) and are candidate generators, not an instruction to compare every window with every other window. + +### Near duplicates + +Default threshold: **90% similarity**, configurable globally/per workspace with a bounded conservative range. + +Require a minimum meaningful document size before near-duplicate analysis. Generate candidates from fingerprints/buckets/inverted term evidence first, then run exact similarity only on candidates. Never perform full all-pairs O(n²) comparison. + +Exact duplicates are independent of the near-duplicate threshold. + +### Suppression + +Users may suppress a noisy duplicate group/pattern. Suppressions are reversible/visible and affect presentation only; they never modify documents or relationship indexes. + +## Backlinks and focused Knowledge Graph + +### Graph model + +Primary nodes: Markdown/MDX documents. + +Primary directed edges: + +- resolved explicit Markdown/HTML/Wiki document links; +- resolved embed/transclusion edges, visually/type-distinct from normal links. + +Optional secondary layers: + +- tags; +- significant headings. + +Inferred relationship edges are **off by default** and shown only through an explicit toggle. They must be visually/type-distinct from explicit links. + +Local same-document fragment links do not create document-to-document graph edges. + +### Focused rendering + +Do not render a full workspace force graph in v1. + +Default graph is a focused neighborhood centered on the active/selected document: + +- direct explicit neighbors on the inner ring; +- secondary neighbors on outer rings; +- optional tag/heading nodes in dedicated outer bands; +- backlinks/outbound direction markers; +- embed edges visually distinct; +- deterministic radial placement for the same graph state. + +Default visible-node cap: **100**, configurable (bounded to a practical range). Strongest/relevant neighbors are selected first; explicit link relationships rank ahead of optional inferred edges. When truncated, show how many nodes are hidden. Search/recenter can bring an omitted document into focus without increasing the cap. + +### Accessibility + +Every visible graph node/edge is mirrored in a synchronized, keyboard-navigable structured list. Graph and list selection/focus remain synchronized. Edge type/direction is available in text and never depends on color alone. + +## Document Relationships + +Use deterministic local candidate generation, not embeddings/cloud models and not all-pairs comparison. + +Candidate inverted indexes include: + +- explicit inbound/outbound links; +- tags; +- normalized headings; +- filename/title terms; +- significant terminology signatures. + +Default scoring profile: + +- direct/shared links: 35% +- shared tags: 20% +- shared headings: 15% +- filename/title overlap: 10% +- terminology overlap: 20% + +Normalize final score to 0–100 and omit candidates with no meaningful signal. + +Provide presets: + +- Default +- Link-focused +- Tag-focused +- Terminology-focused +- Custom weights + +Custom weights are normalized before scoring and persisted through the Insights settings model. + +Every result exposes contributing signals. Show actual shared tags/headings/terms when requested. Readable significant terminology is kept only in the live index or reconstructed from source on demand; the persistent cache keeps hashed/signature terminology used for candidate generation. + +## Settings model + +Configuration precedence: + +1. per-workspace override; +2. global user default; +3. built-in default. + +Insights settings include at least: + +- external-link checking enabled (default off); +- external timeout (default 10 s, 3–30 s); +- scan user patterns; +- 10 MiB soft source limit and per-file/pattern oversized overrides; +- duplicate threshold and suppressions; +- lint rule enable/severity and suppressions; +- relationship preset/custom weights; +- graph visible-node cap; +- cache capacity (global default 500 MiB). + +Provide **Reset workspace overrides**. + +Extend the existing settings JSON export/import schema to include Insights global defaults and per-workspace overrides keyed by app-local workspace identity. Import normalization validates sizes, patterns, severities, thresholds, and weights. + +Private-network approvals, external session results, worker/cache bodies, and other transient state are never included in settings export. + +## Reports + +Insights can export analysis snapshots as both: + +- **Markdown** — human-readable review/share format; +- **JSON** — structured automation/CI/tooling format. + +Export scope supports: + +- entire workspace; +- current filtered view; +- explicit selection. + +Report metadata records scope, workspace identity/display path as appropriate, analyzer version, indexing completeness, skipped/warning counts, and whether results were provisional/incomplete. + +External-link export includes summary fields only: + +- original URL; +- status category; +- HTTP status code when available; +- final URL; +- checked timestamp; +- insecure downgrade flag where applicable. + +Do not export private approvals, DNS resolution chains, credentials, cookies, Authorization, or remote response bodies. + +## Performance model + +- Dedicated workspace scan streams metadata in batches. +- Source reads are bounded and batched. +- One parse per document source revision. +- Worker messages are batched; avoid one message per token/finding. +- Incremental deltas update only the affected document and dependent inverted indexes/backlinks/relationships. +- Near duplicates and relationships use indexed candidate generation; no full O(n²) pairwise workspace pass. +- Graph rendering is bounded independently of full index size. +- Local resource existence checks use metadata probes, never binary reads. +- Inactive workspaces have no live worker/watch analysis. +- Browser polling occurs only while Insights is active/visible. + +## Failure and cancellation semantics + +### Indexing + +Cancellation stops queued reads/worker jobs and leaves already completed derived state reusable. Closing the panel marks unfinished indexing resumable rather than complete. + +Individual failures remain explicit entries/reasons. A workspace can finish **with warnings**. + +### Deletion + +Deleted documents are removed immediately from index/cache/graph/duplicate/relationship state. No tombstones or historical graph nodes are retained. Surviving references become broken links. + +### Rename/move + +High-confidence rename detection migrates internal state; source links are never rewritten automatically. + +### Unsupported capabilities + +A runtime that cannot reliably implement a capability reports `unsupported`/degraded status. Do not emulate HTTP status with opaque responses or emulate filesystem existence with binary reads. + +## Security and privacy + +- Workspace canonicalization occurs before reads/probes/navigation. +- Symlink targets outside workspace are rejected. +- Static analysis never evaluates MDX expressions or arbitrary user code. +- External checker never sends application authentication/session credentials. +- External checker validates DNS/IP at every hop and pins a validated destination address. +- Private-network access requires origin-scoped session confirmation. +- Remote media is never auto-loaded. +- Persistent cache stores derived data only and minimizes readable terminology. +- External/session/private state is non-persistent. +- No repository files are written for cache, settings, workspace identity, or suppressions. + +## Localization + +All new user-visible strings must be added to the existing typed translation catalog and every supported locale, including: + +- Workspace Insights entry/panel/view names; +- statuses, errors, progress and provisional/completeness messages; +- external checker setting/confirmation/status text; +- settings labels/descriptions/reset actions; +- lint/duplicate suppression controls; +- graph accessibility labels; +- export labels/messages; +- Wiki Link ambiguity/missing/cycle/depth placeholders where renderer UI exposes text. + +No English-only fallback strings should be introduced in shared user-facing components where the translation model already requires locale coverage. + +## Testing strategy + +### Shared parser/resolver tests + +Cover: + +- renderer-consistent headings/anchors including duplicate suffixes and Setext/ATX; +- static HTML anchors; +- standard Markdown local/query/fragment behavior; +- Wiki Link parser escaping, path separators, aliases/titles, relative paths, extension expansion, ambiguity, case folding, same-document fragments; +- directory index target behavior; +- malformed Wiki Links; +- transclusion cycle/depth/source-relative resolution; +- static HTML/MDX refs and dynamic-expression classification; +- Markdown-aware tags. + +### Worker/index tests + +Cover: + +- initial batched index; +- active unsaved overlay replacement/removal; +- add/update/delete/rename deltas; +- backlinks/graph/embed edge maintenance; +- incremental candidate index maintenance; +- exact/repeated/near duplicate behavior; +- relationship presets/custom scoring/evidence; +- partial failure/completeness state; +- cancellation/resume. + +### Host contract tests per runtime + +Cover: + +- dedicated scan is not constrained by legacy file-list caps; +- pattern precedence and canonical symlink boundaries; +- source size soft/hard limits; +- `probeWorkspaceResource` uses metadata only and does not read/base64 binaries; +- watcher or polling capability reporting; +- document conversion reuse for supported embeds. + +### External checker security/behavior tests + +Cover: + +- real status categories (not opaque response behavior); +- HEAD -> bounded GET fallback; +- no Cookie/Authorization credentials; +- DNS public/private classification; +- IP pinning/rebinding defense; +- redirect revalidation/private confirmation; +- 5-redirect cap; +- HTTPS downgrade flag; +- TLS failure classification; +- 4-global/2-origin concurrency; +- timeout/retry/Retry-After behavior; +- cancellation of queued and in-flight requests; +- session cache/Recheck; +- unsupported runtime behavior. + +### UI/settings/accessibility tests + +Cover: + +- lazy first-open startup; +- streamed provisional results/progress; +- dedicated resizable panel navigation; +- filtered/full external check scope; +- suppressions/show-suppressed; +- global/per-workspace settings precedence/reset; +- settings export/import normalization; +- cache clear/usage states; +- Markdown+JSON report scope/completeness metadata; +- graph/list keyboard synchronization and non-color edge semantics; +- all typed translation keys/locales. + +## Acceptance criteria + +1. Workspace Insights opens from the sidebar into a dedicated resizable panel with Gallery, Links, Lint, Duplicates, Graph, and Related views. +2. First open performs a dedicated full Insights scan with streamed progress and no silent workspace file cap. +3. The shared parser/renderer semantics are used for headings/anchors/structures; Insights does not maintain an incompatible structural Markdown parser. +4. Wiki Links and `![[...]]` embeds render/navigate through a shared resolver with explicit ambiguity, case-insensitive matching, YAML title/aliases, relative paths, `.md/.mdx` extensionless resolution, depth-5 transclusion, and cycle protection. +5. Gallery lists referenced media/documents only; missing local resources are metadata-probed without binary reads; remote media is never auto-loaded; invalid Mermaid remains visible and linted. +6. Broken Links correctly distinguishes missing, invalid-anchor, ambiguous, outside-workspace, dynamic/uncheckable, unsupported, and non-checkable references. +7. External checking is off by default, explicit, anonymous, host-backed with real status, SSRF protections, private-origin confirmation, redirect/IP revalidation, bounded GET fallback, cancellation, and session-only caching. +8. Lint diagnostics are configurable and suppressible without modifying source. +9. Duplicate detection distinguishes normalized exact duplicates, repeated sections/passages, and candidate-bucketed near duplicates with default 90% threshold and no full all-pairs workspace pass. +10. Backlinks and graph derive from the same resolved link/embed index. The graph is focused, deterministic radial, capped by default at 100 visible nodes, explicit-link-first, and paired with an accessible synchronized list. +11. Related documents use inverted candidate indexes, deterministic 0–100 scoring, presets/custom weights, and human-readable contributing evidence. +12. Active unsaved Markdown/MDX content overlays the disk index and updates all derived views incrementally. +13. Deletions remove state immediately; high-confidence renames migrate app-local state without rewriting Markdown source. +14. Derived cache is app-local, versioned, source-body-free, capped at 500 MiB by default, and uses hybrid eviction. Only the active workspace keeps live analysis state. +15. Manual Refresh hashes all eligible Markdown/MDX sources but makes zero external HTTP requests. +16. Settings export/import includes Insights global defaults and per-workspace overrides but excludes private/network session state. +17. Markdown and JSON report export support full workspace and filtered/selected scopes with explicit completeness metadata. +18. All new user-visible text is localized through the existing translation system. +19. Unit/integration/runtime tests cover parser/resolver, incremental index, cache, security-sensitive host contracts, external checker, UI accessibility, and settings/report behavior. + +## Non-goals for this implementation + +- cloud embeddings/AI similarity services; +- full-workspace force/WebGL graph mode; +- automatic source rewriting after rename; +- linter auto-fixes/formatting; +- executing dynamic MDX expressions for static analysis; +- automatic remote media loading; +- continuous external-link monitoring; +- persistent deleted-document/tombstone history; +- arbitrary filesystem access outside the active workspace; +- TOML/JSON frontmatter metadata interpretation; +- repository-local Insights cache/config files. diff --git a/electron/core/ipc-handlers.js b/electron/core/ipc-handlers.js index fd11d85b..a94f0412 100644 --- a/electron/core/ipc-handlers.js +++ b/electron/core/ipc-handlers.js @@ -34,6 +34,27 @@ function registerIpcHandlers({ ipcMain, clipboard, fs, handlers, getMainWindow, case "loadWorkspaceSearchIndexes": handlers.loadWorkspaceSearchIndexes(msg); break; + case "scanInsightsWorkspace": + await handlers.scanInsightsWorkspace(msg); + break; + case "cancelInsightsScan": + handlers.cancelInsightsScan(msg); + break; + case "readInsightsDocumentSource": + await handlers.readInsightsDocumentSource(msg); + break; + case "probeWorkspaceResource": + await handlers.probeWorkspaceResource(msg); + break; + case "setInsightsWatchState": + handlers.setInsightsWatchState(msg); + break; + case "checkExternalLinks": + await handlers.checkExternalLinks(msg); + break; + case "cancelExternalLinkChecks": + handlers.cancelExternalLinkChecks(msg); + break; case "confirmOpenPath": handlers.confirmOpenPath(msg.path); break; @@ -164,4 +185,4 @@ function registerIpcHandlers({ ipcMain, clipboard, fs, handlers, getMainWindow, }); } -module.exports = { registerIpcHandlers }; +module.exports = { registerIpcHandlers }; \ No newline at end of file diff --git a/electron/core/main-bootstrap.js b/electron/core/main-bootstrap.js index fe6089fe..e206ca83 100644 --- a/electron/core/main-bootstrap.js +++ b/electron/core/main-bootstrap.js @@ -44,6 +44,8 @@ function createAppBootstrap({ createHtmlPreviewServerFn = require("./html-preview-server").createHtmlPreviewServer, createExportResourceHandlersFn = require("./runtime-export-resources").createExportResourceHandlers, createExportSaveHandlerFn = require("./runtime-export-save").createExportSaveHandler, + createInsightsWorkspaceHostFn = require("./runtime-insights").createInsightsWorkspaceHost, + createExternalLinkHostFn = require("./runtime-insights-external").createExternalLinkHost, externalOpenQueue = null, } = {}) { let mainWindowRef = null; @@ -71,6 +73,11 @@ function createAppBootstrap({ } } + function isSameOrInsidePath(basePath, targetPath) { + const relative = pathImpl.relative(pathImpl.resolve(basePath), pathImpl.resolve(targetPath)); + return relative === "" || (!relative.startsWith("..") && !pathImpl.isAbsolute(relative)); + } + const sendHostMessage = (message) => { mainWindowRef?.webContents.send("host-message", message); }; @@ -81,6 +88,14 @@ function createAppBootstrap({ sendHostMessage, getWorkspaceBaseDir, }); + const insightsHost = createInsightsWorkspaceHostFn({ + fs: fsImpl, + pathApi: pathImpl, + sendHostMessage, + getWorkspaceBaseDir, + isSameOrInsidePath, + }); + const externalLinkHost = createExternalLinkHostFn({ sendHostMessage }); const saveExportFile = createExportSaveHandlerFn({ dialog: dialogImpl, fs: fsImpl, @@ -174,6 +189,13 @@ function createAppBootstrap({ loadSearchPreview: runtimeImpl.handleLoadSearchPreview, indexWorkspaceSearchItems: runtimeImpl.handleIndexWorkspaceSearchItems, loadWorkspaceSearchIndexes: runtimeImpl.handleLoadWorkspaceSearchIndexes, + scanInsightsWorkspace: insightsHost.scanInsightsWorkspace, + cancelInsightsScan: insightsHost.cancelInsightsScan, + readInsightsDocumentSource: insightsHost.readInsightsDocumentSource, + probeWorkspaceResource: insightsHost.probeWorkspaceResource, + setInsightsWatchState: insightsHost.setInsightsWatchState, + checkExternalLinks: externalLinkHost.checkExternalLinks, + cancelExternalLinkChecks: externalLinkHost.cancelExternalLinkChecks, confirmOpenPath: runtimeImpl.handleConfirmOpenPath, openRecent: runtimeImpl.handleOpenRecent, deleteRecentWorkspace: runtimeImpl.handleDeleteRecentWorkspace, @@ -213,6 +235,8 @@ function createAppBootstrap({ }); appImpl.on("before-quit", () => { + externalLinkHost.dispose(); + insightsHost.dispose(); runtimeImpl.dispose(); void htmlPreviewServer.dispose(); if (updateManagerRef) { @@ -223,4 +247,4 @@ function createAppBootstrap({ return { createWindow, getMainWindow, getUpdateManager, deliverExternalOpenPath }; } -module.exports = { createAppBootstrap, configureApplicationMenu }; +module.exports = { createAppBootstrap, configureApplicationMenu }; \ No newline at end of file diff --git a/electron/core/runtime-insights-external.js b/electron/core/runtime-insights-external.js new file mode 100644 index 00000000..ac1caf8f --- /dev/null +++ b/electron/core/runtime-insights-external.js @@ -0,0 +1,322 @@ +const dns = require('node:dns').promises; +const http = require('node:http'); +const https = require('node:https'); +const net = require('node:net'); + +const MAX_REDIRECTS = 5; +const GLOBAL_CONCURRENCY = 4; +const ORIGIN_CONCURRENCY = 2; +const DEFAULT_TIMEOUT_MS = 10_000; +const ANONYMOUS_HEADERS = Object.freeze({ accept: '*/*', 'user-agent': 'Markdown Explorer/Insights' }); + +function normalizeOrigin(url) { + const parsed = url instanceof URL ? url : new URL(url); + return parsed.origin; +} + +function isPrivateIpv4(address) { + const parts = address.split('.').map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + const [a, b] = parts; + return a === 0 + || a === 10 + || a === 127 + || (a === 100 && b >= 64 && b <= 127) + || (a === 169 && b === 254) + || (a === 172 && b >= 16 && b <= 31) + || (a === 192 && b === 168) + || (a === 198 && (b === 18 || b === 19)) + || a >= 224; +} + +function isPrivateIpv6(address) { + const value = String(address || '').toLowerCase().split('%', 1)[0]; + if (value === '::' || value === '::1') return true; + if (value.startsWith('fc') || value.startsWith('fd')) return true; + if (/^fe[89ab]/.test(value)) return true; + const mapped = value.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + return mapped ? isPrivateIpv4(mapped[1]) : false; +} + +function isPrivateAddress(address) { + const family = net.isIP(address); + if (family === 4) return isPrivateIpv4(address); + if (family === 6) return isPrivateIpv6(address); + return true; +} + +function isPrivateHostname(hostname) { + const value = String(hostname || '').toLowerCase().replace(/\.$/, ''); + return value === 'localhost' || value.endsWith('.localhost'); +} + +async function defaultResolveHost(hostname) { + const entries = await dns.lookup(hostname, { all: true, verbatim: true }); + return entries.map((entry) => entry.address); +} + +function headersToRecord(headers) { + const result = {}; + for (const [key, value] of Object.entries(headers || {})) { + if (Array.isArray(value)) result[key.toLowerCase()] = value.join(', '); + else if (value != null) result[key.toLowerCase()] = String(value); + } + return result; +} + +function defaultRequest({ url, method, address, headers, timeoutMs, signal, maxBodyBytes = 0 }) { + return new Promise((resolve, reject) => { + const parsed = new URL(url); + const transport = parsed.protocol === 'https:' ? https : http; + const family = net.isIP(address); + let settled = false; + const finishReject = (error) => { + if (settled) return; + settled = true; + reject(error); + }; + const request = transport.request({ + protocol: parsed.protocol, + hostname: parsed.hostname, + port: parsed.port || undefined, + path: `${parsed.pathname}${parsed.search}`, + method, + headers: { ...ANONYMOUS_HEADERS, ...(headers || {}) }, + servername: parsed.protocol === 'https:' ? parsed.hostname : undefined, + lookup: (_hostname, _options, callback) => callback(null, address, family), + agent: false, + }, (response) => { + if (settled) return; + settled = true; + const payload = { status: response.statusCode || 0, headers: headersToRecord(response.headers) }; + if (maxBodyBytes <= 0) response.destroy(); + else response.resume(); + resolve(payload); + }); + request.setTimeout(Math.max(1, Number(timeoutMs) || DEFAULT_TIMEOUT_MS), () => { + const error = new Error('ETIMEDOUT'); + error.code = 'ETIMEDOUT'; + request.destroy(error); + }); + request.on('error', finishReject); + if (signal) { + if (signal.aborted) { + const error = new Error('ABORT_ERR'); + error.name = 'AbortError'; + request.destroy(error); + return; + } + signal.addEventListener('abort', () => { + const error = new Error('ABORT_ERR'); + error.name = 'AbortError'; + request.destroy(error); + }, { once: true }); + } + request.end(); + }); +} + +function parseRetryAfter(value, now = Date.now()) { + if (!value) return undefined; + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000); + const at = Date.parse(value); + return Number.isFinite(at) ? Math.max(0, at - now) : undefined; +} + +function classifyStatus(status) { + if ((status >= 200 && status < 400)) return 'reachable'; + if (status === 401 || status === 403) return 'reachable-auth-required'; + if (status === 404 || status === 410) return 'broken'; + if (status === 429) return 'rate-limited'; + if (status >= 500 && status <= 599) return 'server-error'; + return 'unreachable'; +} + +function createExternalLinkChecker(deps = {}) { + const resolveHost = deps.resolveHost || defaultResolveHost; + const request = deps.request || defaultRequest; + + async function checkedRequest(url, address, method, session) { + return request({ + url: url.toString(), + method, + address, + headers: { ...ANONYMOUS_HEADERS }, + timeoutMs: session.timeoutMs || DEFAULT_TIMEOUT_MS, + signal: session.signal, + maxBodyBytes: method === 'GET' ? 0 : undefined, + }); + } + + async function check(input, session = {}) { + const originalUrl = String(input || ''); + const checkedAt = new Date().toISOString(); + let current; + try { current = new URL(originalUrl); } + catch { return { url: originalUrl, status: 'unsupported', checkedAt, reason: 'invalid-url' }; } + if (current.protocol !== 'http:' && current.protocol !== 'https:') { + return { url: originalUrl, status: 'unsupported', checkedAt, reason: 'unsupported-scheme' }; + } + + const approved = new Set((session.approvedPrivateOrigins || []).map(String)); + let insecureDowngrade = false; + let redirects = 0; + let transientRetries = 0; + + while (true) { + if (session.signal?.aborted) return { url: originalUrl, status: 'unchecked', checkedAt, reason: 'cancelled' }; + const origin = normalizeOrigin(current); + let addresses; + try { addresses = await resolveHost(current.hostname); } + catch (error) { + return { url: originalUrl, status: 'unreachable', finalUrl: current.toString(), checkedAt, insecureDowngrade, reason: String(error?.message || error || 'dns-failure') }; + } + if (!Array.isArray(addresses) || addresses.length === 0) { + return { url: originalUrl, status: 'unreachable', finalUrl: current.toString(), checkedAt, insecureDowngrade, reason: 'dns-empty' }; + } + const privateDestination = isPrivateHostname(current.hostname) || addresses.some(isPrivateAddress); + if (privateDestination && !approved.has(origin)) { + return { + url: originalUrl, + status: 'unchecked', + finalUrl: current.toString(), + checkedAt, + insecureDowngrade, + privateOrigin: origin, + requiresPrivateOriginConfirmation: true, + reason: 'private-origin-confirmation-required', + }; + } + const address = String(addresses[0]); + + let response; + try { + response = await checkedRequest(current, address, 'HEAD', session); + if (response.status === 405 || response.status === 501) { + response = await checkedRequest(current, address, 'GET', session); + } + } catch (error) { + if (session.signal?.aborted || error?.name === 'AbortError') { + return { url: originalUrl, status: 'unchecked', finalUrl: current.toString(), checkedAt, insecureDowngrade, reason: 'cancelled' }; + } + return { url: originalUrl, status: 'unreachable', finalUrl: current.toString(), checkedAt, insecureDowngrade, reason: String(error?.message || error || 'network-error') }; + } + + const status = Number(response.status) || 0; + const headers = headersToRecord(response.headers); + const location = headers.location; + if (status >= 300 && status < 400 && location) { + if (redirects >= MAX_REDIRECTS) { + return { url: originalUrl, status: 'unreachable', httpStatus: status, finalUrl: current.toString(), checkedAt, insecureDowngrade, reason: 'redirect-limit' }; + } + let next; + try { next = new URL(location, current); } + catch { return { url: originalUrl, status: 'unreachable', httpStatus: status, finalUrl: current.toString(), checkedAt, insecureDowngrade, reason: 'invalid-redirect' }; } + if (next.protocol !== 'http:' && next.protocol !== 'https:') { + return { url: originalUrl, status: 'unsupported', httpStatus: status, finalUrl: next.toString(), checkedAt, insecureDowngrade, reason: 'unsupported-redirect-scheme' }; + } + if (current.protocol === 'https:' && next.protocol === 'http:') insecureDowngrade = true; + current = next; + redirects += 1; + transientRetries = 0; + continue; + } + + if (status >= 500 && status <= 599 && transientRetries < 1) { + transientRetries += 1; + continue; + } + + return { + url: originalUrl, + status: classifyStatus(status), + httpStatus: status || undefined, + finalUrl: current.toString(), + checkedAt, + insecureDowngrade, + retryAfterMs: status === 429 || status >= 500 ? parseRetryAfter(headers['retry-after']) : undefined, + }; + } + } + + return { check }; +} + +function createExternalLinkHost({ sendHostMessage, ...checkerDeps } = {}) { + const checker = createExternalLinkChecker(checkerDeps); + const requests = new Map(); + + async function checkExternalLinks(message) { + const requestId = String(message.requestId || ''); + const controller = new AbortController(); + requests.get(requestId)?.abort(); + requests.set(requestId, controller); + const urls = [...new Set((Array.isArray(message.urls) ? message.urls : []).map(String))]; + const queue = urls.map((url, index) => ({ url, index, origin: (() => { try { return new URL(url).origin; } catch { return ''; } })() })); + const activeOrigins = new Map(); + let active = 0; + let cursor = 0; + + await new Promise((resolve) => { + const schedule = () => { + if (controller.signal.aborted || (cursor >= queue.length && active === 0)) { resolve(); return; } + let started = false; + while (active < GLOBAL_CONCURRENCY && cursor < queue.length) { + let selected = -1; + for (let index = cursor; index < queue.length; index += 1) { + const count = activeOrigins.get(queue[index].origin) || 0; + if (count < ORIGIN_CONCURRENCY) { selected = index; break; } + } + if (selected < 0) break; + const [item] = queue.splice(selected, 1); + if (selected <= cursor && cursor > 0) cursor -= 1; + active += 1; + activeOrigins.set(item.origin, (activeOrigins.get(item.origin) || 0) + 1); + started = true; + void checker.check(item.url, { + requestId, + timeoutMs: message.timeoutMs, + approvedPrivateOrigins: message.approvedPrivateOrigins, + signal: controller.signal, + }).then((result) => { + if (!controller.signal.aborted) sendHostMessage?.({ command: 'externalLinkCheckResult', requestId, ...result }); + }).finally(() => { + active -= 1; + const nextCount = (activeOrigins.get(item.origin) || 1) - 1; + if (nextCount <= 0) activeOrigins.delete(item.origin); else activeOrigins.set(item.origin, nextCount); + schedule(); + }); + } + if (!started && active === 0) resolve(); + }; + schedule(); + }); + + const cancelled = controller.signal.aborted; + if (requests.get(requestId) === controller) requests.delete(requestId); + sendHostMessage?.({ command: 'externalLinkCheckComplete', requestId, cancelled }); + } + + function cancelExternalLinkChecks(message) { + requests.get(String(message.requestId || ''))?.abort(); + } + + function dispose() { + for (const controller of requests.values()) controller.abort(); + requests.clear(); + } + + return { checkExternalLinks, cancelExternalLinkChecks, dispose }; +} + +module.exports = { + createExternalLinkChecker, + createExternalLinkHost, + isPrivateAddress, + classifyStatus, + parseRetryAfter, + MAX_REDIRECTS, + GLOBAL_CONCURRENCY, + ORIGIN_CONCURRENCY, +}; diff --git a/electron/core/runtime-insights.js b/electron/core/runtime-insights.js new file mode 100644 index 00000000..9a8907c2 --- /dev/null +++ b/electron/core/runtime-insights.js @@ -0,0 +1,366 @@ +const crypto = require('node:crypto'); + +const DEFAULT_SOFT_LIMIT_BYTES = 10 * 1024 * 1024; +const DEFAULT_HARD_LIMIT_BYTES = 64 * 1024 * 1024; +const SCAN_BATCH_SIZE = 200; +const HARD_EXCLUDED_SEGMENTS = new Set(['.git', '.hg', '.svn']); +const DEFAULT_EXCLUDED_SEGMENTS = new Set(['node_modules', '.next', 'dist', 'build', 'coverage', '.cache']); + +const MIME_TYPES = { + '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', + '.webp': 'image/webp', '.svg': 'image/svg+xml', '.avif': 'image/avif', + '.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime', '.m4v': 'video/x-m4v', + '.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg', '.m4a': 'audio/mp4', + '.md': 'text/markdown', '.mdx': 'text/mdx', '.txt': 'text/plain', '.pdf': 'application/pdf', +}; + +function normalizeRelativePath(pathApi, root, target) { + return pathApi.relative(root, target).split(pathApi.sep).join('/'); +} + +function isRemoteReference(value) { + return /^(?:https?:|data:|blob:|javascript:)/i.test(String(value || '').trim()); +} + +function globToRegExp(pattern) { + let source = String(pattern || '').trim(); + if (!source || source.startsWith('#')) return null; + const negated = source.startsWith('!'); + if (negated) source = source.slice(1); + const directoryOnly = source.endsWith('/'); + if (directoryOnly) source = source.slice(0, -1); + const anchored = source.startsWith('/'); + if (anchored) source = source.slice(1); + let re = ''; + for (let i = 0; i < source.length; i += 1) { + const char = source[i]; + if (char === '*') { + if (source[i + 1] === '*') { re += '.*'; i += 1; } + else re += '[^/]*'; + } else if (char === '?') re += '[^/]'; + else re += char.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); + } + const prefix = anchored ? '^' : '(^|.*/)'; + const suffix = directoryOnly ? '(?:/.*)?$' : '$'; + return { negated, regex: new RegExp(prefix + re + suffix) }; +} + +function applyPatterns(relativePath, initialExcluded, patterns) { + let excluded = initialExcluded; + for (const raw of patterns || []) { + const compiled = globToRegExp(raw); + if (!compiled || !compiled.regex.test(relativePath)) continue; + excluded = !compiled.negated; + } + return excluded; +} + +function readGitignore(fs, pathApi, root) { + try { + return fs.readFileSync(pathApi.join(root, '.gitignore'), 'utf8') + .split(/\r?\n/) + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')); + } catch { + return []; + } +} + +function shouldExclude(relativePath, gitignorePatterns, userPatterns) { + const segments = relativePath.split('/').filter(Boolean); + if (segments.some(segment => HARD_EXCLUDED_SEGMENTS.has(segment))) return true; + let excluded = segments.some(segment => DEFAULT_EXCLUDED_SEGMENTS.has(segment)); + excluded = applyPatterns(relativePath, excluded, gitignorePatterns); + excluded = applyPatterns(relativePath, excluded, userPatterns); + return excluded; +} + +function mimeTypeFor(pathApi, filePath) { + return MIME_TYPES[pathApi.extname(filePath).toLowerCase()] || 'application/octet-stream'; +} + +function createInsightsWorkspaceHost({ + fs, + pathApi, + sendHostMessage, + isSameOrInsidePath, + getWorkspaceBaseDir, +}) { + const cancelledScans = new Set(); + const watchDisposers = new Set(); + let watchRequestId = ''; + let watchWorkspaceOperationId; + let watchGeneration = 0; + + function workspaceRoot() { + const base = getWorkspaceBaseDir(); + if (!base) return null; + try { + return fs.realpathSync(base); + } catch { + return null; + } + } + + function containedRealPath(candidate, root) { + try { + const real = fs.realpathSync(candidate); + return isSameOrInsidePath(root, real, pathApi) ? real : null; + } catch { + return null; + } + } + + function resolveResourcePath(message, root) { + const raw = String(message.resourcePath || '').split(/[?#]/, 1)[0]; + if (!raw || isRemoteReference(raw) || /^file:\/\//i.test(raw)) return { status: 'outside-workspace' }; + let candidate; + if (raw.startsWith('/')) candidate = pathApi.resolve(root, `.${raw}`); + else if (pathApi.isAbsolute(raw)) candidate = pathApi.normalize(raw); + else { + const documentPath = String(message.documentPath || ''); + const documentAbsolute = pathApi.isAbsolute(documentPath) + ? documentPath + : pathApi.resolve(root, documentPath); + candidate = pathApi.resolve(pathApi.dirname(documentAbsolute), raw); + } + if (!isSameOrInsidePath(root, candidate, pathApi)) return { status: 'outside-workspace' }; + return { candidate }; + } + + async function probeWorkspaceResource(message = {}) { + const requestId = String(message.requestId || ''); + const respond = payload => sendHostMessage({ command: 'workspaceResourceProbeResult', requestId, ...payload }); + const root = workspaceRoot(); + if (!root) { respond({ status: 'missing' }); return; } + const resolved = resolveResourcePath(message, root); + if (!resolved.candidate) { respond({ status: resolved.status || 'outside-workspace' }); return; } + try { + if (!fs.existsSync(resolved.candidate)) { respond({ status: 'missing' }); return; } + const real = containedRealPath(resolved.candidate, root); + if (!real) { respond({ status: 'outside-workspace' }); return; } + const stat = fs.statSync(real); + respond({ + status: 'exists', + relativePath: normalizeRelativePath(pathApi, root, real), + kind: stat.isDirectory() ? 'directory' : 'file', + sizeBytes: stat.isFile() ? stat.size : undefined, + mimeType: stat.isFile() ? mimeTypeFor(pathApi, real) : undefined, + }); + } catch { + respond({ status: 'unreadable' }); + } + } + + async function readInsightsDocumentSource(message = {}) { + const requestId = String(message.requestId || ''); + const relativePath = String(message.relativePath || ''); + const respond = payload => sendHostMessage({ command: 'insightsDocumentSourceResult', requestId, relativePath, ...payload }); + const root = workspaceRoot(); + if (!root || !relativePath) { respond({ status: 'missing' }); return; } + const candidate = pathApi.resolve(root, relativePath); + if (!isSameOrInsidePath(root, candidate, pathApi)) { respond({ status: 'outside-workspace' }); return; } + if (!/\.mdx?$/i.test(candidate)) { respond({ status: 'unsupported' }); return; } + try { + const real = containedRealPath(candidate, root); + if (!real) { respond({ status: fs.existsSync(candidate) ? 'outside-workspace' : 'missing' }); return; } + const stat = fs.statSync(real); + if (!stat.isFile()) { respond({ status: 'missing' }); return; } + const softLimit = Math.max(1, Number(message.softLimitBytes) || DEFAULT_SOFT_LIMIT_BYTES); + const requestedHard = Math.max(1, Number(message.hardLimitBytes) || DEFAULT_HARD_LIMIT_BYTES); + const hardLimit = Math.min(requestedHard, DEFAULT_HARD_LIMIT_BYTES); + if (stat.size > hardLimit) { + respond({ status: 'too-large', sizeBytes: stat.size, mtimeMs: stat.mtimeMs, hardLimit: true }); + return; + } + if (stat.size > softLimit) { + respond({ status: 'too-large', sizeBytes: stat.size, mtimeMs: stat.mtimeMs, hardLimit: false }); + return; + } + const source = fs.readFileSync(real, 'utf8'); + const contentHash = crypto.createHash('sha256').update(source).digest('hex'); + respond({ status: 'ok', source, sizeBytes: stat.size, mtimeMs: stat.mtimeMs, contentHash }); + } catch { + respond({ status: 'unreadable' }); + } + } + + async function scanInsightsWorkspace(message = {}) { + const requestId = String(message.requestId || ''); + cancelledScans.delete(requestId); + const root = workspaceRoot(); + if (!root) { + sendHostMessage({ command: 'insightsScanComplete', requestId, totalEntries: 0, excludedEntries: 0, skippedEntries: 0, truncated: false }); + return; + } + const gitignorePatterns = readGitignore(fs, pathApi, root); + const userPatterns = Array.isArray(message.userPatterns) ? message.userPatterns : []; + const batch = []; + let totalEntries = 0; + let excludedEntries = 0; + let skippedEntries = 0; + const visitedDirectories = new Set(); + const visitedFiles = new Set(); + + const flush = () => { + if (!batch.length) return; + sendHostMessage({ + command: 'insightsScanBatch', requestId, + entries: batch.splice(0, batch.length), + scannedEntries: totalEntries, + excludedEntries, + }); + }; + + const walk = absoluteDir => { + if (cancelledScans.has(requestId)) return; + let realDir; + try { realDir = fs.realpathSync(absoluteDir); } catch { skippedEntries += 1; return; } + if (!isSameOrInsidePath(root, realDir, pathApi) || visitedDirectories.has(realDir)) return; + visitedDirectories.add(realDir); + let entries; + try { entries = fs.readdirSync(absoluteDir, { withFileTypes: true }); } + catch { skippedEntries += 1; return; } + for (const dirent of entries) { + if (cancelledScans.has(requestId)) break; + const absolute = pathApi.join(absoluteDir, dirent.name); + const displayRelative = normalizeRelativePath(pathApi, root, absolute); + if (shouldExclude(displayRelative, gitignorePatterns, userPatterns)) { excludedEntries += 1; continue; } + let real; + let stat; + try { + real = fs.realpathSync(absolute); + if (!isSameOrInsidePath(root, real, pathApi)) { excludedEntries += 1; continue; } + stat = fs.statSync(real); + } catch { skippedEntries += 1; continue; } + if (stat.isDirectory()) { walk(absolute); continue; } + if (!stat.isFile() || visitedFiles.has(real)) continue; + visitedFiles.add(real); + totalEntries += 1; + batch.push({ + relativePath: displayRelative, + canonicalRelativePath: normalizeRelativePath(pathApi, root, real), + kind: 'file', + sizeBytes: stat.size, + mtimeMs: stat.mtimeMs, + extension: pathApi.extname(displayRelative).toLowerCase() || undefined, + isSymlink: dirent.isSymbolicLink ? dirent.isSymbolicLink() : false, + }); + if (batch.length >= SCAN_BATCH_SIZE) flush(); + } + }; + + walk(root); + flush(); + const cancelled = cancelledScans.delete(requestId); + sendHostMessage({ + command: 'insightsScanComplete', requestId, totalEntries, excludedEntries, skippedEntries, + truncated: false, + ...(cancelled ? { cancelled: true } : {}), + }); + } + + function cancelInsightsScan(message = {}) { + const requestId = String(message.requestId || ''); + if (requestId) cancelledScans.add(requestId); + } + + function stopWatch() { + watchGeneration += 1; + for (const dispose of watchDisposers) { + try { dispose(); } catch { /* noop */ } + } + watchDisposers.clear(); + } + + function startWatch(root) { + stopWatch(); + const generation = watchGeneration; + const seen = new Set(); + const attach = dir => { + let real; + try { real = fs.realpathSync(dir); } catch { return; } + if (seen.has(real) || !isSameOrInsidePath(root, real, pathApi)) return; + seen.add(real); + let watcher; + try { + watcher = fs.watch(dir, { persistent: false }, (_eventType, name) => { + if (generation !== watchGeneration || !name) return; + const absolute = pathApi.join(dir, String(name)); + const relativePath = normalizeRelativePath(pathApi, root, absolute); + let delta; + try { + if (!fs.existsSync(absolute)) delta = { kind: 'delete', relativePath }; + else { + const realTarget = fs.realpathSync(absolute); + if (!isSameOrInsidePath(root, realTarget, pathApi)) return; + const stat = fs.statSync(realTarget); + if (stat.isDirectory()) { attach(absolute); return; } + delta = { + kind: 'update', + entry: { + relativePath, + canonicalRelativePath: normalizeRelativePath(pathApi, root, realTarget), + kind: 'file', sizeBytes: stat.size, mtimeMs: stat.mtimeMs, + extension: pathApi.extname(relativePath).toLowerCase() || undefined, + }, + }; + } + } catch { delta = { kind: 'delete', relativePath }; } + sendHostMessage({ + command: 'insightsFsDelta', requestId: watchRequestId, + ...(watchWorkspaceOperationId ? { workspaceOperationId: watchWorkspaceOperationId } : {}), + deltas: [delta], + }); + }); + watchDisposers.add(() => watcher.close()); + } catch { return; } + try { + for (const child of fs.readdirSync(dir, { withFileTypes: true })) { + if (child.isDirectory()) attach(pathApi.join(dir, child.name)); + } + } catch { /* noop */ } + }; + attach(root); + } + + function setInsightsWatchState(message = {}) { + watchRequestId = String(message.requestId || ''); + watchWorkspaceOperationId = message.workspaceOperationId; + const root = workspaceRoot(); + const active = message.active === true && Boolean(root); + if (!active) stopWatch(); + else startWatch(root); + sendHostMessage({ + command: 'insightsRuntimeCapabilities', + requestId: watchRequestId, + capabilities: { + fileChanges: active && watchDisposers.size > 0 ? 'native' : 'unsupported', + externalLinkChecking: true, + documentPreviewReuse: true, + }, + }); + } + + function dispose() { + stopWatch(); + cancelledScans.clear(); + } + + return { + capabilities: { fileChanges: 'native', externalLinkChecking: true, documentPreviewReuse: true }, + scanInsightsWorkspace, + cancelInsightsScan, + readInsightsDocumentSource, + probeWorkspaceResource, + setInsightsWatchState, + dispose, + }; +} + +module.exports = { + DEFAULT_SOFT_LIMIT_BYTES, + DEFAULT_HARD_LIMIT_BYTES, + SCAN_BATCH_SIZE, + createInsightsWorkspaceHost, +}; diff --git a/package.json b/package.json index a099b246..82f7c71c 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "test:coverage": "vitest run --coverage", "test:electron": "vitest run --project electron", "test:vscode": "vitest run --project vscode", - "test:ui": "vitest run --project ui", + "test:ui": "vitest run --project ui --shard=1/4 && vitest run --project ui --shard=2/4 && vitest run --project ui --shard=3/4 && vitest run --project ui --shard=4/4", "test:chromium": "vitest run --project chromium", "test:contracts": "vitest run --project contracts --project manifest --project build", "test:translations": "vitest run tests/contracts/translations-coverage.test.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40c14506..357846ec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,7 +36,7 @@ importers: version: 14.6.1(@testing-library/dom@10.4.1) '@vitejs/plugin-react': specifier: ^6.0.3 - version: 6.0.3(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)) + version: 6.0.3(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) '@vitest/coverage-istanbul': specifier: ^4.1.9 version: 4.1.9(supports-color@7.2.0)(vitest@4.1.9) @@ -60,19 +60,19 @@ importers: version: 29.1.1(@noble/hashes@2.2.0) vitest: specifier: ^4.1.9 - version: 4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)) + version: 4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) chromium-xtension: devDependencies: '@vitejs/plugin-react': specifier: ^6.0.3 - version: 6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)) + version: 6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) typescript: specifier: ~5.8.3 version: 5.8.3 vite: specifier: ^8.1.3 - version: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0) + version: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) electron: dependencies: @@ -118,6 +118,9 @@ importers: react-dom: specifier: ^19.2.7 version: 19.2.7(react@19.2.7) + yaml: + specifier: ^2.8.0 + version: 2.9.0 devDependencies: '@types/react': specifier: ^19.2.17 @@ -127,13 +130,13 @@ importers: version: 19.2.3(@types/react@19.2.17) '@vitejs/plugin-react': specifier: ^6.0.3 - version: 6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)) + version: 6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) typescript: specifier: ~5.8.3 version: 5.8.3 vite: specifier: ^8.1.3 - version: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0) + version: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) vscode: dependencies: @@ -161,13 +164,13 @@ importers: devDependencies: '@vitejs/plugin-react': specifier: ^6.0.3 - version: 6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)) + version: 6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) typescript: specifier: ~5.8.3 version: 5.8.3 vite: specifier: ^8.1.3 - version: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0) + version: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) packages: @@ -4006,6 +4009,11 @@ packages: resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} engines: {node: '>=18'} + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -5260,15 +5268,15 @@ snapshots: d3-selection: 3.0.0 d3-transition: 3.0.1(d3-selection@3.0.0) - '@vitejs/plugin-react@6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0))': + '@vitejs/plugin-react@6.0.3(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': dependencies: '@rolldown/pluginutils': 1.0.1 - vite: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0) + vite: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) - '@vitejs/plugin-react@6.0.3(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0))': + '@vitejs/plugin-react@6.0.3(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': dependencies: '@rolldown/pluginutils': 1.0.1 - vite: 8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0) + vite: 8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) '@vitest/coverage-istanbul@4.1.9(supports-color@7.2.0)(vitest@4.1.9)': dependencies: @@ -5282,7 +5290,7 @@ snapshots: magicast: 0.5.3 obug: 2.1.3 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)) + vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) transitivePeerDependencies: - supports-color @@ -5298,7 +5306,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)) + vitest: 4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) '@vitest/expect@4.1.9': dependencies: @@ -5309,13 +5317,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.9(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0))': + '@vitest/mocker@4.1.9(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.9 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0) + vite: 8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) '@vitest/pretty-format@4.1.9': dependencies: @@ -7933,7 +7941,7 @@ snapshots: version-range@4.15.0: {} - vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0): + vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0): dependencies: lightningcss: 1.33.0 picomatch: 4.0.5 @@ -7945,8 +7953,9 @@ snapshots: esbuild: 0.28.1 fsevents: 2.3.3 jiti: 2.7.0 + yaml: 2.9.0 - vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0): + vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0): dependencies: lightningcss: 1.33.0 picomatch: 4.0.5 @@ -7958,11 +7967,12 @@ snapshots: esbuild: 0.28.1 fsevents: 2.3.3 jiti: 2.7.0 + yaml: 2.9.0 - vitest@4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)): + vitest@4.1.9(@types/node@24.13.2)(@vitest/coverage-istanbul@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@29.1.1(@noble/hashes@2.2.0))(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 - '@vitest/mocker': 4.1.9(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)) + '@vitest/mocker': 4.1.9(vite@8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.9 '@vitest/runner': 4.1.9 '@vitest/snapshot': 4.1.9 @@ -7979,7 +7989,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0) + vite: 8.1.3(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 24.13.2 @@ -8088,6 +8098,8 @@ snapshots: yallist@5.0.0: {} + yaml@2.9.0: {} + yargs-parser@21.1.1: {} yargs@17.7.3: diff --git a/tauri/src/app_state.rs b/tauri/src/app_state.rs index b26e7253..e7490c1b 100644 --- a/tauri/src/app_state.rs +++ b/tauri/src/app_state.rs @@ -35,6 +35,9 @@ pub struct AppStateInner { pub search_worker: Option, pub search_preview_paths: HashSet, pub watch_controller: Option, + pub insights_watch_controller: Option, + pub insights_cancelled_scans: HashSet, + pub insights_cancelled_external_checks: HashSet, pub runtime_state: RuntimeState, pub workspace_scan_generation: u64, pub workspace_operation_id: Option, @@ -69,4 +72,4 @@ impl AppState { pub fn new() -> Self { Self::default() } -} +} \ No newline at end of file diff --git a/tauri/src/dispatcher/commands.rs b/tauri/src/dispatcher/commands.rs index ca7a9c2b..b22eee9e 100644 --- a/tauri/src/dispatcher/commands.rs +++ b/tauri/src/dispatcher/commands.rs @@ -20,6 +20,12 @@ impl Dispatcher { if self.handle_workspace_command(cmd, &msg).await? { return Ok(()); } + if crate::insights_external_host::handle_command(&self.app, &self.state, cmd, &msg).await? { + return Ok(()); + } + if crate::insights::handle_command(&self.app, &self.state, cmd, &msg).await? { + return Ok(()); + } if crate::runtime::export_resources::handle_command(&self.app, &self.state, cmd, &msg)? { return Ok(()); } @@ -32,4 +38,4 @@ impl Dispatcher { eprintln!("[dispatcher] unknown command: {cmd}"); Ok(()) } -} +} \ No newline at end of file diff --git a/tauri/src/insights/external.rs b/tauri/src/insights/external.rs new file mode 100644 index 00000000..dcd6ad65 --- /dev/null +++ b/tauri/src/insights/external.rs @@ -0,0 +1,342 @@ +use serde_json::{json, Value}; +use std::collections::HashSet; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}; +use std::str::FromStr; +use std::time::Duration; + +pub const MAX_REDIRECTS: usize = 5; +pub const GLOBAL_CONCURRENCY: usize = 4; +pub const ORIGIN_CONCURRENCY: usize = 2; +pub const DEFAULT_TIMEOUT_MS: u64 = 10_000; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ParsedHttpUrl { + raw: String, + scheme: String, + authority: String, + host: String, + port: u16, + path_and_query: String, +} + +impl ParsedHttpUrl { + fn parse(input: &str) -> Result { + let uri = http::Uri::from_str(input).map_err(|_| "invalid-url".to_string())?; + let scheme = uri.scheme_str().ok_or_else(|| "missing-scheme".to_string())?.to_ascii_lowercase(); + if scheme != "http" && scheme != "https" { + return Err("unsupported-scheme".into()); + } + let authority = uri.authority().ok_or_else(|| "missing-authority".to_string())?; + if authority.as_str().contains('@') { + return Err("embedded-credentials-unsupported".into()); + } + let host = authority.host().to_string(); + if host.is_empty() { + return Err("missing-host".into()); + } + let port = authority.port_u16().unwrap_or(if scheme == "https" { 443 } else { 80 }); + let path_and_query = uri + .path_and_query() + .map(|value| value.as_str().to_string()) + .unwrap_or_else(|| "/".to_string()); + Ok(Self { + raw: input.to_string(), + scheme, + authority: authority.as_str().to_string(), + host, + port, + path_and_query, + }) + } + + fn origin(&self) -> String { + let default_port = (self.scheme == "https" && self.port == 443) || (self.scheme == "http" && self.port == 80); + if default_port && !self.authority.rsplit_once(':').is_some_and(|(_, port)| port.parse::().is_ok()) { + format!("{}://{}", self.scheme, self.host) + } else { + format!("{}://{}", self.scheme, self.authority) + } + } +} + +fn normalize_path(path: &str) -> String { + let mut output: Vec<&str> = Vec::new(); + for part in path.split('/') { + match part { + "" | "." => {} + ".." => { + output.pop(); + } + other => output.push(other), + } + } + format!("/{}", output.join("/")) +} + +fn resolve_redirect(base: &ParsedHttpUrl, location: &str) -> Result { + let location = location.trim(); + let target = if location.starts_with("http://") || location.starts_with("https://") { + location.to_string() + } else if location.starts_with("//") { + format!("{}:{}", base.scheme, location) + } else if location.starts_with('/') { + format!("{}://{}{}", base.scheme, base.authority, location) + } else if location.starts_with('?') { + let path = base.path_and_query.split('?').next().unwrap_or("/"); + format!("{}://{}{}{}", base.scheme, base.authority, path, location) + } else { + let base_path = base.path_and_query.split('?').next().unwrap_or("/"); + let directory = base_path.rsplit_once('/').map(|(dir, _)| dir).unwrap_or(""); + let path = normalize_path(&format!("{directory}/{location}")); + format!("{}://{}{}", base.scheme, base.authority, path) + }; + ParsedHttpUrl::parse(&target) +} + +fn is_private_ipv4(ip: Ipv4Addr) -> bool { + let [a, b, _, _] = ip.octets(); + a == 0 + || a == 10 + || a == 127 + || (a == 100 && (64..=127).contains(&b)) + || (a == 169 && b == 254) + || (a == 172 && (16..=31).contains(&b)) + || (a == 192 && b == 168) + || (a == 198 && (b == 18 || b == 19)) + || a >= 224 +} + +fn is_private_ipv6(ip: Ipv6Addr) -> bool { + if ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() { + return true; + } + let first = ip.segments()[0]; + (first & 0xfe00) == 0xfc00 || (first & 0xffc0) == 0xfe80 +} + +pub fn is_private_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(value) => is_private_ipv4(value), + IpAddr::V6(value) => value.to_ipv4_mapped().map(is_private_ipv4).unwrap_or_else(|| is_private_ipv6(value)), + } +} + +fn is_private_hostname(host: &str) -> bool { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + host == "localhost" || host.ends_with(".localhost") +} + +pub fn classify_status(status: u16) -> &'static str { + match status { + 200..=399 => "reachable", + 401 | 403 => "reachable-auth-required", + 404 | 410 => "broken", + 429 => "rate-limited", + 500..=599 => "server-error", + _ => "unreachable", + } +} + +fn resolve_addresses(parsed: &ParsedHttpUrl) -> Result, String> { + let addresses = (parsed.host.as_str(), parsed.port) + .to_socket_addrs() + .map_err(|error| error.to_string())? + .map(|socket| socket.ip()) + .collect::>(); + if addresses.is_empty() { + Err("dns-empty".into()) + } else { + Ok(addresses) + } +} + +#[derive(Debug)] +struct ResponseMeta { + status: u16, + location: Option, + retry_after: Option, +} + +fn request_once(parsed: &ParsedHttpUrl, ip: IpAddr, method: &str, timeout_ms: u64) -> Result { + let socket = SocketAddr::new(ip, parsed.port); + let agent = ureq::AgentBuilder::new() + .redirects(0) + .timeout(Duration::from_millis(timeout_ms.max(1))) + .resolver(move |_addr: &str| Ok(vec![socket])) + .build(); + let outcome = agent + .request(method, &parsed.raw) + .set("Accept", "*/*") + .set("User-Agent", "Markdown Explorer/Insights") + .call(); + let response = match outcome { + Ok(response) => response, + Err(ureq::Error::Status(_, response)) => response, + Err(error) => return Err(error.to_string()), + }; + Ok(ResponseMeta { + status: response.status(), + location: response.header("Location").map(ToOwned::to_owned), + retry_after: response.header("Retry-After").map(ToOwned::to_owned), + }) +} + +fn parse_retry_after(value: Option<&str>) -> Option { + let raw = value?.trim(); + raw.parse::().ok().map(|seconds| seconds.saturating_mul(1000)) +} + +pub fn origin_for_url(url: &str) -> String { + ParsedHttpUrl::parse(url).map(|parsed| parsed.origin()).unwrap_or_default() +} + +pub fn check_url(url: &str, timeout_ms: u64, approved_private_origins: &HashSet) -> Value { + let original_url = url.to_string(); + let mut current = match ParsedHttpUrl::parse(url) { + Ok(value) => value, + Err(reason) => return json!({"url": original_url, "status": "unsupported", "reason": reason}), + }; + let mut redirects = 0usize; + let mut transient_retries = 0usize; + let mut insecure_downgrade = false; + + loop { + let origin = current.origin(); + let addresses = match resolve_addresses(¤t) { + Ok(value) => value, + Err(reason) => return json!({ + "url": original_url, + "status": "unreachable", + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + "reason": reason, + }), + }; + if (is_private_hostname(¤t.host) || addresses.iter().copied().any(is_private_ip)) + && !approved_private_origins.contains(&origin) + { + return json!({ + "url": original_url, + "status": "unchecked", + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + "privateOrigin": origin, + "requiresPrivateOriginConfirmation": true, + "reason": "private-origin-confirmation-required", + }); + } + let ip = addresses[0]; + let mut response = match request_once(¤t, ip, "HEAD", timeout_ms) { + Ok(value) => value, + Err(reason) => return json!({ + "url": original_url, + "status": "unreachable", + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + "reason": reason, + }), + }; + if response.status == 405 || response.status == 501 { + response = match request_once(¤t, ip, "GET", timeout_ms) { + Ok(value) => value, + Err(reason) => return json!({ + "url": original_url, + "status": "unreachable", + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + "reason": reason, + }), + }; + } + + if (300..400).contains(&response.status) { + if let Some(location) = response.location.as_deref() { + if redirects >= MAX_REDIRECTS { + return json!({ + "url": original_url, + "status": "unreachable", + "httpStatus": response.status, + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + "reason": "redirect-limit", + }); + } + let next = match resolve_redirect(¤t, location) { + Ok(value) => value, + Err(reason) => return json!({ + "url": original_url, + "status": "unreachable", + "httpStatus": response.status, + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + "reason": reason, + }), + }; + if current.scheme == "https" && next.scheme == "http" { + insecure_downgrade = true; + } + current = next; + redirects += 1; + transient_retries = 0; + continue; + } + } + + if (500..600).contains(&response.status) && transient_retries < 1 { + transient_retries += 1; + continue; + } + + let mut result = json!({ + "url": original_url, + "status": classify_status(response.status), + "httpStatus": response.status, + "finalUrl": current.raw, + "insecureDowngrade": insecure_downgrade, + }); + if let Some(retry_after_ms) = parse_retry_after(response.retry_after.as_deref()) { + if let Some(object) = result.as_object_mut() { + object.insert("retryAfterMs".into(), retry_after_ms.into()); + } + } + return result; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_http_statuses() { + assert_eq!(classify_status(200), "reachable"); + assert_eq!(classify_status(302), "reachable"); + assert_eq!(classify_status(401), "reachable-auth-required"); + assert_eq!(classify_status(404), "broken"); + assert_eq!(classify_status(429), "rate-limited"); + assert_eq!(classify_status(503), "server-error"); + } + + #[test] + fn rejects_private_and_local_addresses() { + assert!(is_private_ip("127.0.0.1".parse().unwrap())); + assert!(is_private_ip("10.1.2.3".parse().unwrap())); + assert!(is_private_ip("169.254.2.3".parse().unwrap())); + assert!(is_private_ip("::1".parse().unwrap())); + assert!(is_private_ip("fd00::1".parse().unwrap())); + assert!(!is_private_ip("203.0.113.15".parse().unwrap())); + } + + #[test] + fn keeps_private_approval_origin_scoped() { + assert_eq!(origin_for_url("https://example.test/path"), "https://example.test"); + assert_eq!(origin_for_url("http://example.test:8080/path"), "http://example.test:8080"); + } + + #[test] + fn resolves_relative_redirects_without_losing_origin() { + let base = ParsedHttpUrl::parse("https://example.test/a/b/index.md?x=1").unwrap(); + assert_eq!(resolve_redirect(&base, "../next").unwrap().raw, "https://example.test/a/next"); + assert_eq!(resolve_redirect(&base, "/root").unwrap().raw, "https://example.test/root"); + } +} diff --git a/tauri/src/insights/external_host.rs b/tauri/src/insights/external_host.rs new file mode 100644 index 00000000..8da98ab4 --- /dev/null +++ b/tauri/src/insights/external_host.rs @@ -0,0 +1,140 @@ +use crate::app_state::AppState; +use crate::host_message; +use crate::insights_external::{ + check_url, origin_for_url, DEFAULT_TIMEOUT_MS, GLOBAL_CONCURRENCY, ORIGIN_CONCURRENCY, +}; +use serde_json::{json, Map, Value}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use tauri::AppHandle; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; + +fn emit_value(app: &AppHandle, command: &str, value: Value) { + let extra = value.as_object().cloned().unwrap_or_else(Map::new); + host_message::emit(app, command, extra); +} + +pub async fn handle_command( + app: &AppHandle, + state: &AppState, + command: &str, + message: &Value, +) -> Result { + match command { + "checkExternalLinks" => { + let request_id = message + .get("requestId") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let timeout_ms = message + .get("timeoutMs") + .and_then(Value::as_u64) + .unwrap_or(DEFAULT_TIMEOUT_MS); + let approved = message + .get("approvedPrivateOrigins") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .map(ToOwned::to_owned) + .collect::>() + }) + .unwrap_or_default(); + let mut urls = message + .get("urls") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .map(ToOwned::to_owned) + .collect::>() + }) + .unwrap_or_default(); + let mut seen = HashSet::new(); + urls.retain(|url| seen.insert(url.clone())); + state + .inner + .write() + .insights_cancelled_external_checks + .remove(&request_id); + + let global = Arc::new(Semaphore::new(GLOBAL_CONCURRENCY)); + let mut by_origin: HashMap> = HashMap::new(); + let mut tasks = JoinSet::new(); + + for url in urls { + let origin = origin_for_url(&url); + let origin_limit = by_origin + .entry(origin) + .or_insert_with(|| Arc::new(Semaphore::new(ORIGIN_CONCURRENCY))) + .clone(); + let global_limit = global.clone(); + let approved = approved.clone(); + let state_for_task = state.clone(); + let request_for_task = request_id.clone(); + tasks.spawn(async move { + let _global_permit = global_limit.acquire_owned().await.ok()?; + let _origin_permit = origin_limit.acquire_owned().await.ok()?; + if state_for_task + .inner + .read() + .insights_cancelled_external_checks + .contains(&request_for_task) + { + return None; + } + let result = tauri::async_runtime::spawn_blocking(move || { + check_url(&url, timeout_ms, &approved) + }) + .await + .ok()?; + Some(result) + }); + } + + while let Some(joined) = tasks.join_next().await { + let cancelled = state + .inner + .read() + .insights_cancelled_external_checks + .contains(&request_id); + if cancelled { + continue; + } + if let Ok(Some(mut result)) = joined { + if let Some(object) = result.as_object_mut() { + object.insert("requestId".into(), request_id.clone().into()); + } + emit_value(app, "externalLinkCheckResult", result); + } + } + + let cancelled = state + .inner + .write() + .insights_cancelled_external_checks + .remove(&request_id); + emit_value( + app, + "externalLinkCheckComplete", + json!({"requestId": request_id, "cancelled": cancelled}), + ); + Ok(true) + } + "cancelExternalLinkChecks" => { + if let Some(request_id) = message.get("requestId").and_then(Value::as_str) { + state + .inner + .write() + .insights_cancelled_external_checks + .insert(request_id.to_string()); + } + Ok(true) + } + _ => Ok(false), + } +} diff --git a/tauri/src/insights/mod.rs b/tauri/src/insights/mod.rs new file mode 100644 index 00000000..b34966af --- /dev/null +++ b/tauri/src/insights/mod.rs @@ -0,0 +1,246 @@ +pub mod scan; + +#[cfg(not(test))] +use crate::app_state::AppState; +#[cfg(not(test))] +use crate::host_message; +#[cfg(not(test))] +use crate::workspace::watch::WorkspaceWatchController; +#[cfg(not(test))] +use serde_json::{json, Map, Value}; +#[cfg(not(test))] +use std::path::{Path, PathBuf}; +#[cfg(not(test))] +use tauri::AppHandle; + +#[cfg(not(test))] +fn workspace_root(state: &AppState) -> Option { + let workspace = state.inner.read().workspace_path.clone()?; + if !workspace.exists() { + return None; + } + let base = if workspace.is_file() { + workspace.parent().unwrap_or(&workspace).to_path_buf() + } else { + workspace + }; + std::fs::canonicalize(base).ok() +} + +#[cfg(not(test))] +fn emit_value(app: &AppHandle, command: &str, value: Value) { + let extra = value.as_object().cloned().unwrap_or_else(Map::new); + host_message::emit(app, command, extra); +} + +#[cfg(not(test))] +fn entry_for_path(root: &Path, path: &Path) -> Option { + let root_real = std::fs::canonicalize(root).ok()?; + let real = std::fs::canonicalize(path).ok()?; + if !scan::same_or_inside(&root_real, &real) { + return None; + } + let metadata = std::fs::metadata(&real).ok()?; + if !metadata.is_file() { + return None; + } + let relative_path = pathdiff::diff_paths(path, &root_real)? + .to_string_lossy() + .replace('\\', "/"); + let canonical_relative_path = pathdiff::diff_paths(&real, &root_real)? + .to_string_lossy() + .replace('\\', "/"); + let mtime_ms = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64) + .unwrap_or(0); + Some(scan::InsightsWorkspaceEntry { + relative_path, + canonical_relative_path, + kind: "file", + size_bytes: metadata.len(), + mtime_ms, + extension: real + .extension() + .and_then(|value| value.to_str()) + .map(|value| format!(".{}", value.to_lowercase())), + is_symlink: std::fs::symlink_metadata(path) + .map(|meta| meta.file_type().is_symlink()) + .unwrap_or(false), + }) +} + +#[cfg(not(test))] +pub async fn handle_command( + app: &AppHandle, + state: &AppState, + command: &str, + message: &Value, +) -> Result { + match command { + "scanInsightsWorkspace" => { + let request_id = message.get("requestId").and_then(Value::as_str).unwrap_or("").to_string(); + let user_patterns = message + .get("userPatterns") + .and_then(Value::as_array) + .map(|items| items.iter().filter_map(Value::as_str).map(ToOwned::to_owned).collect::>()) + .unwrap_or_default(); + { + state.inner.write().insights_cancelled_scans.remove(&request_id); + } + let Some(root) = workspace_root(state) else { + emit_value(app, "insightsScanComplete", json!({ + "requestId": request_id, + "totalEntries": 0, + "excludedEntries": 0, + "skippedEntries": 0, + "truncated": false + })); + return Ok(true); + }; + let state_for_cancel = state.clone(); + let request_for_cancel = request_id.clone(); + let root_for_scan = root.clone(); + let result = tauri::async_runtime::spawn_blocking(move || { + scan::scan_workspace(&root_for_scan, &user_patterns, || { + state_for_cancel + .inner + .read() + .insights_cancelled_scans + .contains(&request_for_cancel) + }) + }) + .await + .map_err(|error| format!("insights scan task failed: {error}"))?; + + for chunk in result.entries.chunks(scan::SCAN_BATCH_SIZE) { + emit_value(app, "insightsScanBatch", json!({ + "requestId": request_id, + "entries": chunk, + "scannedEntries": result.entries.len(), + "excludedEntries": result.excluded_entries + })); + } + state.inner.write().insights_cancelled_scans.remove(&request_id); + emit_value(app, "insightsScanComplete", json!({ + "requestId": request_id, + "totalEntries": result.entries.len(), + "excludedEntries": result.excluded_entries, + "skippedEntries": result.skipped_entries, + "truncated": false, + "cancelled": result.cancelled + })); + Ok(true) + } + "cancelInsightsScan" => { + if let Some(request_id) = message.get("requestId").and_then(Value::as_str) { + state.inner.write().insights_cancelled_scans.insert(request_id.to_string()); + } + Ok(true) + } + "readInsightsDocumentSource" => { + let request_id = message.get("requestId").and_then(Value::as_str).unwrap_or(""); + let relative_path = message.get("relativePath").and_then(Value::as_str).unwrap_or(""); + let Some(root) = workspace_root(state) else { + emit_value(app, "insightsDocumentSourceResult", json!({ + "requestId": request_id, + "relativePath": relative_path, + "status": "missing" + })); + return Ok(true); + }; + let soft_limit = message + .get("softLimitBytes") + .and_then(Value::as_u64) + .unwrap_or(scan::DEFAULT_SOFT_LIMIT_BYTES); + let hard_limit = message + .get("hardLimitBytes") + .and_then(Value::as_u64) + .unwrap_or(scan::DEFAULT_HARD_LIMIT_BYTES); + let mut result = scan::read_document_source(&root, relative_path, soft_limit, hard_limit) + .as_object() + .cloned() + .unwrap_or_default(); + result.insert("requestId".into(), request_id.into()); + result.insert("relativePath".into(), relative_path.into()); + host_message::emit(app, "insightsDocumentSourceResult", result); + Ok(true) + } + "probeWorkspaceResource" => { + let request_id = message.get("requestId").and_then(Value::as_str).unwrap_or(""); + let document_path = message.get("documentPath").and_then(Value::as_str).unwrap_or(""); + let resource_path = message.get("resourcePath").and_then(Value::as_str).unwrap_or(""); + let Some(root) = workspace_root(state) else { + emit_value(app, "workspaceResourceProbeResult", json!({ + "requestId": request_id, + "status": "missing" + })); + return Ok(true); + }; + let mut result = scan::probe_resource(&root, document_path, resource_path) + .as_object() + .cloned() + .unwrap_or_default(); + result.insert("requestId".into(), request_id.into()); + host_message::emit(app, "workspaceResourceProbeResult", result); + Ok(true) + } + "setInsightsWatchState" => { + let request_id = message.get("requestId").and_then(Value::as_str).unwrap_or("").to_string(); + let workspace_operation_id = message + .get("workspaceOperationId") + .and_then(Value::as_str) + .map(ToOwned::to_owned); + let active = message.get("active").and_then(Value::as_bool).unwrap_or(false); + + if let Some(existing) = state.inner.write().insights_watch_controller.take() { + existing.dispose(); + } + + let Some(root) = workspace_root(state) else { + emit_value(app, "insightsRuntimeCapabilities", json!({ + "requestId": request_id, + "capabilities": {"fileChanges": "unsupported", "externalLinkChecking": true, "documentPreviewReuse": true} + })); + return Ok(true); + }; + + if active { + let app_for_watch = app.clone(); + let request_for_watch = request_id.clone(); + let operation_for_watch = workspace_operation_id.clone(); + let root_for_watch = root.clone(); + let controller = WorkspaceWatchController::new(120, move |_workspace, change| { + let Some(change) = change else { return; }; + let path = PathBuf::from(&change.fs_path); + let delta = if !change.fs_path.is_empty() && !path.exists() { + json!({"kind": "delete", "relativePath": change.relative_path}) + } else if let Some(entry) = entry_for_path(&root_for_watch, &path) { + json!({"kind": "update", "entry": entry}) + } else { + return; + }; + let mut payload = json!({ + "requestId": request_for_watch, + "deltas": [delta] + }); + if let (Some(operation), Some(object)) = (operation_for_watch.as_ref(), payload.as_object_mut()) { + object.insert("workspaceOperationId".into(), operation.clone().into()); + } + emit_value(&app_for_watch, "insightsFsDelta", payload); + }); + controller.watch_workspace(Some(&root)); + state.inner.write().insights_watch_controller = Some(controller); + } + + emit_value(app, "insightsRuntimeCapabilities", json!({ + "requestId": request_id, + "capabilities": {"fileChanges": if active {"native"} else {"native"}, "externalLinkChecking": true, "documentPreviewReuse": true} + })); + Ok(true) + } + _ => Ok(false), + } +} diff --git a/tauri/src/insights/scan.rs b/tauri/src/insights/scan.rs new file mode 100644 index 00000000..3a2ca70c --- /dev/null +++ b/tauri/src/insights/scan.rs @@ -0,0 +1,322 @@ +use serde::Serialize; +use serde_json::{json, Value}; +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; +use walkdir::WalkDir; + +pub const DEFAULT_SOFT_LIMIT_BYTES: u64 = 10 * 1024 * 1024; +pub const DEFAULT_HARD_LIMIT_BYTES: u64 = 64 * 1024 * 1024; +pub const SCAN_BATCH_SIZE: usize = 200; + +const HARD_EXCLUDED: &[&str] = &[".git", ".hg", ".svn"]; +const DEFAULT_EXCLUDED: &[&str] = &["node_modules", ".next", "dist", "build", "coverage", ".cache"]; + +#[derive(Clone, Debug, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct InsightsWorkspaceEntry { + pub relative_path: String, + pub canonical_relative_path: String, + pub kind: &'static str, + pub size_bytes: u64, + pub mtime_ms: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub extension: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub is_symlink: bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ScanResult { + pub entries: Vec, + pub excluded_entries: usize, + pub skipped_entries: usize, + pub cancelled: bool, +} + +fn normalized_relative(root: &Path, target: &Path) -> Option { + pathdiff::diff_paths(target, root).map(|path| path.to_string_lossy().replace('\\', "/")) +} + +pub fn same_or_inside(root: &Path, target: &Path) -> bool { + normalized_relative(root, target) + .map(|relative| { + relative.is_empty() + || (relative != ".." + && !relative.starts_with("../") + && !Path::new(&relative).is_absolute()) + }) + .unwrap_or(false) +} + +fn has_excluded_segment(relative: &str, segments: &[&str]) -> bool { + relative + .split('/') + .any(|segment| segments.iter().any(|candidate| segment == *candidate)) +} + +fn matches_simple_pattern(relative: &str, pattern: &str) -> bool { + let pattern = pattern.trim().trim_start_matches('/').trim_end_matches('/'); + if pattern.is_empty() { + return false; + } + if !pattern.contains('*') && !pattern.contains('?') { + return relative == pattern + || relative.starts_with(&format!("{pattern}/")) + || (!pattern.contains('/') && relative.split('/').any(|part| part == pattern)); + } + let mut regex = String::from("(?i)^"); + for ch in pattern.chars() { + match ch { + '*' => regex.push_str(".*"), + '?' => regex.push('.'), + '.' => regex.push_str("\\."), + '/' => regex.push('/'), + other if "()[]{}+^$|\\".contains(other) => { + regex.push('\\'); + regex.push(other); + } + other => regex.push(other), + } + } + regex.push('$'); + regex_lite::Regex::new(®ex) + .map(|compiled| compiled.is_match(relative)) + .unwrap_or(false) +} + +fn should_exclude(relative: &str, gitignore: &[String], user_patterns: &[String]) -> bool { + if has_excluded_segment(relative, HARD_EXCLUDED) { + return true; + } + let mut excluded = has_excluded_segment(relative, DEFAULT_EXCLUDED); + for raw in gitignore.iter().chain(user_patterns.iter()) { + let trimmed = raw.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let (negated, pattern) = trimmed + .strip_prefix('!') + .map(|value| (true, value)) + .unwrap_or((false, trimmed)); + if matches_simple_pattern(relative, pattern) { + excluded = !negated; + } + } + excluded +} + +fn read_gitignore(root: &Path) -> Vec { + fs::read_to_string(root.join(".gitignore")) + .map(|source| { + source + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(ToOwned::to_owned) + .collect() + }) + .unwrap_or_default() +} + +fn modified_ms(metadata: &fs::Metadata) -> u64 { + metadata + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64) + .unwrap_or(0) +} + +pub fn scan_workspace(root: &Path, user_patterns: &[String], cancelled: F) -> ScanResult +where + F: Fn() -> bool, +{ + let Ok(root_real) = fs::canonicalize(root) else { + return ScanResult { entries: vec![], excluded_entries: 0, skipped_entries: 1, cancelled: false }; + }; + let gitignore = read_gitignore(&root_real); + let mut entries = Vec::new(); + let mut excluded_entries = 0usize; + let mut skipped_entries = 0usize; + let mut seen_files = HashSet::::new(); + let mut seen_dirs = HashSet::::new(); + seen_dirs.insert(root_real.clone()); + + for item in WalkDir::new(&root_real).follow_links(true).into_iter() { + if cancelled() { + return ScanResult { entries, excluded_entries, skipped_entries, cancelled: true }; + } + let entry = match item { + Ok(entry) => entry, + Err(_) => { skipped_entries += 1; continue; } + }; + if entry.path() == root_real { + continue; + } + let display_relative = normalized_relative(&root_real, entry.path()).unwrap_or_default(); + if should_exclude(&display_relative, &gitignore, user_patterns) { + excluded_entries += 1; + continue; + } + let Ok(real) = fs::canonicalize(entry.path()) else { + skipped_entries += 1; + continue; + }; + if !same_or_inside(&root_real, &real) { + excluded_entries += 1; + continue; + } + let Ok(metadata) = fs::metadata(&real) else { + skipped_entries += 1; + continue; + }; + if metadata.is_dir() { + if !seen_dirs.insert(real) { + continue; + } + continue; + } + if !metadata.is_file() || !seen_files.insert(real.clone()) { + continue; + } + let canonical_relative = normalized_relative(&root_real, &real).unwrap_or_else(|| display_relative.clone()); + entries.push(InsightsWorkspaceEntry { + relative_path: display_relative, + canonical_relative_path: canonical_relative, + kind: "file", + size_bytes: metadata.len(), + mtime_ms: modified_ms(&metadata), + extension: real + .extension() + .and_then(|value| value.to_str()) + .map(|value| format!(".{}", value.to_lowercase())), + is_symlink: entry.path_is_symlink(), + }); + } + + ScanResult { entries, excluded_entries, skipped_entries, cancelled: false } +} + +fn resolve_relative(root: &Path, relative_path: &str) -> Option { + if relative_path.is_empty() { + return None; + } + let candidate = root.join(relative_path); + let root_real = fs::canonicalize(root).ok()?; + let real = fs::canonicalize(candidate).ok()?; + same_or_inside(&root_real, &real).then_some(real) +} + +pub fn read_document_source(root: &Path, relative_path: &str, soft_limit: u64, hard_limit: u64) -> Value { + if !relative_path.to_lowercase().ends_with(".md") && !relative_path.to_lowercase().ends_with(".mdx") { + return json!({"status": "unsupported"}); + } + let Some(real) = resolve_relative(root, relative_path) else { + return json!({"status": "missing"}); + }; + let Ok(metadata) = fs::metadata(&real) else { + return json!({"status": "unreadable"}); + }; + if !metadata.is_file() { + return json!({"status": "missing"}); + } + let hard_limit = hard_limit.clamp(1, DEFAULT_HARD_LIMIT_BYTES); + let soft_limit = soft_limit.max(1); + if metadata.len() > hard_limit { + return json!({"status": "too-large", "sizeBytes": metadata.len(), "mtimeMs": modified_ms(&metadata), "hardLimit": true}); + } + if metadata.len() > soft_limit { + return json!({"status": "too-large", "sizeBytes": metadata.len(), "mtimeMs": modified_ms(&metadata), "hardLimit": false}); + } + match fs::read_to_string(&real) { + Ok(source) => json!({ + "status": "ok", + "source": source, + "sizeBytes": metadata.len(), + "mtimeMs": modified_ms(&metadata) + }), + Err(_) => json!({"status": "unreadable"}), + } +} + +fn mime_type(path: &Path) -> &'static str { + match path.extension().and_then(|value| value.to_str()).unwrap_or("").to_lowercase().as_str() { + "png" => "image/png", "jpg" | "jpeg" => "image/jpeg", "gif" => "image/gif", "webp" => "image/webp", + "svg" => "image/svg+xml", "avif" => "image/avif", "mp4" => "video/mp4", "webm" => "video/webm", + "mp3" => "audio/mpeg", "wav" => "audio/wav", "ogg" => "audio/ogg", "m4a" => "audio/mp4", + "md" => "text/markdown", "mdx" => "text/mdx", "pdf" => "application/pdf", _ => "application/octet-stream", + } +} + +pub fn probe_resource(root: &Path, document_path: &str, resource_path: &str) -> Value { + let raw = resource_path.split(['?', '#']).next().unwrap_or(""); + if raw.is_empty() || raw.starts_with("http:") || raw.starts_with("https:") || raw.starts_with("data:") || raw.starts_with("blob:") || raw.starts_with("file:") { + return json!({"status": "outside-workspace"}); + } + let Ok(root_real) = fs::canonicalize(root) else { + return json!({"status": "missing"}); + }; + let doc = Path::new(document_path); + let doc_absolute = if doc.is_absolute() { doc.to_path_buf() } else { root_real.join(doc) }; + let candidate = if raw.starts_with('/') { + root_real.join(raw.trim_start_matches('/')) + } else { + doc_absolute.parent().unwrap_or(&root_real).join(raw) + }; + let Ok(real) = fs::canonicalize(candidate) else { + return json!({"status": "missing"}); + }; + if !same_or_inside(&root_real, &real) { + return json!({"status": "outside-workspace"}); + } + let Ok(metadata) = fs::metadata(&real) else { + return json!({"status": "unreadable"}); + }; + json!({ + "status": "exists", + "relativePath": normalized_relative(&root_real, &real), + "kind": if metadata.is_dir() { "directory" } else { "file" }, + "sizeBytes": if metadata.is_file() { Some(metadata.len()) } else { None }, + "mimeType": if metadata.is_file() { Some(mime_type(&real)) } else { None } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn scans_more_than_sidebar_cap() { + let dir = tempfile::tempdir().unwrap(); + for index in 0..1005 { + fs::write(dir.path().join(format!("{index}.md")), format!("# {index}")).unwrap(); + } + let result = scan_workspace(dir.path(), &[], || false); + assert_eq!(result.entries.len(), 1005); + assert!(!result.cancelled); + } + + #[test] + fn source_limits_are_checked_before_read() { + let dir = tempfile::tempdir().unwrap(); + let mut file = fs::File::create(dir.path().join("large.md")).unwrap(); + file.write_all(&vec![b'x'; 2048]).unwrap(); + let value = read_document_source(dir.path(), "large.md", 1024, DEFAULT_HARD_LIMIT_BYTES); + assert_eq!(value["status"], "too-large"); + assert_eq!(value["hardLimit"], false); + } + + #[test] + fn probe_returns_metadata() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("docs")).unwrap(); + fs::create_dir_all(dir.path().join("img")).unwrap(); + fs::write(dir.path().join("docs/a.md"), "# A").unwrap(); + fs::write(dir.path().join("img/a.png"), "PNG").unwrap(); + let value = probe_resource(dir.path(), "docs/a.md", "../img/a.png"); + assert_eq!(value["status"], "exists"); + assert_eq!(value["sizeBytes"], 3); + } +} diff --git a/tauri/src/lib.rs b/tauri/src/lib.rs index 53278b8e..222a5f4f 100644 --- a/tauri/src/lib.rs +++ b/tauri/src/lib.rs @@ -9,6 +9,12 @@ pub mod error; pub mod fonts; #[cfg(not(test))] pub mod host_message; +pub mod insights; +#[path = "insights/external.rs"] +pub mod insights_external; +#[cfg(not(test))] +#[path = "insights/external_host.rs"] +pub mod insights_external_host; pub mod local_file; pub mod perf; #[cfg(not(test))] @@ -24,4 +30,4 @@ pub mod youtube; #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { crate::core::bootstrap::boot(); -} +} \ No newline at end of file diff --git a/tests/manifest/coverage-manifest.test.ts b/tests/manifest/coverage-manifest.test.ts index 9a81cc17..853a5b30 100644 --- a/tests/manifest/coverage-manifest.test.ts +++ b/tests/manifest/coverage-manifest.test.ts @@ -3,10 +3,12 @@ import fg from 'fast-glob'; import { describe, expect, test } from 'vitest'; import { coverageManifest, productionGlobs, productionIgnore } from './coverage-manifest'; import { exportScopeCoverageManifest } from './export-scope-coverage-manifest'; +import { workspaceInsightsCoverageManifest } from './workspace-insights-coverage-manifest'; const effectiveCoverageManifest = { ...coverageManifest, ...exportScopeCoverageManifest, + ...workspaceInsightsCoverageManifest, }; describe('coverage manifest', () => { diff --git a/tests/manifest/workspace-insights-coverage-manifest.ts b/tests/manifest/workspace-insights-coverage-manifest.ts new file mode 100644 index 00000000..5d774f5f --- /dev/null +++ b/tests/manifest/workspace-insights-coverage-manifest.ts @@ -0,0 +1,43 @@ +export const workspaceInsightsCoverageManifest: Record = { + 'chromium-xtension/src/chrome-host-insights.ts': ['tests/unit/chromium/chrome-host-insights.test.ts'], + 'chromium-xtension/src/insights-host-router.ts': ['tests/unit/chromium/chrome-host-insights.test.ts'], + 'electron/core/runtime-insights-external.js': ['tests/unit/electron/insights-external.test.ts'], + 'electron/core/runtime-insights.js': ['tests/unit/electron/runtime-insights.test.ts'], + 'ui/src/components/Insights/DuplicatesView.tsx': ['tests/unit/ui/components/insights-lint-duplicates.test.tsx', 'tests/unit/ui/components/insights-settings-integration.test.tsx'], + 'ui/src/components/Insights/GalleryView.tsx': ['tests/unit/ui/components/insights-gallery-links.test.tsx'], + 'ui/src/components/Insights/GraphView.tsx': ['tests/unit/ui/components/insights-graph-related-reports.test.tsx'], + 'ui/src/components/Insights/InsightsSettings.tsx': ['tests/unit/ui/components/workspace-insights-panel.test.tsx', 'tests/unit/ui/components/insights-settings-integration.test.tsx'], + 'ui/src/components/Insights/LinksView.tsx': ['tests/unit/ui/components/insights-gallery-links.test.tsx', 'tests/unit/ui/components/insights-graph-related-reports.test.tsx'], + 'ui/src/components/Insights/LintView.tsx': ['tests/unit/ui/components/insights-lint-duplicates.test.tsx', 'tests/unit/ui/components/insights-settings-integration.test.tsx'], + 'ui/src/components/Insights/RelatedView.tsx': ['tests/unit/ui/components/insights-graph-related-reports.test.tsx'], + 'ui/src/components/Insights/WorkspaceInsightsEntry.tsx': ['tests/unit/ui/components/workspace-insights-panel.test.tsx'], + 'ui/src/components/Insights/WorkspaceInsightsPanel.tsx': ['tests/unit/ui/components/workspace-insights-panel.test.tsx', 'tests/unit/ui/components/insights-settings-integration.test.tsx'], + 'ui/src/contexts/auditedUiTranslationsBase.ts': ['tests/contracts/translations-coverage.test.ts'], + 'ui/src/contexts/insightsTranslations.ts': ['tests/unit/ui/contexts/insights-translations.test.ts'], + 'ui/src/insights/analyzeDocument.ts': ['tests/unit/ui/insights/analyze-document.test.ts'], + 'ui/src/insights/cache.ts': ['tests/unit/ui/insights/cache.test.ts'], + 'ui/src/insights/config.ts': ['tests/unit/ui/insights/contracts.test.ts', 'tests/unit/ui/insights/settings-store.test.ts'], + 'ui/src/insights/contracts.ts': ['tests/unit/ui/insights/contracts.test.ts'], + 'ui/src/insights/duplicates.ts': ['tests/unit/ui/insights/duplicates.test.ts'], + 'ui/src/insights/graph.ts': ['tests/unit/ui/insights/graph.test.ts'], + 'ui/src/insights/index.ts': ['tests/unit/ui/insights/index.test.ts'], + 'ui/src/insights/insights.worker.ts': ['tests/unit/ui/insights/worker-client.test.ts'], + 'ui/src/insights/lint.ts': ['tests/unit/ui/insights/lint.test.ts'], + 'ui/src/insights/patterns.ts': ['tests/unit/ui/insights/patterns.test.ts'], + 'ui/src/insights/relationships.ts': ['tests/unit/ui/insights/relationships.test.ts'], + 'ui/src/insights/reports.ts': ['tests/unit/ui/components/insights-graph-related-reports.test.tsx'], + 'ui/src/insights/settingsStore.ts': ['tests/unit/ui/insights/settings-store.test.ts'], + 'ui/src/insights/useWorkspaceInsights.ts': ['tests/unit/ui/insights/use-workspace-insights.test.tsx'], + 'ui/src/insights/workerClient.ts': ['tests/unit/ui/insights/worker-client.test.ts'], + 'ui/src/insights/workerProtocol.ts': ['tests/unit/ui/insights/worker-client.test.ts'], + 'ui/src/insights/workspaceIdentity.ts': ['tests/unit/ui/insights/workspace-identity.test.ts'], + 'ui/src/insights/workspaceInsightsSession.ts': ['tests/unit/ui/insights/use-workspace-insights.test.tsx'], + 'ui/src/markdown/anchors.ts': ['tests/unit/ui/markdown/anchors.test.ts'], + 'ui/src/markdown/frontmatter.ts': ['tests/unit/ui/markdown/frontmatter.test.ts'], + 'ui/src/markdown/references.ts': ['tests/unit/ui/markdown/references.test.ts'], + 'ui/src/markdown/transclusion.ts': ['tests/unit/ui/markdown/transclusion.test.ts'], + 'ui/src/markdown/wikiLinks.ts': ['tests/unit/ui/markdown/wiki-links.test.ts'], + 'ui/src/settings/settingsImportExportBase.ts': ['tests/unit/ui/settings-import-export.test.ts'], + 'vscode/src/core/panelInsights.ts': ['tests/unit/vscode/panel-insights.test.ts'], + 'vscode/src/core/panelInsightsExternal.ts': ['tests/unit/vscode/insights-external.test.ts'], +}; diff --git a/tests/unit/chromium/chrome-host-insights.test.ts b/tests/unit/chromium/chrome-host-insights.test.ts new file mode 100644 index 00000000..fbd820e0 --- /dev/null +++ b/tests/unit/chromium/chrome-host-insights.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createChromeInsightsHost } from '../../../chromium-xtension/src/chrome-host-insights'; + +function fileHandle(name: string, text: string, type = 'text/markdown') { + const arrayBuffer = vi.fn(async () => new TextEncoder().encode(text).buffer); + return { + kind: 'file', + name, + getFile: vi.fn(async () => ({ + name, + size: text.length, + type, + lastModified: 123, + text: async () => text, + arrayBuffer, + })), + _arrayBuffer: arrayBuffer, + } as any; +} + +function directoryHandle(name: string, files: Record) { + const entries = Object.entries(files); + return { + kind: 'directory', + name, + async *entries() { + for (const entry of entries) yield entry; + }, + async getFileHandle(fileName: string) { + const handle = files[fileName]; + if (!handle || handle.kind !== 'file') throw new DOMException('missing', 'NotFoundError'); + return handle; + }, + async getDirectoryHandle(dirName: string) { + const handle = files[dirName]; + if (!handle || handle.kind !== 'directory') throw new DOMException('missing', 'NotFoundError'); + return handle; + }, + } as any; +} + +describe('Chromium Insights host', () => { + it('probes metadata without reading binary bytes and reports polling capability', async () => { + const image = fileHandle('a.png', 'PNG', 'image/png'); + const docs = directoryHandle('docs', { 'a.md': fileHandle('a.md', '# A') }); + const img = directoryHandle('img', { 'a.png': image }); + const root = directoryHandle('root', { docs, img }); + const sent: any[] = []; + const host = createChromeInsightsHost({ getActiveHandle: () => root, send: m => sent.push(m) }); + + await host.probeWorkspaceResource({ requestId: 'p', documentPath: 'docs/a.md', resourcePath: '../img/a.png' }); + + expect(image._arrayBuffer).not.toHaveBeenCalled(); + expect(sent.at(-1)).toMatchObject({ status: 'exists', sizeBytes: 3, mimeType: 'image/png' }); + expect(host.capabilities).toMatchObject({ fileChanges: 'polling', externalLinkChecking: false }); + }); + + it('enforces source limits and scans beyond 1000 files', async () => { + const files: Record = {}; + for (let i = 0; i < 1005; i += 1) files[`${i}.md`] = fileHandle(`${i}.md`, `# ${i}`); + files['large.md'] = fileHandle('large.md', 'x'.repeat(2048)); + const root = directoryHandle('root', files); + const sent: any[] = []; + const host = createChromeInsightsHost({ getActiveHandle: () => root, send: m => sent.push(m) }); + + await host.readInsightsDocumentSource({ requestId: 'source', relativePath: 'large.md', softLimitBytes: 1024 }); + expect(sent.at(-1)).toMatchObject({ status: 'too-large' }); + + await host.scanInsightsWorkspace({ requestId: 'scan' }); + const entries = sent.filter(m => m.command === 'insightsScanBatch').flatMap(m => m.entries); + expect(entries.filter((entry: any) => entry.extension === '.md')).toHaveLength(1006); + expect(sent.find(m => m.command === 'insightsScanComplete')).toMatchObject({ truncated: false }); + }); +}); diff --git a/tests/unit/electron/insights-external.test.ts b/tests/unit/electron/insights-external.test.ts new file mode 100644 index 00000000..c8208018 --- /dev/null +++ b/tests/unit/electron/insights-external.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createRequire } from 'node:module'; + +const require = createRequire(import.meta.url); +const { createExternalLinkChecker } = require('../../../electron/core/runtime-insights-external.js'); + +function response(status: number, headers: Record = {}) { + return { status, headers }; +} + +describe('Electron Insights external link checker', () => { + it('refuses a private DNS result before opening a connection', async () => { + const resolveHost = vi.fn(async () => ['127.0.0.1']); + const request = vi.fn(); + const checker = createExternalLinkChecker({ resolveHost, request }); + + const result = await checker.check('http://example.test/', { + requestId: 'private', timeoutMs: 1000, approvedPrivateOrigins: [], + }); + + expect(result).toMatchObject({ + status: 'unchecked', + requiresPrivateOriginConfirmation: true, + privateOrigin: 'http://example.test', + }); + expect(request).not.toHaveBeenCalled(); + }); + + it('pins the validated address and revalidates every redirect target', async () => { + const resolveHost = vi.fn(async (host: string) => host === 'public.test' ? ['203.0.113.10'] : ['10.0.0.8']); + const request = vi.fn(async ({ url, address }: any) => { + expect(address).toBe('203.0.113.10'); + expect(url).toBe('https://public.test/'); + return response(302, { location: 'http://private.test/' }); + }); + const checker = createExternalLinkChecker({ resolveHost, request }); + + const result = await checker.check('https://public.test/', { + requestId: 'redirect', timeoutMs: 1000, approvedPrivateOrigins: [], + }); + + expect(result).toMatchObject({ + status: 'unchecked', + requiresPrivateOriginConfirmation: true, + privateOrigin: 'http://private.test', + }); + expect(resolveHost).toHaveBeenCalledTimes(2); + expect(request).toHaveBeenCalledTimes(1); + }); + + it.each([ + [200, 'reachable'], [302, 'reachable'], [401, 'reachable-auth-required'], [403, 'reachable-auth-required'], + [404, 'broken'], [410, 'broken'], [429, 'rate-limited'], [503, 'server-error'], + ])('classifies HTTP %i as %s', async (status, expected) => { + const checker = createExternalLinkChecker({ + resolveHost: vi.fn(async () => ['203.0.113.10']), + request: vi.fn(async () => response(status as number)), + }); + expect((await checker.check('https://public.test/', { requestId: String(status), timeoutMs: 1000 })).status).toBe(expected); + }); + + it('uses anonymous HEAD and falls back to bounded GET when HEAD is unsupported', async () => { + const request = vi.fn() + .mockResolvedValueOnce(response(405)) + .mockResolvedValueOnce(response(200)); + const checker = createExternalLinkChecker({ + resolveHost: vi.fn(async () => ['203.0.113.10']), + request, + }); + + const result = await checker.check('https://public.test/path', { requestId: 'fallback', timeoutMs: 1000 }); + + expect(result.status).toBe('reachable'); + expect(request).toHaveBeenNthCalledWith(1, expect.objectContaining({ + method: 'HEAD', address: '203.0.113.10', headers: expect.not.objectContaining({ cookie: expect.anything(), authorization: expect.anything() }), + })); + expect(request).toHaveBeenNthCalledWith(2, expect.objectContaining({ method: 'GET', maxBodyBytes: 0 })); + }); + + it('marks HTTPS to HTTP redirects as an insecure downgrade', async () => { + const request = vi.fn() + .mockResolvedValueOnce(response(302, { location: 'http://public.test/final' })) + .mockResolvedValueOnce(response(200)); + const checker = createExternalLinkChecker({ resolveHost: vi.fn(async () => ['203.0.113.10']), request }); + const result = await checker.check('https://public.test/', { requestId: 'downgrade', timeoutMs: 1000 }); + expect(result).toMatchObject({ status: 'reachable', insecureDowngrade: true, finalUrl: 'http://public.test/final' }); + }); +}); diff --git a/tests/unit/electron/runtime-insights.test.ts b/tests/unit/electron/runtime-insights.test.ts new file mode 100644 index 00000000..b2129bed --- /dev/null +++ b/tests/unit/electron/runtime-insights.test.ts @@ -0,0 +1,98 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createRequire } from 'node:module'; +import * as nodeFs from 'node:fs'; +import * as nodePath from 'node:path'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; + +const require = createRequire(import.meta.url); +const { createInsightsWorkspaceHost } = require('../../../electron/core/runtime-insights.js'); + +const roots: string[] = []; +afterEach(() => { + while (roots.length) rmSync(roots.pop()!, { recursive: true, force: true }); +}); + +function makeWorkspace() { + const root = mkdtempSync(nodePath.join(tmpdir(), 'mdn-insights-')); + roots.push(root); + const write = (relativePath: string, data: string) => { + const target = nodePath.join(root, relativePath); + mkdirSync(nodePath.dirname(target), { recursive: true }); + writeFileSync(target, data); + return target; + }; + return { root, write }; +} + +function isSameOrInsidePath(base: string, target: string) { + const rel = nodePath.relative(nodePath.resolve(base), nodePath.resolve(target)); + return rel === '' || (!rel.startsWith('..') && !nodePath.isAbsolute(rel)); +} + +function harness(root: string, fsImpl: typeof nodeFs = nodeFs) { + const sent: any[] = []; + const host = createInsightsWorkspaceHost({ + fs: fsImpl, + pathApi: nodePath, + getWorkspaceBaseDir: () => root, + isSameOrInsidePath, + sendHostMessage: (message: any) => sent.push(message), + }); + return { sent, host }; +} + +describe('Electron Insights workspace host', () => { + it('probes resource metadata without reading binary contents', async () => { + const ws = makeWorkspace(); + const documentPath = ws.write('docs/a.md', '# A'); + ws.write('img/a.png', 'PNG'); + const readFileSync = vi.fn(nodeFs.readFileSync); + const fsImpl = { ...nodeFs, readFileSync } as typeof nodeFs; + const { sent, host } = harness(ws.root, fsImpl); + + await host.probeWorkspaceResource({ + requestId: 'probe-1', + documentPath, + resourcePath: '../img/a.png', + }); + + expect(readFileSync).not.toHaveBeenCalled(); + expect(sent.at(-1)).toMatchObject({ + command: 'workspaceResourceProbeResult', + requestId: 'probe-1', + status: 'exists', + relativePath: 'img/a.png', + kind: 'file', + sizeBytes: 3, + }); + }); + + it('enforces soft and hard Markdown source limits explicitly', async () => { + const ws = makeWorkspace(); + ws.write('ok.md', '# ok'); + ws.write('large.md', 'x'.repeat(2048)); + const { sent, host } = harness(ws.root); + + await host.readInsightsDocumentSource({ requestId: 'ok', relativePath: 'ok.md', softLimitBytes: 1024 }); + await host.readInsightsDocumentSource({ requestId: 'soft', relativePath: 'large.md', softLimitBytes: 1024 }); + await host.readInsightsDocumentSource({ requestId: 'hard', relativePath: 'large.md', softLimitBytes: 4096, hardLimitBytes: 1024 }); + + expect(sent.find(m => m.requestId === 'ok')).toMatchObject({ status: 'ok', source: '# ok' }); + expect(sent.find(m => m.requestId === 'soft')).toMatchObject({ status: 'too-large' }); + expect(sent.find(m => m.requestId === 'hard')).toMatchObject({ status: 'too-large' }); + }); + + it('does not silently stop at 1000 eligible files', async () => { + const ws = makeWorkspace(); + for (let i = 0; i < 1005; i += 1) ws.write(`notes/${i}.md`, `# ${i}`); + const { sent, host } = harness(ws.root); + + await host.scanInsightsWorkspace({ requestId: 'scan-1', userPatterns: [] }); + + const batches = sent.filter(m => m.command === 'insightsScanBatch'); + const complete = sent.find(m => m.command === 'insightsScanComplete'); + expect(batches.flatMap(m => m.entries)).toHaveLength(1005); + expect(complete).toMatchObject({ requestId: 'scan-1', totalEntries: 1005, truncated: false }); + }); +}); diff --git a/tests/unit/ui/components/insights-gallery-links.test.tsx b/tests/unit/ui/components/insights-gallery-links.test.tsx new file mode 100644 index 00000000..eed5ba12 --- /dev/null +++ b/tests/unit/ui/components/insights-gallery-links.test.tsx @@ -0,0 +1,62 @@ +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, it, vi } from 'vitest'; +import { analyzeDocument } from '../../../../ui/src/insights/analyzeDocument'; +import { WorkspaceInsightsIndex } from '../../../../ui/src/insights/index'; +import { GalleryView } from '../../../../ui/src/components/Insights/GalleryView'; +import { LinksView } from '../../../../ui/src/components/Insights/LinksView'; + +function doc(path: string, source: string) { + return analyzeDocument({ path, source, revision: `${path}-1` }); +} + +describe('Insights Gallery and Links views', () => { + it('shows referenced media and Mermaid diagrams, probes local metadata, and never auto-loads remote media', async () => { + const probeResource = vi.fn(async () => ({ status: 'exists' as const, kind: 'file' as const, sizeBytes: 123 })); + const document = doc('docs/guide.md', [ + '# Guide', + '![Local diagram](../assets/diagram.png)', + '![Remote image](https://example.test/remote.png)', + '[Audio](../media/clip.mp3)', + '```mermaid', + 'this is not a diagram declaration', + '```', + ].join('\n')); + + render(); + + expect(screen.getByText('../assets/diagram.png')).toBeVisible(); + expect(screen.getByText(/diagram · invalid/i)).toBeVisible(); + expect(screen.queryByRole('img', { name: /remote image/i })).toBeNull(); + await waitFor(() => expect(probeResource).toHaveBeenCalledWith('docs/guide.md', '../assets/diagram.png')); + expect(probeResource).not.toHaveBeenCalledWith('docs/guide.md', 'https://example.test/remote.png'); + + const user = userEvent.setup(); + await user.click(screen.getByRole('button', { name: /load preview.*remote image/i })); + expect(screen.getByRole('img', { name: /remote image/i })).toHaveAttribute('src', 'https://example.test/remote.png'); + }); + + it('distinguishes missing, invalid anchor, outside, dynamic, and unchecked external links', () => { + const index = new WorkspaceInsightsIndex(); + const source = doc('docs/source.md', [ + '# Source', + '[[Missing]]', + '[[Target#Nope]]', + '[outside](file:///tmp/secret.md)', + '[remote](https://example.test)', + 'Dynamic', + ].join('\n')); + const target = doc('docs/Target.md', '# Target\n## Exists'); + index.applyDocument(source); + index.applyDocument(target); + + render(); + + expect(screen.getByText('missing')).toBeVisible(); + expect(screen.getByText('invalid-anchor')).toBeVisible(); + expect(screen.getByText('outside-workspace')).toBeVisible(); + expect(screen.getByText('dynamic')).toBeVisible(); + expect(screen.getByText('unchecked')).toBeVisible(); + expect(screen.getByTestId('broken-link-count')).toHaveTextContent('3'); + }); +}); diff --git a/tests/unit/ui/components/insights-graph-related-reports.test.tsx b/tests/unit/ui/components/insights-graph-related-reports.test.tsx new file mode 100644 index 00000000..a8a3ca4c --- /dev/null +++ b/tests/unit/ui/components/insights-graph-related-reports.test.tsx @@ -0,0 +1,73 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, it, vi } from 'vitest'; +import { analyzeDocument } from '../../../../ui/src/insights/analyzeDocument'; +import { WorkspaceInsightsIndex } from '../../../../ui/src/insights/index'; +import { GraphView } from '../../../../ui/src/components/Insights/GraphView'; +import { RelatedView } from '../../../../ui/src/components/Insights/RelatedView'; +import { LinksView } from '../../../../ui/src/components/Insights/LinksView'; +import { createInsightsJsonReport, createInsightsMarkdownReport } from '../../../../ui/src/insights/reports'; + +function doc(path: string, source: string) { + return analyzeDocument({ path, source, revision: `${path}-1` }); +} + +function fixture() { + const a = doc('a.md', '---\ntags: [docs, api]\n---\n# API Guide\n[[b]]\nAuthentication refresh tokens.'); + const b = doc('b.md', '---\ntags: [docs, api]\n---\n# API Reference\nAuthentication refresh tokens and sessions.'); + const c = doc('c.md', '# Unrelated\nCompletely different material.'); + const index = new WorkspaceInsightsIndex(); + index.applyDocument(a); index.applyDocument(b); index.applyDocument(c); + return { documents: [a, b, c], snapshot: index.snapshot() }; +} + +describe('Insights Graph, Related, external checks, and reports', () => { + it('keeps SVG graph selection synchronized with the accessible list', async () => { + const { snapshot } = fixture(); + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole('button', { name: /api reference/i })); + expect(screen.getByRole('button', { name: /api reference/i })).toHaveAttribute('aria-current', 'true'); + expect(screen.getByTestId('graph-node-b.md')).toHaveAttribute('data-selected', 'true'); + }); + + it('ranks related documents with explainable evidence', () => { + const { documents } = fixture(); + render(); + expect(screen.getByText('API Reference')).toBeVisible(); + expect(screen.getByText(/shared tags.*api.*docs/i)).toBeVisible(); + expect(screen.getByText(/score/i)).toBeVisible(); + }); + + it('checks unique external URLs only after explicit user action', async () => { + const remote = doc('remote.md', '# Remote\n[x](https://example.test/path) [again](https://example.test/path)'); + const index = new WorkspaceInsightsIndex(); index.applyDocument(remote); + const onCheckExternalLinks = vi.fn(async () => []); + const user = userEvent.setup(); + render(); + expect(onCheckExternalLinks).not.toHaveBeenCalled(); + await user.click(screen.getByRole('button', { name: /check external links/i })); + expect(onCheckExternalLinks).toHaveBeenCalledWith(['https://example.test/path']); + }); + + it('exports scoped privacy-safe Markdown and JSON snapshots with completeness metadata', () => { + const { snapshot } = fixture(); + const jsonText = createInsightsJsonReport(snapshot, { + scope: { kind: 'paths', paths: ['a.md'] }, + completeness: { provisional: true, warnings: ['b.md unreadable'] }, + }); + const parsed = JSON.parse(jsonText); + expect(parsed.documents.map((item: any) => item.path)).toEqual(['a.md']); + expect(parsed.completeness.provisional).toBe(true); + expect(jsonText).not.toContain('Authentication refresh tokens.'); + + const markdown = createInsightsMarkdownReport(snapshot, { + scope: { kind: 'paths', paths: ['a.md'] }, + completeness: { provisional: true, warnings: ['b.md unreadable'] }, + }); + expect(markdown).toContain('Workspace Insights'); + expect(markdown).toContain('Provisional'); + expect(markdown).toContain('a.md'); + expect(markdown).not.toContain('Authentication refresh tokens.'); + }); +}); diff --git a/tests/unit/ui/components/insights-lint-duplicates.test.tsx b/tests/unit/ui/components/insights-lint-duplicates.test.tsx new file mode 100644 index 00000000..7dc30d37 --- /dev/null +++ b/tests/unit/ui/components/insights-lint-duplicates.test.tsx @@ -0,0 +1,41 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, it } from 'vitest'; +import { analyzeDocument } from '../../../../ui/src/insights/analyzeDocument'; +import { LintView } from '../../../../ui/src/components/Insights/LintView'; +import { DuplicatesView } from '../../../../ui/src/components/Insights/DuplicatesView'; +import { INSIGHTS_TRANSLATIONS } from '../../../../ui/src/contexts/insightsTranslations'; + +function doc(path: string, source: string) { + return analyzeDocument({ path, source, revision: `${path}-1` }); +} + +describe('Insights Lint and Duplicates views', () => { + it('filters lint findings and supports reversible finding suppression', async () => { + const document = doc('guide.md', '# A \n### C\n# A\n'); + const user = userEvent.setup(); + render(); + + expect(screen.getByText(/heading level jumps/i)).toBeVisible(); + await user.click(screen.getAllByRole('button', { name: INSIGHTS_TRANSLATIONS.en.suppress })[0]); + expect(screen.getByRole('button', { name: INSIGHTS_TRANSLATIONS.en.showSuppressed })).toBeVisible(); + await user.click(screen.getByRole('button', { name: INSIGHTS_TRANSLATIONS.en.showSuppressed })); + expect(screen.getByText(INSIGHTS_TRANSLATIONS.en.suppressed)).toBeVisible(); + }); + + it('shows exact and near duplicate groups and reversible suppression', async () => { + const exactA = doc('a.md', '# Same\nUseful duplicate body with enough repeated terminology here.\n'); + const exactB = doc('b.md', '# Same \r\nUseful duplicate body with enough repeated terminology here.'); + const nearA = doc('near-a.md', '# Auth\nRefresh token rotation access token session authentication policy behavior.'); + const nearB = doc('near-b.md', '# Auth guide\nRefresh token rotation access token session authentication policy behavior details.'); + const user = userEvent.setup(); + + render(); + expect(screen.getByText(INSIGHTS_TRANSLATIONS.en.exactDuplicate)).toBeVisible(); + expect(screen.getByText(INSIGHTS_TRANSLATIONS.en.nearDuplicate)).toBeVisible(); + await user.click(screen.getAllByRole('button', { name: `${INSIGHTS_TRANSLATIONS.en.suppress} ${INSIGHTS_TRANSLATIONS.en.duplicates}` })[0]); + expect(screen.getByRole('button', { name: INSIGHTS_TRANSLATIONS.en.showSuppressed })).toBeVisible(); + await user.click(screen.getByRole('button', { name: INSIGHTS_TRANSLATIONS.en.showSuppressed })); + expect(screen.getByText(INSIGHTS_TRANSLATIONS.en.suppressed)).toBeVisible(); + }); +}); diff --git a/tests/unit/ui/components/insights-settings-integration.test.tsx b/tests/unit/ui/components/insights-settings-integration.test.tsx new file mode 100644 index 00000000..379d07d1 --- /dev/null +++ b/tests/unit/ui/components/insights-settings-integration.test.tsx @@ -0,0 +1,141 @@ +import { fireEvent, render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, it, vi } from 'vitest'; +import { WorkspaceInsightsPanel } from '../../../../ui/src/components/Insights/WorkspaceInsightsPanel'; +import { DuplicatesView } from '../../../../ui/src/components/Insights/DuplicatesView'; +import { LintView } from '../../../../ui/src/components/Insights/LintView'; +import { INSIGHTS_TRANSLATIONS } from '../../../../ui/src/contexts/insightsTranslations'; +import { analyzeDocument } from '../../../../ui/src/insights/analyzeDocument'; +import { normalizeInsightsSettings } from '../../../../ui/src/insights/config'; +import type { WorkspaceInsightsSessionViewModel } from '../../../../ui/src/insights/useWorkspaceInsights'; + +const MIB = 1024 * 1024; + +function session(): WorkspaceInsightsSessionViewModel { + return { + panelOpen: true, + status: 'ready', + snapshot: { documents: new Map(), outboundLinks: new Map(), backlinks: new Map(), brokenLinks: [], tags: new Map(), headings: new Map(), titles: new Map(), revision: 1 }, + progress: { completed: 0, total: 0, provisional: false }, + warnings: [], workerMode: 'worker', externalResults: new Map(), approvedPrivateOrigins: new Set(), + open: vi.fn(async () => {}), closePanel: vi.fn(), refreshLocal: vi.fn(async () => {}), pause: vi.fn(), dispose: vi.fn(), + applyActiveOverlay: vi.fn(async () => {}), clearActiveOverlay: vi.fn(async () => {}), checkExternalLinks: vi.fn(async () => []), + cancelExternalChecks: vi.fn(), approvePrivateOrigin: vi.fn(), getWikiResolverContext: vi.fn(async sourceDocumentPath => ({ sourceDocumentPath, documents: [] })), + }; +} + +function doc(path: string, source: string) { + return analyzeDocument({ path, source, revision: `${path}-1` }); +} + +describe('Workspace Insights persisted settings UI', () => { + it('drives panel tools from resolved settings and exposes editable controls', async () => { + const user = userEvent.setup(); + const settings = normalizeInsightsSettings({ externalLinks: { enabled: true, timeoutMs: 12_000 }, nearDuplicateThreshold: 0.94, graphNodeCap: 175 }); + const onSettingsChange = vi.fn(); + render(); + + await user.click(screen.getByRole('button', { name: `${INSIGHTS_TRANSLATIONS.en.entry} ${INSIGHTS_TRANSLATIONS.en.settings}` })); + expect(screen.getByRole('checkbox', { name: INSIGHTS_TRANSLATIONS.en.externalLinksLabel })).toBeChecked(); + expect(screen.getByRole('spinbutton', { name: INSIGHTS_TRANSLATIONS.en.externalTimeout })).toHaveValue(12000); + expect(screen.getByRole('spinbutton', { name: /near-duplicate threshold/i })).toHaveValue(94); + + await user.click(screen.getByRole('tab', { name: INSIGHTS_TRANSLATIONS.en.duplicates })); + expect(screen.getByText(/near-duplicate threshold: 94%/i)).toBeVisible(); + }); + + it('edits global defaults separately from workspace overrides', async () => { + const user = userEvent.setup(); + const globalSettings = normalizeInsightsSettings({ externalLinks: { enabled: false, timeoutMs: 10_000 }, graphNodeCap: 80 }); + const workspaceSettings = normalizeInsightsSettings({ externalLinks: { enabled: true, timeoutMs: 12_000 }, graphNodeCap: 175 }); + const onGlobalSettingsChange = vi.fn(); + const onSettingsChange = vi.fn(); + + render( + , + ); + + await user.click(screen.getByRole('button', { name: `${INSIGHTS_TRANSLATIONS.en.entry} ${INSIGHTS_TRANSLATIONS.en.settings}` })); + const scope = screen.getByRole('combobox', { name: INSIGHTS_TRANSLATIONS.en.settingsScope }); + expect(scope).toHaveValue('workspace'); + expect(screen.getByRole('checkbox', { name: INSIGHTS_TRANSLATIONS.en.externalLinksLabel })).toBeChecked(); + + await user.selectOptions(scope, 'global'); + expect(screen.getByRole('checkbox', { name: INSIGHTS_TRANSLATIONS.en.externalLinksLabel })).not.toBeChecked(); + expect(screen.getByRole('spinbutton', { name: INSIGHTS_TRANSLATIONS.en.graphNodeCap })).toHaveValue(80); + await user.click(screen.getByRole('checkbox', { name: INSIGHTS_TRANSLATIONS.en.externalLinksLabel })); + expect(onGlobalSettingsChange).toHaveBeenCalledWith(expect.objectContaining({ externalLinks: { enabled: true } })); + expect(onSettingsChange).not.toHaveBeenCalled(); + }); + + it('exposes custom relationship weights, lint rule severity, and global cache cap', async () => { + const user = userEvent.setup(); + const settings = normalizeInsightsSettings({ + relationshipPreset: 'custom', + relationshipWeights: { links: 40, tags: 20, headings: 15, title: 10, terminology: 15 }, + lintRules: { 'heading/duplicate': { enabled: true, severity: 'error' } }, + }); + const onGlobalSettingsChange = vi.fn(); + const onSettingsChange = vi.fn(); + render( + , + ); + await user.click(screen.getByRole('button', { name: `${INSIGHTS_TRANSLATIONS.en.entry} ${INSIGHTS_TRANSLATIONS.en.settings}` })); + + expect(screen.getByRole('spinbutton', { name: INSIGHTS_TRANSLATIONS.en.directLinks })).toHaveValue(40); + fireEvent.change(screen.getByRole('spinbutton', { name: INSIGHTS_TRANSLATIONS.en.directLinks }), { target: { value: '55' } }); + expect(onSettingsChange).toHaveBeenCalledWith(expect.objectContaining({ relationshipWeights: expect.objectContaining({ links: 55 }) })); + + expect(screen.getByRole('checkbox', { name: `${INSIGHTS_TRANSLATIONS.en.rule}: heading/duplicate` })).toBeChecked(); + expect(screen.getByRole('combobox', { name: `${INSIGHTS_TRANSLATIONS.en.severity}: heading/duplicate` })).toHaveValue('error'); + + await user.selectOptions(screen.getByRole('combobox', { name: INSIGHTS_TRANSLATIONS.en.settingsScope }), 'global'); + expect(screen.getByRole('spinbutton', { name: INSIGHTS_TRANSLATIONS.en.cacheCap })).toHaveValue(500); + fireEvent.change(screen.getByRole('spinbutton', { name: INSIGHTS_TRANSLATIONS.en.cacheCap }), { target: { value: '600' } }); + expect(onGlobalSettingsChange).toHaveBeenCalledWith(expect.objectContaining({ cacheCapBytes: 600 * MIB })); + }); + + it('rejects invalid include/exclude patterns with a visible localized error', async () => { + const user = userEvent.setup(); + const onSettingsChange = vi.fn(); + render(); + await user.click(screen.getByRole('button', { name: `${INSIGHTS_TRANSLATIONS.en.entry} ${INSIGHTS_TRANSLATIONS.en.settings}` })); + const patterns = screen.getByRole('textbox', { name: INSIGHTS_TRANSLATIONS.en.includeExcludePatterns }); + fireEvent.change(patterns, { target: { value: 'docs/[' } }); + expect(screen.getByRole('alert')).toHaveTextContent(INSIGHTS_TRANSLATIONS.en.error); + expect(onSettingsChange).not.toHaveBeenCalledWith(expect.objectContaining({ userPatterns: ['docs/['] })); + }); + + it('reports duplicate and lint suppression changes to persisted settings callbacks', async () => { + const user = userEvent.setup(); + const duplicateChange = vi.fn(); + const lintChange = vi.fn(); + const exactA = doc('a.md', '# Same\nUseful duplicate body with enough repeated terminology here.\n'); + const exactB = doc('b.md', '# Same \r\nUseful duplicate body with enough repeated terminology here.'); + const lintDoc = doc('guide.md', '# A \n### C\n# A\n'); + + const { unmount } = render(); + await user.click(screen.getByRole('button', { name: `${INSIGHTS_TRANSLATIONS.en.suppress} ${INSIGHTS_TRANSLATIONS.en.duplicates}` })); + expect(duplicateChange).toHaveBeenCalledWith(expect.arrayContaining([expect.stringMatching(/^exact:/)])); + unmount(); + + render(); + await user.click(screen.getAllByRole('button', { name: INSIGHTS_TRANSLATIONS.en.suppress })[0]); + expect(lintChange).toHaveBeenCalledWith(expect.arrayContaining([expect.objectContaining({ scope: 'finding' })])); + }); +}); diff --git a/tests/unit/ui/components/scope-view-modal.test.tsx b/tests/unit/ui/components/scope-view-modal.test.tsx index e74c1816..49b885d3 100644 --- a/tests/unit/ui/components/scope-view-modal.test.tsx +++ b/tests/unit/ui/components/scope-view-modal.test.tsx @@ -179,9 +179,10 @@ describe('ScopeViewModal', () => { it('offers Open as scope when an internal link is right-clicked inside Scope View', async () => { render( {}} />); - await screen.findByText('Doc 1'); + const link = await screen.findByRole('link', { name: 'Open 2' }); + await waitFor(() => expect(mocks.scheduleEnhancements).toHaveBeenCalled()); - fireEvent.contextMenu(screen.getByText('Open 2'), { clientX: 24, clientY: 30 }); + fireEvent.contextMenu(link, { clientX: 24, clientY: 30 }); expect(await screen.findByText('Open as scope')).toBeTruthy(); fireEvent.click(screen.getByText('Open as scope')); await screen.findByText('Doc 2'); @@ -234,11 +235,12 @@ describe('ScopeViewModal', () => { it('renders Scope View navigation and link actions in the selected application language', async () => { mocks.appState.settings.language = 'vi'; render( {}} />); - await screen.findByText('Doc 1'); + const link = await screen.findByRole('link', { name: 'Open 2' }); + await waitFor(() => expect(mocks.scheduleEnhancements).toHaveBeenCalled()); expect(screen.getByRole('dialog', { name: 'Chế độ xem phạm vi' })).toBeTruthy(); expect(screen.getByLabelText('Phạm vi trước')).toBeTruthy(); - fireEvent.contextMenu(screen.getByText('Open 2'), { clientX: 24, clientY: 30 }); + fireEvent.contextMenu(link, { clientX: 24, clientY: 30 }); expect(await screen.findByText('Mở dưới dạng phạm vi')).toBeTruthy(); }); @@ -319,8 +321,9 @@ describe('ScopeViewModal', () => { window.addEventListener(ACTION_NOTICE_EVENT, onNotice); try { const { container } = render( {}} />); - await screen.findByText('Open self'); - fireEvent.click(screen.getByText('Open self')); + const link = await screen.findByRole('link', { name: 'Open self' }); + await waitFor(() => expect(mocks.scheduleEnhancements).toHaveBeenCalled()); + fireEvent.click(link); await waitFor(() => expect(notices).toContain('You are currently viewing this document.')); expect(mocks.loadDocumentSnapshot).toHaveBeenCalledTimes(1); diff --git a/tests/unit/ui/components/sidebar-render.test.tsx b/tests/unit/ui/components/sidebar-render.test.tsx index 03b739b1..009bc8e6 100644 --- a/tests/unit/ui/components/sidebar-render.test.tsx +++ b/tests/unit/ui/components/sidebar-render.test.tsx @@ -48,6 +48,10 @@ vi.mock('../../../../ui/src/components/Sidebar/SidebarSearch', () => ({ SidebarSearch: ({ isVisible }: any) =>
, })); +vi.mock('../../../../ui/src/components/Insights/WorkspaceInsightsEntry', () => ({ + WorkspaceInsightsEntry: () => } + {!visible.length &&
{labels.noDuplicateGroups}
} +
    + {visible.map(group => { + const hidden = suppressed.has(group.key); + return ( +
  • +
    {labelFor(group, labels)}{group.paths.join(' · ')}{group.score !== undefined ? ` · ${Math.round(group.score * 100)}%` : ''}{hidden && {labels.suppressed}}
    + {hidden ? ( + + ) : ( + + )} +
    • + ); + })} +
+
+ ); +} diff --git a/ui/src/components/Insights/GalleryView.tsx b/ui/src/components/Insights/GalleryView.tsx new file mode 100644 index 00000000..75da791c --- /dev/null +++ b/ui/src/components/Insights/GalleryView.tsx @@ -0,0 +1,120 @@ +import { useEffect, useMemo, useState } from 'react'; +import { INSIGHTS_TRANSLATIONS, type InsightsTranslations } from '../../contexts/insightsTranslations'; +import type { AnalyzedDocument } from '../../insights/analyzeDocument'; +import type { WorkspaceResourceProbeResult } from '../../insights/contracts'; + +export type GalleryCategory = 'image' | 'diagram' | 'video' | 'audio' | 'document'; + +interface GalleryItem { + readonly key: string; + readonly documentPath: string; + readonly target: string; + readonly label: string; + readonly category: GalleryCategory; + readonly remote: boolean; + readonly status?: 'valid' | 'invalid'; +} + +export interface GalleryViewProps { + readonly documents: readonly AnalyzedDocument[]; + readonly labels?: InsightsTranslations; + readonly probeResource?: (documentPath: string, resourcePath: string) => Promise; +} + +const IMAGE = /\.(?:png|jpe?g|gif|webp|svg|avif|bmp)$/i; +const VIDEO = /\.(?:mp4|webm|mov|m4v|ogv)$/i; +const AUDIO = /\.(?:mp3|wav|ogg|m4a|aac|flac|opus)$/i; +const DOCUMENT = /\.(?:pdf|docx?|xlsx?|pptx?|html?|rtf)$/i; + +function categoryFor(target: string): GalleryCategory | null { + const path = target.split(/[?#]/, 1)[0]; + if (IMAGE.test(path)) return 'image'; + if (VIDEO.test(path)) return 'video'; + if (AUDIO.test(path)) return 'audio'; + if (DOCUMENT.test(path)) return 'document'; + return null; +} + +function collectItems(documents: readonly AnalyzedDocument[]): GalleryItem[] { + const items: GalleryItem[] = []; + for (const document of documents) { + for (const reference of document.references) { + const category = categoryFor(reference.target); + if (!category) continue; + items.push({ + key: `${document.path}:${reference.sourceStart}:${reference.target}`, + documentPath: document.path, + target: reference.target, + label: reference.label || reference.target.split('/').pop() || reference.target, + category, + remote: reference.remote, + }); + } + for (const diagram of document.diagrams) { + items.push({ + key: `${document.path}:mermaid:${diagram.sourceStart}`, + documentPath: document.path, + target: `Mermaid · ${diagram.sourceStart}`, + label: `${document.path} · Mermaid`, + category: 'diagram', + remote: false, + status: diagram.status, + }); + } + } + return items; +} + +export function GalleryView({ documents, labels = INSIGHTS_TRANSLATIONS.en, probeResource }: GalleryViewProps) { + const items = useMemo(() => collectItems(documents), [documents]); + const [probes, setProbes] = useState>(() => new Map()); + const [loadedRemote, setLoadedRemote] = useState>(() => new Set()); + + useEffect(() => { + if (!probeResource) return; + let cancelled = false; + for (const item of items) { + if (item.remote || item.category === 'diagram') continue; + void probeResource(item.documentPath, item.target).then(result => { + if (cancelled) return; + setProbes(current => new Map(current).set(item.key, result)); + }); + } + return () => { cancelled = true; }; + }, [items, probeResource]); + + if (!items.length) return
{labels.noMedia}
; + + return ( +
+ {items.map(item => { + const loaded = loadedRemote.has(item.key); + const probe = probes.get(item.key); + return ( +
+
+ {item.category === 'diagram' ? `${item.category} · ${item.status ?? ''}` : item.category} + {!item.remote && item.category !== 'diagram' && {probe?.status ?? '…'}} + {item.remote && {labels.remoteMedia}} +
+
{item.label}
+
{item.target}
+ {item.remote && !loaded && ( + + )} + {item.remote && loaded && item.category === 'image' && {item.label}} + {item.remote && loaded && item.category === 'video' &&
+ ); + })} +
+ ); +} diff --git a/ui/src/components/Insights/GraphView.tsx b/ui/src/components/Insights/GraphView.tsx new file mode 100644 index 00000000..d4eebc24 --- /dev/null +++ b/ui/src/components/Insights/GraphView.tsx @@ -0,0 +1,134 @@ +import { useMemo, useState } from 'react'; +import { INSIGHTS_TRANSLATIONS, type InsightsTranslations } from '../../contexts/insightsTranslations'; +import { buildFocusedGraph, type FocusedGraphNode } from '../../insights/graph'; +import type { WorkspaceInsightsSnapshot } from '../../insights/index'; + +export interface GraphViewProps { + readonly snapshot: WorkspaceInsightsSnapshot; + readonly labels?: InsightsTranslations; + readonly nodeCap?: number; + readonly centerPath?: string; + readonly includeInferred?: boolean; + readonly showTags?: boolean; + readonly showHeadings?: boolean; + readonly onSelectPath?: (path: string) => void; +} + +function nodeRadius(node: FocusedGraphNode): number { + if (node.kind === 'document') return 18; + return node.kind === 'tag' ? 13 : 11; +} + +function format(value: string, key: string, replacement: string | number): string { + return value.replace(`{${key}}`, String(replacement)); +} + +export function GraphView({ + snapshot, + labels = INSIGHTS_TRANSLATIONS.en, + nodeCap = 100, + centerPath, + includeInferred = false, + showTags = false, + showHeadings = false, + onSelectPath, +}: GraphViewProps) { + const graph = useMemo(() => buildFocusedGraph(snapshot, { + centerPath, + nodeCap, + includeInferred, + showTags, + showHeadings, + }), [centerPath, includeInferred, nodeCap, showHeadings, showTags, snapshot]); + const documentNodes = useMemo( + () => graph.nodes.filter((node): node is FocusedGraphNode & { kind: 'document' } => node.kind === 'document'), + [graph.nodes], + ); + const [selectedId, setSelectedId] = useState(() => graph.centerPath ?? documentNodes[0]?.id); + const selected = graph.nodes.some(node => node.id === selectedId) + ? selectedId + : graph.centerPath ?? documentNodes[0]?.id; + + const select = (node: FocusedGraphNode) => { + setSelectedId(node.id); + if (node.kind === 'document') onSelectPath?.(node.id); + }; + + if (!graph.nodes.length) return
{labels.noGraphConnections}
; + + const xs = graph.nodes.map(node => node.x); + const ys = graph.nodes.map(node => node.y); + const minX = Math.min(...xs, -180) - 70; + const maxX = Math.max(...xs, 180) + 70; + const minY = Math.min(...ys, -180) - 70; + const maxY = Math.max(...ys, 180) + 70; + + return ( +
+
+ {format(labels.documentsShown, 'count', documentNodes.length)} + {graph.hiddenCount > 0 ? ` · ${format(labels.hiddenByNodeCap, 'count', graph.hiddenCount)}` : ''} + {includeInferred ? ` · ${labels.inferredShown}` : ` · ${labels.explicitOnly}`} +
+ + + + {graph.nodes.map(node => { + const isSelected = selected === node.id; + return ( + select(node)} + > + + {node.label} + + ); + })} + + + +
+ {documentNodes.map(node => ( + + ))} +
+
+ ); +} diff --git a/ui/src/components/Insights/InsightsSettings.tsx b/ui/src/components/Insights/InsightsSettings.tsx new file mode 100644 index 00000000..ade67808 --- /dev/null +++ b/ui/src/components/Insights/InsightsSettings.tsx @@ -0,0 +1,287 @@ +import { useEffect, useMemo, useState } from 'react'; +import { INSIGHTS_TRANSLATIONS, type InsightsTranslations } from '../../contexts/insightsTranslations'; +import type { + InsightsRelationshipWeights, + InsightsSettings, + InsightsSettingsInput, + InsightsWorkspaceOverrides, +} from '../../insights/config'; +import { INSIGHTS_LINT_RULE_DEFAULTS } from '../../insights/lint'; +import { createInsightsPathMatcher } from '../../insights/patterns'; +import type { WorkspaceInsightsSessionViewModel } from '../../insights/useWorkspaceInsights'; + +export interface InsightsSettingsProps { + readonly session: WorkspaceInsightsSessionViewModel; + readonly labels?: InsightsTranslations; + readonly settings: InsightsSettings; + readonly globalSettings?: InsightsSettings; + readonly onGlobalSettingsChange?: (patch: InsightsSettingsInput) => void; + readonly onSettingsChange?: (patch: InsightsWorkspaceOverrides) => void; + readonly onResetWorkspaceOverrides?: () => void; +} + +function numericValue(value: string, fallback: number): number { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +function controlLabel(value: string): string { + return value.replace(/\s*[:：]?\s*\{[^}]+\}%?\s*$/, '').trim(); +} + +function patternLines(value: string): string[] { + return value.split(/\r?\n/).map(item => item.trim()).filter(Boolean); +} + +export function InsightsSettings({ + session, + labels = INSIGHTS_TRANSLATIONS.en, + settings, + globalSettings, + onGlobalSettingsChange, + onSettingsChange, + onResetWorkspaceOverrides, +}: InsightsSettingsProps) { + const canEditGlobal = Boolean(globalSettings && onGlobalSettingsChange); + const [scope, setScope] = useState<'workspace' | 'global'>('workspace'); + const effectiveScope = scope === 'global' && canEditGlobal ? 'global' : 'workspace'; + const effectiveSettings = effectiveScope === 'global' ? globalSettings ?? settings : settings; + const changeSettings = useMemo(() => ( + effectiveScope === 'global' + ? (patch: InsightsSettingsInput) => onGlobalSettingsChange?.(patch) + : (patch: InsightsWorkspaceOverrides) => onSettingsChange?.(patch) + ), [effectiveScope, onGlobalSettingsChange, onSettingsChange]); + const nearDuplicateLabel = controlLabel(labels.nearDuplicateThreshold); + const [userPatternsDraft, setUserPatternsDraft] = useState(() => effectiveSettings.userPatterns.join('\n')); + const [oversizedPatternsDraft, setOversizedPatternsDraft] = useState(() => effectiveSettings.oversizedPatterns.join('\n')); + const [patternError, setPatternError] = useState(null); + + useEffect(() => { + setUserPatternsDraft(effectiveSettings.userPatterns.join('\n')); + setOversizedPatternsDraft(effectiveSettings.oversizedPatterns.join('\n')); + setPatternError(null); + }, [effectiveScope, effectiveSettings.oversizedPatterns, effectiveSettings.userPatterns]); + + const updatePatterns = (kind: 'userPatterns' | 'oversizedPatterns', value: string) => { + if (kind === 'userPatterns') setUserPatternsDraft(value); + else setOversizedPatternsDraft(value); + const patterns = patternLines(value); + try { + createInsightsPathMatcher({ userPatterns: patterns }); + setPatternError(null); + changeSettings({ [kind]: patterns }); + } catch { + const label = kind === 'userPatterns' ? labels.includeExcludePatterns : labels.oversizedPatterns; + setPatternError(`${labels.error}: ${label}`); + } + }; + + const updateRelationshipWeight = (key: keyof InsightsRelationshipWeights, value: string) => { + const current = effectiveSettings.relationshipWeights[key]; + changeSettings({ + relationshipWeights: { + ...effectiveSettings.relationshipWeights, + [key]: numericValue(value, current), + }, + }); + }; + + const updateLintRule = ( + ruleId: string, + patch: Partial<{ enabled: boolean; severity: 'info' | 'warning' | 'error' }>, + ) => { + const current = effectiveSettings.lintRules[ruleId] ?? { + enabled: true, + severity: INSIGHTS_LINT_RULE_DEFAULTS[ruleId] ?? 'warning', + }; + changeSettings({ + lintRules: { + ...effectiveSettings.lintRules, + [ruleId]: { ...current, ...patch }, + }, + }); + }; + + return ( +
+
{labels.settings}
+

{labels.externalLinksDescription}

+
+ {canEditGlobal && ( + + )} + + + + + + {effectiveScope === 'global' && ( + + )} + + {effectiveSettings.relationshipPreset === 'custom' && ( +
+ {([ + ['links', labels.directLinks], + ['tags', labels.sharedTags], + ['headings', labels.sharedHeadings], + ['title', labels.sharedTitleTerms], + ['terminology', labels.sharedTerminology], + ] as const).map(([key, label]) => ( + + ))} +
+ )} +