From 121be5833ff5c21c3407e8aa4638b233cb72fb90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki?= Date: Wed, 10 Jun 2026 17:11:39 -0300 Subject: [PATCH] =?UTF-8?q?feat(vision):=20Tier-2=20AI=20screen=20evaluati?= =?UTF-8?q?on=20=E2=80=94=20deepens=20the=20score=20to=20core+vision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the proprietary AI vision Tier-2: a vision model evaluates each screen and emits graded Metric[] for 6 subjective dimensions (clutter, saliency, feedback, consistency, affordance, guidance) that feed the 0-100 score and flip it to `core+vision`. The free deterministic Tier-1 score is unchanged. - core: 4 new Tier-2 dimensions across the 5 catalog sites (types, dimensions x3, metric DIMENSION_ISSUE_META) + free-score invariant test (core-only score is independent of Tier-2 catalog size). - vision: provider abstraction (providers/: VisionProvider + OpenAIVisionProvider + Claude stub + selectProvider). Holistic prompt returns {findings, dimensions} (temperature 0 + JSON mode). Per-dimension score 0-100 or null=na (model abstains when it cannot judge a dim from one static screenshot). Defensive parse (sanitizeVisionResponse) drops malformed items. analyze* return VisionResult {issues, metrics, coverage}. COST_PER_SCREEN 0.005 -> 0.009. - cli: --vision implies --score, --provider flag, wires additionalMetrics. - mcp: vision arg (Pro-gated like simulate) + additionalMetrics + force score. - server proxy (apps/web/api/vision.ts): holistic prompt + {findings,dimensions} (additive — old clients read .findings; old servers degrade to metrics:[]). - reporters: radar <3-dim fallback; "core deterministic vs vision AI-assessed (may vary)" note in terminal + HTML. 100 tests (93 core + 7 vision), tsc strict clean across core/vision/cli/mcp. EDET deterministic score byte-identical (70/100, 8 dims) — no free-tier regression. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/index.ts | 26 ++- apps/web/api/vision.ts | 58 ++++-- packages/core/src/metrics/metric.ts | 4 + packages/core/src/reporters/html.ts | 3 +- packages/core/src/reporters/terminal.ts | 1 + packages/core/src/scoring/dimensions.ts | 24 ++- packages/core/src/scoring/score.test.ts | 24 ++- packages/core/src/types.ts | 6 +- packages/mcp/src/index.ts | 34 +++- packages/vision/src/analyzer.test.ts | 96 +++++++--- packages/vision/src/analyzer.ts | 221 +++++++++------------- packages/vision/src/index.ts | 26 ++- packages/vision/src/prompts.ts | 139 +++++++++++--- packages/vision/src/providers/claude.ts | 16 ++ packages/vision/src/providers/index.ts | 17 ++ packages/vision/src/providers/openai.ts | 35 ++++ packages/vision/src/providers/provider.ts | 17 ++ 17 files changed, 534 insertions(+), 213 deletions(-) create mode 100644 packages/vision/src/providers/claude.ts create mode 100644 packages/vision/src/providers/index.ts create mode 100644 packages/vision/src/providers/openai.ts create mode 100644 packages/vision/src/providers/provider.ts diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index ab1aee7..4fd380e 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node import { program } from 'commander'; -import { FigmaClient, FigmaApiError, scan, formatJson, formatTerminal, formatHtml, buildGraph, validateLicenseKey, type Issue } from '@protoscan/core'; +import { FigmaClient, FigmaApiError, scan, formatJson, formatTerminal, formatHtml, buildGraph, validateLicenseKey, type Issue, type Metric } from '@protoscan/core'; import { writeFileSync, existsSync } from 'node:fs'; program @@ -35,7 +35,8 @@ program .option('--simulate', 'Run headless Playwright simulator to detect runtime nav failures (slow, requires @protoscan/simulator)') .option('--record [dir]', 'Record simulator walkthrough video (requires --simulate, saves .webm)') .option('--upload', 'Upload HTML report to GitHub Gist and return shareable URL (requires GITHUB_TOKEN)') - .option('--vision', 'Run AI vision analysis on each screen with GPT-4o (requires PROTOSCAN_API_KEY or OPENAI_API_KEY)') + .option('--vision', 'Run AI vision Tier-2 analysis on each screen (requires PROTOSCAN_API_KEY or OPENAI_API_KEY). Deepens the score to core+vision; implies --score') + .option('--provider ', 'Vision provider: openai (default, GPT-4o) | claude (not yet implemented)') // PROTOSCAN_API_KEY via env var only — never pass keys as CLI args (shell history exposure) .option('--max-vision-cost ', 'Maximum USD to spend on vision analysis', '5') .option('--metrics', 'Run ProtoScan Science metrics (WCAG contrast, Hick, Miller, Fitts, typography, readability, Gestalt, emphasis, balance)') @@ -133,11 +134,12 @@ program } let visionIssues: Issue[] = []; + let visionMetrics: Metric[] = []; if (options.vision) { const protoscanKey = process.env.PROTOSCAN_API_KEY; const openaiKey = process.env.OPENAI_API_KEY; const screenCount = graph.nodes.size; - const COST_PER_SCREEN = 0.005; + const COST_PER_SCREEN = 0.009; const estimatedCost = (screenCount * COST_PER_SCREEN).toFixed(2); const maxCost = parseFloat(options.maxVisionCost); const cappedScreens = Math.min(screenCount, Math.floor(maxCost / COST_PER_SCREEN)); @@ -154,14 +156,16 @@ program try { console.error('Running AI vision analysis via ProtoScan API...'); const { analyzeVisionProxy } = await import('@protoscan/vision'); - visionIssues = await analyzeVisionProxy(graph, { + const vr = await analyzeVisionProxy(graph, { protoscanApiKey: protoscanKey, figmaToken: token, fileKey, maxCost, maxScreens: 200, }); - console.error(`Vision: ${visionIssues.length} issue(s) found.`); + visionIssues = vr.issues; + visionMetrics = vr.metrics; + console.error(`Vision: ${vr.issues.length} issue(s), ${vr.metrics.length} metric(s) across ${vr.coverage.analyzed}/${vr.coverage.total} screens.`); } catch (err: unknown) { if (isModuleNotFound(err, '@protoscan/vision')) { console.error('Error: --vision is not available in this distribution.'); @@ -182,14 +186,17 @@ program try { console.error('Running AI vision analysis (this may take a few minutes)...'); const { analyzeVision } = await import('@protoscan/vision'); - visionIssues = await analyzeVision(graph, { + const vr = await analyzeVision(graph, { figmaToken: token, openaiApiKey: openaiKey, fileKey, maxCost, maxScreens: 200, + provider: options.provider as 'openai' | 'claude' | undefined, }); - console.error(`Vision: ${visionIssues.length} issue(s) found.`); + visionIssues = vr.issues; + visionMetrics = vr.metrics; + console.error(`Vision: ${vr.issues.length} issue(s), ${vr.metrics.length} metric(s) across ${vr.coverage.analyzed}/${vr.coverage.total} screens.`); } catch (err: unknown) { if (isModuleNotFound(err, '@protoscan/vision')) { console.error('Error: --vision is not available in this distribution.'); @@ -212,8 +219,9 @@ program skip: options.skip?.split(',').map((s: string) => s.trim()), pageIds: pageIds.length ? pageIds : undefined, additionalIssues: [...simulatorIssues, ...visionIssues], - metrics: !!(options.metrics || options.score || options.ergonomics), - score: !!(options.score || options.ergonomics), + additionalMetrics: visionMetrics, + metrics: !!(options.metrics || options.score || options.ergonomics || options.vision), + score: !!(options.score || options.ergonomics || options.vision), ignorePatterns: options.ignore?.split(',').map((s: string) => s.trim()).filter(Boolean), }); diff --git a/apps/web/api/vision.ts b/apps/web/api/vision.ts index b6f7398..b782796 100644 --- a/apps/web/api/vision.ts +++ b/apps/web/api/vision.ts @@ -8,18 +8,33 @@ const OPENAI_API_KEY = process.env.OPENAI_API_KEY!; const MAX_IMAGE_SIZE = 5 * 1024 * 1024; // 5 MB base64 (~3.75 MB image) -const SYSTEM_PROMPT = `You are a UX visual analyst. Analyze this Figma screen screenshot and identify visual UX issues. +// Keep in sync with packages/vision/src/prompts.ts SYSTEM_PROMPT. +const SYSTEM_PROMPT = `You are a senior UX/UI designer doing a visual QA audit of ONE mobile app prototype screen (a single static screenshot). -Return a JSON array of findings. Each finding has: -- category: "vision-contrast" | "vision-clarity" | "vision-empty-state" | "vision-overload" -- severity: "high" | "medium" | "low" -- message: brief description of the issue -- area: where on the screen (e.g., "top CTA button", "card list area") +Return STRICT JSON (no markdown) shaped exactly: +{ + "findings": [ { "category": "...", "severity": "...", "message": "...", "area": "..." } ], + "dimensions": [ { "dimension": "...", "score": 0-100 | null, "note": "..." } ] +} -Rules: -- Return ONLY a JSON array, no markdown, no explanation -- Maximum 3 findings per screen -- Only flag genuine issues, not style preferences`; +FINDINGS (qualitative issues, max 3, [] if the screen is fine) — categories ONLY: +- vision-contrast: text/important elements with insufficient contrast +- vision-clarity: ambiguous CTAs, unclear labels, confusing hierarchy +- vision-empty-state: empty/broken/incomplete screen +- vision-overload: too many competing elements, no clear focal point +severity: high | medium | low. Only confident issues; no style preferences. + +DIMENSIONS (grade each 0-100, higher = better) — score these 6: +- clutter: low visual density / few competing elements +- saliency: clear focal point / visual priority for the primary action +- feedback: the screen visibly communicates state/result (loading, success, error, selected) +- consistency: repeated elements (buttons, headers, spacing) look uniform +- affordance: interactive elements look tappable; what's actionable is obvious +- guidance: labels/prompts/empty-states orient the user to the next step + +CRITICAL: If you CANNOT fairly judge a dimension from a SINGLE STATIC screenshot +(feedback, affordance and guidance often need interaction), set its "score" to null +instead of guessing. Include all 6 dimension objects; use null where you cannot assess.`; function hashKey(key: string): string { return createHash('sha256').update(key).digest('hex'); @@ -131,28 +146,39 @@ export default async function handler(req: VercelRequest, res: VercelResponse) { const openai = new OpenAI({ apiKey: OPENAI_API_KEY }); const completion = await openai.chat.completions.create({ model: 'gpt-4o', - max_tokens: 500, + max_tokens: 900, + temperature: 0, + response_format: { type: 'json_object' }, messages: [ { role: 'system', content: SYSTEM_PROMPT }, { role: 'user', content: [ - { type: 'text', text: `Analyze this screen: "${screenName ?? 'Unknown'}"` }, + { type: 'text', text: `Audit this screen: "${screenName ?? 'Unknown'}". Return the JSON object.` }, { type: 'image_url', image_url: { url: `data:image/png;base64,${image}`, detail: 'low' } }, ], }, ], }); - const raw = completion.choices[0]?.message?.content ?? '[]'; - let findings: unknown[]; + const raw = completion.choices[0]?.message?.content ?? '{}'; + let findings: unknown[] = []; + let dimensions: unknown[] = []; try { - findings = JSON.parse(raw); + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + findings = parsed; // legacy/defensive: model returned a bare array + } else if (parsed && typeof parsed === 'object') { + findings = Array.isArray(parsed.findings) ? parsed.findings : []; + dimensions = Array.isArray(parsed.dimensions) ? parsed.dimensions : []; + } } catch { findings = []; + dimensions = []; } - return res.status(200).json({ findings }); + // Additive response: old clients read `.findings` and ignore `.dimensions`. + return res.status(200).json({ findings, dimensions }); } catch (error) { // Refund the credit since OpenAI failed — user shouldn't pay for failed analysis await fetch(`${SUPABASE_URL}/rest/v1/rpc/refund_credit`, { diff --git a/packages/core/src/metrics/metric.ts b/packages/core/src/metrics/metric.ts index cbb19df..599bb7a 100644 --- a/packages/core/src/metrics/metric.ts +++ b/packages/core/src/metrics/metric.ts @@ -84,6 +84,10 @@ const DIMENSION_ISSUE_META: Record< balance: { severity: 'low', confidence: 'probable', category: 'balance' }, clutter: { severity: 'low', confidence: 'probable', category: 'vision' }, saliency: { severity: 'low', confidence: 'probable', category: 'vision' }, + feedback: { severity: 'low', confidence: 'probable', category: 'vision' }, + consistency: { severity: 'low', confidence: 'probable', category: 'vision' }, + affordance: { severity: 'low', confidence: 'probable', category: 'vision' }, + guidance: { severity: 'low', confidence: 'probable', category: 'vision' }, }; function round(n: number): number { diff --git a/packages/core/src/reporters/html.ts b/packages/core/src/reporters/html.ts index 44cacbf..76575ac 100644 --- a/packages/core/src/reporters/html.ts +++ b/packages/core/src/reporters/html.ts @@ -292,9 +292,10 @@ function renderScoreHero(result: ScanResult): string {
${esc(bandText)}
${tierText} · coverage ${score.coverage}%
-
${renderRadar(dims)}
+
${renderRadar(dims) || `
Radar needs ≥3 scored dimensions (have ${dims.length}). See the per-dimension bars.
`}
${bars}
+
core = deterministic (WCAG + peer-reviewed HCI science).${score.tier === 'core+vision' ? ' vision = AI-assessed — may vary a few points run-to-run.' : ''}
${fixes ? `

Top fixes → points

    ${fixes}
` : ''}`; } diff --git a/packages/core/src/reporters/terminal.ts b/packages/core/src/reporters/terminal.ts index cdee280..e7896ea 100644 --- a/packages/core/src/reporters/terminal.ts +++ b/packages/core/src/reporters/terminal.ts @@ -113,6 +113,7 @@ function renderScoreBlock(result: ScanResult): string[] { ? `${score.band.emoji} ${score.band.label}` : colors.dim('(alpha — methodology in calibration)'); out.push(colors.bold(` Usability Score: ${score.global}/100 `) + band); + out.push(colors.dim(` ${score.tier === 'core+vision' ? 'core (deterministic) + vision (AI-assessed, may vary run-to-run)' : 'core — deterministic (WCAG + HCI science)'} · coverage ${score.coverage}%`)); out.push(''); for (const d of score.byDimension) { if (d.score === null) continue; diff --git a/packages/core/src/scoring/dimensions.ts b/packages/core/src/scoring/dimensions.ts index 5860d58..b77797e 100644 --- a/packages/core/src/scoring/dimensions.ts +++ b/packages/core/src/scoring/dimensions.ts @@ -1,9 +1,10 @@ import type { MetricDimension, ScoreBand } from '../types.js'; /** - * FIXED catalog of the 10 score dimensions. The scorer ALWAYS iterates this - * full list so the global score is comparable whether or not Tier-2 ran. - * Order is the display order. + * FIXED catalog of the score dimensions (8 Tier-1 + 6 Tier-2). The scorer ALWAYS + * iterates this full list so the global score is comparable whether or not Tier-2 + * ran. Order is the display order. NOTE: the free (core-only) score iterates + * TIER1_DIMENSIONS, so adding Tier-2 dims here does NOT change the free score. */ export const DIMENSIONS: MetricDimension[] = [ 'contrast', @@ -16,6 +17,10 @@ export const DIMENSIONS: MetricDimension[] = [ 'balance', 'clutter', 'saliency', + 'feedback', + 'consistency', + 'affordance', + 'guidance', ]; export const TIER1_DIMENSIONS: MetricDimension[] = [ @@ -29,7 +34,14 @@ export const TIER1_DIMENSIONS: MetricDimension[] = [ 'balance', ]; -export const TIER2_DIMENSIONS: MetricDimension[] = ['clutter', 'saliency']; +export const TIER2_DIMENSIONS: MetricDimension[] = [ + 'clutter', + 'saliency', + 'feedback', + 'consistency', + 'affordance', + 'guidance', +]; export const DIMENSION_LABELS: Record = { contrast: 'Contrast', @@ -42,6 +54,10 @@ export const DIMENSION_LABELS: Record = { balance: 'Balance', clutter: 'Visual Clutter', saliency: 'Saliency', + feedback: 'Feedback', + consistency: 'Consistency', + affordance: 'Affordance', + guidance: 'Guidance', }; interface BandCutoff extends ScoreBand { diff --git a/packages/core/src/scoring/score.test.ts b/packages/core/src/scoring/score.test.ts index 293b530..ba40106 100644 --- a/packages/core/src/scoring/score.test.ts +++ b/packages/core/src/scoring/score.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { computeScore, annotatePointsImpact, topFixGroups } from './score.js'; -import { getBand, BANDS_CALIBRATED } from './dimensions.js'; +import { getBand, BANDS_CALIBRATED, TIER1_DIMENSIONS, TIER2_DIMENSIONS } from './dimensions.js'; import type { Metric, MetricDimension, MetricStatus, MetricTier } from '../types.js'; let seq = 0; @@ -69,6 +69,28 @@ describe('computeScore', () => { expect(computeScore([]).global).toBe(100); expect(computeScore([]).coverage).toBe(0); }); + + it('free-score invariant: core-only score is independent of Tier-2 catalog size', () => { + // Core-only metrics across 3 Tier-1 dimensions (3 pass / 4 applicable). + const metrics = [m('contrast', 'pass'), m('contrast', 'fail'), m('fitts', 'pass'), m('typography', 'pass')]; + const score = computeScore(metrics, { includeVision: false }); + const dims = score.byDimension.map((d) => d.dimension); + // Only the 8 Tier-1 dims are iterated — NO Tier-2 dim (clutter/saliency/feedback/…) leaks in, + // so growing TIER2_DIMENSIONS from 2 → 6 cannot change the free score. + expect(dims).toEqual(TIER1_DIMENSIONS); + expect(dims.some((d) => TIER2_DIMENSIONS.includes(d))).toBe(false); + expect(score.tier).toBe('core'); + expect(score.global).toBe(75); // 3 pass / 4 applicable + expect(score.coverage).toBe(Math.round((3 / TIER1_DIMENSIONS.length) * 100)); // 3 evaluated / 8 + }); + + it('a single vision metric flips the tier to core+vision and fills its dimension', () => { + const metrics = [m('contrast', 'pass'), m('feedback', 'fail', 'vision')]; + const score = computeScore(metrics); // auto-detects includeVision + expect(score.tier).toBe('core+vision'); + const fb = score.byDimension.find((d) => d.dimension === 'feedback'); + expect(fb?.score).toBe(0); // 0 pass / 1 applicable + }); }); describe('topFixGroups', () => { diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 7efdb09..0c72396 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -246,7 +246,11 @@ export type MetricDimension = | 'emphasis' | 'balance' | 'clutter' - | 'saliency'; + | 'saliency' + | 'feedback' + | 'consistency' + | 'affordance' + | 'guidance'; export type MetricStatus = 'pass' | 'fail' | 'na'; /** 'standard' = normative spec (WCAG). 'signal' = research-grounded heuristic. */ diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 97873f6..357b15a 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -3,7 +3,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { z } from 'zod'; import { FigmaClient, FigmaApiError, scan, buildGraph, formatTerminal, formatJson, formatHtml, validateLicenseKey, topFixGroups } from '@protoscan/core'; -import type { Issue, ScanResult } from '@protoscan/core'; +import type { Issue, Metric, ScanResult } from '@protoscan/core'; import { registerAppTool, registerAppResource, @@ -77,6 +77,7 @@ registerAppTool( file_key: z.string().describe('Figma file key or full URL (e.g. "abc123" or "https://figma.com/design/abc123/Name?node-id=7-2")'), token: z.string().optional().describe('Figma Personal Access Token. Falls back to FIGMA_TOKEN env var.'), simulate: z.boolean().optional().default(false).describe('Run headless browser simulator to verify prototype navigation actually works. Slower (~2 min) but catches runtime issues static analysis misses.'), + vision: z.boolean().optional().default(false).describe('AI vision Tier-2 evaluation (Pro, requires PROTOSCAN_API_KEY): grades each screen across subjective dimensions (clutter, saliency, feedback, consistency, affordance, guidance), deepening the score to core+vision. Implies score.'), score: z.boolean().optional().default(false).describe('Compute the 0-100 usability score (ProtoScan Science) with a per-dimension breakdown and top fixes. Implies metrics.'), metrics: z.boolean().optional().default(false).describe('Run the science metric analyzers without the aggregate score.'), ergonomics: z.boolean().optional().default(false).describe('Alias for score — emphasises the per-dimension usability breakdown. Implies metrics.'), @@ -132,14 +133,39 @@ registerAppTool( } } + // Run AI vision Tier-2 if requested (Pro — requires PROTOSCAN_API_KEY) + let visionIssues: Issue[] = []; + let visionMetrics: Metric[] = []; + if (args.vision) { + const proKey = process.env.PROTOSCAN_API_KEY; + const license = proKey ? await validateLicenseKey(proKey) : { valid: false }; + if (!proKey || !license.valid) { + console.error(`[protoscan] vision requires valid PROTOSCAN_API_KEY — ${!proKey ? 'not set' : 'invalid key'}`); + } else { + try { + const { analyzeVisionProxy } = await import('@protoscan/vision'); + const graph = buildGraph(file, { pageIds: pageIds.length ? pageIds : undefined }); + console.error('[protoscan] Running AI vision analysis via ProtoScan API...'); + const vr = await analyzeVisionProxy(graph, { protoscanApiKey: proKey, figmaToken: token, fileKey }); + visionIssues = vr.issues; + visionMetrics = vr.metrics; + console.error(`[protoscan] Vision: ${vr.issues.length} issue(s), ${vr.metrics.length} metric(s) across ${vr.coverage.analyzed}/${vr.coverage.total} screens`); + } catch (visionError) { + console.error(`[protoscan] Vision error: ${visionError instanceof Error ? visionError.message : String(visionError)}`); + } + } + } + + const mergedIssues = [...simulatorIssues, ...visionIssues]; const result = await scan(file, { fileKey, minTouchTarget: args.min_touch_target, skip: args.skip, pageIds: pageIds.length ? pageIds : undefined, - additionalIssues: simulatorIssues.length ? simulatorIssues : undefined, - metrics: !!(args.metrics || args.score || args.ergonomics), - score: !!(args.score || args.ergonomics), + additionalIssues: mergedIssues.length ? mergedIssues : undefined, + additionalMetrics: visionMetrics.length ? visionMetrics : undefined, + metrics: !!(args.metrics || args.score || args.ergonomics || args.vision), + score: !!(args.score || args.ergonomics || args.vision), ignorePatterns: args.ignore?.filter(Boolean), }); diff --git a/packages/vision/src/analyzer.test.ts b/packages/vision/src/analyzer.test.ts index f2274f3..86c475a 100644 --- a/packages/vision/src/analyzer.test.ts +++ b/packages/vision/src/analyzer.test.ts @@ -1,34 +1,84 @@ import { describe, it, expect } from 'vitest'; -import { SYSTEM_PROMPT, type VisionFinding } from './prompts.js'; +import { SYSTEM_PROMPT, parseVisionResponse } from './prompts.js'; +import { dimensionsToMetrics } from './analyzer.js'; -describe('VisionFinding schema', () => { - it('SYSTEM_PROMPT is a non-empty string', () => { - expect(typeof SYSTEM_PROMPT).toBe('string'); - expect(SYSTEM_PROMPT.length).toBeGreaterThan(0); +describe('parseVisionResponse', () => { + it('parses the {findings, dimensions} object', () => { + const r = parseVisionResponse( + JSON.stringify({ + findings: [{ category: 'vision-overload', severity: 'high', message: 'too dense', area: 'list' }], + dimensions: [ + { dimension: 'clutter', score: 80, note: 'clean' }, + { dimension: 'saliency', score: 40 }, + ], + }), + ); + expect(r.findings).toHaveLength(1); + expect(r.dimensions).toHaveLength(2); + expect(r.dimensions[0]).toMatchObject({ dimension: 'clutter', score: 80 }); }); - it('valid VisionFinding categories are accepted', () => { - const categories: VisionFinding['category'][] = [ - 'vision-contrast', - 'vision-clarity', - 'vision-empty-state', - 'vision-overload', - ]; - expect(categories).toHaveLength(4); + it('tolerates the legacy bare-array (findings only) → dimensions []', () => { + const r = parseVisionResponse(JSON.stringify([{ category: 'vision-clarity', severity: 'low', message: 'x', area: 'y' }])); + expect(r.findings).toHaveLength(1); + expect(r.dimensions).toEqual([]); }); - it('valid VisionFinding severities', () => { - const severities: VisionFinding['severity'][] = ['high', 'medium', 'low']; - expect(severities).toHaveLength(3); + it('missing/invalid JSON → empty result (no throw)', () => { + expect(parseVisionResponse('not json')).toEqual({ findings: [], dimensions: [] }); + expect(parseVisionResponse('{}')).toEqual({ findings: [], dimensions: [] }); + }); + + it('keeps null score (abstention) but drops malformed dimension items', () => { + const r = parseVisionResponse( + JSON.stringify({ + findings: [], + dimensions: [ + { dimension: 'feedback', score: null }, // abstention — kept + { dimension: 'affordance', score: 'high' }, // malformed score — dropped + { dimension: 'guidance', score: 150 }, // out of range — dropped + { dimension: 'not-a-dim', score: 50 }, // unknown dimension — dropped + { dimension: 'clutter', score: 90 }, // valid + { dimension: 'clutter', score: 10 }, // duplicate — dropped + ], + }), + ); + const byDim = Object.fromEntries(r.dimensions.map((d) => [d.dimension, d.score])); + expect(byDim).toEqual({ feedback: null, clutter: 90 }); }); }); -describe('cost estimate guard', () => { - it('$0.005 per screen × 100 screens stays under $5 budget', () => { - const COST_PER_SCREEN = 0.005; - const maxCost = 5; - const maxScreens = Math.floor(maxCost / COST_PER_SCREEN); - expect(maxScreens).toBe(1000); - expect(100 * COST_PER_SCREEN).toBeLessThanOrEqual(maxCost); +describe('dimensionsToMetrics', () => { + it('maps score → status (null=na, >=70 pass, <70 fail) with correct metadata', () => { + const ctr = { n: 0 }; + const metrics = dimensionsToMetrics( + [ + { dimension: 'clutter', score: 85 }, + { dimension: 'saliency', score: 40 }, + { dimension: 'feedback', score: null, note: 'needs interaction' }, + ], + { nodeId: '2:1', screenName: 'Home' }, + 'gpt-4o', + 'vision-metric', + ctr, + ); + expect(metrics.map((m) => m.status)).toEqual(['pass', 'fail', 'na']); + expect(metrics.every((m) => m.tier === 'vision' && m.claimType === 'signal')).toBe(true); + expect(metrics[0].framework).toBe('AI Vision (gpt-4o)'); + expect(metrics[2].value).toBe(0); // null score → value 0, status na + expect(new Set(metrics.map((m) => m.id)).size).toBe(3); // unique ids + expect(metrics[2].whyItMatters).toBe('needs interaction'); + }); +}); + +describe('prompt + cost', () => { + it('SYSTEM_PROMPT mentions both findings and dimensions', () => { + expect(SYSTEM_PROMPT).toContain('findings'); + expect(SYSTEM_PROMPT).toContain('dimensions'); + expect(SYSTEM_PROMPT.toLowerCase()).toContain('null'); // abstention instruction + }); + + it('holistic cost ~$0.009/screen stays well under a $5 budget for 100 screens', () => { + expect(100 * 0.009).toBeLessThanOrEqual(5); }); }); diff --git a/packages/vision/src/analyzer.ts b/packages/vision/src/analyzer.ts index c2b13b8..db4e9ff 100644 --- a/packages/vision/src/analyzer.ts +++ b/packages/vision/src/analyzer.ts @@ -1,27 +1,41 @@ /** - * AI Vision analyzer — fetches Figma screen renders and analyzes them with GPT-4o Vision. + * AI Vision analyzer — fetches Figma screen renders and evaluates them with a + * vision model (GPT-4o now; Claude behind the same provider seam later). * - * Cost model (GPT-4o, May 2026): - * ~$0.00255 per image (1024x1024 detail:low) + ~$0.002 per response - * → ~$0.005 per screen → $0.50 per 100 screens + * Returns BOTH qualitative Issues (UX findings) AND graded Tier-2 Metrics + * (clutter, saliency, feedback, consistency, affordance, guidance) that feed + * the @protoscan/core 0-100 score and flip it to `core+vision`. + * + * Cost model (GPT-4o, holistic prompt, 2026): ~$0.009 per screen + * (~$0.00255 image detail:low + ~$0.006 output for findings + 6 dimensions). */ -import OpenAI from 'openai'; -import type { Issue, PrototypeGraph } from '@protoscan/core'; -import { SYSTEM_PROMPT, type VisionFinding } from './prompts.js'; +import type { Issue, Metric, PrototypeGraph } from '@protoscan/core'; +import { sanitizeVisionResponse, type VisionDimensionScore } from './prompts.js'; +import { selectProvider, type VisionProviderId } from './providers/index.js'; const FIGMA_API_BASE = 'https://api.figma.com'; const IMAGES_BATCH = 50; // Figma images API max per request -const COST_PER_SCREEN = 0.005; +const COST_PER_SCREEN = 0.009; // holistic prompt (findings + 6 dimensions) // Retry config const MAX_RETRIES = 3; const BASE_DELAY_MS = 5_000; // 5s initial backoff +/** Combined result fed to the scanner: issues (additionalIssues) + metrics (additionalMetrics). */ +export interface VisionResult { + issues: Issue[]; + metrics: Metric[]; + /** Screens actually analyzed vs. total candidates (for coverage reporting). */ + coverage: { analyzed: number; total: number }; +} + export interface VisionOptions { figmaToken: string; openaiApiKey: string; fileKey: string; + /** Vision provider. Default 'openai' (GPT-4o). 'claude' is not yet implemented. */ + provider?: VisionProviderId; /** Max USD to spend. Stops after budget is reached. Default: 5 */ maxCost?: number; /** Max screens to analyze. Default: 200 */ @@ -87,7 +101,7 @@ async function fetchImageUrls( throw new Error(`Figma images API returned ${response.status}`); } - const data = await response.json() as { images: Record }; + const data = (await response.json()) as { images: Record }; const map = new Map(); for (const [id, imgUrl] of Object.entries(data.images)) { if (imgUrl) map.set(id, imgUrl); @@ -108,74 +122,53 @@ async function fetchBase64(url: string): Promise { return Buffer.from(buffer).toString('base64'); } -/** Analyze a single screen image with GPT-4o Vision */ -async function analyzeScreen( - openai: OpenAI, - base64Image: string, - screenName: string, -): Promise { - const response = await openai.chat.completions.create({ - model: 'gpt-4o', - max_tokens: 500, - messages: [ - { role: 'system', content: SYSTEM_PROMPT }, - { - role: 'user', - content: [ - { - type: 'text', - text: `Screen name: "${screenName}"\n\nAnalyze this prototype screen for UX issues:`, - }, - { - type: 'image_url', - image_url: { - url: `data:image/png;base64,${base64Image}`, - detail: 'low', - }, - }, - ], - }, - ], +/** Map AI dimension scores → Tier-2 vision Metrics (null score → 'na', never a hallucinated fail). */ +export function dimensionsToMetrics( + dimensions: VisionDimensionScore[], + loc: { nodeId: string; screenName: string }, + model: string, + source: 'vision-metric' | 'vision-metric-proxy', + ctr: { n: number }, +): Metric[] { + return dimensions.map((d) => { + const status: Metric['status'] = d.score == null ? 'na' : d.score >= 70 ? 'pass' : 'fail'; + return { + id: `vision-metric-${++ctr.n}`, + framework: `AI Vision (${model})`, + dimension: d.dimension, + citation: 'Bastien & Scapin 1993 (AI-assessed)', + claimType: 'signal', + tier: 'vision', + value: d.score ?? 0, + threshold: 70, + status, + screenId: loc.nodeId, + screenName: loc.screenName, + nodeId: loc.nodeId, + whyItMatters: d.note, + evidence: { model, aiAssessment: true, score: d.score, source }, + }; }); +} - const content = response.choices[0]?.message?.content ?? '[]'; - try { - const parsed = JSON.parse(content); - if (!Array.isArray(parsed)) return []; - return parsed as VisionFinding[]; - } catch { - return []; - } +/** Collect the screen node IDs that have a render-able bounding box. */ +function screenIdsFrom(graph: PrototypeGraph, maxScreens: number): string[] { + return [...graph.nodes.values()].filter((n) => n.boundingBox).slice(0, maxScreens).map((n) => n.id); } /** - * Run AI vision analysis on all screens in the prototype graph. - * Returns Issue[] compatible with @protoscan/core scan pipeline. + * Run AI vision analysis (BYOK) on all screens in the prototype graph. + * Returns { issues, metrics, coverage }. */ -export async function analyzeVision( - graph: PrototypeGraph, - options: VisionOptions, -): Promise { - const { - figmaToken, - openaiApiKey, - fileKey, - maxCost = 5, - maxScreens = 200, - } = options; - - const openai = new OpenAI({ apiKey: openaiApiKey }); - - const screenIds = [...graph.nodes.values()] - .filter((n) => n.boundingBox) - .slice(0, maxScreens) - .map((n) => n.id); +export async function analyzeVision(graph: PrototypeGraph, options: VisionOptions): Promise { + const { figmaToken, openaiApiKey, fileKey, maxCost = 5, maxScreens = 200, provider } = options; + const visionProvider = selectProvider({ provider, openaiApiKey }); + const screenIds = screenIdsFrom(graph, maxScreens); const total = screenIds.length; const startTime = Date.now(); - console.error(`[vision] Analyzing ${total} screens...`); + console.error(`[vision] Analyzing ${total} screens with ${visionProvider.model}...`); - // Fetch image URLs in batches const imageUrlMap = new Map(); for (let i = 0; i < screenIds.length; i += IMAGES_BATCH) { const batch = screenIds.slice(i, i + IMAGES_BATCH); @@ -185,7 +178,9 @@ export async function analyzeVision( } const issues: Issue[] = []; - let issueCounter = 0; + const metrics: Metric[] = []; + const issueCtr = { n: 0 }; + const metricCtr = { n: 0 }; let totalCost = 0; let analyzed = 0; @@ -194,18 +189,16 @@ export async function analyzeVision( console.error(`[vision] Cost limit $${maxCost} reached after ${analyzed}/${total} screens.`); break; } - const imgUrl = imageUrlMap.get(nodeId); if (!imgUrl) continue; - const node = graph.nodes.get(nodeId)!; analyzed++; try { - const findings = await withRetry( + const result = await withRetry( async () => { const base64 = await fetchBase64(imgUrl); - return analyzeScreen(openai, base64, node.name); + return visionProvider.analyzeScreen(base64, node.name); }, node.name, (attempt, delayMs) => { @@ -214,9 +207,9 @@ export async function analyzeVision( ); totalCost += COST_PER_SCREEN; - for (const f of findings) { + for (const f of result.findings) { issues.push({ - id: `vision-${++issueCounter}`, + id: `vision-${++issueCtr.n}`, category: 'vision', severity: f.severity, confidence: 'probable', @@ -224,25 +217,20 @@ export async function analyzeVision( screenName: node.name, nodeId, message: f.message, - evidence: { visionCategory: f.category, area: f.area, model: 'gpt-4o' }, + evidence: { visionCategory: f.category, area: f.area, model: visionProvider.model, source: 'vision-finding' }, }); } + metrics.push(...dimensionsToMetrics(result.dimensions, { nodeId, screenName: node.name }, visionProvider.model, 'vision-metric', metricCtr)); const elapsed = formatTime(Date.now() - startTime); - const elapsedSec = (Date.now() - startTime) / 1000; - const eta = analyzed >= 3 && elapsedSec > 0 - ? formatTime(((total - analyzed) / (analyzed / elapsedSec)) * 1000) - : ''; - const etaStr = eta ? `, ~${eta} remaining` : ''; - const issueStr = findings.length > 0 ? ` → ${findings.length} issue(s)` : ''; - console.error(`[vision] [${analyzed}/${total}] ${node.name}${issueStr} (${elapsed} elapsed${etaStr})`); + console.error(`[vision] [${analyzed}/${total}] ${node.name} → ${result.findings.length} finding(s), ${result.dimensions.length} dim(s) (${elapsed})`); } catch (err) { console.error(`[vision] ⚠ Skipped "${node.name}" after ${MAX_RETRIES} retries: ${err instanceof Error ? err.message : err}`); } } - console.error(`[vision] Done. ${issues.length} issues found. Cost: ~$${totalCost.toFixed(2)}`); - return issues; + console.error(`[vision] Done. ${issues.length} issues, ${metrics.length} metrics across ${analyzed}/${total} screens. Cost ~$${totalCost.toFixed(2)}`); + return { issues, metrics, coverage: { analyzed, total } }; } // --- Proxy mode: calls ProtoScan server-side vision API instead of OpenAI directly --- @@ -251,23 +239,17 @@ export interface VisionProxyOptions { protoscanApiKey: string; figmaToken: string; fileKey: string; - /** Max USD to spend. Stops after budget is reached. Default: 5 */ maxCost?: number; - /** Max screens to analyze. Default: 200 */ maxScreens?: number; - /** Vision proxy URL. Default: https://web-five-beige-24.vercel.app/api/vision */ proxyUrl?: string; } /** - * Run AI vision analysis via ProtoScan's server-side proxy. - * The user doesn't need an OpenAI key — ProtoScan's key is used server-side. - * Includes retry with exponential backoff for rate limits. + * Run AI vision analysis via ProtoScan's server-side proxy (server holds the + * OpenAI key + GPT-4o). The server returns { findings, dimensions } — old + * servers without `dimensions` degrade gracefully to metrics: []. */ -export async function analyzeVisionProxy( - graph: PrototypeGraph, - options: VisionProxyOptions, -): Promise { +export async function analyzeVisionProxy(graph: PrototypeGraph, options: VisionProxyOptions): Promise { const { protoscanApiKey, figmaToken, @@ -277,16 +259,11 @@ export async function analyzeVisionProxy( proxyUrl = 'https://web-five-beige-24.vercel.app/api/vision', } = options; - const screenIds = [...graph.nodes.values()] - .filter((n) => n.boundingBox) - .slice(0, maxScreens) - .map((n) => n.id); - + const screenIds = screenIdsFrom(graph, maxScreens); const total = screenIds.length; const startTime = Date.now(); console.error(`[vision-proxy] Analyzing ${total} screens via ProtoScan API...`); - // Fetch image URLs in batches const imageUrlMap = new Map(); for (let i = 0; i < screenIds.length; i += IMAGES_BATCH) { const batch = screenIds.slice(i, i + IMAGES_BATCH); @@ -296,7 +273,9 @@ export async function analyzeVisionProxy( } const issues: Issue[] = []; - let issueCounter = 0; + const metrics: Metric[] = []; + const issueCtr = { n: 0 }; + const metricCtr = { n: 0 }; let totalCost = 0; let analyzed = 0; @@ -305,10 +284,8 @@ export async function analyzeVisionProxy( console.error(`[vision-proxy] Cost limit $${maxCost} reached after ${analyzed}/${total} screens.`); break; } - const imgUrl = imageUrlMap.get(nodeId); if (!imgUrl) continue; - const node = graph.nodes.get(nodeId)!; analyzed++; @@ -318,35 +295,28 @@ export async function analyzeVisionProxy( const base64 = await fetchBase64(imgUrl); const response = await fetch(proxyUrl, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${protoscanApiKey}`, - }, + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${protoscanApiKey}` }, body: JSON.stringify({ image: base64, screenName: node.name }), }); - if (!response.ok) { - const err = await response.json().catch(() => ({ error: `HTTP ${response.status}` })) as { error?: string }; - if (response.status === 402) { - throw new Error('NO_CREDITS'); - } + const err = (await response.json().catch(() => ({ error: `HTTP ${response.status}` }))) as { error?: string }; + if (response.status === 402) throw new Error('NO_CREDITS'); throw new Error(err.error ?? `Proxy returned ${response.status}`); } - - return response.json() as Promise<{ findings: VisionFinding[] }>; + return response.json() as Promise; }, node.name, (attempt, delayMs) => { console.error(`[vision-proxy] ⏳ "${node.name}" rate limited, retry ${attempt}/${MAX_RETRIES} in ${delayMs / 1000}s...`); }, ); - - // Check for credit exhaustion (non-retryable) totalCost += COST_PER_SCREEN; - for (const f of data.findings) { + // Validate the server's JSON (tolerates old {findings}-only servers → dimensions []). + const clean = sanitizeVisionResponse(data); + for (const f of clean.findings) { issues.push({ - id: `vision-${++issueCounter}`, + id: `vision-${++issueCtr.n}`, category: 'vision', severity: f.severity, confidence: 'probable', @@ -354,18 +324,13 @@ export async function analyzeVisionProxy( screenName: node.name, nodeId, message: f.message, - evidence: { visionCategory: f.category, area: f.area, model: 'gpt-4o', via: 'proxy' }, + evidence: { visionCategory: f.category, area: f.area, model: 'gpt-4o', source: 'vision-finding', via: 'proxy' }, }); } + metrics.push(...dimensionsToMetrics(clean.dimensions, { nodeId, screenName: node.name }, 'gpt-4o', 'vision-metric-proxy', metricCtr)); const elapsed = formatTime(Date.now() - startTime); - const elapsedSec = (Date.now() - startTime) / 1000; - const eta = analyzed >= 3 && elapsedSec > 0 - ? formatTime(((total - analyzed) / (analyzed / elapsedSec)) * 1000) - : ''; - const etaStr = eta ? `, ~${eta} remaining` : ''; - const issueStr = data.findings.length > 0 ? ` → ${data.findings.length} issue(s)` : ''; - console.error(`[vision-proxy] [${analyzed}/${total}] ${node.name}${issueStr} (${elapsed} elapsed${etaStr})`); + console.error(`[vision-proxy] [${analyzed}/${total}] ${node.name} → ${clean.findings.length} finding(s), ${clean.dimensions.length} dim(s) (${elapsed})`); } catch (err) { if (err instanceof Error && err.message === 'NO_CREDITS') { console.error(`[vision-proxy] No credits remaining. Purchase more at https://protoscan.dev/pricing`); @@ -376,6 +341,6 @@ export async function analyzeVisionProxy( } const elapsed = formatTime(Date.now() - startTime); - console.error(`[vision-proxy] Done. ${issues.length} issues in ${analyzed}/${total} screens. Cost: ~$${totalCost.toFixed(2)} (${elapsed})`); - return issues; + console.error(`[vision-proxy] Done. ${issues.length} issues, ${metrics.length} metrics in ${analyzed}/${total} screens. Cost ~$${totalCost.toFixed(2)} (${elapsed})`); + return { issues, metrics, coverage: { analyzed, total } }; } diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.ts index 2489393..f2e010a 100644 --- a/packages/vision/src/index.ts +++ b/packages/vision/src/index.ts @@ -1 +1,25 @@ -export { analyzeVision, analyzeVisionProxy, type VisionOptions, type VisionProxyOptions } from './analyzer.js'; +export { + analyzeVision, + analyzeVisionProxy, + type VisionOptions, + type VisionProxyOptions, + type VisionResult, +} from './analyzer.js'; +export { + type VisionFinding, + type VisionDimensionScore, + type VisionResponse, + type VisionDimensionId, + VISION_DIMENSIONS, + SYSTEM_PROMPT, + parseVisionResponse, + sanitizeVisionResponse, +} from './prompts.js'; +export { + selectProvider, + type VisionProvider, + type VisionProviderId, + type VisionScreenResult, + OpenAIVisionProvider, + ClaudeVisionProvider, +} from './providers/index.js'; diff --git a/packages/vision/src/prompts.ts b/packages/vision/src/prompts.ts index 28230df..8942c94 100644 --- a/packages/vision/src/prompts.ts +++ b/packages/vision/src/prompts.ts @@ -1,31 +1,20 @@ /** - * GPT-4o Vision prompts and response types for UX screen analysis. + * Vision prompts + response types for holistic AI screen evaluation. + * One call per screen returns BOTH qualitative findings (issues) AND graded + * Tier-2 dimension scores (0-100, or null when the model cannot judge a + * dimension from a single static screenshot). */ -export const SYSTEM_PROMPT = `You are a senior UX/UI designer performing a visual QA audit of a mobile app prototype screen. - -Analyze the screenshot and identify UX issues in these categories ONLY: -- vision-contrast: Text or important UI elements with insufficient contrast (text on image, similar colors, white on light bg) -- vision-clarity: Ambiguous CTAs, unclear button labels, confusing visual hierarchy, missing affordances -- vision-empty-state: Screen appears empty, broken, or incomplete (missing content, skeleton not replaced, blank areas) -- vision-overload: Too many competing visual elements, overwhelming density, unclear focal point - -Rules: -- Return ONLY a valid JSON array, no markdown, no explanation outside JSON -- Only report issues you are confident about (avoid nitpicking) -- Max 3 issues per screen -- If the screen looks fine, return [] -- Do not flag design style preferences (color choices, typography style, etc.) - -Response format (strict JSON array): -[ - { - "category": "vision-contrast" | "vision-clarity" | "vision-empty-state" | "vision-overload", - "severity": "high" | "medium" | "low", - "message": "One sentence describing the issue clearly", - "area": "Brief description of where on screen (e.g. 'top CTA button', 'card list area')" - } -]`; +/** The 6 AI-judged Tier-2 dimensions (must match @protoscan/core TIER2_DIMENSIONS). */ +export const VISION_DIMENSIONS = [ + 'clutter', + 'saliency', + 'feedback', + 'consistency', + 'affordance', + 'guidance', +] as const; +export type VisionDimensionId = (typeof VISION_DIMENSIONS)[number]; export interface VisionFinding { category: 'vision-contrast' | 'vision-clarity' | 'vision-empty-state' | 'vision-overload'; @@ -33,3 +22,103 @@ export interface VisionFinding { message: string; area: string; } + +/** A graded Tier-2 dimension. score is 0-100, or null = "cannot judge from this static screen". */ +export interface VisionDimensionScore { + dimension: VisionDimensionId; + score: number | null; + note?: string; +} + +export interface VisionResponse { + findings: VisionFinding[]; + dimensions: VisionDimensionScore[]; +} + +export const SYSTEM_PROMPT = `You are a senior UX/UI designer doing a visual QA audit of ONE mobile app prototype screen (a single static screenshot). + +Return STRICT JSON (no markdown) shaped exactly: +{ + "findings": [ { "category": "...", "severity": "...", "message": "...", "area": "..." } ], + "dimensions": [ { "dimension": "...", "score": 0-100 | null, "note": "..." } ] +} + +FINDINGS (qualitative issues, max 3, [] if the screen is fine) — categories ONLY: +- vision-contrast: text/important elements with insufficient contrast +- vision-clarity: ambiguous CTAs, unclear labels, confusing hierarchy +- vision-empty-state: empty/broken/incomplete screen +- vision-overload: too many competing elements, no clear focal point +severity: high | medium | low. Only confident issues; no style preferences. + +DIMENSIONS (grade each 0-100, higher = better) — score these 6: +- clutter: low visual density / few competing elements (100 = clean, 0 = overwhelming) +- saliency: clear focal point / visual priority for the primary action +- feedback: the screen visibly communicates state/result (loading, success, error, selected) +- consistency: repeated elements (buttons, headers, spacing) look uniform +- affordance: interactive elements look tappable; what's actionable is obvious +- guidance: labels/prompts/empty-states orient the user to the next step + +CRITICAL: If you CANNOT fairly judge a dimension from a SINGLE STATIC screenshot +(feedback, affordance and guidance often need interaction or multiple states), +set its "score" to null instead of guessing. Do NOT hallucinate a number. +Include all 6 dimension objects; use null where you cannot assess.`; + +/** Defensive parse from a raw model string: tolerates legacy bare-array + missing/malformed dimensions. */ +export function parseVisionResponse(content: string): VisionResponse { + let raw: unknown; + try { + raw = JSON.parse(content); + } catch { + return { findings: [], dimensions: [] }; + } + return sanitizeVisionResponse(raw); +} + +/** Validate an already-parsed response object (used for the proxy server's JSON too). */ +export function sanitizeVisionResponse(raw: unknown): VisionResponse { + if (Array.isArray(raw)) return { findings: sanitizeFindings(raw), dimensions: [] }; + const obj = (raw ?? {}) as { findings?: unknown; dimensions?: unknown }; + return { findings: sanitizeFindings(obj.findings), dimensions: sanitizeDimensions(obj.dimensions) }; +} + +const FINDING_CATEGORIES = new Set(['vision-contrast', 'vision-clarity', 'vision-empty-state', 'vision-overload']); +const SEVERITIES = new Set(['high', 'medium', 'low']); + +function sanitizeFindings(input: unknown): VisionFinding[] { + if (!Array.isArray(input)) return []; + const out: VisionFinding[] = []; + for (const it of input) { + if (!it || typeof it !== 'object') continue; + const f = it as Record; + if (typeof f.message !== 'string' || !FINDING_CATEGORIES.has(f.category as string)) continue; + out.push({ + category: f.category as VisionFinding['category'], + severity: SEVERITIES.has(f.severity as string) ? (f.severity as VisionFinding['severity']) : 'medium', + message: f.message, + area: typeof f.area === 'string' ? f.area : '', + }); + } + return out; +} + +const DIMENSION_IDS = new Set(VISION_DIMENSIONS); + +function sanitizeDimensions(input: unknown): VisionDimensionScore[] { + if (!Array.isArray(input)) return []; + const out: VisionDimensionScore[] = []; + const seen = new Set(); + for (const it of input) { + if (!it || typeof it !== 'object') continue; + const d = it as Record; + if (!DIMENSION_IDS.has(d.dimension as string) || seen.has(d.dimension as string)) continue; + let score: number | null = null; + if (typeof d.score === 'number' && Number.isFinite(d.score) && d.score >= 0 && d.score <= 100) { + score = Math.round(d.score); + } else if (d.score !== null && d.score !== undefined) { + continue; // malformed score (string/NaN/out-of-range) → drop the item entirely + } + seen.add(d.dimension as string); + out.push({ dimension: d.dimension as VisionDimensionId, score, note: typeof d.note === 'string' ? d.note : undefined }); + } + return out; +} diff --git a/packages/vision/src/providers/claude.ts b/packages/vision/src/providers/claude.ts new file mode 100644 index 0000000..9d797a6 --- /dev/null +++ b/packages/vision/src/providers/claude.ts @@ -0,0 +1,16 @@ +import type { VisionProvider, VisionScreenResult } from './provider.js'; + +/** + * Claude (Anthropic) vision provider — STUB. Wired behind the same seam so the + * CLI/MCP can offer `provider:'claude'` later. When implemented, use + * @anthropic-ai/sdk + claude-opus-4-8 with base64 image blocks (see the + * claude-api skill) and the same parseVisionResponse contract. + */ +export class ClaudeVisionProvider implements VisionProvider { + readonly id = 'claude' as const; + readonly model = 'claude-opus-4-8'; + + async analyzeScreen(_base64Png: string, _screenName: string): Promise { + throw new Error('Claude vision provider not yet implemented — use provider "openai" (GPT-4o) for now.'); + } +} diff --git a/packages/vision/src/providers/index.ts b/packages/vision/src/providers/index.ts new file mode 100644 index 0000000..d448d43 --- /dev/null +++ b/packages/vision/src/providers/index.ts @@ -0,0 +1,17 @@ +import type { VisionProvider } from './provider.js'; +import { OpenAIVisionProvider } from './openai.js'; +import { ClaudeVisionProvider } from './claude.js'; + +export type { VisionProvider, VisionScreenResult } from './provider.js'; +export { OpenAIVisionProvider } from './openai.js'; +export { ClaudeVisionProvider } from './claude.js'; + +export type VisionProviderId = 'openai' | 'claude'; + +/** Select a vision provider. Default 'openai' (GPT-4o). */ +export function selectProvider(opts: { provider?: VisionProviderId; openaiApiKey?: string }): VisionProvider { + const id = opts.provider ?? 'openai'; + if (id === 'claude') return new ClaudeVisionProvider(); + if (!opts.openaiApiKey) throw new Error('OpenAI vision provider requires an OpenAI API key.'); + return new OpenAIVisionProvider(opts.openaiApiKey); +} diff --git a/packages/vision/src/providers/openai.ts b/packages/vision/src/providers/openai.ts new file mode 100644 index 0000000..da32c27 --- /dev/null +++ b/packages/vision/src/providers/openai.ts @@ -0,0 +1,35 @@ +import OpenAI from 'openai'; +import { SYSTEM_PROMPT, parseVisionResponse } from '../prompts.js'; +import type { VisionProvider, VisionScreenResult } from './provider.js'; + +/** GPT-4o vision provider. temperature 0 + JSON mode to minimize run-to-run variance. */ +export class OpenAIVisionProvider implements VisionProvider { + readonly id = 'openai' as const; + readonly model = 'gpt-4o'; + private client: OpenAI; + + constructor(apiKey: string) { + this.client = new OpenAI({ apiKey }); + } + + async analyzeScreen(base64Png: string, screenName: string): Promise { + const response = await this.client.chat.completions.create({ + model: this.model, + max_tokens: 900, + temperature: 0, + response_format: { type: 'json_object' }, + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { + role: 'user', + content: [ + { type: 'text', text: `Screen name: "${screenName}"\n\nAudit this prototype screen. Return the JSON object.` }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Png}`, detail: 'low' } }, + ], + }, + ], + }); + const content = response.choices[0]?.message?.content ?? '{}'; + return parseVisionResponse(content); + } +} diff --git a/packages/vision/src/providers/provider.ts b/packages/vision/src/providers/provider.ts new file mode 100644 index 0000000..36b6b2f --- /dev/null +++ b/packages/vision/src/providers/provider.ts @@ -0,0 +1,17 @@ +import type { VisionFinding, VisionDimensionScore } from '../prompts.js'; + +/** Per-screen result from a vision provider: qualitative findings + graded dimensions. */ +export interface VisionScreenResult { + findings: VisionFinding[]; + dimensions: VisionDimensionScore[]; +} + +/** + * A vision model behind a stable seam. OpenAI (GPT-4o) is implemented now; + * Claude is a stub. The server proxy is hardwired to GPT-4o (server-side key). + */ +export interface VisionProvider { + readonly id: 'openai' | 'claude'; + readonly model: string; // e.g. 'gpt-4o' — used in the Metric framework string + evidence + analyzeScreen(base64Png: string, screenName: string): Promise; +}