From c6630bd2c4d37e9f1e8a60926b0c09525ff8e181 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 00:44:36 +0000 Subject: [PATCH 1/9] feat(bench): add a VLM provider bench for the shelf scanner (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 6 of #10: a manual bench that pits the two ShelfScannerPort adapters (Gemini, Qwen/OpenRouter) against each other on real shelf photos, to pick the production default — the phase-3 instrumentation ADR 0005 calls for. - libs/shared/text-match: reusable string normalization + fuzzy equality (Levenshtein ratio, threshold 0.85). A shared primitive: the bench scores a detection against the ground truth with it, and bibliographic reconciliation will match reads against the reference the same way. - tools/bench: dedicated Nx project (type:app / scope:api, so it may know infrastructure and wire the real adapters). Pure, CI-tested logic — ground-truth YAML loading (zod), scoring (recall, precision, per-field accuracy, structuring errors, high-confidence hallucination), Markdown rendering — and a live runner that instruments fetch for latency and token usage. The runner is never in CI: it costs a paid call per photo. - Add tools/* to the Yarn workspaces so the project is linked and keeps its Nx tags (hence the module boundaries). - Docs: bench protocol (tools/bench/README.md), ground-truth template, and a lower-level decision note (docs/decisions/0001), updated from a live run. Live run (2026-09-02): Gemini reads end to end (9/10 photos, one transient 503, ~0.27c/scan, ~20s median). Qwen is unreachable from this environment — its network policy blocks openrouter.ai (403 at the proxy); the adapter correctly surfaces that as ShelfScanFailed. Quality metrics (recall/precision/hallucination) stay pending: they need a human-verified ground truth, which by design a VLM draft cannot stand in for. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .gitignore | 4 + CLAUDE.md | 8 +- .../0001-fournisseur-vlm-par-defaut.md | 75 ++++++ libs/shared/text-match/README.md | 24 ++ libs/shared/text-match/eslint.config.mjs | 25 ++ libs/shared/text-match/package.json | 28 ++ libs/shared/text-match/src/index.ts | 1 + .../text-match/src/lib/text-match.spec.ts | 79 ++++++ libs/shared/text-match/src/lib/text-match.ts | 91 +++++++ libs/shared/text-match/tsconfig.json | 13 + libs/shared/text-match/tsconfig.lib.json | 14 + libs/shared/text-match/tsconfig.spec.json | 14 + libs/shared/text-match/vitest.config.mts | 17 ++ package.json | 3 +- tools/bench/README.md | 78 ++++++ tools/bench/eslint.config.mjs | 25 ++ tools/bench/ground-truth.template.yaml | 45 ++++ tools/bench/package.json | 22 ++ tools/bench/src/config.ts | 87 ++++++ tools/bench/src/io.ts | 45 ++++ tools/bench/src/lib/ground-truth.spec.ts | 77 ++++++ tools/bench/src/lib/ground-truth.ts | 46 ++++ tools/bench/src/lib/render.spec.ts | 68 +++++ tools/bench/src/lib/render.ts | 85 ++++++ tools/bench/src/lib/scoring.spec.ts | 156 +++++++++++ tools/bench/src/lib/scoring.ts | 248 ++++++++++++++++++ tools/bench/src/main.ts | 49 ++++ tools/bench/src/runner.ts | 158 +++++++++++ tools/bench/src/usage.ts | 101 +++++++ tools/bench/tsconfig.json | 13 + tools/bench/tsconfig.lib.json | 25 ++ tools/bench/tsconfig.spec.json | 15 ++ tools/bench/vitest.config.mts | 19 ++ tsconfig.json | 6 + yarn.lock | 23 +- 35 files changed, 1784 insertions(+), 3 deletions(-) create mode 100644 docs/decisions/0001-fournisseur-vlm-par-defaut.md create mode 100644 libs/shared/text-match/README.md create mode 100644 libs/shared/text-match/eslint.config.mjs create mode 100644 libs/shared/text-match/package.json create mode 100644 libs/shared/text-match/src/index.ts create mode 100644 libs/shared/text-match/src/lib/text-match.spec.ts create mode 100644 libs/shared/text-match/src/lib/text-match.ts create mode 100644 libs/shared/text-match/tsconfig.json create mode 100644 libs/shared/text-match/tsconfig.lib.json create mode 100644 libs/shared/text-match/tsconfig.spec.json create mode 100644 libs/shared/text-match/vitest.config.mts create mode 100644 tools/bench/README.md create mode 100644 tools/bench/eslint.config.mjs create mode 100644 tools/bench/ground-truth.template.yaml create mode 100644 tools/bench/package.json create mode 100644 tools/bench/src/config.ts create mode 100644 tools/bench/src/io.ts create mode 100644 tools/bench/src/lib/ground-truth.spec.ts create mode 100644 tools/bench/src/lib/ground-truth.ts create mode 100644 tools/bench/src/lib/render.spec.ts create mode 100644 tools/bench/src/lib/render.ts create mode 100644 tools/bench/src/lib/scoring.spec.ts create mode 100644 tools/bench/src/lib/scoring.ts create mode 100644 tools/bench/src/main.ts create mode 100644 tools/bench/src/runner.ts create mode 100644 tools/bench/src/usage.ts create mode 100644 tools/bench/tsconfig.json create mode 100644 tools/bench/tsconfig.lib.json create mode 100644 tools/bench/tsconfig.spec.json create mode 100644 tools/bench/vitest.config.mts diff --git a/.gitignore b/.gitignore index 791d19e..c34bb2e 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,7 @@ override.tf.json # Photos de référence du bench (issue #10) : jamais commitées — poids, contexte ressourcerie. # Source de vérité : le bucket GCS pick-a-book-505922-reference-photos. fixtures/reference-photos/ + +# Sortie du bench (#10) : détections brutes + tableau, dérivées des photos privées et d'appels +# live. Le tableau final validé vit dans la note de décision commitée, pas ici. +tools/bench/output/ diff --git a/CLAUDE.md b/CLAUDE.md index a6af9e6..0a6870c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,13 +130,19 @@ apps/api/ # NestJS : composition root, orchestration inte apps/web/ # React : feature-slice libs/recognition/domain/ # entités, value objects, ports — zéro dépendance technique libs/recognition/application/ # use cases, parlent aux ports -libs/recognition/infrastructure/ # adapters +libs/recognition/infrastructure/ # adapters (Gemini, Qwen, stub) derrière ShelfScannerPort libs/shared/result/ # contenu partagé, une lib par sujet nommé +libs/shared/text-match/ # normalisation + comparaison floue de chaînes (bench, réconciliation) +tools/bench/ # départage manuel des adapters VLM sur photos réelles (#10) — hors CI docker/ # Dockerfile des deux apps — contexte de build : la racine docs/adr/ +docs/decisions/ # notes de décision de niveau inférieur (pas des ADR) infra/ # infrastructure GCP en Terraform — voir infra/README.md ``` +Le glob des workspaces Yarn couvre `apps/*`, `libs/*/*` et `tools/*` — un projet Nx hors de ces +trois emplacements n'est pas lié et perd ses tags (donc les frontières de modules). + `recognition` est le seul bounded context fondé aujourd'hui (ADR 0005). Les autres attendent leur ADR de découpage — ne pas en créer au jugé. diff --git a/docs/decisions/0001-fournisseur-vlm-par-defaut.md b/docs/decisions/0001-fournisseur-vlm-par-defaut.md new file mode 100644 index 0000000..ee1fc46 --- /dev/null +++ b/docs/decisions/0001-fournisseur-vlm-par-defaut.md @@ -0,0 +1,75 @@ +# Décision — fournisseur VLM par défaut du scan d'étagère + +> **Niveau inférieur, pas un ADR.** L'ADR [0005](../adr/0005-reconnaissance-livres-photo-etagere.md) +> tranche le *quoi* (VLM seul, option B, derrière `ShelfScannerPort`) et délègue explicitement le +> choix du fournisseur précis à une décision de niveau inférieur, instrumentée par un bench sur +> photos réelles (phase 3). Cette note est cette décision. Elle se révise sans nouvel ADR. + +## Statut + +**En attente de la vérité terrain.** Le harnais de bench, les deux adapters et l'endpoint sont +livrés (#10, étapes 1-5). Le run live sur les 10 photos de référence tourne et produit déjà le +contrat, le coût et la latence. **Le gagnant qualité — donc le défaut de prod — ne peut pas être +posé tant que la vérité terrain n'est pas saisie et vérifiée à la main** : sans elle, ni rappel, +ni précision, ni hallucination ne sont mesurables (voir le commentaire du 31/08 sur #10 et +`tools/bench/README.md`). + +Défaut actuel de `SHELF_SCANNER_PROVIDER` : `stub`. Il le reste jusqu'à ce que le bench qualité +départage `gemini` et `qwen`. + +## Candidats + +Les deux configurations déployables (commentaire du 15/08 sur #10) : + +1. **Gemini 2.5/3.6 Flash** via `GEMINI_API_KEY` — décodage JSON contraint par schéma natif. +2. **Qwen3-VL** via **OpenRouter** (`OPENROUTER_API_KEY`) — client OpenAI-compatible. + +Claude reste « non construit en V1 ». Un troisième candidat ne se justifierait que si le tableau +sort serré. + +## Méthode + +`tools/bench` envoie chaque photo de référence aux deux fournisseurs (appels live, hors CI), +confronte la lecture à la vérité terrain avec une comparaison tolérante aux fautes +(`shared-text-match`, seuil 0.85), et micro-moyenne les compteurs. Un couple `(auteur, titre)` est +correct si les **deux** champs correspondent ; une tranche illisible non lue est un faux négatif, +une photo sans livre lisible se lit en tableau vide. + +## Mesures + +### Contrat, coût, latence (run du 2026-09-02, sans vérité terrain) + +| Métrique | gemini | qwen | +|---|---|---| +| Modèle | `gemini-3.6-flash` | `qwen/qwen3-vl-235b-a22b-instruct` | +| Photos scannées | 10 | 10 | +| Échecs adapter | 1 | 10 | +| Latence médiane | 19,7 s | — | +| Tokens (prompt / complétion) | 11808 / 9389 | 0 / 0 | +| Coût total | $0,0270 | n/a | +| Coût / scan | $0,0027 | n/a | + +Gemini a lu 9 des 10 photos (4 à 62 livres par étagère selon la densité), un échec sur un `503` +transitoire de l'API (« high demand ») — l'adapter l'a bien remonté en `ShelfScanFailed` et le run +a continué. Coût mesuré ~0,27 ¢/scan, dans l'ordre de grandeur annoncé (~0,37 ¢). + +> **Qwen n'a pas pu être mesuré depuis cet environnement.** Les 10 appels ont échoué en +> « unreachable (fetch failed) » : la **politique réseau** de l'environnement d'exécution autorise +> Google (Gemini passe) mais **bloque `openrouter.ai`** (le proxy répond `403` au `CONNECT`). Ce +> n'est pas un bug de l'adapter — il a correctement levé `ShelfScanFailed` sur l'échec de connexion. +> Le run Qwen doit se faire depuis un environnement dont la politique réseau autorise l'egress vers +> OpenRouter (ou un endpoint OpenAI-compatible joignable, cf. `QWEN_BASE_URL`). + +Ces chiffres valident la chaîne de bout en bout côté Gemini et donnent son coût et sa latence. Ils +ne disent **rien** de la qualité : un fournisseur peut détecter beaucoup de livres et en inventer +autant. C'est la vérité terrain qui tranche. + +### Qualité (rappel, précision, hallucination) + +**En attente de la vérité terrain.** Une fois `tools/bench/ground-truth.yaml` saisi et vérifié, +relancer le bench et coller ici le tableau complet, puis désigner le gagnant. + +## Décision + +_À écrire une fois la qualité mesurée : fournisseur gagnant + `SHELF_SCANNER_PROVIDER` posé sur +lui par défaut, dans `apps/api/src/config/environment.ts` / `.env.example`._ diff --git a/libs/shared/text-match/README.md b/libs/shared/text-match/README.md new file mode 100644 index 0000000..4539dbe --- /dev/null +++ b/libs/shared/text-match/README.md @@ -0,0 +1,24 @@ +# shared-text-match + +Comparaison de chaînes courtes — noms d'auteur, titres — **tolérante aux différences qui ne +changent pas le sens** : casse, accents, ponctuation, espaces, et la faute d'OCR isolée. + +Lib partagée (`type:shared`, `context:none`) : importable par tous, n'important aucun contexte +(ADR 0002). Elle existe pour ne pas être recopiée. + +- Le **bench de reconnaissance** (#10) s'en sert pour confronter une détection `(auteur, titre)` + à la vérité terrain — un titre correct lu avec une lettre en trop reste correct. +- La **réconciliation bibliographique** (contexte futur) confrontera de la même façon un titre lu + au référentiel. Même geste, même normalisation. + +## API + +| Fonction | Rôle | +|---|---| +| `normalizeText(raw)` | Forme canonique : minuscules, sans diacritiques, alphanumériques séparés par une espace simple. | +| `similarity(a, b)` | Ressemblance dans `[0, 1]` (1 = identique après normalisation). Distance de Levenshtein rapportée à la longueur. | +| `fuzzyEquals(a, b, threshold?)` | `similarity(a, b) >= threshold` — seuil par défaut `0.85`. | + +Le seuil par défaut laisse passer une ou deux lettres de glissement sur un titre de longueur +normale tout en séparant deux œuvres réellement différentes. Un appelant qui a besoin d'un autre +compromis passe le sien. diff --git a/libs/shared/text-match/eslint.config.mjs b/libs/shared/text-match/eslint.config.mjs new file mode 100644 index 0000000..e659c69 --- /dev/null +++ b/libs/shared/text-match/eslint.config.mjs @@ -0,0 +1,25 @@ +import baseConfig from '../../../eslint.config.mjs'; + +export default [ + ...baseConfig, + { + files: ['**/*.json'], + rules: { + '@nx/dependency-checks': [ + 'error', + { + ignoredFiles: [ + '{projectRoot}/eslint.config.{js,cjs,mjs,ts,cts,mts}', + '{projectRoot}/vitest.config.{js,cjs,mjs,ts,cts,mts}', + ], + }, + ], + }, + languageOptions: { + parser: await import('jsonc-eslint-parser'), + }, + }, + { + ignores: ['**/out-tsc'], + }, +]; diff --git a/libs/shared/text-match/package.json b/libs/shared/text-match/package.json new file mode 100644 index 0000000..f272321 --- /dev/null +++ b/libs/shared/text-match/package.json @@ -0,0 +1,28 @@ +{ + "name": "@pick-a-book/shared-text-match", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "nx": { + "name": "shared-text-match", + "tags": [ + "type:shared", + "context:none", + "scope:shared" + ] + }, + "dependencies": { + "tslib": "^2.3.0" + } +} diff --git a/libs/shared/text-match/src/index.ts b/libs/shared/text-match/src/index.ts new file mode 100644 index 0000000..c16a61b --- /dev/null +++ b/libs/shared/text-match/src/index.ts @@ -0,0 +1 @@ +export * from './lib/text-match.js'; diff --git a/libs/shared/text-match/src/lib/text-match.spec.ts b/libs/shared/text-match/src/lib/text-match.spec.ts new file mode 100644 index 0000000..01362f3 --- /dev/null +++ b/libs/shared/text-match/src/lib/text-match.spec.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; + +import { fuzzyEquals, normalizeText, similarity } from './text-match.js'; + +describe('normalizeText', () => { + it('lowercases', () => { + expect(normalizeText('Le Rouge et le Noir')).toBe('le rouge et le noir'); + }); + + it('strips diacritics', () => { + expect(normalizeText('À la recherche du temps perdu')).toBe('a la recherche du temps perdu'); + expect(normalizeText('Émile Zola')).toBe('emile zola'); + }); + + it('folds punctuation and separators to single spaces', () => { + expect(normalizeText('Voyage au bout de la nuit.')).toBe('voyage au bout de la nuit'); + expect(normalizeText('Saint-Exupéry')).toBe('saint exupery'); + expect(normalizeText('Fleurs du mal')).toBe('fleurs du mal'); + }); + + it('trims surrounding whitespace', () => { + expect(normalizeText(' Proust ')).toBe('proust'); + }); + + it('keeps digits', () => { + expect(normalizeText('1984')).toBe('1984'); + }); + + it('returns an empty string for input with no alphanumerics', () => { + expect(normalizeText('— … !')).toBe(''); + }); +}); + +describe('similarity', () => { + it('is 1 for strings equal once normalized', () => { + expect(similarity('Céline', 'celine')).toBe(1); + }); + + it('is 1 for two empty strings', () => { + expect(similarity('', '')).toBe(1); + }); + + it('is 0 when one side is empty and the other is not', () => { + expect(similarity('', 'Zola')).toBe(0); + }); + + it('tolerates a single-character typo', () => { + // "Céline" vs "Célnie": one transposition read as two edits over six characters. + expect(similarity('Céline', 'Célnie')).toBeGreaterThan(0.6); + }); + + it('is low for unrelated strings', () => { + expect(similarity('Zola', 'Proust')).toBeLessThan(0.4); + }); + + it('is symmetric', () => { + expect(similarity('Flaubert', 'Flauber')).toBe(similarity('Flauber', 'Flaubert')); + }); +}); + +describe('fuzzyEquals', () => { + it('accepts an exact match after normalization', () => { + expect(fuzzyEquals('Marcel PROUST', 'marcel proust')).toBe(true); + }); + + it('accepts an OCR-level typo', () => { + expect(fuzzyEquals('Le Rouge et le Noir', 'Le Rouge et le Noire')).toBe(true); + }); + + it('rejects a different title', () => { + expect(fuzzyEquals('La Peste', 'La Chute')).toBe(false); + }); + + it('honours a caller-supplied threshold', () => { + // A stricter threshold turns a borderline match into a rejection. + expect(fuzzyEquals('Flaubert', 'Flauber', 1)).toBe(false); + expect(fuzzyEquals('Flaubert', 'Flaubert', 1)).toBe(true); + }); +}); diff --git a/libs/shared/text-match/src/lib/text-match.ts b/libs/shared/text-match/src/lib/text-match.ts new file mode 100644 index 0000000..9601d71 --- /dev/null +++ b/libs/shared/text-match/src/lib/text-match.ts @@ -0,0 +1,91 @@ +/** + * Comparison of short human-typed or machine-read strings — author names, titles — that + * tolerates the differences that do not change meaning: case, accents, punctuation, spacing, + * and the odd OCR typo. + * + * A shared primitive on purpose (ADR 0002): the recognition bench (#10) uses it to score a + * detection against the ground truth, and bibliographic reconciliation, when it lands, will + * match a read title against a reference the same way. Neither should carry its own copy. + */ + +/** + * The default similarity above which two strings are considered the same thing. + * + * 0.85 leaves room for a one- or two-character slip on a normal-length title while still + * separating genuinely different works. Callers that need a different bar pass their own. + */ +const DEFAULT_THRESHOLD = 0.85; + +/** + * Folds a string to its comparable core: lower case, no diacritics, alphanumerics separated + * by single spaces. Everything else — punctuation, symbols, runs of space — collapses. + * + * Exposed, not private: reconciliation will want the same canonical form to key on. + */ +export function normalizeText(raw: string): string { + return ( + raw + .normalize('NFD') + // Strip the combining marks NFD split off, so "é" becomes "e". + .replaceAll(/\p{Diacritic}/gu, '') + .toLowerCase() + // Anything that is not a letter or a digit becomes a separator. + .replaceAll(/[^\p{Letter}\p{Number}]+/gu, ' ') + .trim() + ); +} + +/** + * How alike two strings are once normalized, in [0, 1]: 1 is identical, 0 is nothing in + * common. Built on the Levenshtein edit distance, scaled by the longer length so that one + * edit weighs less on a long title than on a short surname. + */ +export function similarity(a: string, b: string): number { + const left = normalizeText(a); + const right = normalizeText(b); + + if (left.length === 0 && right.length === 0) { + return 1; + } + + const distance = levenshtein(left, right); + const longest = Math.max(left.length, right.length); + + return 1 - distance / longest; +} + +/** Whether two strings name the same thing, allowing for OCR-level noise. */ +export function fuzzyEquals(a: string, b: string, threshold: number = DEFAULT_THRESHOLD): boolean { + return similarity(a, b) >= threshold; +} + +/** + * Levenshtein edit distance with a rolling pair of rows — O(min length) memory rather than + * the full matrix, which is all this comparison of short strings needs. + */ +function levenshtein(a: string, b: string): number { + if (a.length === 0) { + return b.length; + } + if (b.length === 0) { + return a.length; + } + + let previous = Array.from({ length: b.length + 1 }, (_unused, index) => index); + + for (let i = 1; i <= a.length; i++) { + const current = [i]; + for (let j = 1; j <= b.length; j++) { + const substitutionCost = a[i - 1] === b[j - 1] ? 0 : 1; + // Insertion, deletion, substitution — the cheapest of the three. + current[j] = Math.min( + current[j - 1] + 1, + previous[j] + 1, + previous[j - 1] + substitutionCost, + ); + } + previous = current; + } + + return previous[b.length]; +} diff --git a/libs/shared/text-match/tsconfig.json b/libs/shared/text-match/tsconfig.json new file mode 100644 index 0000000..667a346 --- /dev/null +++ b/libs/shared/text-match/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/shared/text-match/tsconfig.lib.json b/libs/shared/text-match/tsconfig.lib.json new file mode 100644 index 0000000..f0d9e62 --- /dev/null +++ b/libs/shared/text-match/tsconfig.lib.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo", + "emitDeclarationOnly": false, + "forceConsistentCasingInFileNames": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "references": [], + "exclude": ["vitest.config.mts", "src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/libs/shared/text-match/tsconfig.spec.json b/libs/shared/text-match/tsconfig.spec.json new file mode 100644 index 0000000..e7e3ceb --- /dev/null +++ b/libs/shared/text-match/tsconfig.spec.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./out-tsc/vitest", + "types": ["node"], + "forceConsistentCasingInFileNames": true + }, + "include": ["vitest.config.mts", "src/**/*.test.ts", "src/**/*.spec.ts", "src/**/*.d.ts"], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} diff --git a/libs/shared/text-match/vitest.config.mts b/libs/shared/text-match/vitest.config.mts new file mode 100644 index 0000000..3aedc0f --- /dev/null +++ b/libs/shared/text-match/vitest.config.mts @@ -0,0 +1,17 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + root: import.meta.dirname, + cacheDir: '../../../node_modules/.vite/libs/shared/text-match', + test: { + name: 'shared-text-match', + watch: false, + environment: 'node', + include: ['src/**/*.{test,spec}.ts'], + reporters: ['default'], + coverage: { + reportsDirectory: './test-output/vitest/coverage', + provider: 'v8', + }, + }, +}); diff --git a/package.json b/package.json index 07e8bad..3f22f55 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "private": true, "workspaces": [ "apps/*", - "libs/*/*" + "libs/*/*", + "tools/*" ], "scripts": { "build": "nx run-many -t build", diff --git a/tools/bench/README.md b/tools/bench/README.md new file mode 100644 index 0000000..11ef430 --- /dev/null +++ b/tools/bench/README.md @@ -0,0 +1,78 @@ +# bench — départage des adapters de reconnaissance (#10, étape 6) + +Bench **manuel** qui départage les deux adapters `ShelfScannerPort` sur de vraies photos +d'étagère, pour poser le **défaut de prod** (ADR 0005, phase 3 : le bench *est* l'instrumentation +prévue). Il n'est **jamais en CI** — chaque run coûte un appel payant par photo. + +Projet Nx dédié, tagué `type:app` / `scope:api` : comme `apps/api`, il a le droit de connaître +`infrastructure`, parce qu'il instancie les adapters réels. La logique pure (normalisation, +matching, métriques) vit soit dans la lib partagée [`shared-text-match`](../../libs/shared/text-match), +réutilisable par la réconciliation, soit dans `src/lib/` (testée en CI, sans réseau). Seul +`src/main.ts` et ses modules de run appellent le réseau. + +## Ce que le bench mesure + +Pour chaque fournisseur, sur le jeu de référence : + +| Métrique | Définition | +|---|---| +| **Rappel** | couples `(auteur, titre)` corrects / livres réellement présents | +| **Précision** | corrects / détectés | +| **Exactitude auteur / titre** | par champ, sur les détections rattachées à un vrai livre | +| **Erreurs de structuration** | auteur/titre permutés — cible ≈ 0 | +| **Hallucination haute confiance** | détections inventées avec `confidence ≥ seuil` (ADR 0005 pt 2) | +| **Coût / scan** | tokens facturés (coût rendu par OpenRouter, sinon estimé au prix du token) | +| **Latence médiane** | temps mur par scan | + +Un couple est « correct » si **auteur ET titre** correspondent, à la faute d'OCR près +(`shared-text-match`, seuil 0.85). Une tranche illisible non détectée est un **faux négatif**, +pas une erreur ; une photo sans livre lisible se lit en **tableau vide**. + +## Pré-requis + +1. **Photos de référence** dans `fixtures/reference-photos/` (gitignoré). Source de vérité : le + bucket GCS `pick-a-book-505922-reference-photos` (voir le commentaire du 31/08 sur #10). Les + 10 JPEG font ~54 Mo. +2. **Vérité terrain** dans `tools/bench/ground-truth.yaml` — copier + [`ground-truth.template.yaml`](ground-truth.template.yaml) et la remplir **à la main**. Sans + elle, le bench tourne quand même mais ne mesure que le contrat, le coût et la latence : ni + rappel, ni précision, ni hallucination, donc **aucune sélection possible**. + + > La vérité terrain **doit être vérifiée par un humain**. Un brouillon produit par un VLM pour + > départager deux VLM mesure leur accord, pas leur exactitude. La sortie `output/.json` + > d'un premier run fait gagner de la saisie — elle ne dispense pas de la vérification. +3. **Clés** dans l'environnement : `GEMINI_API_KEY` et `OPENROUTER_API_KEY`. +4. **Egress réseau** vers les hôtes des fournisseurs (`generativelanguage.googleapis.com`, + `openrouter.ai`). Certains environnements d'exécution restreignent l'egress par politique + réseau : un fournisseur injoignable fait échouer ses scans en `ShelfScanFailed` + (« unreachable ») — ce n'est pas un bug de l'adapter (constaté sur OpenRouter, cf. la note de + décision `docs/decisions/0001`). + +## Lancer + +```bash +yarn nx build bench # bundle le runner + ses dépendances de workspace +node tools/bench/dist/main.js # appels live, depuis la racine du repo +``` + +Sortie dans `tools/bench/output/` (gitignoré, dérivé des photos privées) : +`report.md` (le tableau) et `.json` (les détections brutes, par photo). + +## Configuration (variables d'environnement) + +| Variable | Défaut | Rôle | +|---|---|---| +| `BENCH_PROVIDERS` | `gemini,qwen` | fournisseurs à départager | +| `GEMINI_MODEL` / `QWEN_MODEL` | modèles de prod | épingler un modèle précis pour ce run | +| `QWEN_BASE_URL` | OpenRouter | pointer un endpoint OpenAI-compatible (ex. Ollama local pour **itérer le prompt** — jamais pour produire les chiffres, cf. #10) | +| `BENCH_HIGH_CONFIDENCE` | `0.8` | seuil de l'hallucination « haute confiance » | +| `BENCH_PHOTOS_DIR` | `fixtures/reference-photos` | dossier des photos | +| `BENCH_GROUND_TRUTH` | `tools/bench/ground-truth.yaml` | vérité terrain | +| `BENCH_OUTPUT_DIR` | `tools/bench/output` | sortie | + +## Après le run + +Consigner le tableau et le fournisseur gagnant dans +[`docs/decisions/0001-fournisseur-vlm-par-defaut.md`](../../docs/decisions/0001-fournisseur-vlm-par-defaut.md), +puis **poser le gagnant comme défaut** de `SHELF_SCANNER_PROVIDER`. Ce n'est pas un ADR : la +sélection de fournisseur est un niveau inférieur, tranché par l'ADR 0005. diff --git a/tools/bench/eslint.config.mjs b/tools/bench/eslint.config.mjs new file mode 100644 index 0000000..8caf44f --- /dev/null +++ b/tools/bench/eslint.config.mjs @@ -0,0 +1,25 @@ +import baseConfig from '../../eslint.config.mjs'; + +export default [ + ...baseConfig, + { + files: ['**/*.json'], + rules: { + '@nx/dependency-checks': [ + 'error', + { + ignoredFiles: [ + '{projectRoot}/eslint.config.{js,cjs,mjs,ts,cts,mts}', + '{projectRoot}/vitest.config.{js,cjs,mjs,ts,cts,mts}', + ], + }, + ], + }, + languageOptions: { + parser: await import('jsonc-eslint-parser'), + }, + }, + { + ignores: ['**/out-tsc', '**/dist', 'output'], + }, +]; diff --git a/tools/bench/ground-truth.template.yaml b/tools/bench/ground-truth.template.yaml new file mode 100644 index 0000000..e423e3d --- /dev/null +++ b/tools/bench/ground-truth.template.yaml @@ -0,0 +1,45 @@ +# Vérité terrain du jeu de référence — MODÈLE À COMPLÉTER PAR UN HUMAIN. +# +# Copier ce fichier en `tools/bench/ground-truth.yaml`, puis, pour chaque photo, saisir la +# liste réelle des couples (auteur, titre) présents et lisibles sur l'étagère. Sans cette +# vérité vérifiée à l'œil, ni rappel, ni précision, ni hallucination ne sont mesurables, donc +# aucun fournisseur ne peut être départagé (#10, ADR 0005). +# +# Réserve de méthode (commentaire du 31/08 sur #10) : cette vérité DOIT être vérifiée par un +# humain. Un brouillon produit par un VLM pour départager deux VLM mesure leur accord, pas leur +# exactitude. La sortie brute d'un run (`tools/bench/output/.json`) fait gagner de la +# saisie — elle ne dispense pas de la vérification. +# +# Règles de saisie : +# - un livre = un couple { author, title }, tel qu'il est lu sur la tranche (avant toute +# réconciliation) ; ne pas confondre l'auteur/titre avec l'éditeur ou la collection ; +# - une tranche illisible n'est pas saisie (elle deviendra un faux négatif, ce qui est voulu) ; +# - une étagère sans aucun livre lisible : `books: []`. +# +# Le fichier réel `ground-truth.yaml` est commité (texte, aucune image), à côté de ce modèle. + +version: 1 +photos: + - file: 20260801_113334.jpg + books: + # - author: Albert Camus + # title: La Peste + [] + - file: 20260801_113338.jpg + books: [] + - file: 20260801_113352.jpg + books: [] + - file: 20260801_113355.jpg + books: [] + - file: 20260801_113405.jpg + books: [] + - file: 20260801_113811.jpg + books: [] + - file: 20260801_114113.jpg + books: [] + - file: 20260801_114237.jpg + books: [] + - file: 20260801_114252.jpg + books: [] + - file: 20260801_114307.jpg + books: [] diff --git a/tools/bench/package.json b/tools/bench/package.json new file mode 100644 index 0000000..a173c32 --- /dev/null +++ b/tools/bench/package.json @@ -0,0 +1,22 @@ +{ + "name": "@pick-a-book/bench", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "./dist/main.js", + "nx": { + "name": "bench", + "tags": [ + "type:app", + "scope:api" + ] + }, + "dependencies": { + "@pick-a-book/recognition-domain": "workspace:*", + "@pick-a-book/recognition-infrastructure": "workspace:*", + "@pick-a-book/shared-text-match": "workspace:*", + "tslib": "^2.3.0", + "yaml": "^2.9.0", + "zod": "^4.5.4" + } +} diff --git a/tools/bench/src/config.ts b/tools/bench/src/config.ts new file mode 100644 index 0000000..771b0eb --- /dev/null +++ b/tools/bench/src/config.ts @@ -0,0 +1,87 @@ +import { extname } from 'node:path'; + +import type { ShelfScannerPort } from '@pick-a-book/recognition-domain'; +import { + GeminiShelfScannerAdapter, + QwenShelfScannerAdapter, +} from '@pick-a-book/recognition-infrastructure'; + +/** + * Reads the bench configuration off the environment and turns it into the list of providers to + * run. Kept apart from the live runner so it carries no network and no clock — the choices, not + * the calls. + */ +export interface ProviderSpec { + readonly name: string; + readonly model: string; + readonly adapter: (transport: typeof fetch) => ShelfScannerPort; +} + +// The real production defaults, copied here on purpose: a bench pins the exact model it tested +// rather than inheriting whatever an adapter would fall back to, so the report names a truth. +const DEFAULT_MODELS: Partial> = { + gemini: 'gemini-3.6-flash', + qwen: 'qwen/qwen3-vl-235b-a22b-instruct', +}; + +const MEDIA_TYPES: Partial> = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.webp': 'image/webp', + '.heic': 'image/heic', +}; + +export function mediaTypeOf(file: string): string | undefined { + return MEDIA_TYPES[extname(file).toLowerCase()]; +} + +export function env(name: string): string | undefined { + const value = process.env[name]; + return value !== undefined && value.trim().length > 0 ? value.trim() : undefined; +} + +export function requireEnv(name: string): string { + const value = env(name); + if (value === undefined) { + throw new Error(`Bench needs ${name} in the environment`); + } + return value; +} + +function modelFor(provider: string): string { + return env(`${provider.toUpperCase()}_MODEL`) ?? DEFAULT_MODELS[provider] ?? 'unknown'; +} + +export function selectedProviders(): ProviderSpec[] { + const requested = (env('BENCH_PROVIDERS') ?? 'gemini,qwen').split(',').map((name) => name.trim()); + return requested.map((name) => providerSpec(name)); +} + +function providerSpec(name: string): ProviderSpec { + const model = modelFor(name); + + if (name === 'gemini') { + return { + name, + model, + adapter: (transport) => + new GeminiShelfScannerAdapter( + { apiKey: requireEnv('GEMINI_API_KEY'), model, baseUrl: env('GEMINI_BASE_URL') }, + transport, + ), + }; + } + if (name === 'qwen') { + return { + name, + model, + adapter: (transport) => + new QwenShelfScannerAdapter( + { apiKey: requireEnv('OPENROUTER_API_KEY'), model, baseUrl: env('QWEN_BASE_URL') }, + transport, + ), + }; + } + throw new Error(`Unknown provider "${name}" — expected gemini or qwen`); +} diff --git a/tools/bench/src/io.ts b/tools/bench/src/io.ts new file mode 100644 index 0000000..62ddee3 --- /dev/null +++ b/tools/bench/src/io.ts @@ -0,0 +1,45 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; + +import { ShelfPhoto } from '@pick-a-book/recognition-domain'; + +import { mediaTypeOf } from './config.js'; +import { parseGroundTruth } from './lib/ground-truth.js'; +import type { GroundTruth } from './lib/ground-truth.js'; + +/** Reading the reference photos and their ground truth off disk. */ +export interface PhotoItem { + readonly file: string; + readonly photo: ShelfPhoto; +} + +export async function loadPhotos(dir: string): Promise { + const entries = (await readdir(dir)).toSorted((left, right) => left.localeCompare(right)); + const loaded = await Promise.all(entries.map(async (entry) => loadPhoto(dir, entry))); + return loaded.filter((item): item is PhotoItem => item !== undefined); +} + +async function loadPhoto(dir: string, entry: string): Promise { + const mediaType = mediaTypeOf(entry); + if (mediaType === undefined) { + return undefined; + } + const bytes = new Uint8Array(await readFile(join(dir, entry))); + return { file: basename(entry), photo: ShelfPhoto.of(bytes, mediaType) }; +} + +/** The ground truth, or `undefined` when the file is absent — a bench without quality scoring. */ +export async function loadGroundTruth(path: string): Promise { + try { + return parseGroundTruth(await readFile(path, 'utf8')); + } catch (error) { + if (isMissingFile(error)) { + return undefined; + } + throw error; + } +} + +function isMissingFile(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; +} diff --git a/tools/bench/src/lib/ground-truth.spec.ts b/tools/bench/src/lib/ground-truth.spec.ts new file mode 100644 index 0000000..61a9524 --- /dev/null +++ b/tools/bench/src/lib/ground-truth.spec.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest'; +import { ZodError } from 'zod'; + +import { booksFor, parseGroundTruth } from './ground-truth.js'; + +const VALID = ` +version: 1 +photos: + - file: shelf-a.jpg + books: + - author: Albert Camus + title: La Peste + - author: Franz Kafka + title: Le Procès + - file: shelf-b.jpg + books: [] +`; + +describe('parseGroundTruth', () => { + it('reads photos and their books', () => { + const truth = parseGroundTruth(VALID); + + expect(truth.photos).toHaveLength(2); + expect(truth.photos[0].file).toBe('shelf-a.jpg'); + expect(truth.photos[0].books).toStrictEqual([ + { author: 'Albert Camus', title: 'La Peste' }, + { author: 'Franz Kafka', title: 'Le Procès' }, + ]); + }); + + it('accepts a shelf with no readable book as an empty list', () => { + const truth = parseGroundTruth(VALID); + + expect(truth.photos[1].books).toStrictEqual([]); + }); + + it('rejects a wrong version rather than guessing', () => { + expect(() => parseGroundTruth('version: 2\nphotos: []\n')).toThrow(/version/u); + }); + + it('rejects a book missing a field', () => { + const missing = ` +version: 1 +photos: + - file: shelf.jpg + books: + - author: Albert Camus +`; + expect(() => parseGroundTruth(missing)).toThrow(ZodError); + }); + + it('rejects an empty author, which is never a real reading', () => { + const empty = ` +version: 1 +photos: + - file: shelf.jpg + books: + - author: "" + title: La Peste +`; + expect(() => parseGroundTruth(empty)).toThrow(ZodError); + }); +}); + +describe('booksFor', () => { + it('returns the books of a known photo', () => { + const truth = parseGroundTruth(VALID); + + expect(booksFor(truth, 'shelf-a.jpg')).toHaveLength(2); + }); + + it('returns undefined for a photo absent from the ground truth', () => { + const truth = parseGroundTruth(VALID); + + expect(booksFor(truth, 'unknown.jpg')).toBeUndefined(); + }); +}); diff --git a/tools/bench/src/lib/ground-truth.ts b/tools/bench/src/lib/ground-truth.ts new file mode 100644 index 0000000..f05d899 --- /dev/null +++ b/tools/bench/src/lib/ground-truth.ts @@ -0,0 +1,46 @@ +import { parse as parseYaml } from 'yaml'; +import { z } from 'zod'; + +import type { BookRef } from './scoring.js'; + +/** + * The human-verified ground truth of the reference set (#10). + * + * It lives in YAML, committed as text next to the bench protocol — no image ever is. Each + * photo names the books actually on the shelf, so recall and precision can be measured. The + * schema is strict on purpose: a silently dropped field would corrupt the very numbers that + * pick a provider, so a malformed truth fails loudly instead. + * + * ADR 0005 and the issue insist this truth be checked by a human. A VLM draft used to arbitrate + * two VLMs measures their agreement, not their accuracy — the draft saves typing, not judgement. + */ +const bookSchema = z.object({ + author: z.string().trim().min(1), + title: z.string().trim().min(1), +}); + +const photoSchema = z.object({ + file: z.string().trim().min(1), + books: z.array(bookSchema), +}); + +const groundTruthSchema = z.object({ + version: z.literal(1), + photos: z.array(photoSchema), +}); + +export type GroundTruth = z.infer; +export type GroundTruthPhoto = z.infer; + +/** Parses and validates ground-truth YAML. Throws on anything off-schema. */ +export function parseGroundTruth(yaml: string): GroundTruth { + return groundTruthSchema.parse(parseYaml(yaml)); +} + +/** + * The books listed for one photo, or `undefined` when the photo is absent from the truth — + * told apart from a shelf truly listed as empty, which returns `[]`. + */ +export function booksFor(truth: GroundTruth, file: string): readonly BookRef[] | undefined { + return truth.photos.find((photo) => photo.file === file)?.books; +} diff --git a/tools/bench/src/lib/render.spec.ts b/tools/bench/src/lib/render.spec.ts new file mode 100644 index 0000000..0ef13ef --- /dev/null +++ b/tools/bench/src/lib/render.spec.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; + +import { renderReport } from './render.js'; +import type { ProviderRun } from './render.js'; + +const run: ProviderRun = { + provider: 'gemini', + model: 'gemini-3.6-flash', + photosScanned: 10, + photosScored: 10, + failures: 0, + score: { + truthCount: 100, + detectedCount: 95, + truePositives: 75, + falsePositives: 20, + falseNegatives: 25, + correspondences: 90, + authorCorrect: 82, + titleCorrect: 80, + swapped: 1, + highConfidenceHallucinations: 3, + recall: 0.75, + precision: 75 / 95, + authorAccuracy: 82 / 90, + titleAccuracy: 80 / 90, + }, + medianLatencyMs: 4200, + totalPromptTokens: 12000, + totalCompletionTokens: 3000, + estimatedCostUsd: 0.037, +}; + +describe('renderReport', () => { + it('names every provider benched', () => { + const markdown = renderReport([run, { ...run, provider: 'qwen', model: 'qwen3-vl' }]); + + expect(markdown).toContain('gemini'); + expect(markdown).toContain('qwen'); + }); + + it('shows recall and precision as percentages', () => { + const markdown = renderReport([run]); + + // Recall 0.75, then precision 75/95. + expect(markdown).toContain('75.0%'); + expect(markdown).toContain('78.9%'); + }); + + it('surfaces the metrics ADR 0005 watches: structuring errors and high-confidence hallucination', () => { + const markdown = renderReport([run]); + + expect(markdown).toMatch(/swap|structur/iu); + expect(markdown).toMatch(/hallucinat/iu); + }); + + it('reports an unknown cost as such rather than as zero', () => { + const markdown = renderReport([{ ...run, estimatedCostUsd: null }]); + + expect(markdown).toContain('n/a'); + }); + + it('notes when a run has no ground truth to score against', () => { + const markdown = renderReport([{ ...run, photosScored: 0 }]); + + expect(markdown).toMatch(/ground truth|vérité/iu); + }); +}); diff --git a/tools/bench/src/lib/render.ts b/tools/bench/src/lib/render.ts new file mode 100644 index 0000000..1e3f9ff --- /dev/null +++ b/tools/bench/src/lib/render.ts @@ -0,0 +1,85 @@ +import type { AggregateScore } from './scoring.js'; + +/** + * Rendering of the bench result as a Markdown table — the artefact the decision note carries + * (#10). A document, so it is written in French, unlike the code around it; the winner is + * still a human call, this only lays the numbers side by side. + */ +export interface ProviderRun { + readonly provider: string; + readonly model: string; + readonly photosScanned: number; + /** Photos with a ground truth to score against — quality is undefined when this is zero. */ + readonly photosScored: number; + /** Photos where the adapter threw `ShelfScanFailed`. */ + readonly failures: number; + readonly score: AggregateScore; + readonly medianLatencyMs: number; + readonly totalPromptTokens: number; + readonly totalCompletionTokens: number; + /** Total USD across the run, or `null` when the provider did not report usable usage. */ + readonly estimatedCostUsd: number | null; +} + +export function renderReport(runs: readonly ProviderRun[]): string { + const header = `| Métrique | ${runs.map((run) => run.provider).join(' | ')} |`; + const divider = `|---|${runs.map(() => '---').join('|')}|`; + + const rows: { label: string; cell: (run: ProviderRun) => string }[] = [ + { label: 'Modèle', cell: (run) => `\`${run.model}\`` }, + { label: 'Photos scannées', cell: (run) => String(run.photosScanned) }, + { label: 'Photos notées (vérité terrain)', cell: (run) => String(run.photosScored) }, + { label: 'Échecs adapter', cell: (run) => String(run.failures) }, + { label: 'Rappel', cell: (run) => quality(run, percent(run.score.recall)) }, + { label: 'Précision', cell: (run) => quality(run, percent(run.score.precision)) }, + { label: 'Exactitude auteur', cell: (run) => quality(run, percent(run.score.authorAccuracy)) }, + { label: 'Exactitude titre', cell: (run) => quality(run, percent(run.score.titleAccuracy)) }, + { + label: 'Erreurs de structuration (auteur/titre permutés)', + cell: (run) => quality(run, String(run.score.swapped)), + }, + { + label: 'Hallucination haute confiance', + cell: (run) => quality(run, String(run.score.highConfidenceHallucinations)), + }, + { label: 'Latence médiane', cell: (run) => `${(run.medianLatencyMs / 1000).toFixed(1)} s` }, + { + label: 'Tokens (prompt / complétion)', + cell: (run) => `${run.totalPromptTokens} / ${run.totalCompletionTokens}`, + }, + { label: 'Coût total', cell: (run) => cost(run.estimatedCostUsd) }, + { + label: 'Coût / scan', + cell: (run) => + run.estimatedCostUsd === null || run.photosScanned === 0 + ? 'n/a' + : cost(run.estimatedCostUsd / run.photosScanned), + }, + ]; + + const body = rows + .map((row) => `| ${row.label} | ${runs.map((run) => row.cell(run)).join(' | ')} |`) + .join('\n'); + + const warnings = runs + .filter((run) => run.photosScored === 0) + .map( + (run) => + `> ⚠️ **${run.provider}** : aucune photo notée — vérité terrain absente, qualité non mesurée.`, + ); + + return [header, divider, body, ...warnings].join('\n'); +} + +/** Quality cells collapse to a dash when there is no ground truth to compute them against. */ +function quality(run: ProviderRun, value: string): string { + return run.photosScored === 0 ? '—' : value; +} + +function percent(rate: number): string { + return `${(rate * 100).toFixed(1)}%`; +} + +function cost(usd: number | null): string { + return usd === null ? 'n/a' : `$${usd.toFixed(4)}`; +} diff --git a/tools/bench/src/lib/scoring.spec.ts b/tools/bench/src/lib/scoring.spec.ts new file mode 100644 index 0000000..e6455a2 --- /dev/null +++ b/tools/bench/src/lib/scoring.spec.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from 'vitest'; + +import { aggregate, scorePhoto } from './scoring.js'; +import type { BookRef, DetectionRecord } from './scoring.js'; + +const HIGH = 0.8; + +function det(author: string, title: string, confidence: number): DetectionRecord { + return { author, title, confidence }; +} + +function ref(author: string, title: string): BookRef { + return { author, title }; +} + +describe('scorePhoto — a correct read', () => { + it('counts a perfect read as all true positives', () => { + const truth = [ref('Albert Camus', 'La Peste'), ref('Marcel Proust', 'Du côté de chez Swann')]; + const detected = [ + det('Albert Camus', 'La Peste', 0.9), + det('Marcel Proust', 'Du côté de chez Swann', 0.95), + ]; + + expect(scorePhoto(detected, truth, HIGH)).toStrictEqual({ + truthCount: 2, + detectedCount: 2, + truePositives: 2, + falsePositives: 0, + falseNegatives: 0, + correspondences: 2, + authorCorrect: 2, + titleCorrect: 2, + swapped: 0, + highConfidenceHallucinations: 0, + }); + }); + + it('forgives OCR-level typos when matching a pair', () => { + const truth = [ref('Céline', 'Voyage au bout de la nuit')]; + const detected = [det('Celine', 'Voyage au bout de la nuit.', 0.7)]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.truePositives).toBe(1); + expect(score.falsePositives).toBe(0); + expect(score.falseNegatives).toBe(0); + }); +}); + +describe('scorePhoto — misses', () => { + it('treats an undetected book as a false negative, not an error', () => { + const truth = [ref('Albert Camus', 'La Peste'), ref('Franz Kafka', 'Le Procès')]; + const detected = [det('Albert Camus', 'La Peste', 0.9)]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.truePositives).toBe(1); + expect(score.falseNegatives).toBe(1); + expect(score.falsePositives).toBe(0); + }); + + it('reads an empty detection of an empty shelf as nothing wrong', () => { + const score = scorePhoto([], [], HIGH); + + expect(score.truePositives).toBe(0); + expect(score.falsePositives).toBe(0); + expect(score.falseNegatives).toBe(0); + }); +}); + +describe('scorePhoto — hallucination', () => { + it('counts an invented book as a false positive', () => { + const truth = [ref('Albert Camus', 'La Peste')]; + const detected = [ + det('Albert Camus', 'La Peste', 0.9), + det('Jean Ficelle', 'Le Grand Néant', 0.6), + ]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.truePositives).toBe(1); + expect(score.falsePositives).toBe(1); + }); + + it('flags a high-confidence invention specifically', () => { + const truth = [ref('Albert Camus', 'La Peste')]; + // One invention sure of itself, one hedged: only the confident one is the watched failure. + const detected = [ + det('Jean Ficelle', 'Le Grand Néant', 0.95), + det('Paul Brume', 'Ombres', 0.2), + ]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.falsePositives).toBe(2); + expect(score.highConfidenceHallucinations).toBe(1); + }); +}); + +describe('scorePhoto — structuring', () => { + it('detects an author/title swap as a structuring error rather than a match', () => { + const truth = [ref('Albert Camus', 'La Peste')]; + const detected = [det('La Peste', 'Albert Camus', 0.9)]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.truePositives).toBe(0); + expect(score.swapped).toBe(1); + expect(score.correspondences).toBe(1); + }); + + it('credits the correct field when only one is right', () => { + const truth = [ref('Albert Camus', 'La Peste')]; + const detected = [det('Albert Camus', 'La Chute', 0.9)]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.truePositives).toBe(0); + expect(score.authorCorrect).toBe(1); + expect(score.titleCorrect).toBe(0); + expect(score.correspondences).toBe(1); + }); +}); + +describe('aggregate', () => { + it('micro-averages counts and derives the rates from the totals', () => { + const first = scorePhoto( + [det('Albert Camus', 'La Peste', 0.9)], + [ref('Albert Camus', 'La Peste'), ref('Franz Kafka', 'Le Procès')], + HIGH, + ); + const second = scorePhoto( + [det('Franz Kafka', 'Le Procès', 0.9), det('Jean Ficelle', 'Le Néant', 0.95)], + [ref('Franz Kafka', 'Le Procès')], + HIGH, + ); + + const totals = aggregate([first, second]); + + expect(totals.truthCount).toBe(3); + expect(totals.detectedCount).toBe(3); + expect(totals.truePositives).toBe(2); + expect(totals.recall).toBeCloseTo(2 / 3, 5); + expect(totals.precision).toBeCloseTo(2 / 3, 5); + expect(totals.highConfidenceHallucinations).toBe(1); + }); + + it('reports zero rates rather than dividing by zero on an empty run', () => { + const totals = aggregate([]); + + expect(totals.recall).toBe(0); + expect(totals.precision).toBe(0); + expect(totals.authorAccuracy).toBe(0); + expect(totals.titleAccuracy).toBe(0); + }); +}); diff --git a/tools/bench/src/lib/scoring.ts b/tools/bench/src/lib/scoring.ts new file mode 100644 index 0000000..5e3b759 --- /dev/null +++ b/tools/bench/src/lib/scoring.ts @@ -0,0 +1,248 @@ +import { fuzzyEquals, similarity } from '@pick-a-book/shared-text-match'; + +/** + * Scoring of a shelf reading against a human-verified ground truth (#10). + * + * The unit under measure is the couple `(author, title)`: a detection counts as correct only + * when both fields match the same real book, allowing for OCR-level noise (`shared-text-match`). + * This is where recall, precision, per-field accuracy, structuring errors and high-confidence + * hallucination come from — the metrics ADR 0005 wants in hand before a provider is chosen. + * + * Pure and free of any provider: it scores records, so the same code scores Gemini and Qwen, + * and it runs in CI with no network and no key. + */ + +/** One book read off a photo, as the adapter returned it — plain fields, no domain object. */ +export interface DetectionRecord { + readonly author: string; + readonly title: string; + readonly confidence: number; +} + +/** One book actually on the shelf, from the ground truth. */ +export interface BookRef { + readonly author: string; + readonly title: string; +} + +/** Raw counts for a single photo. Rates are derived only once, on the aggregate. */ +export interface PhotoScore { + readonly truthCount: number; + readonly detectedCount: number; + readonly truePositives: number; + readonly falsePositives: number; + readonly falseNegatives: number; + /** Detections paired with a real book, right or wrong — the denominator of field accuracy. */ + readonly correspondences: number; + readonly authorCorrect: number; + readonly titleCorrect: number; + readonly swapped: number; + readonly highConfidenceHallucinations: number; +} + +export interface AggregateScore extends PhotoScore { + readonly recall: number; + readonly precision: number; + readonly authorAccuracy: number; + readonly titleAccuracy: number; +} + +/** + * A detection and the truth book it most plausibly refers to. + * + * Establishing this correspondence — before judging whether the read is correct — is what + * lets a half-right detection credit the field it got right, and lets a swap be told apart + * from a plain miss. + */ +interface Correspondence { + readonly detection: DetectionRecord; + readonly truth: BookRef; +} + +/** How close a detection sits to a truth book, ignoring which field is which. */ +function affinity(detection: DetectionRecord, truth: BookRef): number { + const aligned = + similarity(detection.author, truth.author) + similarity(detection.title, truth.title); + // A swap still refers to the same book; count its best reading so it corresponds rather + // than being mistaken for a hallucination. + const swapped = + similarity(detection.author, truth.title) + similarity(detection.title, truth.author); + + return Math.max(aligned, swapped); +} + +/** + * Greedily pairs detections to truth books by descending affinity, one to one. Greedy rather + * than optimal: on a real shelf the strong pairs are unambiguous, and the cost of an exact + * assignment is not worth it for a measurement tool. + */ +function correspond( + detected: readonly DetectionRecord[], + truth: readonly BookRef[], +): Correspondence[] { + const candidates: { affinity: number; detectionIndex: number; truthIndex: number }[] = []; + detected.forEach((detection, detectionIndex) => { + truth.forEach((book, truthIndex) => { + candidates.push({ affinity: affinity(detection, book), detectionIndex, truthIndex }); + }); + }); + candidates.sort((left, right) => right.affinity - left.affinity); + + const usedDetections = new Set(); + const usedTruth = new Set(); + const pairs: Correspondence[] = []; + + for (const candidate of candidates) { + // Below one field's worth of resemblance, this is not the same book: leave the detection + // to stand as a hallucination and the truth book as a miss. + if (candidate.affinity <= 0) { + break; + } + if (usedDetections.has(candidate.detectionIndex) || usedTruth.has(candidate.truthIndex)) { + continue; + } + usedDetections.add(candidate.detectionIndex); + usedTruth.add(candidate.truthIndex); + pairs.push({ + detection: detected[candidate.detectionIndex], + truth: truth[candidate.truthIndex], + }); + } + + return pairs; +} + +export function scorePhoto( + detected: readonly DetectionRecord[], + truth: readonly BookRef[], + highConfidenceThreshold: number, +): PhotoScore { + const pairs = correspond(detected, truth); + const fields = tallyFields(pairs); + + return { + truthCount: truth.length, + detectedCount: detected.length, + truePositives: fields.truePositives, + falsePositives: detected.length - fields.truePositives, + falseNegatives: truth.length - fields.truePositives, + correspondences: pairs.length, + authorCorrect: fields.authorCorrect, + titleCorrect: fields.titleCorrect, + swapped: fields.swapped, + highConfidenceHallucinations: countHighConfidenceHallucinations( + detected, + pairs, + highConfidenceThreshold, + ), + }; +} + +interface FieldTally { + readonly truePositives: number; + readonly authorCorrect: number; + readonly titleCorrect: number; + readonly swapped: number; +} + +/** Judges each correspondence: right pair, right field, or an author/title swap. */ +function tallyFields(pairs: readonly Correspondence[]): FieldTally { + let truePositives = 0; + let authorCorrect = 0; + let titleCorrect = 0; + let swapped = 0; + + for (const { detection, truth: book } of pairs) { + const authorRight = fuzzyEquals(detection.author, book.author); + const titleRight = fuzzyEquals(detection.title, book.title); + + if (authorRight) { + authorCorrect++; + } + if (titleRight) { + titleCorrect++; + } + if (authorRight && titleRight) { + truePositives++; + } else if (isSwap(detection, book)) { + swapped++; + } + } + + return { truePositives, authorCorrect, titleCorrect, swapped }; +} + +function isSwap(detection: DetectionRecord, book: BookRef): boolean { + return fuzzyEquals(detection.author, book.title) && fuzzyEquals(detection.title, book.author); +} + +/** + * A hallucination is a detection that matched no real book as a correct pair; the ones that did + * so while confident are the failure ADR 0005 watches to justify option C one day. + */ +function countHighConfidenceHallucinations( + detected: readonly DetectionRecord[], + pairs: readonly Correspondence[], + threshold: number, +): number { + const correct = new Set( + pairs.filter((pair) => isCorrectPair(pair)).map((pair) => pair.detection), + ); + + return detected.filter( + (detection) => detection.confidence >= threshold && !correct.has(detection), + ).length; +} + +function isCorrectPair(pair: Correspondence): boolean { + return ( + fuzzyEquals(pair.detection.author, pair.truth.author) && + fuzzyEquals(pair.detection.title, pair.truth.title) + ); +} + +/** Micro-averages a set of per-photo scores: sum the counts, then derive the rates once. */ +export function aggregate(scores: readonly PhotoScore[]): AggregateScore { + const summed = scores.reduce( + (totals, score) => ({ + truthCount: totals.truthCount + score.truthCount, + detectedCount: totals.detectedCount + score.detectedCount, + truePositives: totals.truePositives + score.truePositives, + falsePositives: totals.falsePositives + score.falsePositives, + falseNegatives: totals.falseNegatives + score.falseNegatives, + correspondences: totals.correspondences + score.correspondences, + authorCorrect: totals.authorCorrect + score.authorCorrect, + titleCorrect: totals.titleCorrect + score.titleCorrect, + swapped: totals.swapped + score.swapped, + highConfidenceHallucinations: + totals.highConfidenceHallucinations + score.highConfidenceHallucinations, + }), + EMPTY, + ); + + return { + ...summed, + recall: ratio(summed.truePositives, summed.truthCount), + precision: ratio(summed.truePositives, summed.detectedCount), + authorAccuracy: ratio(summed.authorCorrect, summed.correspondences), + titleAccuracy: ratio(summed.titleCorrect, summed.correspondences), + }; +} + +const EMPTY: PhotoScore = { + truthCount: 0, + detectedCount: 0, + truePositives: 0, + falsePositives: 0, + falseNegatives: 0, + correspondences: 0, + authorCorrect: 0, + titleCorrect: 0, + swapped: 0, + highConfidenceHallucinations: 0, +}; + +/** Zero rather than NaN when the denominator is zero — an empty run is not a failure. */ +function ratio(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : numerator / denominator; +} diff --git a/tools/bench/src/main.ts b/tools/bench/src/main.ts new file mode 100644 index 0000000..95dfffb --- /dev/null +++ b/tools/bench/src/main.ts @@ -0,0 +1,49 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +import { env, selectedProviders } from './config.js'; +import { loadGroundTruth, loadPhotos } from './io.js'; +import { renderReport } from './lib/render.js'; +import type { ProviderRun } from './lib/render.js'; +import { benchProvider, forEachSequential } from './runner.js'; + +/** + * The manual quality bench of the two shelf-scanner adapters (#10, step 6). + * + * Live calls, never in CI: it reads the reference photos, sends each to every selected provider, + * scores the reading against the human-verified ground truth, and writes the metrics table the + * decision note carries. The photos and the raw output stay out of git; only the ground truth + * (text) and the decision note are committed. + * + * Run it from the repo root, keys in the environment: + * nx build bench && node tools/bench/dist/main.js + */ +async function main(): Promise { + const photosDir = resolve(env('BENCH_PHOTOS_DIR') ?? 'fixtures/reference-photos'); + const truthPath = resolve(env('BENCH_GROUND_TRUTH') ?? 'tools/bench/ground-truth.yaml'); + const outputDir = resolve(env('BENCH_OUTPUT_DIR') ?? 'tools/bench/output'); + const highConfidence = Number(env('BENCH_HIGH_CONFIDENCE') ?? '0.8'); + + await mkdir(outputDir, { recursive: true }); + + const photos = await loadPhotos(photosDir); + if (photos.length === 0) { + throw new Error(`No reference photo in ${photosDir} — see tools/bench/README.md`); + } + const truth = await loadGroundTruth(truthPath); + process.stdout.write( + `Benching ${photos.length} photos${truth === undefined ? ' (no ground truth — contract only)' : ''}\n`, + ); + + const runs: ProviderRun[] = []; + await forEachSequential(selectedProviders(), async (spec) => { + process.stdout.write(`\n== ${spec.name} ==\n`); + runs.push(await benchProvider(spec, photos, truth, highConfidence, outputDir)); + }); + + const report = renderReport(runs); + await writeFile(join(outputDir, 'report.md'), `${report}\n`, 'utf8'); + process.stdout.write(`\n${report}\n\nWritten to ${join(outputDir, 'report.md')}\n`); +} + +await main(); diff --git a/tools/bench/src/runner.ts b/tools/bench/src/runner.ts new file mode 100644 index 0000000..3424b35 --- /dev/null +++ b/tools/bench/src/runner.ts @@ -0,0 +1,158 @@ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { ShelfScanFailed } from '@pick-a-book/recognition-domain'; +import type { DetectedBook, ShelfScannerPort } from '@pick-a-book/recognition-domain'; + +import type { ProviderSpec } from './config.js'; +import type { GroundTruth } from './lib/ground-truth.js'; +import { booksFor } from './lib/ground-truth.js'; +import type { ProviderRun } from './lib/render.js'; +import { aggregate, scorePhoto } from './lib/scoring.js'; +import type { DetectionRecord, PhotoScore } from './lib/scoring.js'; +import type { PhotoItem } from './io.js'; +import { estimateCost, recordingTransport } from './usage.js'; +import type { CallMetrics } from './usage.js'; + +/** Runs the scans of one provider and gathers everything the report needs from them. */ +interface Accumulator { + readonly scores: PhotoScore[]; + readonly latencies: number[]; + readonly perPhoto: Record; + failures: number; + promptTokens: number; + completionTokens: number; + reportedCostUsd: number | null; + scored: number; +} + +/** Runs items one after another — the bench must not fan out and race the rate limits. */ +export async function forEachSequential( + items: readonly T[], + fn: (item: T) => Promise, +): Promise { + await items.reduce>(async (chain, item) => { + await chain; + await fn(item); + }, Promise.resolve()); +} + +export async function benchProvider( + spec: ProviderSpec, + photos: readonly PhotoItem[], + truth: GroundTruth | undefined, + highConfidence: number, + outputDir: string, +): Promise { + const { transport, last } = recordingTransport(); + const adapter = spec.adapter(transport); + const acc = emptyAccumulator(); + + await forEachSequential(photos, async (item) => { + await scanOne(spec.name, adapter, last, acc, item, truth, highConfidence); + }); + + await writeFile( + join(outputDir, `${spec.name}.json`), + JSON.stringify(acc.perPhoto, null, 2), + 'utf8', + ); + return assembleRun(spec, photos.length, acc); +} + +async function scanOne( + providerName: string, + adapter: ShelfScannerPort, + last: () => CallMetrics | undefined, + acc: Accumulator, + item: PhotoItem, + truth: GroundTruth | undefined, + highConfidence: number, +): Promise { + let records: DetectionRecord[]; + try { + records = toRecords(await adapter.scan(item.photo)); + } catch (error) { + if (error instanceof ShelfScanFailed) { + acc.failures++; + acc.perPhoto[item.file] = { error: error.message }; + process.stdout.write(` ${providerName} ${item.file} FAILED: ${error.message}\n`); + return; + } + throw error; + } + + const metrics = last(); + if (metrics !== undefined) { + applyMetrics(acc, metrics); + } + acc.perPhoto[item.file] = { books: records }; + process.stdout.write(` ${providerName} ${item.file} ${records.length} books\n`); + + const reference = truth === undefined ? undefined : booksFor(truth, item.file); + if (reference !== undefined) { + acc.scores.push(scorePhoto(records, reference, highConfidence)); + acc.scored++; + } +} + +function applyMetrics(acc: Accumulator, metrics: CallMetrics): void { + acc.latencies.push(metrics.latencyMs); + acc.promptTokens += metrics.promptTokens; + acc.completionTokens += metrics.completionTokens; + if (metrics.reportedCostUsd === null) { + acc.reportedCostUsd = null; + } else if (acc.reportedCostUsd !== null) { + acc.reportedCostUsd += metrics.reportedCostUsd; + } +} + +function assembleRun(spec: ProviderSpec, scanned: number, acc: Accumulator): ProviderRun { + return { + provider: spec.name, + model: spec.model, + photosScanned: scanned, + photosScored: acc.scored, + failures: acc.failures, + score: aggregate(acc.scores), + medianLatencyMs: median(acc.latencies), + totalPromptTokens: acc.promptTokens, + totalCompletionTokens: acc.completionTokens, + estimatedCostUsd: estimateCost( + spec.name, + acc.promptTokens, + acc.completionTokens, + acc.reportedCostUsd, + ), + }; +} + +function emptyAccumulator(): Accumulator { + return { + scores: [], + latencies: [], + perPhoto: {}, + failures: 0, + promptTokens: 0, + completionTokens: 0, + reportedCostUsd: 0, + scored: 0, + }; +} + +function toRecords(books: readonly DetectedBook[]): DetectionRecord[] { + return books.map((book) => ({ + author: book.author.value, + title: book.title.value, + confidence: book.confidence.value, + })); +} + +function median(values: readonly number[]): number { + if (values.length === 0) { + return 0; + } + const sorted = values.toSorted((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; +} diff --git a/tools/bench/src/usage.ts b/tools/bench/src/usage.ts new file mode 100644 index 0000000..19b09b1 --- /dev/null +++ b/tools/bench/src/usage.ts @@ -0,0 +1,101 @@ +import { performance } from 'node:perf_hooks'; + +import { z } from 'zod'; + +/** + * Instruments `fetch` so the bench can bill each scan: it times the call and reads token usage + * off a clone of the response, leaving the body for the adapter. One scan is one call, so the + * last recorded metrics are that scan's. + */ +export interface CallMetrics { + readonly latencyMs: number; + readonly promptTokens: number; + readonly completionTokens: number; + readonly reportedCostUsd: number | null; +} + +// Rough token prices, USD per million, so a run without a provider-reported cost still gets an +// estimate. Image tokens dominate and the catalogue moves, so the report labels this "estimé". +const PRICES: Partial> = { + gemini: { input: 0.3, output: 2.5 }, + qwen: { input: 0.2, output: 0.6 }, +}; + +const geminiUsageSchema = z.object({ + usageMetadata: z.object({ + promptTokenCount: z.number().optional(), + candidatesTokenCount: z.number().optional(), + }), +}); + +const openAiUsageSchema = z.object({ + usage: z.object({ + prompt_tokens: z.number().optional(), + completion_tokens: z.number().optional(), + cost: z.number().optional(), + }), +}); + +export function recordingTransport(): { + transport: typeof fetch; + last: () => CallMetrics | undefined; +} { + let latest: CallMetrics | undefined; + + const transport: typeof fetch = async (input, init) => { + const start = performance.now(); + const response = await fetch(input, init); + const latencyMs = performance.now() - start; + latest = { latencyMs, ...(await readUsage(response.clone())) }; + return response; + }; + + return { transport, last: () => latest }; +} + +async function readUsage(response: Response): Promise> { + const empty = { promptTokens: 0, completionTokens: 0, reportedCostUsd: null }; + let body: unknown; + try { + body = await response.json(); + } catch { + return empty; + } + + const gemini = geminiUsageSchema.safeParse(body); + if (gemini.success) { + return { + promptTokens: gemini.data.usageMetadata.promptTokenCount ?? 0, + completionTokens: gemini.data.usageMetadata.candidatesTokenCount ?? 0, + reportedCostUsd: null, + }; + } + + const openai = openAiUsageSchema.safeParse(body); + if (openai.success) { + return { + promptTokens: openai.data.usage.prompt_tokens ?? 0, + completionTokens: openai.data.usage.completion_tokens ?? 0, + reportedCostUsd: openai.data.usage.cost ?? null, + }; + } + + return empty; +} + +/** The provider's own billed amount when it gave one, otherwise a token-price estimate. */ +export function estimateCost( + provider: string, + promptTokens: number, + completionTokens: number, + reportedCostUsd: number | null, +): number | null { + if (reportedCostUsd !== null && reportedCostUsd > 0) { + return reportedCostUsd; + } + const price = PRICES[provider]; + if (price === undefined || promptTokens + completionTokens === 0) { + return null; + } + return (promptTokens * price.input + completionTokens * price.output) / 1_000_000; +} diff --git a/tools/bench/tsconfig.json b/tools/bench/tsconfig.json new file mode 100644 index 0000000..62ebbd9 --- /dev/null +++ b/tools/bench/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/tools/bench/tsconfig.lib.json b/tools/bench/tsconfig.lib.json new file mode 100644 index 0000000..874a8c9 --- /dev/null +++ b/tools/bench/tsconfig.lib.json @@ -0,0 +1,25 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo", + "emitDeclarationOnly": false, + "forceConsistentCasingInFileNames": true, + "types": ["node"], + "lib": ["es2023"] + }, + "include": ["src/**/*.ts"], + "references": [ + { + "path": "../../libs/shared/text-match/tsconfig.lib.json" + }, + { + "path": "../../libs/recognition/infrastructure/tsconfig.lib.json" + }, + { + "path": "../../libs/recognition/domain/tsconfig.lib.json" + } + ], + "exclude": ["vitest.config.mts", "src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/tools/bench/tsconfig.spec.json b/tools/bench/tsconfig.spec.json new file mode 100644 index 0000000..49f8c9a --- /dev/null +++ b/tools/bench/tsconfig.spec.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./out-tsc/vitest", + "types": ["node"], + "forceConsistentCasingInFileNames": true, + "lib": ["es2023"] + }, + "include": ["vitest.config.mts", "src/**/*.test.ts", "src/**/*.spec.ts", "src/**/*.d.ts"], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} diff --git a/tools/bench/vitest.config.mts b/tools/bench/vitest.config.mts new file mode 100644 index 0000000..b2363c3 --- /dev/null +++ b/tools/bench/vitest.config.mts @@ -0,0 +1,19 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + root: import.meta.dirname, + cacheDir: '../../node_modules/.vite/tools/bench', + test: { + name: 'bench', + watch: false, + environment: 'node', + // Only the pure logic is unit-tested; the live runner (`src/main.ts`) is never in CI — + // it costs a paid call per photo. It stays out of the include on purpose. + include: ['src/lib/**/*.{test,spec}.ts'], + reporters: ['default'], + coverage: { + reportsDirectory: './test-output/vitest/coverage', + provider: 'v8', + }, + }, +}); diff --git a/tsconfig.json b/tsconfig.json index 0981135..95131ca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,6 +20,12 @@ }, { "path": "./libs/shared/result" + }, + { + "path": "./libs/shared/text-match" + }, + { + "path": "./tools/bench" } ], "compilerOptions": {} diff --git a/yarn.lock b/yarn.lock index 0f0512c..5089a8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3303,6 +3303,19 @@ __metadata: languageName: unknown linkType: soft +"@pick-a-book/bench@workspace:tools/bench": + version: 0.0.0-use.local + resolution: "@pick-a-book/bench@workspace:tools/bench" + dependencies: + "@pick-a-book/recognition-domain": "workspace:*" + "@pick-a-book/recognition-infrastructure": "workspace:*" + "@pick-a-book/shared-text-match": "workspace:*" + tslib: "npm:^2.3.0" + yaml: "npm:^2.9.0" + zod: "npm:^4.5.4" + languageName: unknown + linkType: soft + "@pick-a-book/recognition-application@workspace:*, @pick-a-book/recognition-application@workspace:libs/recognition/application": version: 0.0.0-use.local resolution: "@pick-a-book/recognition-application@workspace:libs/recognition/application" @@ -3338,6 +3351,14 @@ __metadata: languageName: unknown linkType: soft +"@pick-a-book/shared-text-match@workspace:*, @pick-a-book/shared-text-match@workspace:libs/shared/text-match": + version: 0.0.0-use.local + resolution: "@pick-a-book/shared-text-match@workspace:libs/shared/text-match" + dependencies: + tslib: "npm:^2.3.0" + languageName: unknown + linkType: soft + "@pick-a-book/source@workspace:.": version: 0.0.0-use.local resolution: "@pick-a-book/source@workspace:." @@ -13404,7 +13425,7 @@ __metadata: languageName: node linkType: hard -"yaml@npm:2.9.0, yaml@npm:^2.8.3": +"yaml@npm:2.9.0, yaml@npm:^2.8.3, yaml@npm:^2.9.0": version: 2.9.0 resolution: "yaml@npm:2.9.0" bin: From ebba390c99eb0f671ac851454582f5a8eea3e555 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 10:30:09 +0000 Subject: [PATCH 2/9] docs(bench): record the Qwen run and the proxied-env fetch note (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenRouter is reachable now that its domain is on the environment egress. Both providers ran end to end: near-equal cost (~0.25c vs ~0.23c/scan), Qwen ~7x faster, but Qwen is unstable under the shared json_object prompt — 5 of 10 photos return 0 books, one returns 158, one fails on truncated JSON — while Gemini stays regular via native schema-constrained decoding. An operational signal, not the quality verdict: that still waits on the human-verified ground truth. Also document that Node's fetch needs NODE_USE_ENV_PROXY=1 to honor HTTPS_PROXY in a proxied environment, otherwise calls 403 on the egress allowlist. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .../0001-fournisseur-vlm-par-defaut.md | 47 +++++++++++-------- tools/bench/README.md | 12 +++-- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/docs/decisions/0001-fournisseur-vlm-par-defaut.md b/docs/decisions/0001-fournisseur-vlm-par-defaut.md index ee1fc46..723ddde 100644 --- a/docs/decisions/0001-fournisseur-vlm-par-defaut.md +++ b/docs/decisions/0001-fournisseur-vlm-par-defaut.md @@ -39,30 +39,37 @@ une photo sans livre lisible se lit en tableau vide. ### Contrat, coût, latence (run du 2026-09-02, sans vérité terrain) +Les deux fournisseurs ont tourné de bout en bout (OpenRouter atteint après ajout de son domaine à +l'egress de l'environnement ; en environnement proxifié, le runner a besoin de `NODE_USE_ENV_PROXY=1` +pour que le `fetch` de Node passe par le proxy — cf. `tools/bench/README.md`). + | Métrique | gemini | qwen | |---|---|---| | Modèle | `gemini-3.6-flash` | `qwen/qwen3-vl-235b-a22b-instruct` | | Photos scannées | 10 | 10 | -| Échecs adapter | 1 | 10 | -| Latence médiane | 19,7 s | — | -| Tokens (prompt / complétion) | 11808 / 9389 | 0 / 0 | -| Coût total | $0,0270 | n/a | -| Coût / scan | $0,0027 | n/a | - -Gemini a lu 9 des 10 photos (4 à 62 livres par étagère selon la densité), un échec sur un `503` -transitoire de l'API (« high demand ») — l'adapter l'a bien remonté en `ShelfScanFailed` et le run -a continué. Coût mesuré ~0,27 ¢/scan, dans l'ordre de grandeur annoncé (~0,37 ¢). - -> **Qwen n'a pas pu être mesuré depuis cet environnement.** Les 10 appels ont échoué en -> « unreachable (fetch failed) » : la **politique réseau** de l'environnement d'exécution autorise -> Google (Gemini passe) mais **bloque `openrouter.ai`** (le proxy répond `403` au `CONNECT`). Ce -> n'est pas un bug de l'adapter — il a correctement levé `ShelfScanFailed` sur l'échec de connexion. -> Le run Qwen doit se faire depuis un environnement dont la politique réseau autorise l'egress vers -> OpenRouter (ou un endpoint OpenAI-compatible joignable, cf. `QWEN_BASE_URL`). - -Ces chiffres valident la chaîne de bout en bout côté Gemini et donnent son coût et sa latence. Ils -ne disent **rien** de la qualité : un fournisseur peut détecter beaucoup de livres et en inventer -autant. C'est la vérité terrain qui tranche. +| Échecs adapter | 1 | 1 | +| Latence médiane | 32,7 s | 4,8 s | +| Tokens (prompt / complétion) | 11808 / 8389 | 50138 / 7356 | +| Coût total | $0,0245 | $0,0232 | +| Coût / scan | $0,0025 | $0,0023 | + +Coûts quasi identiques (~0,25 ¢ vs ~0,23 ¢/scan), Qwen ~7× plus rapide. Mais le **nombre** de +détections trahit déjà des régimes très différents — un signal opérationnel, **pas** un verdict +qualité (celui-là attend la vérité terrain) : + +- **Gemini** : détections régulières (5 à 53 livres selon la densité), grâce au décodage contraint + par schéma natif. L'unique échec est un **rejet du domaine** — le modèle a renvoyé un auteur vide, + `Author` l'a refusé, et le payload entier est rejeté en bloc (`ShelfScanFailed`) : le comportement + « rien de partiel ne remonte » voulu par l'ADR 0005, vérifié en vrai. +- **Qwen** : très instable sous le même prompt en `json_object` (sans schéma natif) — **5 photos sur + 10 renvoient 0 livre**, une en renvoie **158** (bien au-delà du réel), et l'unique échec est un + **JSON tronqué** (réponse coupée à ~48 ko). Un modèle qui rend 0 sur une étagère pleine et 158 sur + une autre est un drapeau rouge à confirmer sur la vérité terrain — c'est exactement le genre + d'écart que le rappel et l'hallucination mesureront. + +Ces chiffres valident la chaîne de bout en bout pour les **deux** adapters et donnent coût et +latence. Ils ne disent **rien de définitif** sur la qualité : un fournisseur peut détecter beaucoup +et inventer autant, ou détecter peu et rater le reste. C'est la vérité terrain qui tranche. ### Qualité (rappel, précision, hallucination) diff --git a/tools/bench/README.md b/tools/bench/README.md index 11ef430..e6b0070 100644 --- a/tools/bench/README.md +++ b/tools/bench/README.md @@ -43,10 +43,10 @@ pas une erreur ; une photo sans livre lisible se lit en **tableau vide**. > d'un premier run fait gagner de la saisie — elle ne dispense pas de la vérification. 3. **Clés** dans l'environnement : `GEMINI_API_KEY` et `OPENROUTER_API_KEY`. 4. **Egress réseau** vers les hôtes des fournisseurs (`generativelanguage.googleapis.com`, - `openrouter.ai`). Certains environnements d'exécution restreignent l'egress par politique - réseau : un fournisseur injoignable fait échouer ses scans en `ShelfScanFailed` - (« unreachable ») — ce n'est pas un bug de l'adapter (constaté sur OpenRouter, cf. la note de - décision `docs/decisions/0001`). + `openrouter.ai`). Certains environnements restreignent l'egress par politique réseau : un + fournisseur injoignable fait échouer ses scans en `ShelfScanFailed` (« unreachable » ou + `403 Host not in allowlist`) — ce n'est pas un bug de l'adapter. Autoriser le domaine côté + environnement, et voir la note sur `NODE_USE_ENV_PROXY=1` ci-dessous. ## Lancer @@ -55,6 +55,10 @@ yarn nx build bench # bundle le runner + ses dépendances de wor node tools/bench/dist/main.js # appels live, depuis la racine du repo ``` +En **environnement proxifié** (egress via un proxy, `HTTPS_PROXY` posé), le `fetch` de Node ne lit +pas `HTTPS_PROXY` par défaut : préfixer par `NODE_USE_ENV_PROXY=1` (Node ≥ 22.21), sinon les appels +échouent en `403 Host not in allowlist` ou en « unreachable » quand bien même le domaine est autorisé. + Sortie dans `tools/bench/output/` (gitignoré, dérivé des photos privées) : `report.md` (le tableau) et `.json` (les détections brutes, par photo). From 13fa280aa8320f8f18af47c27271fc844832a66b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 12:49:24 +0000 Subject: [PATCH 3/9] chore(bench): add a `yarn bench` shortcut and document it (#10) Wraps `nx build bench && node tools/bench/dist/main.js` behind `yarn bench`, documented in tools/bench/README.md and the CLAUDE.md command list. Env vars go in front, e.g. `BENCH_PROVIDERS=qwen yarn bench`. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- CLAUDE.md | 1 + package.json | 3 ++- tools/bench/README.md | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0a6870c..8e2c4c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,6 +107,7 @@ yarn format # oxfmt (yarn format:check pour vér yarn api # démarre l'API (http://localhost:3000/health) yarn web # démarre le front (http://localhost:4200) +yarn bench # départage les adapters VLM (appels live, hors CI) — voir tools/bench/README.md docker compose up --build # API + front + Postgres + émulateur de bucket yarn nx run-many -t lint -p api # cibler un projet diff --git a/package.json b/package.json index 3f22f55..f89061d 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "format": "oxfmt", "format:check": "oxfmt --check", "api": "nx serve api", - "web": "nx serve web" + "web": "nx serve web", + "bench": "nx build bench && node tools/bench/dist/main.js" }, "devDependencies": { "@eslint/js": "^9.8.0", diff --git a/tools/bench/README.md b/tools/bench/README.md index e6b0070..9595ee7 100644 --- a/tools/bench/README.md +++ b/tools/bench/README.md @@ -51,10 +51,12 @@ pas une erreur ; une photo sans livre lisible se lit en **tableau vide**. ## Lancer ```bash -yarn nx build bench # bundle le runner + ses dépendances de workspace -node tools/bench/dist/main.js # appels live, depuis la racine du repo +yarn bench # build le runner puis l'exécute (appels live), depuis la racine ``` +Équivaut à `nx build bench && node tools/bench/dist/main.js`. Les variables d'environnement ci-dessous +se placent devant : `BENCH_PROVIDERS=qwen yarn bench`. + En **environnement proxifié** (egress via un proxy, `HTTPS_PROXY` posé), le `fetch` de Node ne lit pas `HTTPS_PROXY` par défaut : préfixer par `NODE_USE_ENV_PROXY=1` (Node ≥ 22.21), sinon les appels échouent en `403 Host not in allowlist` ou en « unreachable » quand bien même le domaine est autorisé. From 4eb49a8fe1708590bedcd1de5910fe7595636643 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 15:11:21 +0000 Subject: [PATCH 4/9] feat(recognition): constrain Qwen to the JSON schema, not free-form JSON (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Qwen adapter asked for `response_format: json_object` — free-form JSON. On dense shelves that let the model truncate its answer mid-structure and emit empty author/title fields, failing whole scans (bench run on #10: 0 books on half the photos for the 235B, one 158-book reply, truncated JSON). Switch to `response_format: json_schema` carrying `SHELF_SCAN_JSON_SCHEMA` — the same contract Gemini already decodes against, now held over Qwen's grammar. All three candidate Qwen-VL models advertise structured-output support on OpenRouter. `strict` narrows the shape, not the meaning: the answer is still validated downstream by the shared response mapper, so an off-contract value still fails closed rather than leaking a bad DetectedBook. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .../lib/qwen-shelf-scanner.adapter.spec.ts | 20 ++++++++++++++++--- .../src/lib/qwen-shelf-scanner.adapter.ts | 15 +++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts index cc4ff17..04ab159 100644 --- a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts +++ b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts @@ -9,7 +9,16 @@ import recorded from './recorded/qwen-shelf-scan.json' with { type: 'json' }; const chatRequestSchema = z.object({ model: z.string(), temperature: z.number(), - response_format: z.object({ type: z.string() }), + response_format: z.object({ + type: z.string(), + json_schema: z + .object({ + name: z.string(), + strict: z.boolean(), + schema: z.object({ required: z.array(z.string()) }), + }) + .optional(), + }), messages: z.array( z.object({ role: z.string(), @@ -102,7 +111,7 @@ describe('QwenShelfScannerAdapter builds its request', () => { expect(request.messages[0]?.content[1]?.image_url?.url).toBe('data:image/jpeg;base64,/9j/4A=='); }); - it('authenticates with a bearer token and asks for a JSON answer', async () => { + it('authenticates with a bearer token and constrains the answer to the schema', async () => { const transport = respondWith(recorded); await adapterWith(transport).scan(photo); const { url, body, headers } = requestOf(transport); @@ -110,8 +119,13 @@ describe('QwenShelfScannerAdapter builds its request', () => { expect(headers.get('authorization')).toBe('Bearer test-key'); expect(url).not.toContain('test-key'); + // json_schema, not json_object: free-form JSON let the model truncate and emit empty + // fields on dense shelves (issue #10). The schema is the same contract Gemini decodes + // against, so the two providers are held to one shape. const request = chatRequestSchema.parse(JSON.parse(body)); - expect(request.response_format.type).toBe('json_object'); + expect(request.response_format.type).toBe('json_schema'); + expect(request.response_format.json_schema?.strict).toBe(true); + expect(request.response_format.json_schema?.schema.required).toContain('books'); expect(request.temperature).toBe(0); }); diff --git a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts index 6b1beaf..066a13f 100644 --- a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts +++ b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts @@ -2,7 +2,7 @@ import { ShelfScanFailed } from '@pick-a-book/recognition-domain'; import type { DetectedBook, ShelfPhoto, ShelfScannerPort } from '@pick-a-book/recognition-domain'; import { z } from 'zod'; -import { SHELF_SCAN_PROMPT } from './shelf-scan-prompt.js'; +import { SHELF_SCAN_JSON_SCHEMA, SHELF_SCAN_PROMPT } from './shelf-scan-prompt.js'; import { toDetectedBooks } from './shelf-scan-response.js'; const DEFAULT_BASE_URL = 'https://openrouter.ai/api/v1'; @@ -77,10 +77,15 @@ export class QwenShelfScannerAdapter implements ShelfScannerPort { ], }, ], - // `json_object` is the OpenAI-compatible way to ask for JSON. Weaker than Gemini's - // schema-constrained decoding, which is exactly why the answer is validated downstream - // rather than trusted. - response_format: { type: 'json_object' }, + // Schema-constrained decoding, the OpenAI-compatible way: the same contract Gemini + // decodes against (`SHELF_SCAN_JSON_SCHEMA`), held over the model's grammar. Plain + // `json_object` let dense shelves truncate the JSON and emit empty author/title fields + // (issue #10) — the schema removes that failure mode. The answer is still validated + // downstream: `strict` narrows the shape, not the meaning. + response_format: { + type: 'json_schema', + json_schema: { name: 'shelf_scan', strict: true, schema: SHELF_SCAN_JSON_SCHEMA }, + }, temperature: 0, }; From ac5f70acd17170ca746e138890b536265d5b18d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 15:46:18 +0000 Subject: [PATCH 5/9] feat(recognition): default Qwen to qwen2.5-vl-72b and cap completion tokens (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench default 235B was unusable on dense shelves — 0 books on half the photos, truncated JSON on others — and json_schema alone did not rescue it. The 72B (the OCR-focused VL line) stays stable, so it becomes the Qwen default in the adapter and the bench. Also send a generous max_tokens so a legitimate long list is not cut off mid-JSON; a runaway repetition still hits the cap and fails, which is the outcome we want. This does not fix the deeper issue that all providers under-detect on these dense ressourcerie shelves — that is the finding logged on #10, and needs the human-verified ground truth to quantify. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .../src/lib/qwen-shelf-scanner.adapter.spec.ts | 17 +++++++++++++++++ .../src/lib/qwen-shelf-scanner.adapter.ts | 14 +++++++++++++- tools/bench/src/config.ts | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts index 04ab159..a9c2197 100644 --- a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts +++ b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts @@ -9,6 +9,7 @@ import recorded from './recorded/qwen-shelf-scan.json' with { type: 'json' }; const chatRequestSchema = z.object({ model: z.string(), temperature: z.number(), + max_tokens: z.number(), response_format: z.object({ type: z.string(), json_schema: z @@ -129,6 +130,22 @@ describe('QwenShelfScannerAdapter builds its request', () => { expect(request.temperature).toBe(0); }); + it('gives the answer room for a full shelf so a long list is not truncated', async () => { + const transport = respondWith(recorded); + await adapterWith(transport).scan(photo); + + expect(chatRequestSchema.parse(JSON.parse(requestOf(transport).body)).max_tokens).toBe(8192); + }); + + it('defaults to the OCR-focused qwen2.5-vl-72b, not the unstable 235B (issue #10)', async () => { + const transport = respondWith(recorded); + await adapterWith(transport).scan(photo); + + expect(chatRequestSchema.parse(JSON.parse(requestOf(transport).body)).model).toBe( + 'qwen/qwen2.5-vl-72b-instruct', + ); + }); + it('reports an empty shelf as an empty array, not a failure', async () => { const empty = { choices: [{ message: { content: '{"books":[]}' } }] }; diff --git a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts index 066a13f..980cd80 100644 --- a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts +++ b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts @@ -6,7 +6,14 @@ import { SHELF_SCAN_JSON_SCHEMA, SHELF_SCAN_PROMPT } from './shelf-scan-prompt.j import { toDetectedBooks } from './shelf-scan-response.js'; const DEFAULT_BASE_URL = 'https://openrouter.ai/api/v1'; -const DEFAULT_MODEL = 'qwen/qwen3-vl-235b-a22b-instruct'; +// qwen2.5-vl-72b, not qwen3-vl-235b: the 235B was unusable on dense shelves in the bench — +// it returned 0 books on half the photos and truncated its JSON on others, where the 72B stays +// stable (issue #10). The 72B is the OCR-focused line; the 235B a general model with vision. +const DEFAULT_MODEL = 'qwen/qwen2.5-vl-72b-instruct'; +// Enough head-room for a full dense shelf (~100 spines) so a legitimate long list is not cut +// off mid-JSON. It does not tame a runaway repetition — that still hits the cap and fails, which +// is the right outcome — it only stops truncating honest answers (issue #10). +const DEFAULT_MAX_TOKENS = 8192; export interface QwenConfiguration { readonly apiKey: string; @@ -18,6 +25,8 @@ export interface QwenConfiguration { * is a different weight class from a quantised local one (issue #10). */ readonly baseUrl?: string; + /** Completion-token ceiling. Overridable, but the default already fits a full shelf. */ + readonly maxTokens?: number; } /** The slice of the chat-completions envelope this adapter depends on, and nothing more. */ @@ -38,6 +47,7 @@ const chatEnvelopeSchema = z.object({ export class QwenShelfScannerAdapter implements ShelfScannerPort { private readonly model: string; private readonly baseUrl: string; + private readonly maxTokens: number; constructor( private readonly configuration: QwenConfiguration, @@ -45,6 +55,7 @@ export class QwenShelfScannerAdapter implements ShelfScannerPort { ) { this.model = configuration.model ?? DEFAULT_MODEL; this.baseUrl = configuration.baseUrl ?? DEFAULT_BASE_URL; + this.maxTokens = configuration.maxTokens ?? DEFAULT_MAX_TOKENS; } async scan(photo: ShelfPhoto): Promise { @@ -86,6 +97,7 @@ export class QwenShelfScannerAdapter implements ShelfScannerPort { type: 'json_schema', json_schema: { name: 'shelf_scan', strict: true, schema: SHELF_SCAN_JSON_SCHEMA }, }, + max_tokens: this.maxTokens, temperature: 0, }; diff --git a/tools/bench/src/config.ts b/tools/bench/src/config.ts index 771b0eb..9211e48 100644 --- a/tools/bench/src/config.ts +++ b/tools/bench/src/config.ts @@ -21,7 +21,7 @@ export interface ProviderSpec { // rather than inheriting whatever an adapter would fall back to, so the report names a truth. const DEFAULT_MODELS: Partial> = { gemini: 'gemini-3.6-flash', - qwen: 'qwen/qwen3-vl-235b-a22b-instruct', + qwen: 'qwen/qwen2.5-vl-72b-instruct', }; const MEDIA_TYPES: Partial> = { From e353967e3d22d4a458704a9faf66405ae8f1b47e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 23:03:41 +0000 Subject: [PATCH 6/9] fix(recognition): keep post() under the line limit and test the bench pure logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review triage on #43. - max-lines-per-function (blocking, CI red): the json_schema + max_tokens block pushed QwenShelfScannerAdapter.post past 50 lines and a spec describe past 50. Extract requestBody() from post, split the request-shape describe. oxlint green. - Untested pure logic: the bench vitest include was src/lib/** only, leaving the cost/usage parsing (usage.ts) and provider selection (config.ts) — which feed the decision-note numbers — outside CI. Widen the include to src/**, export readUsage, and add specs for estimateCost, readUsage (both payload shapes), mediaTypeOf, env/requireEnv and selectedProviders (unknown provider throws). The live runner still has no spec, so nothing paid runs in CI. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .../lib/qwen-shelf-scanner.adapter.spec.ts | 2 + .../src/lib/qwen-shelf-scanner.adapter.ts | 54 +++++++-------- tools/bench/src/config.spec.ts | 68 +++++++++++++++++++ tools/bench/src/usage.spec.ts | 64 +++++++++++++++++ tools/bench/src/usage.ts | 3 +- tools/bench/vitest.config.mts | 10 ++- 6 files changed, 170 insertions(+), 31 deletions(-) create mode 100644 tools/bench/src/config.spec.ts create mode 100644 tools/bench/src/usage.spec.ts diff --git a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts index a9c2197..3cb4810 100644 --- a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts +++ b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.spec.ts @@ -129,7 +129,9 @@ describe('QwenShelfScannerAdapter builds its request', () => { expect(request.response_format.json_schema?.schema.required).toContain('books'); expect(request.temperature).toBe(0); }); +}); +describe('QwenShelfScannerAdapter caps and pins its request', () => { it('gives the answer room for a full shelf so a long list is not truncated', async () => { const transport = respondWith(recorded); await adapterWith(transport).scan(photo); diff --git a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts index 980cd80..b49475f 100644 --- a/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts +++ b/libs/recognition/infrastructure/src/lib/qwen-shelf-scanner.adapter.ts @@ -72,19 +72,39 @@ export class QwenShelfScannerAdapter implements ShelfScannerPort { private async post(photo: ShelfPhoto): Promise { const url = `${this.baseUrl}/chat/completions`; - const body = { + + let response: Response; + try { + response = await this.transport(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${this.configuration.apiKey}`, + }, + body: JSON.stringify(this.requestBody(photo)), + }); + } catch (cause) { + throw new ShelfScanFailed(`Qwen is unreachable (${describe(cause)})`, { cause }); + } + + if (!response.ok) { + throw new ShelfScanFailed(`Qwen answered ${response.status} (${await readText(response)})`); + } + + return response; + } + + private requestBody(photo: ShelfPhoto): object { + const dataUrl = `data:${photo.mediaType};base64,${Buffer.from(photo.bytes).toString('base64')}`; + + return { model: this.model, messages: [ { role: 'user', content: [ { type: 'text', text: SHELF_SCAN_PROMPT }, - { - type: 'image_url', - image_url: { - url: `data:${photo.mediaType};base64,${Buffer.from(photo.bytes).toString('base64')}`, - }, - }, + { type: 'image_url', image_url: { url: dataUrl } }, ], }, ], @@ -100,26 +120,6 @@ export class QwenShelfScannerAdapter implements ShelfScannerPort { max_tokens: this.maxTokens, temperature: 0, }; - - let response: Response; - try { - response = await this.transport(url, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${this.configuration.apiKey}`, - }, - body: JSON.stringify(body), - }); - } catch (cause) { - throw new ShelfScanFailed(`Qwen is unreachable (${describe(cause)})`, { cause }); - } - - if (!response.ok) { - throw new ShelfScanFailed(`Qwen answered ${response.status} (${await readText(response)})`); - } - - return response; } } diff --git a/tools/bench/src/config.spec.ts b/tools/bench/src/config.spec.ts new file mode 100644 index 0000000..2b67e89 --- /dev/null +++ b/tools/bench/src/config.spec.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { env, mediaTypeOf, requireEnv, selectedProviders } from './config.js'; + +describe('mediaTypeOf', () => { + it('maps known image extensions, case-insensitively', () => { + expect(mediaTypeOf('20260801_113334.jpg')).toBe('image/jpeg'); + expect(mediaTypeOf('shelf.JPEG')).toBe('image/jpeg'); + expect(mediaTypeOf('shelf.png')).toBe('image/png'); + }); + + it('returns undefined for anything else, so non-images are skipped', () => { + expect(mediaTypeOf('ground-truth.yaml')).toBeUndefined(); + expect(mediaTypeOf('README')).toBeUndefined(); + }); +}); + +describe('env / requireEnv', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('trims a set value and treats blank as absent', () => { + vi.stubEnv('BENCH_X', ' value '); + expect(env('BENCH_X')).toBe('value'); + + vi.stubEnv('BENCH_X', ' '); + expect(env('BENCH_X')).toBeUndefined(); + }); + + it('requireEnv throws, naming the missing variable', () => { + vi.stubEnv('BENCH_MISSING', ''); + expect(() => requireEnv('BENCH_MISSING')).toThrow(/BENCH_MISSING/u); + }); +}); + +describe('selectedProviders', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('builds the two default providers with their pinned models', () => { + vi.stubEnv('BENCH_PROVIDERS', 'gemini,qwen'); + vi.stubEnv('GEMINI_MODEL', ''); + vi.stubEnv('QWEN_MODEL', ''); + + const specs = selectedProviders(); + + expect(specs.map((spec) => spec.name)).toStrictEqual(['gemini', 'qwen']); + expect(specs.map((spec) => spec.model)).toStrictEqual([ + 'gemini-3.6-flash', + 'qwen/qwen2.5-vl-72b-instruct', + ]); + }); + + it('honours a model override without touching the code', () => { + vi.stubEnv('BENCH_PROVIDERS', 'qwen'); + vi.stubEnv('QWEN_MODEL', 'qwen/qwen3-vl-32b-instruct'); + + expect(selectedProviders()[0]?.model).toBe('qwen/qwen3-vl-32b-instruct'); + }); + + it('refuses an unknown provider rather than falling back silently', () => { + vi.stubEnv('BENCH_PROVIDERS', 'stub'); + + expect(() => selectedProviders()).toThrow(/Unknown provider "stub"/u); + }); +}); diff --git a/tools/bench/src/usage.spec.ts b/tools/bench/src/usage.spec.ts new file mode 100644 index 0000000..23c9175 --- /dev/null +++ b/tools/bench/src/usage.spec.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest'; + +import { estimateCost, readUsage } from './usage.js'; + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { headers: { 'content-type': 'application/json' } }); +} + +describe('estimateCost', () => { + it('trusts the provider-reported cost when it gave one', () => { + expect(estimateCost('qwen', 5000, 500, 0.0012)).toBe(0.0012); + }); + + it('falls back to a token-price estimate when no cost was reported', () => { + // gemini prices: 0.3 in, 2.5 out per million. + expect(estimateCost('gemini', 1_000_000, 1_000_000, null)).toBeCloseTo(2.8, 10); + }); + + it('returns null for a provider without a price table', () => { + expect(estimateCost('unknown', 1000, 1000, null)).toBeNull(); + }); + + it('returns null rather than 0 when nothing was billed and nothing consumed', () => { + expect(estimateCost('gemini', 0, 0, null)).toBeNull(); + }); +}); + +describe('readUsage', () => { + it("reads Gemini's usageMetadata", async () => { + const usage = await readUsage( + jsonResponse({ usageMetadata: { promptTokenCount: 1200, candidatesTokenCount: 300 } }), + ); + + expect(usage).toStrictEqual({ + promptTokens: 1200, + completionTokens: 300, + reportedCostUsd: null, + }); + }); + + it('reads the OpenAI-shaped usage, cost included', async () => { + const usage = await readUsage( + jsonResponse({ usage: { prompt_tokens: 800, completion_tokens: 120, cost: 0.0009 } }), + ); + + expect(usage).toStrictEqual({ + promptTokens: 800, + completionTokens: 120, + reportedCostUsd: 0.0009, + }); + }); + + it('reports zeros for an envelope carrying no usage', async () => { + const usage = await readUsage(jsonResponse({ choices: [] })); + + expect(usage).toStrictEqual({ promptTokens: 0, completionTokens: 0, reportedCostUsd: null }); + }); + + it('reports zeros rather than throwing on a non-JSON body', async () => { + const usage = await readUsage(new Response('not json at all')); + + expect(usage).toStrictEqual({ promptTokens: 0, completionTokens: 0, reportedCostUsd: null }); + }); +}); diff --git a/tools/bench/src/usage.ts b/tools/bench/src/usage.ts index 19b09b1..6087212 100644 --- a/tools/bench/src/usage.ts +++ b/tools/bench/src/usage.ts @@ -53,7 +53,8 @@ export function recordingTransport(): { return { transport, last: () => latest }; } -async function readUsage(response: Response): Promise> { +/** Exported for its own tests: the two usage-payload shapes feed the decision-note costs. */ +export async function readUsage(response: Response): Promise> { const empty = { promptTokens: 0, completionTokens: 0, reportedCostUsd: null }; let body: unknown; try { diff --git a/tools/bench/vitest.config.mts b/tools/bench/vitest.config.mts index b2363c3..a78e813 100644 --- a/tools/bench/vitest.config.mts +++ b/tools/bench/vitest.config.mts @@ -7,9 +7,13 @@ export default defineConfig({ name: 'bench', watch: false, environment: 'node', - // Only the pure logic is unit-tested; the live runner (`src/main.ts`) is never in CI — - // it costs a paid call per photo. It stays out of the include on purpose. - include: ['src/lib/**/*.{test,spec}.ts'], + // Every pure module is unit-tested wherever it lives: the scoring/render/ground-truth of + // `src/lib/`, and the cost, usage-parsing and provider-selection of `src/usage.ts` and + // `src/config.ts` — those feed the numbers the decision note posts, so a regression must + // not slip through (issue #10). Only the live runner (`src/main.ts` and its network I/O in + // `src/runner.ts`/`src/io.ts`) has no spec: it costs a paid call per photo, so it is never + // in CI. A spec is discovered by this glob, never a source module on its own. + include: ['src/**/*.{test,spec}.ts'], reporters: ['default'], coverage: { reportsDirectory: './test-output/vitest/coverage', From b368b2a12d31ea144ddbb59536d2b85953e2b5fb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 16:58:47 +0000 Subject: [PATCH 7/9] feat(recognition): make the scan prompt exhaustive and dedupe detections Follow-up on the bench findings (#10). - Prompt: the shared scan prompt now pushes exhaustiveness (scan the whole shelf band by band, a short list means stopping too early). On a throwaway probe this roughly tripled Qwen's reads on a sparse shelf (7 -> 19) and quadrupled them on a dense one (10 -> 42). To keep that recall without breaking the contract, it also forbids empty author/title (the domain rejects the whole payload on an empty field) and asks for each book once. - Scoring: dedupe detections on the normalized (author, title) before matching, so a spine the model lists twice is not double-counted nor charged as a false positive. Keyed on shared-text-match normalization; the highest-confidence copy survives. The prompt is shared by both adapters on purpose (the bench measures models, not prompts); its quality is validated by the manual bench, not a unit test (ADR 0005). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .../src/lib/shelf-scan-prompt.ts | 19 +++++++---- tools/bench/src/lib/scoring.spec.ts | 24 ++++++++++++++ tools/bench/src/lib/scoring.ts | 33 ++++++++++++++++--- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/libs/recognition/infrastructure/src/lib/shelf-scan-prompt.ts b/libs/recognition/infrastructure/src/lib/shelf-scan-prompt.ts index 8e64046..a4620ad 100644 --- a/libs/recognition/infrastructure/src/lib/shelf-scan-prompt.ts +++ b/libs/recognition/infrastructure/src/lib/shelf-scan-prompt.ts @@ -9,20 +9,27 @@ * padding the list to look thorough. */ export const SHELF_SCAN_PROMPT = [ - 'You are reading the spines of books on a shelf, photographed with a phone.', + 'You are cataloguing every book on a shelf, photographed with a phone. A shelf like this holds', + '50 to 100 books; a short list means you stopped too early.', '', - 'Return one entry per book you can actually read. For each one:', + 'Scan the whole image systematically, band by band, left to right and top to bottom, and return', + 'one entry per book. For each one:', '- "author": the author name printed on the spine.', '- "title": the title printed on the spine.', '- "confidence": your own certainty for that entry, between 0 and 1.', '', 'Rules:', - '- Never invent. If a spine is partly readable, do not complete it towards a work you', - ' expect — report only what is printed, with a low confidence.', + '- Be exhaustive: do not stop until you have swept the entire shelf. Leaving a readable book out', + ' is the main mistake to avoid.', + '- Never invent. Report only what is printed; for a partly legible spine, give your best reading', + ' with a low confidence rather than a work you merely expect.', + '- Every entry needs BOTH an author and a title. If you can read only one of the two, skip the', + ' spine — never emit an empty "author" or "title".', + '- Report each book once. Do not repeat an entry.', '- Do not confuse the author or the title with the publisher or the collection', ' (Gallimard, Folio, Points, Le Livre de Poche and the like are never the author).', - '- Skip any spine you cannot read. An empty list is a valid answer.', - '- Spines may be rotated or upside down; read them anyway.', + '- Spines may be rotated or upside down; read them anyway. A spine you truly cannot read is', + ' skipped, and an empty list is a valid answer.', '', 'Answer with JSON only, of the form {"books": [{"author": "", "title": "", "confidence": 0}]}.', ].join('\n'); diff --git a/tools/bench/src/lib/scoring.spec.ts b/tools/bench/src/lib/scoring.spec.ts index e6455a2..bab88c5 100644 --- a/tools/bench/src/lib/scoring.spec.ts +++ b/tools/bench/src/lib/scoring.spec.ts @@ -122,6 +122,30 @@ describe('scorePhoto — structuring', () => { }); }); +describe('scorePhoto — deduplication', () => { + it('collapses repeated detections of the same book before scoring', () => { + const truth = [ref('Albert Camus', 'La Peste')]; + // The exhaustive prompt makes the model list a spine twice, in an OCR variant. + const detected = [det('Albert Camus', 'La Peste', 0.9), det('albert camus', 'La Peste.', 0.6)]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.detectedCount).toBe(1); + expect(score.truePositives).toBe(1); + expect(score.falsePositives).toBe(0); + }); + + it('keeps genuinely distinct books by the same author', () => { + const truth = [ref('Albert Camus', 'La Peste'), ref('Albert Camus', 'La Chute')]; + const detected = [det('Albert Camus', 'La Peste', 0.9), det('Albert Camus', 'La Chute', 0.9)]; + + const score = scorePhoto(detected, truth, HIGH); + + expect(score.detectedCount).toBe(2); + expect(score.truePositives).toBe(2); + }); +}); + describe('aggregate', () => { it('micro-averages counts and derives the rates from the totals', () => { const first = scorePhoto( diff --git a/tools/bench/src/lib/scoring.ts b/tools/bench/src/lib/scoring.ts index 5e3b759..e0c9ba3 100644 --- a/tools/bench/src/lib/scoring.ts +++ b/tools/bench/src/lib/scoring.ts @@ -1,4 +1,4 @@ -import { fuzzyEquals, similarity } from '@pick-a-book/shared-text-match'; +import { fuzzyEquals, normalizeText, similarity } from '@pick-a-book/shared-text-match'; /** * Scoring of a shelf reading against a human-verified ground truth (#10). @@ -117,27 +117,50 @@ export function scorePhoto( truth: readonly BookRef[], highConfidenceThreshold: number, ): PhotoScore { - const pairs = correspond(detected, truth); + const distinct = dedupe(detected); + const pairs = correspond(distinct, truth); const fields = tallyFields(pairs); return { truthCount: truth.length, - detectedCount: detected.length, + detectedCount: distinct.length, truePositives: fields.truePositives, - falsePositives: detected.length - fields.truePositives, + falsePositives: distinct.length - fields.truePositives, falseNegatives: truth.length - fields.truePositives, correspondences: pairs.length, authorCorrect: fields.authorCorrect, titleCorrect: fields.titleCorrect, swapped: fields.swapped, highConfidenceHallucinations: countHighConfidenceHallucinations( - detected, + distinct, pairs, highConfidenceThreshold, ), }; } +/** + * Collapses repeated detections of the same book. The exhaustive prompt makes the model list a + * spine twice now and then (issue #10); keyed on the normalized `(author, title)`, only the + * highest-confidence copy survives — so a doubled read is neither double-counted nor charged as a + * false positive. It is not a matching step: two genuinely different books stay apart. + */ +function dedupe(detected: readonly DetectionRecord[]): DetectionRecord[] { + const byKey = new Map(); + + for (const detection of detected) { + // Newline separates the fields: normalization never emits one, so "a b" + "c" + // cannot collide with "a" + "b c". + const key = `${normalizeText(detection.author)}\n${normalizeText(detection.title)}`; + const kept = byKey.get(key); + if (kept === undefined || detection.confidence > kept.confidence) { + byKey.set(key, detection); + } + } + + return [...byKey.values()]; +} + interface FieldTally { readonly truePositives: number; readonly authorCorrect: number; From 4fe74022838239b65c4cba972a7cd9b517aa27c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 22:15:45 +0000 Subject: [PATCH 8/9] chore(bench): rename reference photos to shelf-fixture-N Rename the 10 reference-shelf photos from their camera basenames (20260801_HHMMSS.jpg) to shelf-fixture-1..10.jpg, on the GCS bucket and in every tracked reference: the ground-truth template, the config spec example, and the Gemini recorded-response provenance note. The bench discovers photos dynamically, so no runner code changes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- .../infrastructure/src/lib/recorded/README.md | 2 +- tools/bench/ground-truth.template.yaml | 20 +++++++++---------- tools/bench/src/config.spec.ts | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libs/recognition/infrastructure/src/lib/recorded/README.md b/libs/recognition/infrastructure/src/lib/recorded/README.md index 44bda23..7dd4b69 100644 --- a/libs/recognition/infrastructure/src/lib/recorded/README.md +++ b/libs/recognition/infrastructure/src/lib/recorded/README.md @@ -8,7 +8,7 @@ nécessaire non plus — `ShelfPhoto` se construit avec quelques octets valides. | Fichier | Origine | Fiabilité | |---|---|---| -| `gemini-shelf-scan.json` | **Appel réel**, capturé une fois le 2026-08-31 sur `gemini-3.6-flash`, photo de référence `20260801_114252.jpg` | Authentique : l'enveloppe est exactement celle que renvoie l'API | +| `gemini-shelf-scan.json` | **Appel réel**, capturé une fois le 2026-08-31 sur `gemini-3.6-flash`, photo de référence `shelf-fixture-9.jpg` | Authentique : l'enveloppe est exactement celle que renvoie l'API | | `qwen-shelf-scan.json` | **Écrite à la main**, d'après la forme documentée de l'API chat-completions OpenAI | ⚠️ Non authentique — voir ci-dessous | ## Pourquoi la fixture Qwen n'est pas un enregistrement diff --git a/tools/bench/ground-truth.template.yaml b/tools/bench/ground-truth.template.yaml index e423e3d..fadd62f 100644 --- a/tools/bench/ground-truth.template.yaml +++ b/tools/bench/ground-truth.template.yaml @@ -20,26 +20,26 @@ version: 1 photos: - - file: 20260801_113334.jpg + - file: shelf-fixture-1.jpg books: # - author: Albert Camus # title: La Peste [] - - file: 20260801_113338.jpg + - file: shelf-fixture-2.jpg books: [] - - file: 20260801_113352.jpg + - file: shelf-fixture-3.jpg books: [] - - file: 20260801_113355.jpg + - file: shelf-fixture-4.jpg books: [] - - file: 20260801_113405.jpg + - file: shelf-fixture-5.jpg books: [] - - file: 20260801_113811.jpg + - file: shelf-fixture-6.jpg books: [] - - file: 20260801_114113.jpg + - file: shelf-fixture-7.jpg books: [] - - file: 20260801_114237.jpg + - file: shelf-fixture-8.jpg books: [] - - file: 20260801_114252.jpg + - file: shelf-fixture-9.jpg books: [] - - file: 20260801_114307.jpg + - file: shelf-fixture-10.jpg books: [] diff --git a/tools/bench/src/config.spec.ts b/tools/bench/src/config.spec.ts index 2b67e89..9fe3ce4 100644 --- a/tools/bench/src/config.spec.ts +++ b/tools/bench/src/config.spec.ts @@ -4,7 +4,7 @@ import { env, mediaTypeOf, requireEnv, selectedProviders } from './config.js'; describe('mediaTypeOf', () => { it('maps known image extensions, case-insensitively', () => { - expect(mediaTypeOf('20260801_113334.jpg')).toBe('image/jpeg'); + expect(mediaTypeOf('shelf-fixture-1.jpg')).toBe('image/jpeg'); expect(mediaTypeOf('shelf.JPEG')).toBe('image/jpeg'); expect(mediaTypeOf('shelf.png')).toBe('image/png'); }); From bd2fdf04d5c0f9ee9264a6d7aa8edcce906f026f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 23:13:01 +0000 Subject: [PATCH 9/9] chore(bench): drop shelf-fixture-2 (near-duplicate of fixture-1) shelf-fixture-2 framed the same shelf as shelf-fixture-1, only less wide, so it added no coverage to the reference set. Removed from the GCS bucket and from the ground-truth template. Remaining fixtures keep their numbers (1, 3..10) so existing identifiers stay stable. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PGqVYV5GLC8qBTSKqAq19k --- tools/bench/ground-truth.template.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/bench/ground-truth.template.yaml b/tools/bench/ground-truth.template.yaml index fadd62f..873951c 100644 --- a/tools/bench/ground-truth.template.yaml +++ b/tools/bench/ground-truth.template.yaml @@ -25,8 +25,6 @@ photos: # - author: Albert Camus # title: La Peste [] - - file: shelf-fixture-2.jpg - books: [] - file: shelf-fixture-3.jpg books: [] - file: shelf-fixture-4.jpg