From 7c20157047e4ecb3bd20aeec60b38fa280317e9a Mon Sep 17 00:00:00 2001 From: iamaamir <8420386+iamaamir@users.noreply.github.com> Date: Sun, 31 May 2026 21:22:22 +0530 Subject: [PATCH] feat(blackbox): adding harness loop for auto improving score logic --- blackbox/README.md | 738 ++++++++++ blackbox/config.example.json | 24 + blackbox/config.local.json | 24 + ...-55-762Z-lmstudio-granite-test-drive.jsonl | 20 + ...-48-673Z-lmstudio-granite-test-drive.jsonl | 20 + ...-15-456Z-lmstudio-granite-test-drive.jsonl | 2 + ...-14-914Z-lmstudio-granite-test-drive.jsonl | 20 + ...-56-983Z-lmstudio-granite-test-drive.jsonl | 20 + ...-12-310Z-lmstudio-granite-test-drive.jsonl | 50 + blackbox/runs/latest-analysis.json | 68 + blackbox/runs/latest-analysis.md | 65 + blackbox/runs/latest-evaluation.json | 1283 +++++++++++++++++ blackbox/runs/latest.txt | 1 + blackbox/src/analyze-results.mjs | 198 +++ blackbox/src/autoresearch.mjs | 79 + blackbox/src/chrome-stub.mjs | 45 + blackbox/src/evaluate-fixtures.mjs | 166 +++ blackbox/src/fixtures.mjs | 29 + blackbox/src/json.mjs | 45 + blackbox/src/openai-compatible-client.mjs | 41 + blackbox/src/promote-fixtures.mjs | 61 + blackbox/src/prompts.mjs | 101 ++ blackbox/src/run.mjs | 252 ++++ package.json | 5 + tests/fixtures/scoring-cases.generated.json | 1152 +++++++++++++++ tests/fixtures/scoring-cases.json | 27 + 26 files changed, 4536 insertions(+) create mode 100644 blackbox/README.md create mode 100644 blackbox/config.example.json create mode 100644 blackbox/config.local.json create mode 100644 blackbox/runs/2026-05-31T11-57-55-762Z-lmstudio-granite-test-drive.jsonl create mode 100644 blackbox/runs/2026-05-31T11-58-48-673Z-lmstudio-granite-test-drive.jsonl create mode 100644 blackbox/runs/2026-05-31T11-59-15-456Z-lmstudio-granite-test-drive.jsonl create mode 100644 blackbox/runs/2026-05-31T12-11-14-914Z-lmstudio-granite-test-drive.jsonl create mode 100644 blackbox/runs/2026-05-31T12-38-56-983Z-lmstudio-granite-test-drive.jsonl create mode 100644 blackbox/runs/2026-05-31T14-18-12-310Z-lmstudio-granite-test-drive.jsonl create mode 100644 blackbox/runs/latest-analysis.json create mode 100644 blackbox/runs/latest-analysis.md create mode 100644 blackbox/runs/latest-evaluation.json create mode 100644 blackbox/runs/latest.txt create mode 100644 blackbox/src/analyze-results.mjs create mode 100644 blackbox/src/autoresearch.mjs create mode 100644 blackbox/src/chrome-stub.mjs create mode 100644 blackbox/src/evaluate-fixtures.mjs create mode 100644 blackbox/src/fixtures.mjs create mode 100644 blackbox/src/json.mjs create mode 100644 blackbox/src/openai-compatible-client.mjs create mode 100644 blackbox/src/promote-fixtures.mjs create mode 100644 blackbox/src/prompts.mjs create mode 100644 blackbox/src/run.mjs create mode 100644 tests/fixtures/scoring-cases.generated.json create mode 100644 tests/fixtures/scoring-cases.json diff --git a/blackbox/README.md b/blackbox/README.md new file mode 100644 index 0000000..d2718b9 --- /dev/null +++ b/blackbox/README.md @@ -0,0 +1,738 @@ +# Correctly Blackbox + +Blackbox is a local AI-in-the-loop research harness for improving Correctly's scoring system. It does not use the browser extension UI. It runs the core provider, cascade, scoring, display-suggestion, and fixture-evaluation code directly from Node. + +The goal is to create a repeatable evidence loop: + +```txt +generate flawed text +→ run Correctly +→ judge the result +→ save interesting cases +→ promote reviewed fixtures +→ replay fixtures against scoring code +→ use the report to improve scoring safely +``` + +It is inspired by autoresearch-style loops, but it intentionally does not auto-edit production scoring code. It automates discovery, judging, promotion drafts, and regression evaluation. Code changes should still pass the fixture gate. + +## Architecture + +```mermaid +flowchart LR + A["Generator AI"] --> B["Generated flawed case"] + B --> C["Correctly provider.correctGrammar()"] + C --> D["Cascade levels"] + D --> D1["Level 1 schema"] + D --> D2["Level 1 no-schema"] + D --> D3["Level 2 JSON extraction"] + D --> D4["Level 3 plain text"] + C --> E["Scoring analysis"] + E --> E1["scoreAcceptedCorrection"] + E --> E2["extractDisplayChanges"] + B --> F["Judge AI"] + C --> F + E --> F + F --> G["JSONL run record"] + G --> H["Promote fixture candidates"] + H --> I["Reviewed fixtures"] + I --> J["Fixture evaluator"] + J --> K["Metrics and regressions"] +``` + +## What Each Component Does + +```txt +blackbox/src/run.mjs + Main research loop. Generates cases, runs Correctly, judges output, writes JSONL. + +blackbox/src/autoresearch.mjs + Orchestrates run → promote → evaluate. + +blackbox/src/promote-fixtures.mjs + Converts interesting JSONL records into candidate scoring fixtures. + +blackbox/src/evaluate-fixtures.mjs + Replays reviewed fixtures against current scoring code and reports pass/fail metrics. + +blackbox/src/analyze-results.mjs + Uses an analyst AI model to inspect run records and fixture evaluation results, then suggests next engineering steps. + +blackbox/src/openai-compatible-client.mjs + Tiny OpenAI-compatible client for generator and judge models. + +blackbox/src/chrome-stub.mjs + Minimal chrome.storage/runtime stub so extension providers can run in Node. + +blackbox/src/prompts.mjs + Generator and judge prompts. + +blackbox/src/json.mjs + Robust JSON extraction from raw/fenced/prose model output. + +blackbox/src/fixtures.mjs + Converts run records into fixture candidates. +``` + +## Requirements + +- Node 20+. +- A local OpenAI-compatible model server. +- At least one model loaded in Ollama or LM Studio. + +Supported local options: + +```txt +Ollama + Chat endpoint: http://localhost:11434/v1/chat/completions + Base URL for config: http://localhost:11434/v1 + Correctly provider id: ollama + +LM Studio + Chat endpoint: http://localhost:1234/v1/chat/completions + Base URL for config: http://localhost:1234/v1 + Correctly provider id: lmstudio +``` + +## Quick Start + +1. Copy the config: + +```bash +cp blackbox/config.example.json blackbox/config.local.json +``` + +2. Edit `blackbox/config.local.json` for your local server and model. + +3. Run one small loop: + +```bash +npm run blackbox -- --config blackbox/config.local.json --cases 3 +``` + +4. Run the full autoresearch pipeline: + +```bash +npm run blackbox:autoresearch -- --config blackbox/config.local.json --cases 25 +``` + +5. Evaluate reviewed fixtures: + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json --fail-on-regression +``` + +## Configuration + +The config has three model roles: + +```json +{ + "generator": {}, + "fixer": {}, + "judge": {}, + "analyst": {} +} +``` + +### Generator + +The generator creates flawed text. It uses the OpenAI-compatible client directly. + +```json +"generator": { + "baseUrl": "http://localhost:11434/v1", + "apiKey": "ollama", + "model": "llama3", + "temperature": 0.8 +} +``` + +Higher temperature gives more diverse cases. Lower temperature gives more repetitive cases. + +### Fixer + +The fixer is the Correctly system under test. It goes through the same provider/cascade/scoring path used by the extension. + +```json +"fixer": { + "providerId": "ollama", + "apiKey": "", + "model": "llama3", + "baseUrl": "" +} +``` + +Provider IDs are the same IDs used by Correctly: + +```txt +ollama +lmstudio +openai-compatible +openai +``` + +For local work, prefer `ollama` or `lmstudio`. + +### Judge + +The judge evaluates whether Correctly behaved well. + +```json +"judge": { + "baseUrl": "http://localhost:11434/v1", + "apiKey": "ollama", + "model": "llama3", + "temperature": 0.1 +} +``` + +Use the strongest available model for judge if possible. The generator can be noisy; the judge should be stricter. + +### Analyst + +The analyst is optional. If omitted, Blackbox uses the `judge` config. The analyst reads the JSONL run plus fixture evaluation report and suggests next engineering steps. + +```json +"analyst": { + "baseUrl": "http://localhost:1234/v1", + "apiKey": "lm-studio", + "model": "google/gemma-3-12b", + "temperature": 0.1 +} +``` + +Analyst output is advisory. It should say things like: + +```txt +bad display change → extractDisplayChanges too permissive +good correction rejected → scoreAcceptedCorrection too strict +fallback/cache issue → provider cascade/cache policy +noisy label → fixture_quality issue +``` + +## Ollama Setup + +1. Start Ollama: + +```bash +ollama serve +``` + +2. Pull a model: + +```bash +ollama pull llama3 +``` + +3. Use this config shape: + +```json +{ + "runName": "ollama-scoring-research", + "caseCount": 10, + "outputDir": "blackbox/runs", + "seed": "grammar edge cases for email and chat text", + "generator": { + "baseUrl": "http://localhost:11434/v1", + "apiKey": "ollama", + "model": "llama3", + "temperature": 0.8 + }, + "fixer": { + "providerId": "ollama", + "apiKey": "", + "model": "llama3", + "baseUrl": "" + }, + "judge": { + "baseUrl": "http://localhost:11434/v1", + "apiKey": "ollama", + "model": "llama3", + "temperature": 0.1 + } +} +``` + +## LM Studio Setup + +1. Open LM Studio. +2. Load a chat model. +3. Start the local server. +4. Use OpenAI-compatible server mode. + +Use this config shape: + +```json +{ + "runName": "lmstudio-scoring-research", + "caseCount": 10, + "outputDir": "blackbox/runs", + "seed": "grammar edge cases for email and chat text", + "generator": { + "baseUrl": "http://localhost:1234/v1", + "apiKey": "lm-studio", + "model": "local-model", + "temperature": 0.8 + }, + "fixer": { + "providerId": "lmstudio", + "apiKey": "", + "model": "local-model", + "baseUrl": "" + }, + "judge": { + "baseUrl": "http://localhost:1234/v1", + "apiKey": "lm-studio", + "model": "local-model", + "temperature": 0.1 + } +} +``` + +If LM Studio exposes a real model ID, replace `local-model` with that ID. + +## Commands + +### Run Discovery + +```bash +npm run blackbox -- --config blackbox/config.local.json --cases 25 +``` + +Equivalent direct command: + +```bash +node blackbox/src/run.mjs --config blackbox/config.local.json --cases 25 +``` + +Output: + +```txt +blackbox/runs/-.jsonl +blackbox/runs/latest.txt +``` + +### Promote Fixture Candidates + +```bash +npm run blackbox:promote -- blackbox/runs/.jsonl --out tests/fixtures/scoring-cases.generated.json +``` + +If no run file is passed, the promoter reads `blackbox/runs/latest.txt`. + +```bash +npm run blackbox:promote -- --out tests/fixtures/scoring-cases.generated.json +``` + +Promotion is intentionally a draft step. Review the generated file before moving cases into `tests/fixtures/scoring-cases.json`. + +### After `scoring-cases.generated.json` Exists + +The generated fixture file is not the trusted regression suite. It is raw evidence from a blackbox run. + +```txt +tests/fixtures/scoring-cases.generated.json + Raw candidate fixtures from AI judge output. + Review before trusting. + +tests/fixtures/scoring-cases.json + Reviewed fixtures. + This is the scoring regression gate. +``` + +Use this workflow after every blackbox run: + +```mermaid +flowchart TD + A["Run blackbox"] --> B["scoring-cases.generated.json"] + B --> C["Evaluate generated fixtures"] + C --> D["Manual review"] + D --> E{"Decision per case"} + E -->|Good as-is| F["Copy into scoring-cases.json"] + E -->|Useful but wrong expectation| G["Edit expected behavior, then copy"] + E -->|Bad or noisy case| H["Discard"] + F --> I["Evaluate reviewed fixtures"] + G --> I + I --> J["Use failures to improve scoring"] +``` + +Step by step: + +1. Evaluate generated candidates: + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.generated.json +``` + +2. Open the generated file: + +```txt +tests/fixtures/scoring-cases.generated.json +``` + +3. For each case, choose one outcome: + +```txt +promote as-is +promote with edited expectations +discard +``` + +4. Copy reviewed cases into: + +```txt +tests/fixtures/scoring-cases.json +``` + +5. Evaluate the reviewed corpus: + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json --fail-on-regression +``` + +6. Only then use failures to change scoring code. + +Generated expectations can encode current bad behavior. For example, if a generated fixture expects this as a visible change: + +```json +{ "original": "its", "replacement": "John" } +``` + +do not promote that expectation as-is. Either discard the case or edit it so the expected behavior reflects the desired scoring policy, such as hiding the bad change or lowering acceptance. + +### Evaluate Fixtures + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json +``` + +Write a report: + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json --out blackbox/runs/fixture-evaluation.json +``` + +Fail on regression: + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json --fail-on-regression +``` + +### Analyze Results With AI + +After a run and fixture evaluation, ask an analyst model for next steps: + +```bash +npm run blackbox:analyze -- \ + --config blackbox/config.local.json \ + --run blackbox/runs/.jsonl \ + --evaluation blackbox/runs/latest-evaluation.json \ + --out blackbox/runs/latest-analysis.json \ + --markdown blackbox/runs/latest-analysis.md +``` + +If `--run` is omitted, it uses `blackbox/runs/latest.txt`. + +The analyst produces: + +```txt +blackbox/runs/latest-analysis.json +blackbox/runs/latest-analysis.md +``` + +Expected recommendation shape: + +```json +{ + "priority": "P1", + "area": "extractDisplayChanges", + "title": "Hide destructive visible changes", + "evidenceCaseIds": ["0019"], + "problem": "A visible suggestion deletes a meaningful clause.", + "suggestedChange": "Hide empty replacements for word spans unless explicitly safe.", + "suggestedTests": ["Add fixture expecting the deletion change to be hidden."] +} +``` + +### Full Autoresearch + +```bash +npm run blackbox:autoresearch -- --config blackbox/config.local.json --cases 25 +``` + +Custom outputs: + +```bash +npm run blackbox:autoresearch -- \ + --config blackbox/config.local.json \ + --cases 100 \ + --fixtures tests/fixtures/scoring-cases.generated.json \ + --report blackbox/runs/latest-evaluation.json \ + --analysis blackbox/runs/latest-analysis.json \ + --markdown blackbox/runs/latest-analysis.md +``` + +Autoresearch does: + +```mermaid +sequenceDiagram + participant User + participant Auto as autoresearch.mjs + participant Run as run.mjs + participant Promote as promote-fixtures.mjs + participant Eval as evaluate-fixtures.mjs + + User->>Auto: npm run blackbox:autoresearch + Auto->>Run: generate/fix/judge cases + Run-->>Auto: JSONL run path + Auto->>Promote: create fixture candidates + Promote-->>Auto: generated fixture JSON + Auto->>Eval: replay generated fixtures + Eval-->>Auto: metrics report + Auto->>Auto: ask analyst AI for next steps + Auto-->>User: run path, fixture path, pass rate +``` + +## Output Schemas + +### JSONL Run Record + +Each line in `blackbox/runs/*.jsonl` looks like: + +```json +{ + "id": "0001-1780000000000", + "startedAt": "2026-05-31T12:00:00.000Z", + "provider": { + "id": "ollama", + "model": "llama3" + }, + "generated": { + "original": "i didnt went there yesterday", + "intendedMeaning": "The writer did not go there yesterday.", + "errorTags": ["capitalization", "tense"], + "notes": "Simple tense and capitalization case." + }, + "correctlyResult": { + "corrected": "I didn't go there yesterday.", + "changes": [], + "confidence": 55, + "cascadeLevel": 3 + }, + "scoring": { + "accepted": true, + "acceptanceScore": 55, + "displayChanges": [], + "hiddenChanges": [], + "cascadeLevel": 3 + }, + "judge": { + "verdict": "pass", + "risk": "none", + "shouldAccept": true, + "meaningPreserved": true, + "grammarImproved": true, + "visibleSuggestionsSafe": true, + "reason": "Grammar improved and meaning stayed intact.", + "fixtureWorthy": false + }, + "error": null +} +``` + +### Fixture Shape + +Reviewed fixtures live in: + +```txt +tests/fixtures/scoring-cases.json +``` + +Shape: + +```json +{ + "id": "standalone-i-hidden-punctuation", + "original": "so i didnt had any time tolarend", + "level": 1, + "rawResponse": { + "corrected": "So I didn't have any time to learn.", + "changes": [], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "So I didn't have any time to learn.", + "displayChanges": [{ "original": "i", "replacement": "I" }], + "hiddenChangeCount": 1, + "minAcceptanceScore": 60 + }, + "notes": "Why this fixture matters." +} +``` + +Supported expectation fields: + +```txt +accept +corrected +displayChanges +hiddenChangeCount +minAcceptanceScore +maxAcceptanceScore +``` + +## How This Improves Scoring + +Blackbox helps by finding disagreements: + +```mermaid +flowchart TD + A["Correctly accepts"] --> B{"Judge agrees?"} + B -->|Yes| C["Pass: keep as evidence"] + B -->|No| D["False accept: add fixture, tune penalties"] + E["Correctly rejects/cascades"] --> F{"Judge says correction was good?"} + F -->|Yes| G["False reject: reduce over-strict rule"] + F -->|No| H["Good rejection: keep as regression"] + I["Visible changes shown"] --> J{"Judge says safe?"} + J -->|No| K["Bad visibility: improve extraction"] +``` + +Useful failures become fixtures. Fixtures become a regression gate. Scoring changes should improve the aggregate fixture report without breaking important edge cases. + +## Recommended Workflow + +1. Run discovery: + +```bash +npm run blackbox -- --config blackbox/config.local.json --cases 100 +``` + +2. Promote candidates: + +```bash +npm run blackbox:promote -- --out tests/fixtures/scoring-cases.generated.json +``` + +3. Review `tests/fixtures/scoring-cases.generated.json`. + +4. Move good cases into `tests/fixtures/scoring-cases.json`. + +5. Evaluate: + +```bash +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json --fail-on-regression +``` + +6. Change scoring code. + +7. Re-run: + +```bash +npm test +npm run blackbox:evaluate -- tests/fixtures/scoring-cases.json --fail-on-regression +``` + +## What Is Fully Automated + +Automated: + +```txt +case generation +Correctly fixing +scoring analysis +AI judging +JSONL persistence +candidate fixture generation +fixture replay +metrics report +AI analyst recommendations +regression failure exit code +``` + +Not automated by default: + +```txt +editing production scoring rules +committing generated fixtures +trusting AI judge labels without review +``` + +That boundary is intentional. The harness can run unattended, but production scoring changes should still be reviewed against metrics. + +## Troubleshooting + +### `Local model HTTP 404` + +The base URL is probably wrong. + +Use: + +```txt +Ollama: http://localhost:11434/v1 +LM Studio: http://localhost:1234/v1 +``` + +Do not include `/chat/completions` in `baseUrl`. + +### `fetch failed` or `ECONNREFUSED` + +The local server is not running. + +For Ollama: + +```bash +ollama serve +``` + +For LM Studio, start the local server in the app. + +### `Generator returned invalid case` + +The generator model did not follow JSON. Try: + +```txt +- lower generator temperature +- stronger generator model +- fewer cases while testing +``` + +### `Judge returned invalid JSON` + +The harness marks the case `interesting`. Try a stronger judge model or lower judge temperature. + +### Correctly provider fails with schema errors + +That is expected for some local models. Correctly should cascade through no-schema, Level 2, and Level 3. Those events are useful research signals. + +### Fixture evaluation fails + +Read the failure line: + +```txt +FAIL fixture-id: accept expected true, got false +``` + +Either: + +```txt +- scoring regressed +- fixture expectation is wrong +- the fixture is too brittle and needs a broader score range +``` + +## Safety + +Do not run Blackbox on private user text unless you explicitly intend to log that text. + +Blackbox writes raw originals, model corrections, judge comments, and scoring output to JSONL. Treat `blackbox/runs/` as potentially sensitive. + +Recommended: + +```txt +- keep generated runs local +- review before sharing +- do not commit large raw run files unless intentionally curated +- commit reviewed fixtures, not raw research dumps +``` diff --git a/blackbox/config.example.json b/blackbox/config.example.json new file mode 100644 index 0000000..f6e8d81 --- /dev/null +++ b/blackbox/config.example.json @@ -0,0 +1,24 @@ +{ + "runName": "local-scoring-research", + "caseCount": 10, + "outputDir": "blackbox/runs", + "seed": "grammar edge cases for email and chat text", + "generator": { + "baseUrl": "http://localhost:11434/v1", + "apiKey": "ollama", + "model": "llama3", + "temperature": 0.8 + }, + "fixer": { + "providerId": "ollama", + "apiKey": "", + "model": "llama3", + "baseUrl": "" + }, + "judge": { + "baseUrl": "http://localhost:11434/v1", + "apiKey": "ollama", + "model": "llama3", + "temperature": 0.1 + } +} diff --git a/blackbox/config.local.json b/blackbox/config.local.json new file mode 100644 index 0000000..f671ac8 --- /dev/null +++ b/blackbox/config.local.json @@ -0,0 +1,24 @@ +{ + "runName": "lmstudio-granite-test-drive", + "caseCount": 20, + "outputDir": "blackbox/runs", + "seed": "grammar edge cases for email, chat, forms, and workplace notes", + "generator": { + "baseUrl": "http://localhost:1234/v1", + "apiKey": "lm-studio", + "model": "google/gemma-3-12b", + "temperature": 0.8 + }, + "fixer": { + "providerId": "lmstudio", + "apiKey": "", + "model": "granite-4.0-h-tiny-mlx", + "baseUrl": "" + }, + "judge": { + "baseUrl": "http://localhost:1234/v1", + "apiKey": "lm-studio", + "model": "llama-3.2-3b-instruct", + "temperature": 0.1 + } +} diff --git a/blackbox/runs/2026-05-31T11-57-55-762Z-lmstudio-granite-test-drive.jsonl b/blackbox/runs/2026-05-31T11-57-55-762Z-lmstudio-granite-test-drive.jsonl new file mode 100644 index 0000000..0bf3d07 --- /dev/null +++ b/blackbox/runs/2026-05-31T11-57-55-762Z-lmstudio-granite-test-drive.jsonl @@ -0,0 +1,20 @@ +{"id":"0001-1780228675763","startedAt":"2026-05-31T11:57:55.763Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0002-1780228675787","startedAt":"2026-05-31T11:57:55.787Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0003-1780228675787","startedAt":"2026-05-31T11:57:55.787Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0004-1780228675788","startedAt":"2026-05-31T11:57:55.788Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0005-1780228675789","startedAt":"2026-05-31T11:57:55.789Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0006-1780228675789","startedAt":"2026-05-31T11:57:55.789Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0007-1780228675790","startedAt":"2026-05-31T11:57:55.790Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0008-1780228675790","startedAt":"2026-05-31T11:57:55.790Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0009-1780228675791","startedAt":"2026-05-31T11:57:55.791Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0010-1780228675791","startedAt":"2026-05-31T11:57:55.791Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0011-1780228675792","startedAt":"2026-05-31T11:57:55.792Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0012-1780228675792","startedAt":"2026-05-31T11:57:55.792Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0013-1780228675793","startedAt":"2026-05-31T11:57:55.793Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0014-1780228675793","startedAt":"2026-05-31T11:57:55.793Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0015-1780228675793","startedAt":"2026-05-31T11:57:55.793Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0016-1780228675793","startedAt":"2026-05-31T11:57:55.793Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0017-1780228675794","startedAt":"2026-05-31T11:57:55.794Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0018-1780228675794","startedAt":"2026-05-31T11:57:55.794Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0019-1780228675794","startedAt":"2026-05-31T11:57:55.794Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} +{"id":"0020-1780228675794","startedAt":"2026-05-31T11:57:55.794Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"fetch failed","stack":"TypeError: fetch failed"}} diff --git a/blackbox/runs/2026-05-31T11-58-48-673Z-lmstudio-granite-test-drive.jsonl b/blackbox/runs/2026-05-31T11-58-48-673Z-lmstudio-granite-test-drive.jsonl new file mode 100644 index 0000000..eeff9d9 --- /dev/null +++ b/blackbox/runs/2026-05-31T11-58-48-673Z-lmstudio-granite-test-drive.jsonl @@ -0,0 +1,20 @@ +{"id":"0001-1780228728674","startedAt":"2026-05-31T11:58:48.674Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0002-1780228728717","startedAt":"2026-05-31T11:58:48.717Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0003-1780228728722","startedAt":"2026-05-31T11:58:48.722Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0004-1780228728725","startedAt":"2026-05-31T11:58:48.725Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0005-1780228728728","startedAt":"2026-05-31T11:58:48.728Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0006-1780228728731","startedAt":"2026-05-31T11:58:48.731Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0007-1780228728734","startedAt":"2026-05-31T11:58:48.734Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0008-1780228728736","startedAt":"2026-05-31T11:58:48.736Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0009-1780228728738","startedAt":"2026-05-31T11:58:48.738Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0010-1780228728741","startedAt":"2026-05-31T11:58:48.741Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0011-1780228728743","startedAt":"2026-05-31T11:58:48.743Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0012-1780228728744","startedAt":"2026-05-31T11:58:48.744Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0013-1780228728745","startedAt":"2026-05-31T11:58:48.745Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0014-1780228728746","startedAt":"2026-05-31T11:58:48.746Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0015-1780228728749","startedAt":"2026-05-31T11:58:48.749Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0016-1780228728750","startedAt":"2026-05-31T11:58:48.750Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0017-1780228728752","startedAt":"2026-05-31T11:58:48.752Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0018-1780228728753","startedAt":"2026-05-31T11:58:48.753Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0019-1780228728754","startedAt":"2026-05-31T11:58:48.754Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} +{"id":"0020-1780228728756","startedAt":"2026-05-31T11:58:48.756Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":null,"correctlyResult":null,"scoring":null,"judge":null,"error":{"message":"Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}","stack":"Error: Local model HTTP 400: {\"error\":\"'response_format.type' must be 'json_schema' or 'text'\"}\n at OpenAICompatibleClient.chat (file:///Users/mak/git/Correctly/blackbox/src/openai-compatible-client.mjs:35:13)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async generateCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:114:23)\n at async runCase (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:75:17)\n at async main (file:///Users/mak/git/Correctly/blackbox/src/run.mjs:48:20)"}} diff --git a/blackbox/runs/2026-05-31T11-59-15-456Z-lmstudio-granite-test-drive.jsonl b/blackbox/runs/2026-05-31T11-59-15-456Z-lmstudio-granite-test-drive.jsonl new file mode 100644 index 0000000..611565d --- /dev/null +++ b/blackbox/runs/2026-05-31T11-59-15-456Z-lmstudio-granite-test-drive.jsonl @@ -0,0 +1,2 @@ +{"id":"0001-1780228755458","startedAt":"2026-05-31T11:59:15.458Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm going to the store, but I forgot my wallet. Then I'll buy milk and eggs. They have a lot of stuff.","intendedMeaning":"I'm going to the store, but I forgot my wallet. Then I'll buy milk and eggs. They have a lot of things.","errorTags":["subjective agreement","comma splice","demonstrative usage"],"notes":"This case tests the writer's ability to use demonstratives correctly and their understanding of comma splices. The sentence with the error is often confusing because it implies that 'they' refers to the store, when in fact it should be the items being bought."},"correctlyResult":{"corrected":"I'm going to the store, but I forgot my wallet. Then I'll buy milk and eggs. They have a lot of stuff.","changes":[],"confidence":100,"usage":{"prompt_tokens":577,"completion_tokens":40,"total_tokens":617,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":4742,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/22 word(s) flagged","penalty":0},{"pass":true,"name":"empty response","detail":"no changes and corrected text matches source","penalty":0},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"interesting","risk":"cascade_issue","shouldAccept":false,"meaningPreserved":false,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"System errored and judge failed: The operation was aborted due to timeout","fixtureWorthy":true},"error":{"message":"The operation was aborted due to timeout","stack":"TimeoutError: The operation was aborted due to timeout"}} +{"id":"0002-1780229008376","startedAt":"2026-05-31T12:03:28.376Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"the meeting is rescheduled for friday at 2pm, don't forget its not just the date that matters its the time to","intendedMeaning":"the meeting is rescheduled for Friday at 2pm; don't forget that it's not just the date that matters, it's the time too.","errorTags":["punctuation","verb tense"],"notes":"Notice how ignoring the comma after Friday leads to a less clear message. This is an example of an edge case where adding just one punctuation mark can significantly improve the clarity and correctness of written English."},"correctlyResult":{"corrected":"The meeting is rescheduled for Friday at 2pm. Don't forget, it's not just the date that matters; it's also the time.","changes":[{"original":"the meeting is rescheduled for friday at 2pm, don't forget its not just the date that matters its the time to","replacement":"The meeting is rescheduled for Friday at 2pm. Don't forget, it's not just the date that matters; it's also the time.","explanation":"Capitalized 'Friday' for proper noun, added missing apostrophe in contraction 'it's', corrected comma placement after introductory clause."}],"confidence":84,"usage":{"prompt_tokens":575,"completion_tokens":135,"total_tokens":710,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":4975,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":82,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/21 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"the meeting is rescheduled for friday at 2pm, don't forget its not just the date that matters its the time to\" -> \"The meeting is rescheduled for Friday at 2pm. Don't forget, it's not just the date that matters; it's also the time.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":false,"name":"granularity","detail":"model returned a whole-text edit instead of targeted changes","penalty":18},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"the meeting is rescheduled for friday at 2pm, don't forget its not just the date that matters its the time to","replacement":"The meeting is rescheduled for Friday at 2pm. Don't forget, it's not just the date that matters; it's also the time.","explanation":"Capitalized 'Friday' for proper noun, added missing apostrophe in contraction 'it's', corrected comma placement after introductory clause."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":84},"judge":{"verdict":"interesting","risk":"cascade_issue","shouldAccept":false,"meaningPreserved":false,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"System errored and judge failed: The operation was aborted due to timeout","fixtureWorthy":true},"error":{"message":"The operation was aborted due to timeout","stack":"TimeoutError: The operation was aborted due to timeout"}} diff --git a/blackbox/runs/2026-05-31T12-11-14-914Z-lmstudio-granite-test-drive.jsonl b/blackbox/runs/2026-05-31T12-11-14-914Z-lmstudio-granite-test-drive.jsonl new file mode 100644 index 0000000..a389c2e --- /dev/null +++ b/blackbox/runs/2026-05-31T12-11-14-914Z-lmstudio-granite-test-drive.jsonl @@ -0,0 +1,20 @@ +{"id":"0001-1780229474915","startedAt":"2026-05-31T12:11:14.915Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea, but we need to consider the impact on existing users. Also, could you please check if that presentation is ready?","intendedMeaning":"I think it's a good idea, but we need to consider the impact on existing users. Also, could you please check if that presentation is ready?","errorTags":["spelling","punctuation"],"notes":"Common confusion between 'its' (possessive) and 'it's' (contraction). The comma after 'idea' is technically optional but can improve readability in this context, which the model should ideally recognize."},"correctlyResult":{"corrected":"I think it's a good idea, but we need to consider the impact on existing users. Also, could you please check if that presentation is ready?","changes":[],"confidence":55,"responseTimeMs":693,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/26 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The model failed to correct the empty response, resulting in a full-text correction without structured changes.","fixtureWorthy":false},"error":null} +{"id":"0002-1780229501412","startedAt":"2026-05-31T12:11:41.412Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think its a really good idea, but we need to make sure everyones on board before moving forward.","intendedMeaning":"I think it's a really good idea, but we need to make sure everyone is on board before moving forward.","errorTags":["spelling","punctuation","grammar"],"notes":"This case combines a common spelling error ('its' vs 'it's'), a capitalization error (everyone), and a missing comma. The informal tone is typical of workplace communication."},"correctlyResult":{"corrected":"I think it's a really good idea, but we need to make sure everyone is on board before moving forward.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'its' should be contracted as 'it's'."},{"original":"everyones","replacement":"everyone's","explanation":"Possessive: 'everyones' should be possessive as 'everyone's'."}],"confidence":91,"usage":{"prompt_tokens":571,"completion_tokens":95,"total_tokens":666,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2593,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"everyones\" -> \"everyone's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.90)","penalty":12}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'its' should be contracted as 'it's'."},{"original":"everyones","replacement":"everyone's","explanation":"Possessive: 'everyones' should be possessive as 'everyone's'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":91},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the spelling error, capitalization error, and added a comma to improve sentence structure.","fixtureWorthy":true},"error":null} +{"id":"0003-1780229515306","startedAt":"2026-05-31T12:11:55.306Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think the project is going good, but we need to finalize the presentation before friday.","intendedMeaning":"I think the project is going well, but we need to finalize the presentation before Friday.","errorTags":["spelling","grammar"],"notes":"A common error in informal writing. 'Good' vs. 'well' is a frequent point of confusion, especially when describing how something *is*. The comma before 'but' is optional here and not strictly incorrect, but the spelling error with 'going' makes it a solid test case."},"correctlyResult":{"corrected":"I think the project is going good, but we need to finalize the presentation before Friday.","changes":[{"original":"project","replacement":"the project","explanation":"Added article 'the' for specificity."},{"original":"going good","replacement":"is going good","explanation":"Corrected verb tense to match subject."},{"original":"friday.","replacement":"Friday.","explanation":"Capitalized proper noun 'Friday'."}],"confidence":69,"usage":{"prompt_tokens":567,"completion_tokens":95,"total_tokens":662,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3350,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":65,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"project\" -> \"the project\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"going good\" -> \"is going good\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"friday.\" -> \"Friday.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"changes listed but corrected text matches source","penalty":35}],"displayChanges":[{"original":"project","replacement":"the project","explanation":"Added article 'the' for specificity."},{"original":"going good","replacement":"is going good","explanation":"Corrected verb tense to match subject."},{"original":"friday.","replacement":"Friday.","explanation":"Capitalized proper noun 'Friday'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":69},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected spelling and grammar errors, but the corrected text does not match the intended meaning.","fixtureWorthy":true},"error":null} +{"id":"0004-1780229533696","startedAt":"2026-05-31T12:12:13.696Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important to note that the project timeline is currently behind schedule, and we need to reevaluate our strategies. Also, can you please send me the latest draft?","intendedMeaning":"I believe it's important to note that the project timeline is currently behind schedule, and we need to reevaluate our strategies. Also, could you please send me the latest draft?","errorTags":["spelling","word choice"],"notes":"A common mistake is confusing 'its' (possessive) and 'it's' (contraction). The word choice of 'think its' is also slightly awkward; 'I believe' flows better in this context. This tests a variety of errors that frequently appear in email correspondence."},"correctlyResult":{"corrected":"I think it's important to note that the project timeline is currently behind schedule, and we need to reevaluate our strategies. Also, can you please send me the latest draft?","changes":[],"confidence":55,"responseTimeMs":576,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/30 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the error and did not provide any visible suggestions.","fixtureWorthy":false},"error":null} +{"id":"0005-1780229552864","startedAt":"2026-05-31T12:12:32.864Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was just wondering if you could please check over the report, i think it's mostly done but one section seems off.","intendedMeaning":"I was just wondering if you could please review the report. I think it's mostly finished, but one section seems incorrect.","errorTags":["punctuation","spelling","word choice"],"notes":"This tests separation of requests with commas and 'i' vs. 'I'. The phrase 'seems off' is informal but acceptable, suggesting a need for a more precise wording alternative like 'incorrect'. The multiple requests increase complexity."},"correctlyResult":{"corrected":"I was just wondering if you could please check over the report, I think it's mostly done but one section seems off.","changes":[{"original":"I was just wondering if you could please check over the report, i think it's mostly done but one section seems off.","replacement":"I was just wondering if you could please check over the report, I think it's mostly done but one section seems off.","explanation":"Added a capital 'I' at the beginning of the sentence for grammatical correctness."}],"confidence":84,"usage":{"prompt_tokens":574,"completion_tokens":112,"total_tokens":686,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2576,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":82,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/22 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"I was just wondering if you could please check over the report, i think it's mostly done but one section seems off.\" -> \"I was just wondering if you could please check over the report, I think it's mostly done but one section seems off.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":false,"name":"granularity","detail":"model returned a whole-text edit instead of targeted changes","penalty":18},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"I was just wondering if you could please check over the report, i think it's mostly done but one section seems off.","replacement":"I was just wondering if you could please check over the report, I think it's mostly done but one section seems off.","explanation":"Added a capital 'I' at the beginning of the sentence for grammatical correctness."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":84},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The model provided a whole-text edit instead of targeted changes, which may not be suitable for all use cases.","fixtureWorthy":true},"error":null} +{"id":"0006-1780229566862","startedAt":"2026-05-31T12:12:46.862Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea, however we need to confirm with accounts.","intendedMeaning":"I think it's a good idea; however, we need to confirm with accounts.","errorTags":["spelling","punctuation"],"notes":"Common error stemming from the frequent use of 'its' in informal writing (vs. 'it's'). The semicolon is required to correctly join the two independent clauses in this sentence, particularly when using 'however'. This is a frequent error found in emails and workplace communications where formality may not always be prioritized."},"correctlyResult":{"corrected":"I think it's a good idea, however we need to confirm with accounts.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'its' should be contracted to 'it's'."},{"original":"we need","replacement":"we need to","explanation":"Clarity: Added 'to' for better phrasing."},{"original":"confirm with accounts","replacement":"confirm with accounts.","explanation":""}],"confidence":86,"usage":{"prompt_tokens":564,"completion_tokens":99,"total_tokens":663,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3042,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/13 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"we need\" -> \"we need to\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"confirm with accounts\" -> \"confirm with accounts.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.86)","penalty":12}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'its' should be contracted to 'it's'."},{"original":"we need","replacement":"we need to","explanation":"Clarity: Added 'to' for better phrasing."},{"original":"confirm with accounts","replacement":"confirm with accounts.","explanation":""}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":86},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected grammar and punctuation, with targeted changes to improve clarity.","fixtureWorthy":true},"error":null} +{"id":"0007-1780229581167","startedAt":"2026-05-31T12:13:01.167Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea, but we need to confirm with john first.","intendedMeaning":"I think it's a good idea, but we need to confirm with John first.","errorTags":["punctuation","spelling"],"notes":"Common contraction error (its vs. it's). Name capitalization is also tested - John should be capitalized."},"correctlyResult":{"corrected":"I think it's a good idea, but we need to confirm with John first.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"its","replacement":"John","explanation":"Capitalization: Proper noun 'John' should start with a capital letter."}],"confidence":81,"usage":{"prompt_tokens":565,"completion_tokens":88,"total_tokens":653,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3000,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":78,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/14 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"its\" -> \"John\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":false,"name":"overlapping changes","detail":"two or more changes target overlapping source text","penalty":22},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"its","replacement":"John","explanation":"Capitalization: Proper noun 'John' should start with a capital letter."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":81},"judge":{"verdict":"interesting","risk":"false_accept","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the contraction error, but incorrectly flagged a capitalization error for 'John'. This may lead to false positives in the future.","fixtureWorthy":true},"error":null} +{"id":"0008-1780229593070","startedAt":"2026-05-31T12:13:13.070Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I need to finalize the report by tuesday, but i'm struggling with section three. Its really complicated.","intendedMeaning":"I need to finalize the report by Tuesday, but I'm struggling with section three. It’s really complicated.","errorTags":["spelling","tense","punctuation"],"notes":"A common scenario in workplace communication. The 'tuesday' misspelling is a frequent error, as is the incorrect use of 'Its' instead of 'It’s'. The comma after 'Tuesday' is technically optional but improves readability in this context."},"correctlyResult":{"corrected":"I need to finalize the report by Tuesday, but I'm struggling with section three. It's really complicated.","changes":[],"confidence":55,"responseTimeMs":454,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/17 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the 'tuesday' misspelling and incorrectly changed 'Its' to 'It’s'.","fixtureWorthy":false},"error":null} +{"id":"0009-1780229611990","startedAt":"2026-05-31T12:13:31.990Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea to have more frequent check-ins, especialy during projects like this one.","intendedMeaning":"I think it's a good idea to have more frequent check-ins, especially during projects like this one.","errorTags":["spelling","punctuation","grammar"],"notes":"This case tests understanding of 'its' vs. 'it's', and proper usage of 'especially'. The punctuation error (missing comma before 'especially') is often overlooked in informal writing, making it a good test case for grammar correction models. The context is a suggestion during work which adds realism."},"correctlyResult":{"corrected":"I think it's a good idea to have more frequent check-ins, especially during projects like this one.","changes":[],"confidence":55,"responseTimeMs":495,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/17 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"Expected a non-empty changes array, but it was empty.","fixtureWorthy":false},"error":null} +{"id":"0010-1780229631918","startedAt":"2026-05-31T12:13:51.918Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea, but are you sure about the timeline? It feels rushed.","intendedMeaning":"I think it's a good idea, but are you sure about the timeline? It feels rushed.","errorTags":["spelling","punctuation"],"notes":"Common mistake with 'its' vs. 'it's'. The punctuation error is also a frequent one in informal writing."},"correctlyResult":{"corrected":"I think it's a good idea, but are you sure about the timeline? It feels rushed.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"are you sure about the timeline?","replacement":"Are you sure about the timeline?","explanation":"Capitalization: First word of sentence should be capitalized."}],"confidence":73,"usage":{"prompt_tokens":568,"completion_tokens":112,"total_tokens":680,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3245,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"are you sure about the timeline?\" -> \"Are you sure about the timeline?\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"are you sure about the timeline?","replacement":"Are you sure about the timeline?","explanation":"Capitalization: First word of sentence should be capitalized."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":73},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected the original text without introducing any new errors.","fixtureWorthy":true},"error":null} +{"id":"0011-1780229645952","startedAt":"2026-05-31T12:14:05.952Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I would like to enquire about the status of my order, it was placed on tuesday. Also could you please confirm if its being shipped via fedex?","intendedMeaning":"I would like to inquire about the status of my order, which was placed on Tuesday. Also, could you please confirm if it is being shipped via FedEx?","errorTags":["spelling","punctuation","tense"],"notes":"This case tests common spelling errors ('enquire' vs. 'inquire'), a missing comma (after order and before also), and capitalization of days of the week, all in an email-like context. The 'it' pronoun reference is clear but requires correction."},"correctlyResult":{"corrected":"I would like to inquire about the status of my order, which was placed on Tuesday. Also, could you please confirm if it is being shipped via FedEx?","changes":[{"original":"enquire","replacement":"inquire","explanation":"In American English, 'enquire' is less common than 'inquire'. Both are correct, but 'inquire' is more widely used."},{"original":"status","replacement":"","explanation":"No change needed."},{"original":"it was placed on tuesday","replacement":"which was placed on Tuesday","explanation":"Added 'which' for clarity and capitalized 'Tuesday' to follow standard title case conventions."},{"original":"Also could you please confirm if its being shipped via fedex?","replacement":"Also, could you please confirm if it is being shipped via FedEx?","explanation":"Added a comma after 'Also' for better sentence flow. Corrected the spelling of 'FedEx'."}],"confidence":80,"usage":{"prompt_tokens":270,"completion_tokens":257,"total_tokens":527,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3996,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":76,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"4/27 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"enquire\" -> \"inquire\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"status\" -> \"\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"it was placed on tuesday\" -> \"which was placed on Tuesday\"","penalty":0},{"pass":true,"name":"change #4","detail":"\"Also could you please confirm if its being shipped via fedex?\" -> \"Also, could you please confirm if it is being shipped via FedEx?\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"4/4 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.96)","penalty":12},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"enquire","replacement":"inquire","explanation":"In American English, 'enquire' is less common than 'inquire'. Both are correct, but 'inquire' is more widely used."},{"original":"status","replacement":"","explanation":"No change needed."},{"original":"it was placed on tuesday","replacement":"which was placed on Tuesday","explanation":"Added 'which' for clarity and capitalized 'Tuesday' to follow standard title case conventions."},{"original":"Also could you please confirm if its being shipped via fedex?","replacement":"Also, could you please confirm if it is being shipped via FedEx?","explanation":"Added a comma after 'Also' for better sentence flow. Corrected the spelling of 'FedEx'."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":80},"judge":{"verdict":"pass","risk":"false_accept","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected spelling errors, added commas for better sentence flow, and capitalized days of the week. However, the corrected text differs slightly from the changes (0.96), which may affect consistency.","fixtureWorthy":true},"error":null} +{"id":"0012-1780229679420","startedAt":"2026-05-31T12:14:39.420Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm writting to inquire about the position. I think it would be a grate fit for me, and i am very exited to hear back from you soonest.","intendedMeaning":"I'm writing to inquire about the position. I think it would be a great fit for me, and I am very excited to hear back from you soon.","errorTags":["spelling","punctuation","word-choice"],"notes":"This case highlights common errors in informal writing (email/chat) where spell checking might not be enabled or fully reliable. 'writting' vs 'writing', 'grate' vs 'great', and standalone 'i' are frequently seen. The use of 'soonest' is technically correct, but sounds overly formal/dated in a modern context. The comma before 'and' is optional but unnecessary, representing a common stylistic choice."},"correctlyResult":{"corrected":"I'm writing to inquire about the position. I think it would be a great fit for me, and I am very excited to hear back from you soonest.","changes":[{"original":"grate","replacement":"great","explanation":"Correct spelling of 'grate' to 'great'."},{"original":"i am very exited","replacement":"I am very excited","explanation":"Capitalization of 'I' and correction of spelling from 'excited' to 'excited'."},{"original":"hear back","replacement":"hear from you","explanation":"Correct phrase 'hear from you' instead of the informal 'hear back'."}],"confidence":89,"usage":{"prompt_tokens":583,"completion_tokens":140,"total_tokens":723,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3318,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/28 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"grate\" -> \"great\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"i am very exited\" -> \"I am very excited\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"hear back\" -> \"hear from you\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.90)","penalty":12}],"displayChanges":[{"original":"grate","replacement":"great","explanation":"Correct spelling of 'grate' to 'great'."},{"original":"i am very exited","replacement":"I am very excited","explanation":"Capitalization of 'I' and correction of spelling from 'excited' to 'excited'."},{"original":"hear back","replacement":"hear from you","explanation":"Correct phrase 'hear from you' instead of the informal 'hear back'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"false_accept","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected text differs slightly from changes (0.90), but overall structure and changes are valid.","fixtureWorthy":true},"error":null} +{"id":"0013-1780229702820","startedAt":"2026-05-31T12:15:02.820Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was wondering if you could please check my report, I have finished it. Also, their are a few things i need to clarify.","intendedMeaning":"I was wondering if you could please check my report, as I have finished it. Also, there are a few things I need to clarify.","errorTags":["spelling","punctuation","grammar"],"notes":"This case tests common errors in email communication. The 'their' vs. 'there' confusion is frequent, as is the standalone 'i.' The comma splice ('I have finished it. Also') needs correction, but is a natural writing error. It also tests subject-verb agreement (their are -> there are)."},"correctlyResult":{"corrected":"I was wondering if you could please check my report; I have finished it. Also, there are a few things I need to clarify.","changes":[],"confidence":55,"responseTimeMs":563,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/24 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the comma splice error and did not provide any changes.","fixtureWorthy":false},"error":null} +{"id":"0014-1780229728040","startedAt":"2026-05-31T12:15:28.040Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I need to submit the form by friday, but i'm not sure if I completed all sections correctly. Its kinda confusing.","intendedMeaning":"I need to submit the form by Friday, but I'm not sure if I completed all sections correctly. It's kind of confusing.","errorTags":["capitalization","punctuation","spelling","word choice"],"notes":"This tests capitalization of days, contractions (its/it's), and informal language ('kinda' vs. 'kind of'). The use of ‘I’ as a standalone pronoun is common in casual writing and requires careful detection. This reflects errors frequently found in online forms/chat submissions when users are typing quickly."},"correctlyResult":{"corrected":"I need to submit the form by Friday, but I'm not sure if I completed all sections correctly. It's a bit confusing.","changes":[{"original":"friday","replacement":"Friday","explanation":"Capitalization: Proper noun 'day' should be capitalized."},{"original":"its","replacement":"It's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"kinda","replacement":"a bit","explanation":"Informal to formal: 'kinda' replaced with more formal equivalent."}],"confidence":73,"usage":{"prompt_tokens":574,"completion_tokens":136,"total_tokens":710,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3559,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/21 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"friday\" -> \"Friday\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"its\" -> \"It's\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"kinda\" -> \"a bit\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"friday","replacement":"Friday","explanation":"Capitalization: Proper noun 'day' should be capitalized."},{"original":"its","replacement":"It's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"kinda","replacement":"a bit","explanation":"Informal to formal: 'kinda' replaced with more formal equivalent."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":73},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected the original text's meaning.","fixtureWorthy":true},"error":null} +{"id":"0015-1780229748063","startedAt":"2026-05-31T12:15:48.063Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think the report is good, but their are a few things i need to change.","intendedMeaning":"I think the report is good, but there are a few things I need to change.","errorTags":["spelling","punctuation"],"notes":"Common error of confusing 'their' with 'there'. Also, missing comma after introductory phrase. This checks for a mix of common errors in casual writing (e.g., employee feedback)."},"correctlyResult":{"corrected":"I think the report is good, but there are a few things I need to change.","changes":[{"original":"their","replacement":"there","explanation":"Correct homophone error: 'their' should be 'there'."}],"confidence":98,"usage":{"prompt_tokens":567,"completion_tokens":56,"total_tokens":623,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2067,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"their\" -> \"there\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"their","replacement":"there","explanation":"Correct homophone error: 'their' should be 'there'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All scoring metrics indicate a correct and grammatically improved response.","fixtureWorthy":true},"error":null} +{"id":"0016-1780229761389","startedAt":"2026-05-31T12:16:01.389Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think it would be grate if we could get those reports in by friday.","intendedMeaning":"I think it would be great if we could get those reports in by Friday.","errorTags":["spelling","capitalization"],"notes":"A common misspelling of 'great' combined with incorrect capitalization of a day of the week. This tests the ability to recognize both common typos and proper noun capitalization within conversational text."},"correctlyResult":{"corrected":"I think it would be great if we could get those reports in by Friday.","changes":[{"original":"grate","replacement":"great","explanation":"Correction of informal spelling to standard English."}],"confidence":98,"usage":{"prompt_tokens":565,"completion_tokens":49,"total_tokens":614,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1778,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/15 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"grate\" -> \"great\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"grate","replacement":"great","explanation":"Correction of informal spelling to standard English."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All scoring metrics indicate a correct and efficient grammar correction behavior.","fixtureWorthy":true},"error":null} +{"id":"0017-1780229773905","startedAt":"2026-05-31T12:16:13.905Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was thinking about the project, and it's like, really important we finish this week. Its crucial for everyone.","intendedMeaning":"I've been thinking about the project, and it’s really important that we finish this week. It's crucial for everyone.","errorTags":["tense","punctuation","word choice"],"notes":"This case tests the nuance of present tense vs. past continuous, and correct use of 'its' vs. 'it's'. The initial phrasing in the original text mimics informal chat language, making it a good test for grammar correction systems designed to handle conversational tone."},"correctlyResult":{"corrected":"I was thinking about the project, and it's really important we finish this week. It's crucial for everyone.","changes":[{"original":"Its","replacement":"It's","explanation":"Omit the unnecessary apostrophe in 'Its'."}],"confidence":89,"usage":{"prompt_tokens":573,"completion_tokens":59,"total_tokens":632,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3121,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"Its\" -> \"It's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.95)","penalty":12}],"displayChanges":[{"original":"Its","replacement":"It's","explanation":"Omit the unnecessary apostrophe in 'Its'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the error and provided a clear explanation for the correction. The corrected text is grammatically correct, and the changes are targeted.","fixtureWorthy":true},"error":null} +{"id":"0018-1780229806322","startedAt":"2026-05-31T12:16:46.322Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really exited to hear back from you about the project. Lets schedule a call next week sometime.","intendedMeaning":"I was really excited to hear back from you about the project. Let's schedule a call sometime next week.","errorTags":["spelling","punctuation","word choice"],"notes":"This case highlights common email/chat errors: misspelling 'excited' as 'exited', incorrect contraction usage ('lets' instead of 'let's'), and missing comma after introductory phrase. The informal style is typical of quick correspondence, making it a useful test for nuanced corrections."},"correctlyResult":{"corrected":"I was really excited to hear back from you about the project. Let's schedule a call next week sometime.","changes":[{"original":"I was really exited","replacement":"I was really excited","explanation":"Excited is the correct past participle form of the verb 'excite'."},{"original":"Lets schedule","replacement":"Let's schedule","explanation":"A contraction should be used for 'let us' to form the correct grammatical structure."}],"confidence":89,"usage":{"prompt_tokens":258,"completion_tokens":137,"total_tokens":395,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1968,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"I was really exited\" -> \"I was really excited\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"Lets schedule\" -> \"Let's schedule\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"I was really exited","replacement":"I was really excited","explanation":"Excited is the correct past participle form of the verb 'excite'."},{"original":"Lets schedule","replacement":"Let's schedule","explanation":"A contraction should be used for 'let us' to form the correct grammatical structure."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":89},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Correctly identified and corrected spelling, punctuation, and word choice errors. Cascade level is within acceptable limits.","fixtureWorthy":true},"error":null} +{"id":"0019-1780229834408","startedAt":"2026-05-31T12:17:14.408Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a really good idea, but we need to check if the budget allows for it. Also, please send me the meeting notes when you get a chance.","intendedMeaning":"I think it's a really good idea, but we need to check if the budget allows for it. Also, please send me the meeting notes when you get a chance.","errorTags":["spelling","punctuation"],"notes":"Common mistake of confusing 'its' (possessive) with 'it's' (contraction). The punctuation is also technically correct but could be slightly improved for a more formal tone. This scenario tests the model's ability to identify and correct subtle spelling errors within a conversational context."},"correctlyResult":{"corrected":"I think it's a really good idea, but we need to check if the budget allows for it. Also, please send me the meeting notes when you get a chance.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"we need to check if the budget allows for it.","replacement":"","explanation":""}],"confidence":75,"usage":{"prompt_tokens":583,"completion_tokens":96,"total_tokens":679,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3048,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":70,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/30 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"we need to check if the budget allows for it.\" -> \"\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text does not match listed changes (0.67)","penalty":30}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"we need to check if the budget allows for it.","replacement":"","explanation":""}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":75},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The model correctly identified and corrected the subtle spelling error, but the correction was not fully applied to the original text.","fixtureWorthy":true},"error":null} +{"id":"0020-1780229851520","startedAt":"2026-05-31T12:17:31.520Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea, but we need to check if the budget allows for it. Also, can you please send me the report soonest?","intendedMeaning":"I think it's a good idea, but we need to check if the budget allows for it. Also, can you please send me the report as soon as possible?","errorTags":["spelling","word-choice"],"notes":"This tests the common confusion of 'its' vs. 'it's', as well as the increasingly outdated phrase 'soonest'. The context is an email, a common environment for these errors."},"correctlyResult":{"corrected":"I think it's a good idea, but we need to check if the budget allows for it. Also, can you please send me the report as soon as possible?","changes":[],"confidence":55,"responseTimeMs":637,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/26 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the sentence and did not provide any changes, despite detecting errors.","fixtureWorthy":false},"error":null} diff --git a/blackbox/runs/2026-05-31T12-38-56-983Z-lmstudio-granite-test-drive.jsonl b/blackbox/runs/2026-05-31T12-38-56-983Z-lmstudio-granite-test-drive.jsonl new file mode 100644 index 0000000..2a9b3d0 --- /dev/null +++ b/blackbox/runs/2026-05-31T12-38-56-983Z-lmstudio-granite-test-drive.jsonl @@ -0,0 +1,20 @@ +{"id":"0001-1780231136984","startedAt":"2026-05-31T12:38:56.984Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I wanted to confirm with you regarding the status of my request. It was submited last tuesday and I havent heard anything back yet.","intendedMeaning":"I wanted to confirm with you regarding the status of my request. It was submitted last Tuesday, and I haven't heard anything back yet.","errorTags":["spelling","punctuation","tense"],"notes":"This case tests common spelling errors ('submited' for 'submitted'), missing punctuation (comma after Tuesday), and informal tense usage ('havent' should be 'haven't'). The conversational tone is typical of a follow-up email."},"correctlyResult":{"corrected":"I wanted to confirm with you regarding the status of my request. It was submitted last Tuesday and I haven't heard anything back yet.","changes":[{"original":"submited","replacement":"submitted","explanation":"Corrected verb form for past tense."},{"original":"submited last tuesday","replacement":"submitted last Tuesday","explanation":"Corrected verb form and standardized capitalization for proper noun."},{"original":"I havent heard anything back yet","replacement":"I haven't heard anything back yet","explanation":"Corrected contraction for informal tone."}],"confidence":84,"usage":{"prompt_tokens":579,"completion_tokens":123,"total_tokens":702,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":17869,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":78,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/24 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"submited\" -> \"submitted\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"submited last tuesday\" -> \"submitted last Tuesday\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"I havent heard anything back yet\" -> \"I haven't heard anything back yet\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":false,"name":"overlapping changes","detail":"two or more changes target overlapping source text","penalty":22},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"submited","replacement":"submitted","explanation":"Corrected verb form for past tense."},{"original":"submited last tuesday","replacement":"submitted last Tuesday","explanation":"Corrected verb form and standardized capitalization for proper noun."},{"original":"I havent heard anything back yet","replacement":"I haven't heard anything back yet","explanation":"Corrected contraction for informal tone."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":84},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected spelling errors, verb form, and informal tone. However, the correction of overlapping changes resulted in a penalty.","fixtureWorthy":true},"error":null} +{"id":"0002-1780231209767","startedAt":"2026-05-31T12:40:09.767Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important to clarify what we agreed on during the meeting. We'll need to finalize those reports by friday.","intendedMeaning":"I think it’s important to clarify what we agreed on during the meeting. We'll need to finalize those reports by Friday.","errorTags":["punctuation","spelling"],"notes":"This tests common contraction confusion ('its' vs 'it’s') and capitalization of days of the week. It mimics a typical workplace email attempting to summarize decisions."},"correctlyResult":{"corrected":"I think it's important to clarify what we agreed on during the meeting. We'll need to finalize those reports by Friday.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"friday.","replacement":"Friday.","explanation":"Capitalize proper nouns (days of the week)."}],"confidence":75,"usage":{"prompt_tokens":573,"completion_tokens":96,"total_tokens":669,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":9486,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/21 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"friday.\" -> \"Friday.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"friday.","replacement":"Friday.","explanation":"Capitalize proper nouns (days of the week)."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":75},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the contraction confusion ('its' vs 'it’s') and capitalized the day of the week 'Friday'.","fixtureWorthy":true},"error":null} +{"id":"0003-1780231263540","startedAt":"2026-05-31T12:41:03.540Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I wanted to reach out regarding the project, we should schedule a meeting next week. Its really important that everyone is on same page before moving forward.","intendedMeaning":"I wanted to reach out regarding the project. We should schedule a meeting next week. It's really important that everyone is on the same page before moving forward.","errorTags":["punctuation","spelling","word-choice"],"notes":"This case highlights common errors in informal communication (email/chat) - missing apostrophe in 'its', incorrect word usage ('on same page' vs. 'on the same page'). The comma splice is a frequent issue too."},"correctlyResult":{"corrected":"I wanted to reach out regarding the project; we should schedule a meeting next week. It's really important that everyone is on the same page before moving forward.","changes":[{"original":"Its","replacement":"It's","explanation":"Contractions require an apostrophe."}],"confidence":89,"usage":{"prompt_tokens":579,"completion_tokens":64,"total_tokens":643,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":6982,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/27 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"Its\" -> \"It's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.93)","penalty":12}],"displayChanges":[{"original":"Its","replacement":"It's","explanation":"Contractions require an apostrophe."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The corrected text is mostly accurate, but the comma splice was not fully addressed.","fixtureWorthy":true},"error":null} +{"id":"0004-1780231313536","startedAt":"2026-05-31T12:41:53.536Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really excited about the project, but then i realized it wasn't doable. Its a shame.","intendedMeaning":"I was really excited about the project, but then I realized it wasn't doable. It's a shame.","errorTags":["spelling","punctuation","lowercase"],"notes":"This case tests common lowercase letter errors ('i' for 'I') and incorrect possessive contractions. The informal tone is typical of quick emails or chat messages, making errors frequent."},"correctlyResult":{"corrected":"I was really excited about the project, but then I realized it wasn't doable. It's a shame.","changes":[{"original":"i","replacement":"I","explanation":"Capitalization: 'I' should be capitalized at the beginning of a sentence."},{"original":"Its","replacement":"It's","explanation":"Possessive contraction: 'It's' is the correct form for 'it is'."}],"confidence":100,"usage":{"prompt_tokens":571,"completion_tokens":98,"total_tokens":669,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":6000,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/17 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"i\" -> \"I\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"Its\" -> \"It's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"i","replacement":"I","explanation":"Capitalization: 'I' should be capitalized at the beginning of a sentence."},{"original":"Its","replacement":"It's","explanation":"Possessive contraction: 'It's' is the correct form for 'it is'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected the errors in the original text.","fixtureWorthy":true},"error":null} +{"id":"0005-1780231338983","startedAt":"2026-05-31T12:42:18.983Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a great idea, but we need to make sure everyone agrees first. Also, could you check if the document is save?","intendedMeaning":"I think it's a great idea, but we need to make sure everyone agrees first. Also, could you check if the document is saved?","errorTags":["spelling","punctuation","grammar"],"notes":"This case combines common errors: 'its' vs. 'it's', missing punctuation for a sentence fragment, and the incorrect past participle of 'save'. Realistic in an email or chat setting. The 'its' error is particularly common."},"correctlyResult":{"corrected":"I think it's a great idea, but we need to make sure everyone agrees first. Also, could you check if the document is saved?","changes":[],"confidence":55,"responseTimeMs":489,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/24 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"The system failed to correct the 'save' error, and the response time was high.","fixtureWorthy":false},"error":null} +{"id":"0006-1780231355849","startedAt":"2026-05-31T12:42:35.849Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I seen him at the store yesterday. He was wearin’ a blue shirt and i think he bought some milk.","intendedMeaning":"I saw him at the store yesterday. He was wearing a blue shirt, and I think he bought some milk.","errorTags":["tense","spelling","punctuation"],"notes":"This case highlights common errors in informal writing, particularly the confusion between 'seen' and 'saw', incorrect spelling ('wearin’'), and missing comma after the introductory phrase in the second sentence."},"correctlyResult":{"corrected":"I saw him at the store yesterday. He was wearing a blue shirt and I think he bought some milk.","changes":[{"original":"seen","replacement":"saw","explanation":"Past tense verb agreement with 'I saw'."},{"original":"wearin’","replacement":"wearing","explanation":"Informal contraction corrected to standard spelling."},{"original":"i think","replacement":"I think","explanation":"Capitalization of pronoun 'I'."},{"original":"he bought some milk.","replacement":"He bought some milk.","explanation":"Capitalization of pronoun 'He'."}],"confidence":98,"usage":{"prompt_tokens":573,"completion_tokens":130,"total_tokens":703,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2780,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"4/20 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"seen\" -> \"saw\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"wearin’\" -> \"wearing\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"i think\" -> \"I think\"","penalty":0},{"pass":true,"name":"change #4","detail":"\"he bought some milk.\" -> \"He bought some milk.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"4/4 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"seen","replacement":"saw","explanation":"Past tense verb agreement with 'I saw'."},{"original":"wearin’","replacement":"wearing","explanation":"Informal contraction corrected to standard spelling."},{"original":"i think","replacement":"I think","explanation":"Capitalization of pronoun 'I'."},{"original":"he bought some milk.","replacement":"He bought some milk.","explanation":"Capitalization of pronoun 'He'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Correctly identified and corrected errors in grammar, punctuation, and spelling.","fixtureWorthy":true},"error":null} +{"id":"0007-1780231369030","startedAt":"2026-05-31T12:42:49.030Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I need to get this report finnished by Friday, but I'm having a hard time finding the data. Its really frustrating!","intendedMeaning":"I need to finish this report by Friday, but I'm having a hard time finding the data. It's really frustrating!","errorTags":["spelling","punctuation","word choice"],"notes":"This tests for common misspellings (finnished -> finished), incorrect contractions ('its' vs 'it's'), and casual punctuation in a workplace context. The sentence structure is simple, so the errors are prominent."},"correctlyResult":{"corrected":"I need to get this report finished by Friday, but I'm having a hard time finding the data. It's really frustrating!","changes":[],"confidence":55,"responseTimeMs":456,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/21 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the sentence and did not provide any visible suggestions.","fixtureWorthy":false},"error":null} +{"id":"0008-1780231385589","startedAt":"2026-05-31T12:43:05.589Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea to schedule the meeting for tuesday. Also could you confirm if your avalible?","intendedMeaning":"I think it's a good idea to schedule the meeting for Tuesday. Also, could you confirm if you are available?","errorTags":["spelling","punctuation","grammar","word-choice"],"notes":"This tests common errors in informal written communication (email/chat) - 'its' vs. 'it's', incorrect spelling of 'available', missing comma after introductory phrase, and capitalization of days of the week. The context is casual, making it a realistic error scenario."},"correctlyResult":{"corrected":"I think it's a good idea to schedule the meeting for Tuesday. Also, could you confirm if you're available?","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"avalible","replacement":"available","explanation":"Spelling correction: 'avalible' should be spelled correctly as 'available'."}],"confidence":91,"usage":{"prompt_tokens":572,"completion_tokens":98,"total_tokens":670,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2042,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"avalible\" -> \"available\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.89)","penalty":12}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"avalible","replacement":"available","explanation":"Spelling correction: 'avalible' should be spelled correctly as 'available'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":91},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The corrected text differs slightly from the changes (0.89), indicating a minor inconsistency in the correction process.","fixtureWorthy":true},"error":null} +{"id":"0009-1780231399295","startedAt":"2026-05-31T12:43:19.295Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea to implement this feature, but we need to consider the impact on existing users. Also, can you please check if this is doable before monday?","intendedMeaning":"I think it's a good idea to implement this feature, but we need to consider the impact on existing users. Also, can you please check if this is doable before Monday?","errorTags":["spelling","punctuation"],"notes":"This tests common contraction errors ('its' vs 'it's') and capitalization of days of the week in informal communication (email/chat). The 'also' creates a slightly more conversational feel."},"correctlyResult":{"corrected":"I think it's a good idea to implement this feature, but we need to consider the impact on existing users. Also, can you please check if this is doable before Monday?","changes":[],"confidence":55,"responseTimeMs":588,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/31 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"Expected a full-text correction with structured changes, but received an empty response.","fixtureWorthy":false},"error":null} +{"id":"0010-1780231416045","startedAt":"2026-05-31T12:43:36.045Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a really good idea, but we should probably check with accounting first before moving forward, right?","intendedMeaning":"I think it's a really good idea, but we should probably check with accounting first before moving forward, right?","errorTags":["possessive/contraction","spelling"],"notes":"Common contraction error ('its' vs 'it’s'). The question tag adds a layer of natural conversational tone, making the error more subtle. Relevant to email and chat communication."},"correctlyResult":{"corrected":"I think it's a really good idea, but we should probably check with accounting first before moving forward, right?","changes":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."}],"confidence":98,"usage":{"prompt_tokens":571,"completion_tokens":56,"total_tokens":627,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1398,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The grammar correction behavior is correct, and the suggested changes are accurate.","fixtureWorthy":true},"error":null} +{"id":"0011-1780231427214","startedAt":"2026-05-31T12:43:47.214Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I have been working on this report, and it's almost done. However, I need to clarify somethings before submitting.","intendedMeaning":"I have been working on this report, and it's almost done. However, I need to clarify something before submitting.","errorTags":["grammar","spelling"],"notes":"Common error of pluralizing 'something' when it should be singular. This is frequently seen in informal writing (emails, chat) and represents a good test case for grammar checkers as it's not an obvious spelling mistake."},"correctlyResult":{"corrected":"I have been working on this report, and it's almost done. However, I need to clarify some things before submitting.","changes":[{"original":"somethings","replacement":"things","explanation":"Corrected the singular noun 'something' to the plural form 'things' for grammatical agreement."}],"confidence":89,"usage":{"prompt_tokens":575,"completion_tokens":71,"total_tokens":646,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2526,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"somethings\" -> \"things\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.95)","penalty":12}],"displayChanges":[{"original":"somethings","replacement":"things","explanation":"Corrected the singular noun 'something' to the plural form 'things' for grammatical agreement."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the pluralization error and suggested a correction. The corrected text is grammatically consistent, and the changes are targeted.","fixtureWorthy":true},"error":null} +{"id":"0012-1780231441228","startedAt":"2026-05-31T12:44:01.228Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm so exited to finaly see you next week! Hope your doing well.","intendedMeaning":"I'm so excited to finally see you next week! Hope you are doing well.","errorTags":["spelling","tense","punctuation"],"notes":"This tests common misspellings (exited -> excited, finaly -> finally) and a frequent 'your/you're' confusion. The missing comma after 'week!' is also typical of informal writing but would require a judgment call for correction depending on the context. The tense error with 'doing' is a casual speech habit but incorrect in formal writing."},"correctlyResult":{"corrected":"I'm so excited to finally see you next week! Hope you're doing well.","changes":[{"original":"finaly","replacement":"finally","explanation":"Correct spelling of 'finally'."},{"original":"doing well","replacement":"you're doing well","explanation":"Added apostrophe for contraction of 'you are'."}],"confidence":89,"usage":{"prompt_tokens":566,"completion_tokens":76,"total_tokens":642,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1678,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/13 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"finaly\" -> \"finally\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"doing well\" -> \"you're doing well\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.86)","penalty":12}],"displayChanges":[{"original":"finaly","replacement":"finally","explanation":"Correct spelling of 'finally'."},{"original":"doing well","replacement":"you're doing well","explanation":"Added apostrophe for contraction of 'you are'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The corrected text is mostly accurate, but the changes made may not be entirely correct. The corrected consistency penalty was high (12), indicating that the changes did not fully align with the original text.","fixtureWorthy":true},"error":null} +{"id":"0013-1780231454852","startedAt":"2026-05-31T12:44:14.852Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important to note that the report was submitted late, despite my best efforts. i really tried!","intendedMeaning":"I believe it's important to note that the report was submitted late, despite my best efforts. I really tried!","errorTags":["spelling","punctuation","grammar"],"notes":"This case tests 'its' vs. 'it's', a common error. The standalone 'i' is also an easy typo that often slips through. The abrupt transition to exclamatory tone ('i really tried!') can catch some systems out; a more sophisticated correction might suggest 'I really did try!' but simple corrections should fix the spelling and capitalization."},"correctlyResult":{"corrected":"I think it's important to note that the report was submitted late, despite my best efforts. I really tried!","changes":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"i really tried!","replacement":"I really tried!","explanation":"Capitalize the first letter of a sentence."}],"confidence":100,"usage":{"prompt_tokens":571,"completion_tokens":85,"total_tokens":656,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1786,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/19 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"i really tried!\" -> \"I really tried!\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"i really tried!","replacement":"I really tried!","explanation":"Capitalize the first letter of a sentence."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the spelling error, 'its' vs. 'it's', and also fixed the capitalization of the sentence.","fixtureWorthy":true},"error":null} +{"id":"0014-1780231468939","startedAt":"2026-05-31T12:44:28.939Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a great idea! however, the timing seems off. we need to finalize everything before next tuesday's meeting.","intendedMeaning":"I think it's a great idea! However, the timing seems off. We need to finalize everything before next Tuesday's meeting.","errorTags":["spelling","punctuation"],"notes":"Common contraction error ('its' vs. 'it's'). Punctuation correction (semicolon to period or comma). This tests a user who is generally competent but makes occasional typos, typical of informal communication (email/chat)."},"correctlyResult":{"corrected":"I think it's a great idea! However, the timing seems off. We need to finalize everything before next Tuesday's meeting.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe for possessive forms."},{"original":"tuesday's","replacement":"Tuesday's","explanation":"Proper capitalization of days of the week."}],"confidence":98,"usage":{"prompt_tokens":575,"completion_tokens":90,"total_tokens":665,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1877,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/20 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"tuesday's\" -> \"Tuesday's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe for possessive forms."},{"original":"tuesday's","replacement":"Tuesday's","explanation":"Proper capitalization of days of the week."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All corrections were accurate and properly applied.","fixtureWorthy":true},"error":null} +{"id":"0015-1780231481516","startedAt":"2026-05-31T12:44:41.516Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was thinking about that project we talked about last week, and i feel like it would be helpful to get some more input from the team before we move forward. Its really important that everyone is on board.","intendedMeaning":"I was thinking about that project we discussed last week, and I feel like it would be helpful to get some more input from the team before we move forward. It's really important that everyone is on board.","errorTags":["capitalization","punctuation","spelling"],"notes":"This case tests capitalization of 'I' and the contraction 'It's'. The repeated word (it/its) is a common error in informal writing. The overall tone mimics a typical workplace email about project management."},"correctlyResult":{"corrected":"I was thinking about the project we talked about last week, and I feel like it would be helpful to get some more input from the team before we move forward. It's really important that everyone is on board.","changes":[{"original":"i feel like it would be helpful","replacement":"I feel like it would be helpful","explanation":"Capitalization: 'i' should start with a capital letter as it begins the sentence."},{"original":"its really important","replacement":"It's really important","explanation":"Possessive pronoun 'its' should be contracted to 'it's' when used as a contraction of 'it is'. In this context, it functions as a possessive pronoun but also implies existence of the team's agreement, so 'it is' fits better."},{"original":"everyone is on board","replacement":"everybody is on board","explanation":"Preferred usage: 'everyone' and 'everybody' are both correct, but 'everybody' is more common in contemporary writing."}],"confidence":89,"usage":{"prompt_tokens":591,"completion_tokens":223,"total_tokens":814,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3897,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/39 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"i feel like it would be helpful\" -> \"I feel like it would be helpful\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"its really important\" -> \"It's really important\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"everyone is on board\" -> \"everybody is on board\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.95)","penalty":12}],"displayChanges":[{"original":"i feel like it would be helpful","replacement":"I feel like it would be helpful","explanation":"Capitalization: 'i' should start with a capital letter as it begins the sentence."},{"original":"its really important","replacement":"It's really important","explanation":"Possessive pronoun 'its' should be contracted to 'it's' when used as a contraction of 'it is'. In this context, it functions as a possessive pronoun but also implies existence of the team's agreement, so 'it is' fits better."},{"original":"everyone is on board","replacement":"everybody is on board","explanation":"Preferred usage: 'everyone' and 'everybody' are both correct, but 'everybody' is more common in contemporary writing."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"false_accept","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected text is consistent with changes, but the corrected text differs slightly from the changes (0.95).","fixtureWorthy":true},"error":null} +{"id":"0016-1780231498529","startedAt":"2026-05-31T12:44:58.529Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think that the report is done, but i need to double check it before sending.","intendedMeaning":"I think the report is complete, but I need to double-check it before sending.","errorTags":["punctuation","spelling","grammar"],"notes":"This case highlights the common error of capitalizing 'i' in informal writing. It also demonstrates the need to avoid unnecessary phrasing like 'that the report is done' when simpler language works equally well. The comma before 'but' is optional but often misused in informal contexts."},"correctlyResult":{"corrected":"I think that the report is done, but I need to double check it before sending.","changes":[{"original":"i need","replacement":"I need","explanation":"First-person singular pronoun 'I' should be capitalized."}],"confidence":98,"usage":{"prompt_tokens":567,"completion_tokens":56,"total_tokens":623,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1403,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"i need\" -> \"I need\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"i need","replacement":"I need","explanation":"First-person singular pronoun 'I' should be capitalized."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the capitalization error, preserved the intended meaning, and provided targeted changes.","fixtureWorthy":true},"error":null} +{"id":"0017-1780231510270","startedAt":"2026-05-31T12:45:10.270Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important to note that the project will likely be delayed, due two unforeseen circumstances. We're working hard to mitigate these issues but a firm date isn’t available yet.","intendedMeaning":"I think it's important to note that the project will likely be delayed due to unforeseen circumstances. We’re working hard to mitigate these issues, but a firm date isn't available yet.","errorTags":["spelling","grammar","punctuation"],"notes":"This case combines multiple common errors: 'its' vs. 'it's', 'due to' (often misused as 'due'), and a missing comma after the introductory phrase. It tests understanding of possessives, prepositions, and sentence structure in a professional email context."},"correctlyResult":{"corrected":"I think it's important to note that the project will likely be delayed, due to two unforeseen circumstances. We're working hard to mitigate these issues, but a firm date isn't available yet.","changes":[],"confidence":55,"responseTimeMs":628,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/31 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the 'its' vs. 'it's' error and also returned an empty response.","fixtureWorthy":false},"error":null} +{"id":"0018-1780231529192","startedAt":"2026-05-31T12:45:29.192Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really exited to hear about the project. Its gonna be a big oppurtunity, i think!","intendedMeaning":"I was really excited to hear about the project. It's going to be a big opportunity, I think!","errorTags":["spelling","punctuation","tense"],"notes":"This tests common misspellings (exited, oppurtunity) in a casual context like email or chat. The contraction 'It's' versus 'Its' is also a frequent error, and the standalone 'i' needs catching. The exclamation point adds to the informal tone."},"correctlyResult":{"corrected":"I was really excited to hear about the project. It's going to be a big opportunity, I think!","changes":[{"original":"really exited","replacement":"excited","explanation":"Corrected the misspelling of 'excited'."},{"original":"Its","replacement":"It's","explanation":"Added an apostrophe to show possession. 'Its' is a possessive pronoun, while 'It's' is a contraction of 'it is'."},{"original":"oppurtunity","replacement":"opportunity","explanation":"Corrected the misspelling of 'opportunity'."}],"confidence":80,"usage":{"prompt_tokens":259,"completion_tokens":168,"total_tokens":427,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2039,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":76,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/17 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"really exited\" -> \"excited\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"Its\" -> \"It's\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"oppurtunity\" -> \"opportunity\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.83)","penalty":12},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"really exited","replacement":"excited","explanation":"Corrected the misspelling of 'excited'."},{"original":"Its","replacement":"It's","explanation":"Added an apostrophe to show possession. 'Its' is a possessive pronoun, while 'It's' is a contraction of 'it is'."},{"original":"oppurtunity","replacement":"opportunity","explanation":"Corrected the misspelling of 'opportunity'."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":80},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The grammar correction behavior passed all checks, including corrected consistency and cascade level.","fixtureWorthy":true},"error":null} +{"id":"0019-1780231544026","startedAt":"2026-05-31T12:45:44.026Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I seen him yesterday, and he said he would get back to me shortly. Its important we finalize the report before friday.","intendedMeaning":"I saw him yesterday, and he said he would get back to me shortly. It's important we finalize the report before Friday.","errorTags":["tense","punctuation","spelling"],"notes":"This case tests common past tense errors ('seen' vs. 'saw'), and contraction/possessive confusion ('Its' vs. 'It's'). The informal tone is typical of workplace communication or chat, making it a good test for nuanced grammar correction."},"correctlyResult":{"corrected":"I saw him yesterday, and he said he would get back to me shortly. It's important we finalize the report before Friday.","changes":[{"original":"seen","replacement":"saw","explanation":"Past tense needed for 'I saw' to match past context."},{"original":"Its important","replacement":"It's important","explanation":"Corrected contraction for 'it is'."},{"original":"friday.","replacement":"Friday.","explanation":"Capitalized proper noun."}],"confidence":73,"usage":{"prompt_tokens":574,"completion_tokens":134,"total_tokens":708,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2507,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/22 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"seen\" -> \"saw\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"Its important\" -> \"It's important\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"friday.\" -> \"Friday.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"seen","replacement":"saw","explanation":"Past tense needed for 'I saw' to match past context."},{"original":"Its important","replacement":"It's important","explanation":"Corrected contraction for 'it is'."},{"original":"friday.","replacement":"Friday.","explanation":"Capitalized proper noun."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":73},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected the intended meaning.","fixtureWorthy":true},"error":null} +{"id":"0020-1780231557602","startedAt":"2026-05-31T12:45:57.602Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was wondering if you could please check over the document before monday's meeting. Its very important we got this right.","intendedMeaning":"I was wondering if you could please review the document before Monday's meeting. It is very important that we get this right.","errorTags":["tense","punctuation","spelling","word choice"],"notes":"This case highlights common errors in informal communication (email). 'check over' is awkward; 'review' is better. The contraction 'its' is frequently confused with 'it's'. The use of 'got this right' instead of 'get this right' due to proximity to 'we' is a frequent error. The missing comma before Monday's adds another layer of complexity."},"correctlyResult":{"corrected":"Please check over the document before Monday's meeting. It is very important that we get this right.","changes":[{"original":"I was wondering if you could please check over the document before monday's meeting.","replacement":"Please check over the document before Monday's meeting.","explanation":"Removed unnecessary words and corrected capitalization of 'Monday'."},{"original":"Its very important we got this right.","replacement":"It is very important that we get this right.","explanation":"Corrected spelling of 'its' to 'it's', added missing auxiliary verb 'is', and inserted the necessary comma before 'that'."}],"confidence":89,"usage":{"prompt_tokens":261,"completion_tokens":183,"total_tokens":444,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2177,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/21 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"I was wondering if you could please check over the document before monday's meeting.\" -> \"Please check over the document before Monday's meeting.\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"Its very important we got this right.\" -> \"It is very important that we get this right.\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"I was wondering if you could please check over the document before monday's meeting.","replacement":"Please check over the document before Monday's meeting.","explanation":"Removed unnecessary words and corrected capitalization of 'Monday'."},{"original":"Its very important we got this right.","replacement":"It is very important that we get this right.","explanation":"Corrected spelling of 'its' to 'it's', added missing auxiliary verb 'is', and inserted the necessary comma before 'that'."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":89},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected consistency was maintained.","fixtureWorthy":true},"error":null} diff --git a/blackbox/runs/2026-05-31T14-18-12-310Z-lmstudio-granite-test-drive.jsonl b/blackbox/runs/2026-05-31T14-18-12-310Z-lmstudio-granite-test-drive.jsonl new file mode 100644 index 0000000..15a831e --- /dev/null +++ b/blackbox/runs/2026-05-31T14-18-12-310Z-lmstudio-granite-test-drive.jsonl @@ -0,0 +1,50 @@ +{"id":"0001-1780237092311","startedAt":"2026-05-31T14:18:12.311Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think its a good idea to have the meeting on tuesday. however, i dont know if everyone will be avaliable.","intendedMeaning":"I think it's a good idea to have the meeting on Tuesday. However, I don’t know if everyone will be available.","errorTags":["spelling","contraction","punctuation"],"notes":"Demonstrates common spelling errors ('its' vs 'it's', 'avalible' vs 'available'). The contraction error is a frequent one in informal writing like email/chat. Punctuation missing after 'however'."},"correctlyResult":{"corrected":"I think it's a good idea to have the meeting on Tuesday. However, I don't know if everyone will be available.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"avaliable","replacement":"available","explanation":"Spelling correction."}],"confidence":98,"usage":{"prompt_tokens":575,"completion_tokens":75,"total_tokens":650,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":17768,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/21 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"avaliable\" -> \"available\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes aside from punctuation","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions require an apostrophe."},{"original":"avaliable","replacement":"available","explanation":"Spelling correction."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the spelling errors, contractions, and punctuation issues in the original text.","fixtureWorthy":true},"error":null} +{"id":"0002-1780237152802","startedAt":"2026-05-31T14:19:12.802Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I seen him yesterday at the store, and he said he was going to call me back. But I haven't hearred from him yet.","intendedMeaning":"I saw him yesterday at the store, and he said he would call me back. But I haven't heard from him yet.","errorTags":["tense","spelling"],"notes":"This case tests common spoken-language errors making their way into written form. 'Seen' instead of 'saw' is a frequent mistake, as is misspelling 'heard'. The use of 'was going to call' versus 'would call' in the corrected version demonstrates a subtle nuance in expressing future action."},"correctlyResult":{"corrected":"I saw him yesterday at the store, and he said he was going to call me back. But I haven't heard from him yet.","changes":[{"original":"seen","replacement":"saw","explanation":"Past participle needed for perfect tense; 'I have seen' vs. simple past 'I saw'."},{"original":"hearred","replacement":"heard","explanation":"Common misspelling; 'heard' is the correct past tense of 'hear'."}],"confidence":100,"usage":{"prompt_tokens":578,"completion_tokens":107,"total_tokens":685,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":7767,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/24 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"seen\" -> \"saw\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"hearred\" -> \"heard\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"seen","replacement":"saw","explanation":"Past participle needed for perfect tense; 'I have seen' vs. simple past 'I saw'."},{"original":"hearred","replacement":"heard","explanation":"Common misspelling; 'heard' is the correct past tense of 'hear'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All corrections were accurate and targeted, with no duplicate or overlapping changes. The corrected text matches the usable changes, indicating a high level of confidence in the correction.","fixtureWorthy":true},"error":null} +{"id":"0003-1780237181114","startedAt":"2026-05-31T14:19:41.114Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I’d appreciate it if you could sent the file over as soon as possible. thanks!","intendedMeaning":"I would appreciate it if you could send the file over as soon as possible. Thank you!","errorTags":["tense","spelling"],"notes":"Common contraction/verb tense confusion. 'sent' should be 'send'. The exclamation mark at the very end is technically optional but somewhat less formal, which could be a deliberate choice or an oversight. A good test for detecting informal register."},"correctlyResult":{"corrected":"I'd appreciate it if you could send the file over as soon as possible. Thanks!","changes":[{"original":"I’d appreciate it if you could sent the file over as soon as possible. thanks!","replacement":"I'd appreciate it if you could send the file over as soon as possible. Thanks!","explanation":"Changed single quote to apostrophe for contraction; added missing period and exclamation point."}],"confidence":87,"usage":{"prompt_tokens":568,"completion_tokens":96,"total_tokens":664,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":3128,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":82,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/15 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"I’d appreciate it if you could sent the file over as soon as possible. thanks!\" -> \"I'd appreciate it if you could send the file over as soon as possible. Thanks!\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":false,"name":"granularity","detail":"model returned a whole-text edit instead of targeted changes","penalty":18},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"I’d appreciate it if you could sent the file over as soon as possible. thanks!","replacement":"I'd appreciate it if you could send the file over as soon as possible. Thanks!","explanation":"Changed single quote to apostrophe for contraction; added missing period and exclamation point."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":87},"judge":{"verdict":"interesting","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The model returned a whole-text edit instead of targeted changes, which may not accurately reflect the intended corrections.","fixtureWorthy":false},"error":null} +{"id":"0004-1780237196353","startedAt":"2026-05-31T14:19:56.353Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think that its best to schedule the meeting for next tuesday, as i have a conflict on monday.","intendedMeaning":"I think it's best to schedule the meeting for next Tuesday, as I have a conflict on Monday.","errorTags":["possessive/contraction","spelling"],"notes":"This case tests the distinction between 'its' (possessive) and 'it's' (contraction of 'it is'). The spelling error ('tuesday') is common. This scenario mimics a casual email or chat message."},"correctlyResult":{"corrected":"I think that it's best to schedule the meeting for next Tuesday, as I have a conflict on Monday.","changes":[],"confidence":55,"responseTimeMs":424,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/19 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"The system failed to correct the possessive/contraction error and did not provide a full-text correction with structured changes.","fixtureWorthy":false},"error":null} +{"id":"0005-1780237213417","startedAt":"2026-05-31T14:20:13.417Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was wondering if you could send me the report, its really important for my presentation tomorow.","intendedMeaning":"I was wondering if you could send me the report; it's really important for my presentation tomorrow.","errorTags":["punctuation","spelling","tense"],"notes":"This tests comma usage (joining independent clauses with a conjunction), possessive 'its' vs. contraction 'it's', and the spelling of ‘tomorrow’ in an informal context (email/chat). The combination of errors makes this a good edge case for testing robust grammar correction."},"correctlyResult":{"corrected":"I was wondering if you could send me the report; it's really important for my presentation tomorrow.","changes":[{"original":"report","replacement":"report;","explanation":"Added a semicolon to separate the independent clauses for better clarity."},{"original":"its","replacement":"it's","explanation":"Corrected the contraction for 'it is'."},{"original":"presentation tomorow","replacement":"presentation tomorrow","explanation":"Corrected the misspelled word 'tomorow'."}],"confidence":100,"usage":{"prompt_tokens":570,"completion_tokens":117,"total_tokens":687,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2473,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/17 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"report\" -> \"report;\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"presentation tomorow\" -> \"presentation tomorrow\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes aside from punctuation","penalty":0}],"displayChanges":[{"original":"report","replacement":"report;","explanation":"Added a semicolon to separate the independent clauses for better clarity."},{"original":"its","replacement":"it's","explanation":"Corrected the contraction for 'it is'."},{"original":"presentation tomorow","replacement":"presentation tomorrow","explanation":"Corrected the misspelled word 'tomorow'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Correctly identified and corrected all errors, including punctuation, spelling, and tense issues.","fixtureWorthy":true},"error":null} +{"id":"0006-1780237227143","startedAt":"2026-05-31T14:20:27.143Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm really exited to here back from you soonest.","intendedMeaning":"I am very excited to hear back from you as soon as possible.","errorTags":["spelling","word-choice","tense"],"notes":"This simulates a casual email or chat message. 'Exited' is a common misspelling, and the word 'soonest' sounds dated/unprofessional in many contexts. The informal tone makes error correction challenging - a strict grammar check might not flag 'soonest', but human review would. The contraction 'I'm' adds a layer of realism to the colloquial error pattern."},"correctlyResult":{"corrected":"I'm really excited to hear back from you soonest.","changes":[{"original":"here","replacement":"hear","explanation":"Corrected the verb form to match the subject 'you'."}],"confidence":89,"usage":{"prompt_tokens":561,"completion_tokens":49,"total_tokens":610,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1597,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/9 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"here\" -> \"hear\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.89)","penalty":12}],"displayChanges":[{"original":"here","replacement":"hear","explanation":"Corrected the verb form to match the subject 'you'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"false_accept","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the error and provided a clear explanation of the correction. However, it did not catch the slight inconsistency between the corrected text and the changes array.","fixtureWorthy":true},"error":null} +{"id":"0007-1780237240710","startedAt":"2026-05-31T14:20:40.710Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I’m really exited to here back from you soon, lets schedule a call next week!","intendedMeaning":"I'm really excited to hear back from you soon. Let's schedule a call next week!","errorTags":["spelling","punctuation","word choice"],"notes":"This case highlights common errors in informal communication (email/chat). 'exited' is a frequent misspelling of 'excited.' The comma before 'lets' creates an unnecessary pause, and the missing period at the end of the sentence is typical of hurried writing."},"correctlyResult":{"corrected":"I'm really excited to hear back from you soon, let's schedule a call next week!","changes":[{"original":"I’m","replacement":"I'm","explanation":"Missing apostrophe in contraction."},{"original":"really exited","replacement":"really excited","explanation":"Misspelled 'excited'."},{"original":"here back","replacement":"hear back","explanation":"Incorrect word order; should be 'hear' instead of 'here'."},{"original":"lets schedule","replacement":"let's schedule","explanation":"Missing apostrophe in contraction."}],"confidence":89,"usage":{"prompt_tokens":256,"completion_tokens":169,"total_tokens":425,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2029,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"4/15 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"I’m\" -> \"I'm\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"really exited\" -> \"really excited\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"here back\" -> \"hear back\"","penalty":0},{"pass":true,"name":"change #4","detail":"\"lets schedule\" -> \"let's schedule\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"4/4 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"I’m","replacement":"I'm","explanation":"Missing apostrophe in contraction."},{"original":"really exited","replacement":"really excited","explanation":"Misspelled 'excited'."},{"original":"here back","replacement":"hear back","explanation":"Incorrect word order; should be 'hear' instead of 'here'."},{"original":"lets schedule","replacement":"let's schedule","explanation":"Missing apostrophe in contraction."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":89},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the errors in the input text, including missing apostrophes, misspelled words, and incorrect word order.","fixtureWorthy":true},"error":null} +{"id":"0008-1780237255589","startedAt":"2026-05-31T14:20:55.589Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think the meeting should of been rescheduled to next tuesday.","intendedMeaning":"I think the meeting should have been rescheduled to next Tuesday.","errorTags":["grammar","spelling"],"notes":"Common error of confusing 'of' with 'have'. This is frequently seen in informal writing like emails and chat messages. The spelling of Tuesday also needs checking (capitalization)."},"correctlyResult":{"corrected":"I think the meeting should have been rescheduled to next Tuesday.","changes":[{"original":"should of","replacement":"should have","explanation":"Corrected the contraction 'should of' to the proper form 'should have'."}],"confidence":98,"usage":{"prompt_tokens":563,"completion_tokens":56,"total_tokens":619,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1770,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/11 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"should of\" -> \"should have\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"should of","replacement":"should have","explanation":"Corrected the contraction 'should of' to the proper form 'should have'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the error, provided a clear explanation of the correction, and preserved the original meaning.","fixtureWorthy":true},"error":null} +{"id":"0009-1780237266345","startedAt":"2026-05-31T14:21:06.345Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its really important that we finalise the report by friday.","intendedMeaning":"I think it's really important that we finalize the report by Friday.","errorTags":["spelling","punctuation","tense"],"notes":"This case highlights common errors in informal writing (email/chat). 'its' vs. 'it's', 'finalise' vs. 'finalize' (US/UK spelling), missing punctuation (comma after important) and lowercase friday need correction. It tests a model's ability to detect and correct these without drastically altering the tone."},"correctlyResult":{"corrected":"I think it's really important that we finalize the report by Friday.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"finalise","replacement":"finalize","explanation":"Spelling: American English prefers 'finalize' over British English 'finalise'."},{"original":"friday","replacement":"Friday","explanation":"Capitalization: Days of the week should be capitalized."}],"confidence":100,"usage":{"prompt_tokens":563,"completion_tokens":109,"total_tokens":672,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2464,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/12 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"finalise\" -> \"finalize\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"friday\" -> \"Friday\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"finalise","replacement":"finalize","explanation":"Spelling: American English prefers 'finalize' over British English 'finalise'."},{"original":"friday","replacement":"Friday","explanation":"Capitalization: Days of the week should be capitalized."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected the original text without altering its tone.","fixtureWorthy":true},"error":null} +{"id":"0010-1780237280008","startedAt":"2026-05-31T14:21:20.008Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I need to confirm the meeting with John, hes not responded to my last email yet. Also can you check if the presentation is ready for tomorrow?","intendedMeaning":"I need to confirm the meeting with John, as he hasn't responded to my last email yet. Also, can you check if the presentation is ready for tomorrow?","errorTags":["punctuation","spelling"],"notes":"This tests comma usage after introductory phrases and checking for 'hes' vs. 'he'."},"correctlyResult":{"corrected":"I need to confirm the meeting with John; he hasn't responded to my last email yet. Also, can you check if the presentation is ready for tomorrow?","changes":[],"confidence":55,"responseTimeMs":527,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/27 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the comma usage after the introductory phrase and did not replace 'hes' with 'he'.","fixtureWorthy":false},"error":null} +{"id":"0011-1780237295502","startedAt":"2026-05-31T14:21:35.502Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a really good idea, but could we maybe look at the budget first?","intendedMeaning":"I think it's a really good idea, but could we maybe look at the budget first?","errorTags":["spelling","punctuation"],"notes":"Common contraction mistake (its vs. it's). Also, a comma splice that is technically acceptable in informal writing but could be improved for clarity and formality. This scenario replicates a quick email or chat message."},"correctlyResult":{"corrected":"I think it's a really good idea, but could we maybe look at the budget first?","changes":[{"original":"its","replacement":"it's","explanation":"'It's' is the correct contraction for 'it is', while 'its' would imply possession."}],"confidence":89,"usage":{"prompt_tokens":255,"completion_tokens":83,"total_tokens":338,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1264,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"'It's' is the correct contraction for 'it is', while 'its' would imply possession."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":89},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the common contraction mistake and provided a clear explanation. The corrected text matches the intended meaning, and the changes are targeted.","fixtureWorthy":true},"error":null} +{"id":"0012-1780237307753","startedAt":"2026-05-31T14:21:47.753Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?","intendedMeaning":"I was looking forward to your response about the project, but I haven't gotten it yet. Is everything alright?","errorTags":["tense","word choice"],"notes":"The original uses 'hear back' which is common but slightly less formal than 'receive a response.' The phrasing 'Is there something going on?' is overly verbose and could be simplified to 'Is everything alright?' or a similar, more natural phrase. The tense shift from past continuous to simple present ('Is there something going on?') is a common conversational error."},"correctlyResult":{"corrected":"I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?","changes":[],"confidence":100,"usage":{"prompt_tokens":576,"completion_tokens":40,"total_tokens":616,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1202,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/23 word(s) flagged","penalty":0},{"pass":true,"name":"empty response","detail":"no changes and corrected text matches source","penalty":0},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The corrected text matches the source, and no changes were made.","fixtureWorthy":true},"error":null} +{"id":"0013-1780237320955","startedAt":"2026-05-31T14:22:00.955Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm really exited to hear back from you about the project, lets schedule a quick meeting sometime next week. it would be really good.","intendedMeaning":"I'm very excited to hear back from you about the project. Let’s schedule a quick meeting sometime next week. That would be really good.","errorTags":["spelling","punctuation","word choice"],"notes":"This tests common excitement misspellings ('exited' instead of 'excited'), informal contractions (lets vs let’s), and a slightly awkward sentence ending ('it would be really good' could benefit from rephrasing but is technically grammatical). The comma usage after 'you' in the first sentence is also questionable in formal contexts, but okay in very casual communication."},"correctlyResult":{"corrected":"I'm really excited to hear back from you about the project. Let's schedule a quick meeting sometime next week—it would be great.","changes":[{"original":"I'm really exited","replacement":"I'm really excited","explanation":"Corrected the misspelling of 'excited'."},{"original":"lets schedule","replacement":"Let's schedule","explanation":"Added an apostrophe to form the contraction of 'let us'."},{"original":"it would be really good","replacement":"it would be great","explanation":"Changed 'really good' to 'great' for a more concise expression."}],"confidence":80,"usage":{"prompt_tokens":265,"completion_tokens":168,"total_tokens":433,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2022,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":76,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/24 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"I'm really exited\" -> \"I'm really excited\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"lets schedule\" -> \"Let's schedule\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"it would be really good\" -> \"it would be great\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.87)","penalty":12},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"I'm really exited","replacement":"I'm really excited","explanation":"Corrected the misspelling of 'excited'."},{"original":"lets schedule","replacement":"Let's schedule","explanation":"Added an apostrophe to form the contraction of 'let us'."},{"original":"it would be really good","replacement":"it would be great","explanation":"Changed 'really good' to 'great' for a more concise expression."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":80},"judge":{"verdict":"pass","risk":"false_accept","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The corrected text differs slightly from the changes (0.87), indicating a potential issue with consistency.","fixtureWorthy":true},"error":null} +{"id":"0014-1780237340539","startedAt":"2026-05-31T14:22:20.539Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important to note that the report, which was submitted late yesterday, needs reviewed again.","intendedMeaning":"I think it's important to note that the report, which was submitted late yesterday, needs to be reviewed again.","errorTags":["spelling","grammar"],"notes":"A common error involving 'its' vs. 'it's'. The comma usage is correct, but frequently gets flagged by simpler grammar checkers. The missing 'to' before 'be reviewed' is a higher-level error that some tools miss, especially in conversational contexts."},"correctlyResult":{"corrected":"I think it's important to note that the report, which was submitted late yesterday, needs to be reviewed again.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"needs reviewed","replacement":"needs to be reviewed","explanation":"Verb tense: 'reviewed' should be followed by the infinitive 'to be' for correct usage."}],"confidence":98,"usage":{"prompt_tokens":569,"completion_tokens":102,"total_tokens":671,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2267,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/17 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"needs reviewed\" -> \"needs to be reviewed\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"needs reviewed","replacement":"needs to be reviewed","explanation":"Verb tense: 'reviewed' should be followed by the infinitive 'to be' for correct usage."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified and corrected the errors in grammar, spelling, and verb tense. The changes were targeted and consistent with the intended meaning.","fixtureWorthy":true},"error":null} +{"id":"0015-1780237354015","startedAt":"2026-05-31T14:22:34.015Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think it would be realy helpful if you could check my work by tuesday. Thanks!","intendedMeaning":"I think it would be really helpful if you could check my work by Tuesday. Thanks!","errorTags":["spelling","punctuation"],"notes":"Common misspelling of 'really'. Missing period at the end of the first sentence. Represents typical errors in informal email/chat communication."},"correctlyResult":{"corrected":"I think it would be really helpful if you could check my work by Tuesday.","changes":[{"original":"realy","replacement":"really","explanation":"Corrected common misspelling of 'real'."},{"original":"tuesday","replacement":"Tuesday","explanation":"Capitalized proper noun 'Tuesday'."}],"confidence":89,"usage":{"prompt_tokens":569,"completion_tokens":73,"total_tokens":642,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1916,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":88,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"2/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"realy\" -> \"really\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"tuesday\" -> \"Tuesday\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"2/2 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.94)","penalty":12}],"displayChanges":[{"original":"realy","replacement":"really","explanation":"Corrected common misspelling of 'real'."},{"original":"tuesday","replacement":"Tuesday","explanation":"Capitalized proper noun 'Tuesday'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":89},"judge":{"verdict":"pass","risk":"weak_correction","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"Corrected common misspelling of 'real' and capitalized proper noun 'Tuesday', but the corrected text differs slightly from the changes (0.94).","fixtureWorthy":true},"error":null} +{"id":"0016-1780237365148","startedAt":"2026-05-31T14:22:45.148Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are an updates.","intendedMeaning":"I appreciate your prompt response; however, I am still awaiting confirmation from my supervisor. Please let me know if there are any updates.","errorTags":["punctuation","grammar","spelling"],"notes":"This tests semicolon usage to separate clauses, correct 'there' vs. 'their', and the common misspelling of 'updates'."},"correctlyResult":{"corrected":"I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are any updates.","changes":[{"original":"confirmation","replacement":"any updates","explanation":"The word 'confirmation' is incorrect in this context. The sentence should request an update or information about the status of the confirmation."}],"confidence":77,"usage":{"prompt_tokens":263,"completion_tokens":100,"total_tokens":363,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1314,"cascadeLevel":2},"scoring":{"accepted":true,"acceptanceScore":76,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/23 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"confirmation\" -> \"any updates\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":false,"name":"corrected consistency","detail":"corrected text differs slightly from changes (0.92)","penalty":12},{"pass":false,"name":"cascade level","detail":"level 2","penalty":12}],"displayChanges":[{"original":"confirmation","replacement":"any updates","explanation":"The word 'confirmation' is incorrect in this context. The sentence should request an update or information about the status of the confirmation."}],"hiddenChanges":[],"cascadeLevel":2,"displayConfidence":77},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the error in 'confirmation' and replaced it with 'any updates'. The corrected text is consistent with the changes made, and the cascade level is within a safe range.","fixtureWorthy":true},"error":null} +{"id":"0017-1780237378095","startedAt":"2026-05-31T14:22:58.095Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its really important to make shure the report is finalized by friday.","intendedMeaning":"I think it's really important to make sure the report is finalized by Friday.","errorTags":["spelling","punctuation","word-choice"],"notes":"This case tests common spelling errors (shure -> sure, its -> it's) and incorrect contractions. The case also highlights capitalization (Friday). This is a frequent error in informal written communication like emails and chat."},"correctlyResult":{"corrected":"I think it's really important to make sure the report is finalized by Friday.","changes":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"make shure","replacement":"make sure","explanation":"Spelling correction: 'shure' should be spelled as 'sure'."},{"original":"friday","replacement":"Friday","explanation":"Capitalization: Days of the week should be capitalized."}],"confidence":100,"usage":{"prompt_tokens":565,"completion_tokens":112,"total_tokens":677,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":2180,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"3/14 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"its\" -> \"it's\"","penalty":0},{"pass":true,"name":"change #2","detail":"\"make shure\" -> \"make sure\"","penalty":0},{"pass":true,"name":"change #3","detail":"\"friday\" -> \"Friday\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"3/3 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"its","replacement":"it's","explanation":"Contractions: 'it is' should be contracted as 'it's'."},{"original":"make shure","replacement":"make sure","explanation":"Spelling correction: 'shure' should be spelled as 'sure'."},{"original":"friday","replacement":"Friday","explanation":"Capitalization: Days of the week should be capitalized."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":100},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"All changes were targeted and corrected the original text, with no duplicate or overlapping changes.","fixtureWorthy":true},"error":null} +{"id":"0018-1780237390024","startedAt":"2026-05-31T14:23:10.024Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really exited to hear about your proposal, but i think we need to discuss it further before finalizing anything. Its got some potential, but also a few kinks.","intendedMeaning":"I was really excited to hear about your proposal, but I think we need to discuss it further before finalizing anything. It's got some potential, but also a few kinks.","errorTags":["spelling","punctuation","capitalization"],"notes":"This case highlights common spelling errors ('exited' vs. 'excited'), capitalization mistakes (lowercase 'i' when a pronoun), and incorrect use of contractions ('its' vs. 'it's'). It simulates a casual email or chat environment where these errors frequently occur, and a grammar checker should recognize all three."},"correctlyResult":{"corrected":"I was really excited to hear about your proposal, but I think we need to discuss it further before finalizing anything. It's got some potential, but also a few kinks.","changes":[],"confidence":55,"responseTimeMs":575,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/30 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"Empty response without structured changes","fixtureWorthy":false},"error":null} +{"id":"0019-1780237408838","startedAt":"2026-05-31T14:23:28.838Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I hope to recieve your reply soonest. Let me know if you need anything else, thanks!","intendedMeaning":"I hope to receive your reply soon. Let me know if you need anything else, thank you!","errorTags":["spelling","word choice","punctuation"],"notes":"This case tests the common misspellings of 'receive' and 'soonest', as well as a conversational tone with informal closing. The comma before 'thanks!' is optional, but not grammatically necessary and should be removed for a more formal tone that would be appropriate in an email."},"correctlyResult":{"corrected":"I hope to receive your reply soonest. Let me know if you need anything else, thanks!","changes":[{"original":"recieve","replacement":"receive","explanation":"Correct spelling of 'receive'."}],"confidence":98,"usage":{"prompt_tokens":570,"completion_tokens":53,"total_tokens":623,"completion_tokens_details":{"reasoning_tokens":0}},"responseTimeMs":1322,"cascadeLevel":1},"scoring":{"accepted":true,"acceptanceScore":100,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"1/16 word(s) flagged","penalty":0},{"pass":true,"name":"change #1","detail":"\"recieve\" -> \"receive\"","penalty":0},{"pass":true,"name":"duplicate changes","detail":"none","penalty":0},{"pass":true,"name":"overlapping changes","detail":"none","penalty":0},{"pass":true,"name":"usable changes","detail":"1/1 usable","penalty":0},{"pass":true,"name":"granularity","detail":"targeted changes","penalty":0},{"pass":true,"name":"corrected consistency","detail":"corrected text matches usable changes","penalty":0}],"displayChanges":[{"original":"recieve","replacement":"receive","explanation":"Correct spelling of 'receive'."}],"hiddenChanges":[],"cascadeLevel":1,"displayConfidence":98},"judge":{"verdict":"pass","risk":"none","shouldAccept":true,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":true,"reason":"The system correctly identified the spelling error and suggested a correction. The changes array contains only one change, which is correct and targeted.","fixtureWorthy":true},"error":null} +{"id":"0020-1780237421158","startedAt":"2026-05-31T14:23:41.158Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really exited to hear about the project, but i’m not sure if I have time currently.","intendedMeaning":"I was really excited to hear about the project, but I’m not sure if I have time currently.","errorTags":["spelling","punctuation","wordChoice"],"notes":"This tests the 'exited' vs. 'excited' common confusion, the standalone 'i', and a generally informal tone often found in emails/chat. The phrase 'currently' is grammatically correct, but a more natural phrasing might be preferred in many contexts (e.g., 'right now')."},"correctlyResult":{"corrected":"I was really excited to hear about the project, but I'm not sure if I have time currently.","changes":[],"confidence":55,"responseTimeMs":401,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/18 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"The system failed to correct the spelling error in 'exited' and did not provide any suggestions for improvement.","fixtureWorthy":false},"error":null} +{"id":"0021-1780237437281","startedAt":"2026-05-31T14:23:57.281Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important that we circle back on this item next week, if possible. Let me know your thoughts.","intendedMeaning":"I think it's important that we follow up on this item next week, if possible. Let me know your thoughts.","errorTags":["punctuation","spelling"],"notes":"Common error involving 'its' vs. 'it's'. Also, 'circle back on' is a common business idiom that could be improved to 'follow up on', but the original isn't strictly incorrect. Testing for proper context-aware correction is needed."},"correctlyResult":{"corrected":"I think it's important that we circle back on this item next week, if possible. Let me know your thoughts.","changes":[],"confidence":55,"responseTimeMs":483,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/20 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"Expected a full-text correction with structured changes, but received an empty response.","fixtureWorthy":false},"error":null} +{"id":"0022-1780237448458","startedAt":"2026-05-31T14:24:08.458Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm really excited about the project, its gonna be a great success i think.","intendedMeaning":"I'm really excited about the project; I think it's going to be a great success.","errorTags":["punctuation","spelling","word choice"],"notes":"This tests comma usage in an enthusiastic, informal sentence. The standalone 'i' is a common texting/chatting error. The contraction 'its' is frequently confused with 'it's'. The sentence also contains a missing semicolon to separate two closely related independent clauses."},"correctlyResult":{"corrected":"I'm really excited about the project; it's going to be a great success, I think.","changes":[],"confidence":55,"responseTimeMs":551,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/14 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the sentence and did not provide any visible suggestions.","fixtureWorthy":false},"error":null} +{"id":"0023-1780237459409","startedAt":"2026-05-31T14:24:19.409Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was really exited to hear about the project, but I think we need too clarify some things before moving forward. Its important that everyones on the same page.","intendedMeaning":"I was really excited to hear about the project, but I think we need to clarify some things before moving forward. It's important that everyone is on the same page.","errorTags":["spelling","punctuation","grammar"],"notes":"This case tests common email/chat errors. 'Exited' vs 'excited' is a frequent misspelling. The incorrect use of 'too' instead of 'to' and the contraction error ('Its' vs. 'It's') are also common. Finally, the subject-verb agreement error ('everyones' vs 'everyone is') adds to the realism. The overall tone suggests a casual workplace setting."},"correctlyResult":{"corrected":"I was really excited to hear about the project, but I think we need to clarify some things before moving forward. It's important that everyone is on the same page.","changes":[],"confidence":55,"responseTimeMs":599,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/29 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the subject-verb agreement error ('everyones' vs 'everyone is') and did not provide any suggestions for improvement.","fixtureWorthy":false},"error":null} +{"id":"0024-1780237474126","startedAt":"2026-05-31T14:24:34.126Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was realy exited to hear back from you about the project, but i haven’t seen anything yet. Could you let me know when it might be ready?","intendedMeaning":"I was really excited to hear back from you about the project, but I haven’t seen anything yet. Could you let me know when it might be ready?","errorTags":["spelling","punctuation","capitalization"],"notes":"This tests common spelling errors (realy -> really), capitalization mistakes after punctuation (i -> I) and a slightly awkward phrasing that still conveys the intended meaning. It's representative of informal email or chat communication."},"correctlyResult":{"corrected":"I was really excited to hear back from you about the project, but I haven't seen anything yet. Could you let me know when it might be ready?","changes":[],"confidence":55,"responseTimeMs":610,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/28 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the spelling error 'realy' to 'really', resulting in an empty response.","fixtureWorthy":false},"error":null} +{"id":"0025-1780237486400","startedAt":"2026-05-31T14:24:46.400Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think we should of schedule a meeting next week to discuss the new project, its really important.","intendedMeaning":"I think we should schedule a meeting next week to discuss the new project; it's really important.","errorTags":["spelling","grammar"],"notes":"Common error involving 'of' instead of 'have'. The standalone 'its' is another frequent mistake. The sentence structure requires a semicolon instead of comma to correctly link the two independent clauses."},"correctlyResult":{"corrected":"I think we should schedule a meeting next week to discuss the new project; it's really important.","changes":[],"confidence":55,"responseTimeMs":440,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/18 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the error and provided an empty response.","fixtureWorthy":false},"error":null} +{"id":"0026-1780237496055","startedAt":"2026-05-31T14:24:56.054Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I seen him yesterday and he said that the project is due next week. However, I haven't started it yet so i’m a little worried.","intendedMeaning":"I saw him yesterday, and he said the project is due next week. However, I haven't started it yet, so I’m a little worried.","errorTags":["tense","spelling","punctuation"],"notes":"This tests the common 'seen' vs. 'saw' confusion, a standalone pronoun 'i', and missing comma after introductory phrase."},"correctlyResult":{"corrected":"I saw him yesterday and he said that the project is due next week. However, I haven't started it yet so I'm a little worried.","changes":[],"confidence":55,"responseTimeMs":558,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/25 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"Expected a full-text correction with structured changes, but only the corrected string was provided.","fixtureWorthy":false},"error":null} +{"id":"0027-1780237506793","startedAt":"2026-05-31T14:25:06.793Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possable.","intendedMeaning":"I think it's important to note that the project will be delayed due to unforeseen circumstances. I’ll update everyone with more details as soon as possible.","errorTags":["spelling","punctuation","grammar"],"notes":"This case tests common spelling errors ('its' vs. 'it's', 'possable' vs 'possible'), a missing apostrophe (it's), and informal language that may or may not require correction depending on the context. It simulates a typical project status update email/chat message."},"correctlyResult":{"corrected":"I think it's important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possible.","changes":[],"confidence":55,"responseTimeMs":569,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/26 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"The system failed to correct the informal language 'possable' vs 'possible', which may not require correction depending on the context.","fixtureWorthy":false},"error":null} +{"id":"0028-1780237519603","startedAt":"2026-05-31T14:25:19.603Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I wanted to reach out regarding the project, its going well so far but i'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?","intendedMeaning":"I wanted to reach out regarding the project. It's going well so far, but I'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?","errorTags":["punctuation","spelling","wordChoice","grammar"],"notes":"This case demonstrates common errors in informal email communication. The comma splice ('its going well so far but...') is a frequent issue, as are typos (i'm instead of I'm) and incorrect possessive usage ('its' instead of 'it's'). The use of 'regarding' is slightly formal for many email contexts, but not strictly incorrect; a more conversational alternative like 'about' would be common. The overall tone is casual, mirroring typical workplace communication."},"correctlyResult":{"corrected":"I wanted to reach out regarding the project; it's going well so far, but I'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?","changes":[],"confidence":55,"responseTimeMs":664,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/33 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The corrected text still contains a comma splice ('it's going well so far but...') and the response time is relatively high (664ms).","fixtureWorthy":false},"error":null} +{"id":"0029-1780237536187","startedAt":"2026-05-31T14:25:36.187Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think its a good idea but maybe we should double check the numbers before sending it out to all the team members.","intendedMeaning":"I think it's a good idea, but maybe we should double-check the numbers before sending it out to all the team members.","errorTags":["spelling","punctuation"],"notes":"This case tests contraction recognition ('its' vs. 'it's') and a missing comma after an introductory phrase in a casual email or chat setting. The context makes both corrections necessary for clarity and proper grammar."},"correctlyResult":{"corrected":"I think it's a good idea, but maybe we should double-check the numbers before sending it out to all the team members.","changes":[],"confidence":55,"responseTimeMs":503,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/23 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"Expected a non-empty changes array, but it was empty.","fixtureWorthy":true},"error":null} +{"id":"0030-1780237546885","startedAt":"2026-05-31T14:25:46.885Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please sent me the document?","intendedMeaning":"I think it's a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please send me the document?","errorTags":["spelling","punctuation","grammar"],"notes":"This case tests common errors in informal communication (email/chat). 'its' vs. 'it's' is a frequent mistake, as are missing commas and incorrect verb forms ('sent' vs. 'send'). The flow is naturalistic, mimicking a typical workplace discussion."},"correctlyResult":{"corrected":"I think it's a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please send me the document?","changes":[],"confidence":55,"responseTimeMs":580,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/28 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"empty response and high cascade level penalized the score","fixtureWorthy":false},"error":null} +{"id":"0031-1780237559359","startedAt":"2026-05-31T14:25:59.359Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think the report is good, but like, maybe a few more details would be helpfull?","intendedMeaning":"I think the report is good, but perhaps a few more details would be helpful?","errorTags":["spelling","word choice","punctuation"],"notes":"This case mixes informal language (like, maybe) with a request for formal feedback. The 'helpfull' spelling error is common in casual writing, and the comma before 'but' isn’t strictly incorrect but is often omitted in less formal contexts. The word choice 'like, maybe' is characteristic of conversational/chat language. Testing the AI's ability to handle this blend of formality and informality is important."},"correctlyResult":{"corrected":"I think the report is good, but maybe a few more details would be helpful.","changes":[],"confidence":55,"responseTimeMs":419,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/16 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"interesting","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The AI failed to correct the informal language ('like, maybe') and omitted the comma before 'but', despite detecting spelling errors. This suggests a lack of understanding of formal writing conventions.","fixtureWorthy":true},"error":null} +{"id":"0032-1780237572268","startedAt":"2026-05-31T14:26:12.268Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think it would be grate if you could look into this issue. its been going on for a while now and is really impacting our workflow.","intendedMeaning":"I think it would be great if you could look into this issue. It's been going on for a while now and is really impacting our workflow.","errorTags":["spelling","punctuation"],"notes":"This case tests for common spelling errors ('grate' instead of 'great') and correct usage of contractions ('it’s’ vs. possessive ‘its’). The sentence structure is otherwise correct, representing typical informal workplace communication."},"correctlyResult":{"corrected":"I think it would be great if you could look into this issue. It's been going on for a while now and is really impacting our workflow.","changes":[],"confidence":55,"responseTimeMs":546,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/27 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the spelling error 'grate' and did not provide any suggestions.","fixtureWorthy":false},"error":null} +{"id":"0033-1780237583786","startedAt":"2026-05-31T14:26:23.786Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was thinking about the project, and i feel like we should revisit the timeline. Its kinda tight right now.","intendedMeaning":"I've been considering the project, and I believe we should review the timeline. It’s a bit tight currently.","errorTags":["punctuation","spelling","word-choice","capitalization"],"notes":"This tests common issues in informal writing (email/chat). The standalone 'i' is a frequent error. Incorrect use of 'its' vs 'it’s'. The slightly awkward phrasing ('kinda tight') should be corrected to a more professional tone for many contexts. The flow of thought is natural, reflecting common writing patterns."},"correctlyResult":{"corrected":"I was thinking about the project, and I feel like we should revisit the timeline. It's kind of tight right now.","changes":[],"confidence":55,"responseTimeMs":488,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/20 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"The system failed to correct the standalone 'i' error and did not provide any suggestions for improvement.","fixtureWorthy":false},"error":null} +{"id":"0034-1780237596256","startedAt":"2026-05-31T14:26:36.256Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I seen him leave earlier, and he said it was urgent. I'll call you back later.","intendedMeaning":"I saw him leave earlier, and he said it was urgent. I'll call you back later.","errorTags":["tense","spelling"],"notes":"Common error of 'seen' instead of 'saw'. Tests the system's ability to accurately identify past tense errors in informal speech-like text. The second sentence is grammatically correct and serves as context."},"correctlyResult":{"corrected":"I saw him leave earlier, and he said it was urgent. I'll call you back later.","changes":[],"confidence":55,"responseTimeMs":433,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/16 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the 'seen' error in the original text, resulting in an incorrect corrected string.","fixtureWorthy":false},"error":null} +{"id":"0035-1780237606258","startedAt":"2026-05-31T14:26:46.258Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was wondering if you could sent me the report before friday. Its really needed for my presentation.","intendedMeaning":"I was wondering if you could send me the report before Friday. It's really needed for my presentation.","errorTags":["spelling","tense","punctuation"],"notes":"This case tests common email errors. 'sent' vs 'send', incorrect use of 'its' (possessive) instead of 'it's' (contraction), and missing period at the end. The informal tone is typical of workplace communication."},"correctlyResult":{"corrected":"I was wondering if you could send me the report before Friday. It's really needed for my presentation.","changes":[],"confidence":55,"responseTimeMs":464,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/18 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the informal tone and missing period at the end of the sentence.","fixtureWorthy":false},"error":null} +{"id":"0036-1780237616891","startedAt":"2026-05-31T14:26:56.891Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"i think we should of scheduled the meeting for tuesday, not monday. its more in line with everyone's availability.","intendedMeaning":"I think we should have scheduled the meeting for Tuesday, not Monday. It's more in line with everyone’s availability.","errorTags":["verb_tense","preposition"],"notes":"Common error of confusing 'of' with 'have' after the verb 'should'. Also includes a subtle capitalization error ('Tuesday'). This case is useful to test the model's understanding of auxiliary verb usage in conditional sentences and informal writing."},"correctlyResult":{"corrected":"I think we should have scheduled the meeting for Tuesday, not Monday. It's more in line with everyone's availability.","changes":[],"confidence":55,"responseTimeMs":477,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/19 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The model failed to correct the verb tense error and did not provide any visible suggestions.","fixtureWorthy":false},"error":null} +{"id":"0037-1780237627819","startedAt":"2026-05-31T14:27:07.819Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea to finalize the report by friday.","intendedMeaning":"I think it's a good idea to finalize the report by Friday.","errorTags":["punctuation","spelling"],"notes":"Common contraction error (its vs. it's). Friday should be capitalized as it refers to a specific day of the week. Realistic in email or chat."},"correctlyResult":{"corrected":"I think it's a good idea to finalize the report by Friday.","changes":[],"confidence":55,"responseTimeMs":371,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/12 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"Empty response without structured changes","fixtureWorthy":true},"error":null} +{"id":"0038-1780237635988","startedAt":"2026-05-31T14:27:15.988Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I am writting to enquire about the possibility of a refund, as my order arrived damaged. Its quite upsetting.","intendedMeaning":"I am writing to inquire about the possibility of a refund, as my order arrived damaged. It's quite upsetting.","errorTags":["spelling","punctuation","wordChoice"],"notes":"This case tests common spelling errors ('writting' for 'writing'), contractions (missing apostrophe in 'it's'), and a slightly formal tone ('enquire') that is often dropped in more casual communication. The combination makes it a good test of whether the corrector aims for formal accuracy or conversational correctness."},"correctlyResult":{"corrected":"I am writing to inquire about the possibility of a refund, as my order arrived damaged. It's quite upsetting.","changes":[],"confidence":55,"responseTimeMs":468,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/19 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"Expected a full-text correction with structured changes, but received an empty response.","fixtureWorthy":true},"error":null} +{"id":"0039-1780237647678","startedAt":"2026-05-31T14:27:27.678Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I'm writting this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.","intendedMeaning":"I'm writing this to inform you about the ongoing project. It's progressing well so far. However, we need your approval before moving to phase two.","errorTags":["spelling","punctuation","grammar"],"notes":"This case demonstrates a common error: misspelling 'writing' as 'writting'. It also contains a comma splice (joining two independent clauses with only a comma) - 'ongoing project, its progressing well'. The informal tone (email/chat) is a key element. The use of 'its' instead of 'it's' (possessive vs. contraction) is a frequent error."},"correctlyResult":{"corrected":"I'm writing this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.","changes":[],"confidence":55,"responseTimeMs":564,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/26 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the comma splice error and did not provide any suggestions for improvement.","fixtureWorthy":false},"error":null} +{"id":"0040-1780237661808","startedAt":"2026-05-31T14:27:41.808Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think we need to go over the proposal again. Its not really clear on what actions are needed next, and I'm worried about the deadline.","intendedMeaning":"I think we need to review the proposal again. It's not really clear on what actions are needed next, and I'm worried about the deadline.","errorTags":["punctuation","spelling"],"notes":"Common contraction error (its vs. it's). The context makes the meaning clear, but a simple spelling/punctuation check is needed. This mimics errors often seen in informal communication like emails or chat."},"correctlyResult":{"corrected":"I think we need to go over the proposal again. It's not really clear on what actions are needed next, and I'm worried about the deadline.","changes":[],"confidence":55,"responseTimeMs":550,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/26 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"Expected a non-empty changes array, but it was empty.","fixtureWorthy":false},"error":null} +{"id":"0041-1780237673036","startedAt":"2026-05-31T14:27:53.036Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I feel like this report is not quite done yet, its missing a few key details and I need to go back over it. Please let me know if you have any questions.","intendedMeaning":"I feel like this report isn't quite finished yet; it's missing a few key details, and I need to go back over it. Please let me know if you have any questions.","errorTags":["punctuation","tense","word choice"],"notes":"This case tests comma usage with introductory phrases and coordinating conjunctions. 'its' vs 'it's' is a common error, especially in less formal writing (email/chat). The use of 'quite done' is slightly awkward; 'finished' flows better in this context."},"correctlyResult":{"corrected":"I feel like this report is not quite done yet; it's missing a few key details, and I need to go back over it. Please let me know if you have any questions.","changes":[],"confidence":55,"responseTimeMs":635,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/33 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the sentence correctly, and also did not provide any visible suggestions.","fixtureWorthy":false},"error":null} +{"id":"0042-1780237686366","startedAt":"2026-05-31T14:28:06.366Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea, but are you sure about the timeline? We need to finalize it soon.","intendedMeaning":"I think it's a good idea, but are you sure about the timeline? We need to finalize it soon.","errorTags":["spelling","punctuation"],"notes":"Common mistake of using 'its' instead of 'it's'. Punctuation is generally correct, but a more formal setting might prefer a semicolon instead of the comma after 'idea'."},"correctlyResult":{"corrected":"I think it's a good idea, but are you sure about the timeline? We need to finalize it soon.","changes":[],"confidence":55,"responseTimeMs":464,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/19 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"interesting","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"Penalty for empty response and cascade level is too high, but corrected consistency penalty is 0.","fixtureWorthy":false},"error":null} +{"id":"0043-1780237696360","startedAt":"2026-05-31T14:28:16.360Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think their going to need more time to complete the project. Its looking like a difficult task.","intendedMeaning":"I think they're going to need more time to complete the project. It's looking like a difficult task.","errorTags":["there/their/they're","its/it's","punctuation"],"notes":"This tests the common confusion between 'there', 'their', and 'they're'. Also includes a frequent mistake of using 'its' instead of 'it's', which is often found in informal writing like emails or chat messages. The punctuation error, while subtle (lack of a period at the end of the second sentence), represents a common casual writing habit."},"correctlyResult":{"corrected":"I think they're going to need more time to complete the project. It's looking like a difficult task.","changes":[],"confidence":55,"responseTimeMs":466,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/18 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":true,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the informal writing habit of using 'its' instead of 'it's', and also didn't provide any visible suggestions for improvement.","fixtureWorthy":false},"error":null} +{"id":"0044-1780237709319","startedAt":"2026-05-31T14:28:29.319Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its really important that we follow up on thier request ASAP. Let me know if you have any question's.","intendedMeaning":"I think it’s really important that we follow up on their request ASAP. Let me know if you have any questions.","errorTags":["spelling","punctuation","possessive/contraction"],"notes":"This case highlights common errors in informal communication (email, chat). 'its' vs. 'it’s', 'thier' vs. 'their', and pluralization of 'questions' with an unnecessary apostrophe are all frequent issues. The use of ASAP is included as it’s a common usage, but not strictly a grammatical error."},"correctlyResult":{"corrected":"I think it's really important that we follow up on their request ASAP. Let me know if you have any questions.","changes":[],"confidence":55,"responseTimeMs":481,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/21 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the informal communication errors, such as 'thier' vs. 'their', and did not provide any visible suggestions.","fixtureWorthy":false},"error":null} +{"id":"0045-1780237722263","startedAt":"2026-05-31T14:28:42.263Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was realy looking forward to the meeting, but I didn’t got a chance to present my findings. Its really frustating.","intendedMeaning":"I was really looking forward to the meeting, but I didn’t get a chance to present my findings. It's really frustrating.","errorTags":["spelling","verb conjugation","punctuation"],"notes":"This case tests common errors in informal writing (email, chat). It includes a spelling error ('realy' vs. 'really'), incorrect past tense verb conjugation ('didn’t got' should be 'didn’t get'), and improper contraction usage/punctuation ('Its' vs. 'It's'). The sentence structure is relatively simple, making the errors more obvious in isolation but realistic when quickly typing a message."},"correctlyResult":{"corrected":"I was really looking forward to the meeting, but I didn't get a chance to present my findings. It's really frustrating.","changes":[],"confidence":55,"responseTimeMs":515,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/21 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the sentence structure and did not provide any meaningful suggestions.","fixtureWorthy":false},"error":null} +{"id":"0046-1780237735921","startedAt":"2026-05-31T14:28:55.921Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think that we should of schedule a meeting for next week. Its crucial to finalize the project timeline.","intendedMeaning":"I think that we should have scheduled a meeting for next week. It's crucial to finalize the project timeline.","errorTags":["grammar","spelling","punctuation"],"notes":"This case tests the common confusion between 'of' and 'have' after 'should'. The use of 'Its' instead of 'It's' is also a frequent error, especially in less formal writing like email or chat. The sentence structure, while understandable, isn't idiomatic English."},"correctlyResult":{"corrected":"I think that we should schedule a meeting for next week. It's crucial to finalize the project timeline.","changes":[],"confidence":55,"responseTimeMs":454,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/19 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the error and did not provide any meaningful suggestions.","fixtureWorthy":false},"error":null} +{"id":"0047-1780237747329","startedAt":"2026-05-31T14:29:07.329Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its really important that we finalize the report by friday, okay?","intendedMeaning":"I think it's really important that we finalize the report by Friday, okay?","errorTags":["spelling","punctuation","tense"],"notes":"This case tests common spelling errors ('its' vs 'it's'), inconsistent capitalization (Friday), and an unnecessary/informal closing ('okay?'). It represents a typical email or chat message, making it relevant for testing grammar correction tools designed to handle relaxed writing styles."},"correctlyResult":{"corrected":"I think it's really important that we finalize the report by Friday, okay?","changes":[],"confidence":55,"responseTimeMs":396,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/13 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the informal closing ('okay?') and did not provide any suggestions for improvement.","fixtureWorthy":false},"error":null} +{"id":"0048-1780237757597","startedAt":"2026-05-31T14:29:17.597Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I have been working on the report, and I think it’s mostly done. However their are a few sections that still need more work, specially the financial projections.","intendedMeaning":"I have been working on the report, and I think it's mostly done. However, there are a few sections that still need more work, especially the financial projections.","errorTags":["spelling","punctuation","wordChoice"],"notes":"This tests for common 'their/there' confusion, a punctuation error (missing comma after 'However'), and a slight word choice improvement ('specially' -> 'especially'). The sentence structure is otherwise simple, making it a good test case for typical user errors in email or workplace notes."},"correctlyResult":{"corrected":"I have been working on the report, and I think it's mostly done. However, there are a few sections that still need more work, especially the financial projections.","changes":[],"confidence":55,"responseTimeMs":916,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/28 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"Expected a non-empty changes array or more detailed feedback on the correction process.","fixtureWorthy":false},"error":null} +{"id":"0049-1780237771683","startedAt":"2026-05-31T14:29:31.683Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I think its a good idea but are you shore?","intendedMeaning":"I think it's a good idea, but are you sure?","errorTags":["spelling","punctuation","grammar"],"notes":"A common error combining 'its' (possessive) with 'it's' (contraction). Also contains a spelling error ('shore' instead of 'sure') and missing punctuation (comma after 'idea'). This represents a casual online communication style."},"correctlyResult":{"corrected":"I think it's a good idea, but are you sure?","changes":[],"confidence":55,"responseTimeMs":433,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/10 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":false,"reason":"The system failed to correct the grammar error and also returned an empty response.","fixtureWorthy":false},"error":null} +{"id":"0050-1780237781746","startedAt":"2026-05-31T14:29:41.746Z","provider":{"id":"lmstudio","model":"granite-4.0-h-tiny-mlx"},"generated":{"original":"I was wondering if you could please look ovver the proposal i sent you yesterday. Let me know your thoughts as soon as possable.","intendedMeaning":"I was wondering if you could please review the proposal I sent you yesterday. Let me know your thoughts as soon as possible.","errorTags":["spelling","punctuation","word-choice"],"notes":"This tests for a common misspelling ('ovver' should be 'over'), capitalization error (lowercase 'i' when referring to self in an email), and incorrect word choice ('possable' should be 'possible'). It is a realistic email request, involving reviewing a document and seeking feedback quickly."},"correctlyResult":{"corrected":"I was wondering if you could please look over the proposal I sent you yesterday. Let me know your thoughts as soon as possible.","changes":[],"confidence":55,"responseTimeMs":1063,"cascadeLevel":3},"scoring":{"accepted":false,"acceptanceScore":55,"reasons":[{"pass":true,"name":"structure","detail":"valid object with corrected string and changes array","penalty":0},{"pass":true,"name":"change density","detail":"0/24 word(s) flagged","penalty":0},{"pass":false,"name":"empty response","detail":"full-text correction without structured changes","penalty":25},{"pass":false,"name":"cascade level","detail":"level 3","penalty":20},{"pass":true,"name":"corrected consistency","detail":"not applicable for empty changes","penalty":0}],"displayChanges":[],"hiddenChanges":[],"cascadeLevel":3,"displayConfidence":55},"judge":{"verdict":"fail","risk":"false_accept","shouldAccept":false,"meaningPreserved":true,"grammarImproved":false,"visibleSuggestionsSafe":true,"reason":"Expected a more comprehensive correction, including changes to the text and a higher confidence score.","fixtureWorthy":false},"error":null} diff --git a/blackbox/runs/latest-analysis.json b/blackbox/runs/latest-analysis.json new file mode 100644 index 0000000..47e539b --- /dev/null +++ b/blackbox/runs/latest-analysis.json @@ -0,0 +1,68 @@ +{ + "summary": "All tests passed with an average acceptance score of 67.", + "metrics": { + "mainRisks": ["bad_visibility", "cascade_issue"], + "confidence": "high" + }, + "recommendations": [ + { + "priority": "P0", + "area": "extractDisplayChanges", + "title": "Review extract display changes for better visibility and consistency.", + "evidenceCaseIds": ["0001", "0003"], + "problem": "Some changes were not visible or consistent with the intended meaning.", + "suggestedChange": "Review the extract display changes to ensure they are accurate and consistent with the intended meaning.", + "suggestedTests": ["fixture or unit/e2e test to add"] + }, + { + "priority": "P1", + "area": "scoreAcceptedCorrection", + "title": "Review scoring for accepted corrections.", + "evidenceCaseIds": ["0002", "0011"], + "problem": "Some corrections were not scored correctly.", + "suggestedChange": "Review the scoring for accepted corrections to ensure it is accurate and consistent with the intended meaning.", + "suggestedTests": ["fixture or unit/e2e test to add"] + }, + { + "priority": "P1", + "area": "cascade_cache", + "title": "Review cascade cache policy.", + "evidenceCaseIds": ["0004", "0020"], + "problem": "Some changes were not cached correctly.", + "suggestedChange": "Review the cascade cache policy to ensure it is accurate and consistent with the intended meaning.", + "suggestedTests": ["fixture or unit/e2e test to add"] + }, + { + "priority": "P1", + "area": "prompts", + "title": "Review prompts for better visibility and consistency.", + "evidenceCaseIds": ["0005", "0021"], + "problem": "Some prompts were not visible or consistent with the intended meaning.", + "suggestedChange": "Review the prompts to ensure they are accurate and consistent with the intended meaning.", + "suggestedTests": ["fixture or unit/e2e test to add"] + }, + { + "priority": "P1", + "area": "fixture_quality", + "title": "Review fixture quality.", + "evidenceCaseIds": ["0006", "0022"], + "problem": "Some fixtures were not of high quality.", + "suggestedChange": "Review the fixture quality to ensure it is accurate and consistent with the intended meaning.", + "suggestedTests": ["fixture or unit/e2e test to add"] + }, + { + "priority": "P1", + "area": "model_quality", + "title": "Review model quality.", + "evidenceCaseIds": ["0007", "0023"], + "problem": "Some models were not of high quality.", + "suggestedChange": "Review the model quality to ensure it is accurate and consistent with the intended meaning.", + "suggestedTests": ["fixture or unit/e2e test to add"] + } + ], + "fixtureReview": { + "promoteAsIs": ["0008"], + "promoteWithEdits": [], + "discard": [] + } +} diff --git a/blackbox/runs/latest-analysis.md b/blackbox/runs/latest-analysis.md new file mode 100644 index 0000000..95d99e5 --- /dev/null +++ b/blackbox/runs/latest-analysis.md @@ -0,0 +1,65 @@ +# Blackbox Analysis + +Run: `blackbox/runs/2026-05-31T14-18-12-310Z-lmstudio-granite-test-drive.jsonl` +Evaluation: `blackbox/runs/latest-evaluation.json` + +## Summary + +All tests passed with an average acceptance score of 67. + +## Recommendations + +### P0: Review extract display changes for better visibility and consistency. + +- Area: `extractDisplayChanges` +- Evidence: `0001`, `0003` +- Problem: Some changes were not visible or consistent with the intended meaning. +- Suggested change: Review the extract display changes to ensure they are accurate and consistent with the intended meaning. +- Suggested tests: fixture or unit/e2e test to add + +### P1: Review scoring for accepted corrections. + +- Area: `scoreAcceptedCorrection` +- Evidence: `0002`, `0011` +- Problem: Some corrections were not scored correctly. +- Suggested change: Review the scoring for accepted corrections to ensure it is accurate and consistent with the intended meaning. +- Suggested tests: fixture or unit/e2e test to add + +### P1: Review cascade cache policy. + +- Area: `cascade_cache` +- Evidence: `0004`, `0020` +- Problem: Some changes were not cached correctly. +- Suggested change: Review the cascade cache policy to ensure it is accurate and consistent with the intended meaning. +- Suggested tests: fixture or unit/e2e test to add + +### P1: Review prompts for better visibility and consistency. + +- Area: `prompts` +- Evidence: `0005`, `0021` +- Problem: Some prompts were not visible or consistent with the intended meaning. +- Suggested change: Review the prompts to ensure they are accurate and consistent with the intended meaning. +- Suggested tests: fixture or unit/e2e test to add + +### P1: Review fixture quality. + +- Area: `fixture_quality` +- Evidence: `0006`, `0022` +- Problem: Some fixtures were not of high quality. +- Suggested change: Review the fixture quality to ensure it is accurate and consistent with the intended meaning. +- Suggested tests: fixture or unit/e2e test to add + +### P1: Review model quality. + +- Area: `model_quality` +- Evidence: `0007`, `0023` +- Problem: Some models were not of high quality. +- Suggested change: Review the model quality to ensure it is accurate and consistent with the intended meaning. +- Suggested tests: fixture or unit/e2e test to add + +## Fixture Review + +- Promote as-is: `0008` +- Promote with edits: none +- Discard: none + diff --git a/blackbox/runs/latest-evaluation.json b/blackbox/runs/latest-evaluation.json new file mode 100644 index 0000000..d8f5a0b --- /dev/null +++ b/blackbox/runs/latest-evaluation.json @@ -0,0 +1,1283 @@ +{ + "input": "tests/fixtures/scoring-cases.generated.json", + "evaluatedAt": "2026-05-31T14:29:55.331Z", + "summary": { + "total": 50, + "passed": 50, + "failed": 0, + "passRate": 1, + "averageAcceptanceScore": 67 + }, + "results": [ + { + "id": "0001-1780237092311", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I think it's a good idea to have the meeting on Tuesday. However, I don't know if everyone will be available.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "avaliable", + "replacement": "available" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think it's a good idea to have the meeting on Tuesday. However, I don't know if everyone will be available.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "avaliable", + "replacement": "available" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified and corrected the spelling errors, contractions, and punctuation issues in the original text." + }, + { + "id": "0002-1780237152802", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I saw him yesterday at the store, and he said he was going to call me back. But I haven't heard from him yet.", + "displayChanges": [ + { + "original": "seen", + "replacement": "saw" + }, + { + "original": "hearred", + "replacement": "heard" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I saw him yesterday at the store, and he said he was going to call me back. But I haven't heard from him yet.", + "displayChanges": [ + { + "original": "seen", + "replacement": "saw" + }, + { + "original": "hearred", + "replacement": "heard" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "All corrections were accurate and targeted, with no duplicate or overlapping changes. The corrected text matches the usable changes, indicating a high level of confidence in the correction." + }, + { + "id": "0003-1780237181114", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 82, + "corrected": "I'd appreciate it if you could send the file over as soon as possible. Thanks!", + "displayChanges": [ + { + "original": "I’d appreciate it if you could sent the file over as soon as possible. thanks!", + "replacement": "I'd appreciate it if you could send the file over as soon as possible. Thanks!" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I'd appreciate it if you could send the file over as soon as possible. Thanks!", + "displayChanges": [ + { + "original": "I’d appreciate it if you could sent the file over as soon as possible. thanks!", + "replacement": "I'd appreciate it if you could send the file over as soon as possible. Thanks!" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The model returned a whole-text edit instead of targeted changes, which may not accurately reflect the intended corrections." + }, + { + "id": "0004-1780237196353", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think that it's best to schedule the meeting for next Tuesday, as I have a conflict on Monday.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think that it's best to schedule the meeting for next Tuesday, as I have a conflict on Monday.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the possessive/contraction error and did not provide a full-text correction with structured changes." + }, + { + "id": "0005-1780237213417", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I was wondering if you could send me the report; it's really important for my presentation tomorrow.", + "displayChanges": [ + { + "original": "report", + "replacement": "report;" + }, + { + "original": "its", + "replacement": "it's" + }, + { + "original": "presentation tomorow", + "replacement": "presentation tomorrow" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I was wondering if you could send me the report; it's really important for my presentation tomorrow.", + "displayChanges": [ + { + "original": "report", + "replacement": "report;" + }, + { + "original": "its", + "replacement": "it's" + }, + { + "original": "presentation tomorow", + "replacement": "presentation tomorrow" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "Correctly identified and corrected all errors, including punctuation, spelling, and tense issues." + }, + { + "id": "0006-1780237227143", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 88, + "corrected": "I'm really excited to hear back from you soonest.", + "displayChanges": [ + { + "original": "here", + "replacement": "hear" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I'm really excited to hear back from you soonest.", + "displayChanges": [ + { + "original": "here", + "replacement": "hear" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the error and provided a clear explanation of the correction. However, it did not catch the slight inconsistency between the corrected text and the changes array." + }, + { + "id": "0007-1780237240710", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 88, + "corrected": "I'm really excited to hear back from you soon, let's schedule a call next week!", + "displayChanges": [ + { + "original": "I’m", + "replacement": "I'm" + }, + { + "original": "really exited", + "replacement": "really excited" + }, + { + "original": "here back", + "replacement": "hear back" + }, + { + "original": "lets schedule", + "replacement": "let's schedule" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I'm really excited to hear back from you soon, let's schedule a call next week!", + "displayChanges": [ + { + "original": "I’m", + "replacement": "I'm" + }, + { + "original": "really exited", + "replacement": "really excited" + }, + { + "original": "here back", + "replacement": "hear back" + }, + { + "original": "lets schedule", + "replacement": "let's schedule" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified and corrected the errors in the input text, including missing apostrophes, misspelled words, and incorrect word order." + }, + { + "id": "0008-1780237255589", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I think the meeting should have been rescheduled to next Tuesday.", + "displayChanges": [ + { + "original": "should of", + "replacement": "should have" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think the meeting should have been rescheduled to next Tuesday.", + "displayChanges": [ + { + "original": "should of", + "replacement": "should have" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the error, provided a clear explanation of the correction, and preserved the original meaning." + }, + { + "id": "0009-1780237266345", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I think it's really important that we finalize the report by Friday.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "finalise", + "replacement": "finalize" + }, + { + "original": "friday", + "replacement": "Friday" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think it's really important that we finalize the report by Friday.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "finalise", + "replacement": "finalize" + }, + { + "original": "friday", + "replacement": "Friday" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "All changes were targeted and corrected the original text without altering its tone." + }, + { + "id": "0010-1780237280008", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I need to confirm the meeting with John; he hasn't responded to my last email yet. Also, can you check if the presentation is ready for tomorrow?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I need to confirm the meeting with John; he hasn't responded to my last email yet. Also, can you check if the presentation is ready for tomorrow?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the comma usage after the introductory phrase and did not replace 'hes' with 'he'." + }, + { + "id": "0011-1780237295502", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 88, + "corrected": "I think it's a really good idea, but could we maybe look at the budget first?", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think it's a really good idea, but could we maybe look at the budget first?", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the common contraction mistake and provided a clear explanation. The corrected text matches the intended meaning, and the changes are targeted." + }, + { + "id": "0012-1780237307753", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The corrected text matches the source, and no changes were made." + }, + { + "id": "0013-1780237320955", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 76, + "corrected": "I'm really excited to hear back from you about the project. Let's schedule a quick meeting sometime next week—it would be great.", + "displayChanges": [ + { + "original": "I'm really exited", + "replacement": "I'm really excited" + }, + { + "original": "lets schedule", + "replacement": "Let's schedule" + }, + { + "original": "it would be really good", + "replacement": "it would be great" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I'm really excited to hear back from you about the project. Let's schedule a quick meeting sometime next week—it would be great.", + "displayChanges": [ + { + "original": "I'm really exited", + "replacement": "I'm really excited" + }, + { + "original": "lets schedule", + "replacement": "Let's schedule" + }, + { + "original": "it would be really good", + "replacement": "it would be great" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The corrected text differs slightly from the changes (0.87), indicating a potential issue with consistency." + }, + { + "id": "0014-1780237340539", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I think it's important to note that the report, which was submitted late yesterday, needs to be reviewed again.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "needs reviewed", + "replacement": "needs to be reviewed" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think it's important to note that the report, which was submitted late yesterday, needs to be reviewed again.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "needs reviewed", + "replacement": "needs to be reviewed" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified and corrected the errors in grammar, spelling, and verb tense. The changes were targeted and consistent with the intended meaning." + }, + { + "id": "0015-1780237354015", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 88, + "corrected": "I think it would be really helpful if you could check my work by Tuesday.", + "displayChanges": [ + { + "original": "realy", + "replacement": "really" + }, + { + "original": "tuesday", + "replacement": "Tuesday" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think it would be really helpful if you could check my work by Tuesday.", + "displayChanges": [ + { + "original": "realy", + "replacement": "really" + }, + { + "original": "tuesday", + "replacement": "Tuesday" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "Corrected common misspelling of 'real' and capitalized proper noun 'Tuesday', but the corrected text differs slightly from the changes (0.94)." + }, + { + "id": "0016-1780237365148", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 76, + "corrected": "I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are any updates.", + "displayChanges": [ + { + "original": "confirmation", + "replacement": "any updates" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are any updates.", + "displayChanges": [ + { + "original": "confirmation", + "replacement": "any updates" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the error in 'confirmation' and replaced it with 'any updates'. The corrected text is consistent with the changes made, and the cascade level is within a safe range." + }, + { + "id": "0017-1780237378095", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I think it's really important to make sure the report is finalized by Friday.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "make shure", + "replacement": "make sure" + }, + { + "original": "friday", + "replacement": "Friday" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I think it's really important to make sure the report is finalized by Friday.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "make shure", + "replacement": "make sure" + }, + { + "original": "friday", + "replacement": "Friday" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "All changes were targeted and corrected the original text, with no duplicate or overlapping changes." + }, + { + "id": "0018-1780237390024", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was really excited to hear about your proposal, but I think we need to discuss it further before finalizing anything. It's got some potential, but also a few kinks.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear about your proposal, but I think we need to discuss it further before finalizing anything. It's got some potential, but also a few kinks.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Empty response without structured changes" + }, + { + "id": "0019-1780237408838", + "passed": true, + "issues": [], + "actual": { + "accept": true, + "acceptanceScore": 100, + "corrected": "I hope to receive your reply soonest. Let me know if you need anything else, thanks!", + "displayChanges": [ + { + "original": "recieve", + "replacement": "receive" + } + ], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": true, + "corrected": "I hope to receive your reply soonest. Let me know if you need anything else, thanks!", + "displayChanges": [ + { + "original": "recieve", + "replacement": "receive" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the spelling error and suggested a correction. The changes array contains only one change, which is correct and targeted." + }, + { + "id": "0020-1780237421158", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was really excited to hear about the project, but I'm not sure if I have time currently.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear about the project, but I'm not sure if I have time currently.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the spelling error in 'exited' and did not provide any suggestions for improvement." + }, + { + "id": "0021-1780237437281", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's important that we circle back on this item next week, if possible. Let me know your thoughts.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's important that we circle back on this item next week, if possible. Let me know your thoughts.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a full-text correction with structured changes, but received an empty response." + }, + { + "id": "0022-1780237448458", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I'm really excited about the project; it's going to be a great success, I think.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I'm really excited about the project; it's going to be a great success, I think.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the sentence and did not provide any visible suggestions." + }, + { + "id": "0023-1780237459409", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was really excited to hear about the project, but I think we need to clarify some things before moving forward. It's important that everyone is on the same page.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear about the project, but I think we need to clarify some things before moving forward. It's important that everyone is on the same page.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the subject-verb agreement error ('everyones' vs 'everyone is') and did not provide any suggestions for improvement." + }, + { + "id": "0024-1780237474126", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was really excited to hear back from you about the project, but I haven't seen anything yet. Could you let me know when it might be ready?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear back from you about the project, but I haven't seen anything yet. Could you let me know when it might be ready?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the spelling error 'realy' to 'really', resulting in an empty response." + }, + { + "id": "0025-1780237486400", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think we should schedule a meeting next week to discuss the new project; it's really important.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think we should schedule a meeting next week to discuss the new project; it's really important.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the error and provided an empty response." + }, + { + "id": "0026-1780237496055", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I saw him yesterday and he said that the project is due next week. However, I haven't started it yet so I'm a little worried.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I saw him yesterday and he said that the project is due next week. However, I haven't started it yet so I'm a little worried.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a full-text correction with structured changes, but only the corrected string was provided." + }, + { + "id": "0027-1780237506793", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possible.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possible.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal language 'possable' vs 'possible', which may not require correction depending on the context." + }, + { + "id": "0028-1780237519603", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I wanted to reach out regarding the project; it's going well so far, but I'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I wanted to reach out regarding the project; it's going well so far, but I'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The corrected text still contains a comma splice ('it's going well so far but...') and the response time is relatively high (664ms)." + }, + { + "id": "0029-1780237536187", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's a good idea, but maybe we should double-check the numbers before sending it out to all the team members.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea, but maybe we should double-check the numbers before sending it out to all the team members.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a non-empty changes array, but it was empty." + }, + { + "id": "0030-1780237546885", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please send me the document?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please send me the document?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "empty response and high cascade level penalized the score" + }, + { + "id": "0031-1780237559359", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think the report is good, but maybe a few more details would be helpful.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think the report is good, but maybe a few more details would be helpful.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The AI failed to correct the informal language ('like, maybe') and omitted the comma before 'but', despite detecting spelling errors. This suggests a lack of understanding of formal writing conventions." + }, + { + "id": "0032-1780237572268", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it would be great if you could look into this issue. It's been going on for a while now and is really impacting our workflow.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it would be great if you could look into this issue. It's been going on for a while now and is really impacting our workflow.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the spelling error 'grate' and did not provide any suggestions." + }, + { + "id": "0033-1780237583786", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was thinking about the project, and I feel like we should revisit the timeline. It's kind of tight right now.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was thinking about the project, and I feel like we should revisit the timeline. It's kind of tight right now.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the standalone 'i' error and did not provide any suggestions for improvement." + }, + { + "id": "0034-1780237596256", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I saw him leave earlier, and he said it was urgent. I'll call you back later.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I saw him leave earlier, and he said it was urgent. I'll call you back later.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the 'seen' error in the original text, resulting in an incorrect corrected string." + }, + { + "id": "0035-1780237606258", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was wondering if you could send me the report before Friday. It's really needed for my presentation.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was wondering if you could send me the report before Friday. It's really needed for my presentation.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal tone and missing period at the end of the sentence." + }, + { + "id": "0036-1780237616891", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think we should have scheduled the meeting for Tuesday, not Monday. It's more in line with everyone's availability.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think we should have scheduled the meeting for Tuesday, not Monday. It's more in line with everyone's availability.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The model failed to correct the verb tense error and did not provide any visible suggestions." + }, + { + "id": "0037-1780237627819", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's a good idea to finalize the report by Friday.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea to finalize the report by Friday.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Empty response without structured changes" + }, + { + "id": "0038-1780237635988", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I am writing to inquire about the possibility of a refund, as my order arrived damaged. It's quite upsetting.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I am writing to inquire about the possibility of a refund, as my order arrived damaged. It's quite upsetting.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a full-text correction with structured changes, but received an empty response." + }, + { + "id": "0039-1780237647678", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I'm writing this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I'm writing this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the comma splice error and did not provide any suggestions for improvement." + }, + { + "id": "0040-1780237661808", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think we need to go over the proposal again. It's not really clear on what actions are needed next, and I'm worried about the deadline.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think we need to go over the proposal again. It's not really clear on what actions are needed next, and I'm worried about the deadline.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a non-empty changes array, but it was empty." + }, + { + "id": "0041-1780237673036", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I feel like this report is not quite done yet; it's missing a few key details, and I need to go back over it. Please let me know if you have any questions.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I feel like this report is not quite done yet; it's missing a few key details, and I need to go back over it. Please let me know if you have any questions.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the sentence correctly, and also did not provide any visible suggestions." + }, + { + "id": "0042-1780237686366", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's a good idea, but are you sure about the timeline? We need to finalize it soon.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea, but are you sure about the timeline? We need to finalize it soon.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Penalty for empty response and cascade level is too high, but corrected consistency penalty is 0." + }, + { + "id": "0043-1780237696360", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think they're going to need more time to complete the project. It's looking like a difficult task.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think they're going to need more time to complete the project. It's looking like a difficult task.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal writing habit of using 'its' instead of 'it's', and also didn't provide any visible suggestions for improvement." + }, + { + "id": "0044-1780237709319", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's really important that we follow up on their request ASAP. Let me know if you have any questions.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's really important that we follow up on their request ASAP. Let me know if you have any questions.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal communication errors, such as 'thier' vs. 'their', and did not provide any visible suggestions." + }, + { + "id": "0045-1780237722263", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was really looking forward to the meeting, but I didn't get a chance to present my findings. It's really frustrating.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was really looking forward to the meeting, but I didn't get a chance to present my findings. It's really frustrating.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the sentence structure and did not provide any meaningful suggestions." + }, + { + "id": "0046-1780237735921", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think that we should schedule a meeting for next week. It's crucial to finalize the project timeline.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think that we should schedule a meeting for next week. It's crucial to finalize the project timeline.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the error and did not provide any meaningful suggestions." + }, + { + "id": "0047-1780237747329", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's really important that we finalize the report by Friday, okay?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's really important that we finalize the report by Friday, okay?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal closing ('okay?') and did not provide any suggestions for improvement." + }, + { + "id": "0048-1780237757597", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I have been working on the report, and I think it's mostly done. However, there are a few sections that still need more work, especially the financial projections.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I have been working on the report, and I think it's mostly done. However, there are a few sections that still need more work, especially the financial projections.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a non-empty changes array or more detailed feedback on the correction process." + }, + { + "id": "0049-1780237771683", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I think it's a good idea, but are you sure?", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea, but are you sure?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the grammar error and also returned an empty response." + }, + { + "id": "0050-1780237781746", + "passed": true, + "issues": [], + "actual": { + "accept": false, + "acceptanceScore": 55, + "corrected": "I was wondering if you could please look over the proposal I sent you yesterday. Let me know your thoughts as soon as possible.", + "displayChanges": [], + "hiddenChangeCount": 0, + "hiddenReasons": [] + }, + "expected": { + "accept": false, + "corrected": "I was wondering if you could please look over the proposal I sent you yesterday. Let me know your thoughts as soon as possible.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a more comprehensive correction, including changes to the text and a higher confidence score." + } + ] +} diff --git a/blackbox/runs/latest.txt b/blackbox/runs/latest.txt new file mode 100644 index 0000000..6405311 --- /dev/null +++ b/blackbox/runs/latest.txt @@ -0,0 +1 @@ +2026-05-31T14-18-12-310Z-lmstudio-granite-test-drive.jsonl diff --git a/blackbox/src/analyze-results.mjs b/blackbox/src/analyze-results.mjs new file mode 100644 index 0000000..d5b1256 --- /dev/null +++ b/blackbox/src/analyze-results.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { safeJsonParseObject } from "./json.mjs"; +import { OpenAICompatibleClient } from "./openai-compatible-client.mjs"; +import { ANALYST_SYSTEM_PROMPT, analystUserPrompt } from "./prompts.mjs"; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const configPath = args.config || "blackbox/config.local.json"; + const runPath = args.run || (await readLatestRunPath(args.runsDir || "blackbox/runs")); + const evaluationPath = args.evaluation || "blackbox/runs/latest-evaluation.json"; + const out = args.out || "blackbox/runs/latest-analysis.json"; + const markdownOut = args.markdown || out.replace(/\.json$/i, ".md"); + const config = await readJson(configPath); + const records = await readJsonLines(runPath); + const evaluation = await readJson(evaluationPath).catch(() => null); + const analystConfig = config.analyst || config.judge || config.generator; + const analyst = new OpenAICompatibleClient(analystConfig); + + const payload = { + runSummary: summarizeRun(records), + selectedRecords: selectEvidence(records), + evaluation, + }; + + const { content } = await analyst.chat({ + system: ANALYST_SYSTEM_PROMPT, + user: analystUserPrompt(payload), + temperature: analyst.temperature ?? 0.1, + }); + const analysis = safeJsonParseObject(content) || fallbackAnalysis(content, payload); + + await fs.mkdir(path.dirname(out), { recursive: true }); + await fs.writeFile(out, `${JSON.stringify(analysis, null, 2)}\n`); + await fs.writeFile(markdownOut, renderMarkdown({ analysis, runPath, evaluationPath })); + + console.log(`Wrote analysis to ${out}`); + console.log(`Wrote markdown report to ${markdownOut}`); +} + +function summarizeRun(records) { + const byVerdictRisk = {}; + const byCascadeLevel = {}; + let errors = 0; + let fixtureWorthy = 0; + + for (const record of records) { + if (record.error) errors++; + if (record.judge?.fixtureWorthy) fixtureWorthy++; + const key = `${record.judge?.verdict || "error"}/${record.judge?.risk || "unknown"}`; + byVerdictRisk[key] = (byVerdictRisk[key] || 0) + 1; + const level = record.correctlyResult?.cascadeLevel || "none"; + byCascadeLevel[level] = (byCascadeLevel[level] || 0) + 1; + } + + return { + total: records.length, + errors, + fixtureWorthy, + byVerdictRisk, + byCascadeLevel, + }; +} + +function selectEvidence(records) { + return records + .filter( + (record) => + record.error || + record.judge?.fixtureWorthy || + record.judge?.verdict !== "pass" || + record.judge?.risk !== "none", + ) + .slice(0, 25) + .map((record) => ({ + id: record.id, + original: record.generated?.original || null, + corrected: record.correctlyResult?.corrected || null, + changes: record.correctlyResult?.changes || [], + cascadeLevel: record.correctlyResult?.cascadeLevel || null, + scoring: record.scoring + ? { + accepted: record.scoring.accepted, + acceptanceScore: record.scoring.acceptanceScore, + displayChanges: record.scoring.displayChanges, + hiddenChanges: record.scoring.hiddenChanges, + } + : null, + judge: record.judge, + error: record.error ? { message: record.error.message } : null, + })); +} + +function fallbackAnalysis(content, payload) { + return { + summary: "Analyst model did not return valid JSON.", + metrics: { + mainRisks: Object.keys(payload.runSummary.byVerdictRisk), + confidence: "low", + }, + recommendations: [ + { + priority: "P2", + area: "fixture_quality", + title: "Review analyst raw output", + evidenceCaseIds: [], + problem: "The analyst response was not valid JSON.", + suggestedChange: content.slice(0, 1000), + suggestedTests: ["Re-run analysis with a stronger or lower-temperature analyst model."], + }, + ], + fixtureReview: { + promoteAsIs: [], + promoteWithEdits: [], + discard: [], + }, + }; +} + +function renderMarkdown({ analysis, runPath, evaluationPath }) { + const lines = []; + lines.push("# Blackbox Analysis"); + lines.push(""); + lines.push(`Run: \`${runPath}\``); + lines.push(`Evaluation: \`${evaluationPath}\``); + lines.push(""); + lines.push("## Summary"); + lines.push(""); + lines.push(analysis.summary || "No summary."); + lines.push(""); + lines.push("## Recommendations"); + lines.push(""); + for (const rec of analysis.recommendations || []) { + lines.push(`### ${rec.priority || "P?"}: ${rec.title || "Untitled"}`); + lines.push(""); + lines.push(`- Area: \`${rec.area || "unknown"}\``); + lines.push(`- Evidence: ${(rec.evidenceCaseIds || []).map((id) => `\`${id}\``).join(", ") || "none"}`); + lines.push(`- Problem: ${rec.problem || ""}`); + lines.push(`- Suggested change: ${rec.suggestedChange || ""}`); + lines.push(`- Suggested tests: ${(rec.suggestedTests || []).join("; ") || "none"}`); + lines.push(""); + } + lines.push("## Fixture Review"); + lines.push(""); + lines.push( + `- Promote as-is: ${(analysis.fixtureReview?.promoteAsIs || []).map((id) => `\`${id}\``).join(", ") || "none"}`, + ); + lines.push( + `- Promote with edits: ${(analysis.fixtureReview?.promoteWithEdits || []).map((id) => `\`${id}\``).join(", ") || "none"}`, + ); + lines.push(`- Discard: ${(analysis.fixtureReview?.discard || []).map((id) => `\`${id}\``).join(", ") || "none"}`); + lines.push(""); + return `${lines.join("\n")}\n`; +} + +async function readJson(filePath) { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw); +} + +async function readJsonLines(filePath) { + const raw = await fs.readFile(filePath, "utf8"); + return raw + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +async function readLatestRunPath(outputDir) { + const latest = (await fs.readFile(path.join(outputDir, "latest.txt"), "utf8")).trim(); + return path.join(outputDir, latest); +} + +function parseArgs(argv) { + const args = {}; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) continue; + const key = arg.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) args[key] = true; + else { + args[key] = next; + i++; + } + } + return args; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/blackbox/src/autoresearch.mjs b/blackbox/src/autoresearch.mjs new file mode 100644 index 0000000..ca5e24d --- /dev/null +++ b/blackbox/src/autoresearch.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const config = args.config || "blackbox/config.local.json"; + const cases = args.cases || "25"; + const runsDir = args.runsDir || "blackbox/runs"; + const fixturesOut = args.fixtures || "tests/fixtures/scoring-cases.generated.json"; + const reportOut = args.report || "blackbox/runs/latest-evaluation.json"; + const analysisOut = args.analysis || "blackbox/runs/latest-analysis.json"; + const analysisMarkdownOut = args.markdown || "blackbox/runs/latest-analysis.md"; + + await run("node", ["blackbox/src/run.mjs", "--config", config, "--cases", cases]); + const latestRun = await readLatestRunPath(runsDir); + await run("node", ["blackbox/src/promote-fixtures.mjs", latestRun, "--out", fixturesOut]); + await run("node", ["blackbox/src/evaluate-fixtures.mjs", fixturesOut, "--out", reportOut]); + await run("node", [ + "blackbox/src/analyze-results.mjs", + "--config", + config, + "--run", + latestRun, + "--evaluation", + reportOut, + "--out", + analysisOut, + "--markdown", + analysisMarkdownOut, + ]); + + const report = JSON.parse(await fs.readFile(reportOut, "utf8")); + console.log( + `Autoresearch complete: run=${latestRun}, fixtures=${fixturesOut}, passRate=${report.summary.passRate}, analysis=${analysisOut}`, + ); +} + +function run(command, args) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: "inherit" }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`${command} ${args.join(" ")} exited with ${code}`)); + }); + }); +} + +async function readLatestRunPath(outputDir) { + const latest = (await fs.readFile(path.join(outputDir, "latest.txt"), "utf8")).trim(); + return path.join(outputDir, latest); +} + +function parseArgs(argv) { + const args = {}; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) continue; + const key = arg.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) args[key] = true; + else { + args[key] = next; + i++; + } + } + return args; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/blackbox/src/chrome-stub.mjs b/blackbox/src/chrome-stub.mjs new file mode 100644 index 0000000..7e938b8 --- /dev/null +++ b/blackbox/src/chrome-stub.mjs @@ -0,0 +1,45 @@ +function makeStorageArea() { + const store = new Map(); + return { + async get(keys) { + if (Array.isArray(keys)) return Object.fromEntries(keys.map((key) => [key, store.get(key)])); + if (typeof keys === "string") return { [keys]: store.get(keys) }; + if (keys && typeof keys === "object") { + return Object.fromEntries( + Object.entries(keys).map(([key, fallback]) => [key, store.has(key) ? store.get(key) : fallback]), + ); + } + return Object.fromEntries(store.entries()); + }, + async set(values) { + for (const [key, value] of Object.entries(values)) store.set(key, value); + }, + async remove(keys) { + for (const key of Array.isArray(keys) ? keys : [keys]) store.delete(key); + }, + _store: store, + }; +} + +export function installChromeStub() { + if (globalThis.chrome?.storage?.local) return globalThis.chrome; + globalThis.chrome = { + storage: { + local: makeStorageArea(), + session: makeStorageArea(), + onChanged: { addListener() {} }, + }, + runtime: { + sendMessage() {}, + onMessage: { addListener() {} }, + }, + tabs: { + sendMessage: async () => undefined, + }, + action: { + setBadgeText: async () => undefined, + setBadgeBackgroundColor: async () => undefined, + }, + }; + return globalThis.chrome; +} diff --git a/blackbox/src/evaluate-fixtures.mjs b/blackbox/src/evaluate-fixtures.mjs new file mode 100644 index 0000000..3fc6394 --- /dev/null +++ b/blackbox/src/evaluate-fixtures.mjs @@ -0,0 +1,166 @@ +#!/usr/bin/env node + +import fs from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { extractDisplayChanges, scoreAcceptedCorrection } from "../../lib/score.js"; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const input = args._[0] || "tests/fixtures/scoring-cases.json"; + const out = args.out || null; + const failOnRegression = Boolean(args["fail-on-regression"]); + const fixtures = await readJson(input); + const results = fixtures.map(evaluateFixture); + const summary = summarize(results); + const report = { + input, + evaluatedAt: new Date().toISOString(), + summary, + results, + }; + + if (out) { + await fs.mkdir(dirname(out), { recursive: true }); + await fs.writeFile(out, `${JSON.stringify(report, null, 2)}\n`); + } + + printSummary(report); + + if (failOnRegression && summary.failed > 0) { + process.exitCode = 1; + } +} + +export function evaluateFixture(fixture) { + const issues = []; + const original = fixture.original || ""; + const level = fixture.level || 1; + const rawResponse = fixture.rawResponse; + const expected = fixture.expected || {}; + + if (!rawResponse) { + return { + id: fixture.id || "", + passed: false, + issues: ["missing rawResponse"], + actual: null, + expected, + }; + } + + const acceptance = scoreAcceptedCorrection(rawResponse, original, level); + const extraction = extractDisplayChanges(rawResponse, original); + const actual = { + accept: acceptance.accepted, + acceptanceScore: acceptance.acceptanceScore, + corrected: rawResponse.corrected, + displayChanges: extraction.displayChanges.map(({ original: o, replacement }) => ({ original: o, replacement })), + hiddenChangeCount: extraction.hiddenChanges.length, + hiddenReasons: extraction.hiddenChanges.map((change) => change.reason), + }; + + if (typeof expected.accept === "boolean" && actual.accept !== expected.accept) { + issues.push(`accept expected ${expected.accept}, got ${actual.accept}`); + } + + if (typeof expected.corrected === "string" && actual.corrected !== expected.corrected) { + issues.push("corrected text mismatch"); + } + + if (Array.isArray(expected.displayChanges)) { + for (const expectedChange of expected.displayChanges) { + const found = actual.displayChanges.some( + (change) => change.original === expectedChange.original && change.replacement === expectedChange.replacement, + ); + if (!found) { + issues.push(`missing display change ${expectedChange.original} -> ${expectedChange.replacement}`); + } + } + } + + if (typeof expected.hiddenChangeCount === "number" && actual.hiddenChangeCount !== expected.hiddenChangeCount) { + issues.push(`hiddenChangeCount expected ${expected.hiddenChangeCount}, got ${actual.hiddenChangeCount}`); + } + + if (typeof expected.minAcceptanceScore === "number" && actual.acceptanceScore < expected.minAcceptanceScore) { + issues.push(`acceptanceScore expected >= ${expected.minAcceptanceScore}, got ${actual.acceptanceScore}`); + } + + if (typeof expected.maxAcceptanceScore === "number" && actual.acceptanceScore > expected.maxAcceptanceScore) { + issues.push(`acceptanceScore expected <= ${expected.maxAcceptanceScore}, got ${actual.acceptanceScore}`); + } + + return { + id: fixture.id || "", + passed: issues.length === 0, + issues, + actual, + expected, + notes: fixture.notes || "", + }; +} + +function summarize(results) { + const failed = results.filter((result) => !result.passed); + const acceptanceScores = results + .map((result) => result.actual?.acceptanceScore) + .filter((score) => typeof score === "number"); + const averageAcceptanceScore = + acceptanceScores.length > 0 + ? Math.round(acceptanceScores.reduce((sum, score) => sum + score, 0) / acceptanceScores.length) + : null; + + return { + total: results.length, + passed: results.length - failed.length, + failed: failed.length, + passRate: results.length > 0 ? Number(((results.length - failed.length) / results.length).toFixed(3)) : 0, + averageAcceptanceScore, + }; +} + +function printSummary(report) { + const { summary } = report; + console.log( + `Fixture evaluation: ${summary.passed}/${summary.total} passed, passRate=${summary.passRate}, avgScore=${summary.averageAcceptanceScore}`, + ); + for (const result of report.results.filter((item) => !item.passed)) { + console.log(`FAIL ${result.id}: ${result.issues.join("; ")}`); + } +} + +async function readJson(filePath) { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw); +} + +function parseArgs(argv) { + const args = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) { + args._.push(arg); + continue; + } + const key = arg.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) args[key] = true; + else { + args[key] = next; + i++; + } + } + return args; +} + +function dirname(filePath) { + const index = filePath.lastIndexOf("/"); + return index === -1 ? "." : filePath.slice(0, index); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/blackbox/src/fixtures.mjs b/blackbox/src/fixtures.mjs new file mode 100644 index 0000000..7572a11 --- /dev/null +++ b/blackbox/src/fixtures.mjs @@ -0,0 +1,29 @@ +export function toFixtureCandidate(record) { + const result = record.correctlyResult; + const scoring = record.scoring; + return { + id: record.id, + original: record.generated?.original || record.original, + level: result?.cascadeLevel || 1, + rawResponse: result + ? { + corrected: result.corrected, + changes: result.changes || [], + confidence: normalizeConfidenceForFixture(result.confidence), + } + : null, + expected: { + accept: Boolean(scoring?.accepted), + corrected: result?.corrected || "", + displayChanges: scoring?.displayChanges?.map(({ original, replacement }) => ({ original, replacement })) || [], + hiddenChangeCount: scoring?.hiddenChanges?.length || 0, + }, + notes: record.judge?.reason || record.generated?.notes || "", + }; +} + +function normalizeConfidenceForFixture(confidence) { + if (typeof confidence !== "number" || !Number.isFinite(confidence)) return 5; + if (confidence >= 1 && confidence <= 10) return confidence; + return Math.max(1, Math.min(10, Math.round(confidence / 10))); +} diff --git a/blackbox/src/json.mjs b/blackbox/src/json.mjs new file mode 100644 index 0000000..70b0280 --- /dev/null +++ b/blackbox/src/json.mjs @@ -0,0 +1,45 @@ +export function extractJsonObject(text) { + if (typeof text !== "string") throw new Error("Expected text to parse JSON object"); + + const fence = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/i); + if (fence) return JSON.parse(fence[1].trim()); + + const firstBrace = text.indexOf("{"); + if (firstBrace === -1) throw new Error("No JSON object found"); + + let depth = 0; + let inString = false; + let escaped = false; + + for (let i = firstBrace; i < text.length; i++) { + const char = text[i]; + if (escaped) { + escaped = false; + continue; + } + if (char === "\\") { + escaped = true; + continue; + } + if (char === '"') { + inString = !inString; + continue; + } + if (inString) continue; + if (char === "{") depth++; + if (char === "}") { + depth--; + if (depth === 0) return JSON.parse(text.slice(firstBrace, i + 1)); + } + } + + throw new Error("Unclosed JSON object"); +} + +export function safeJsonParseObject(text, fallback = null) { + try { + return extractJsonObject(text); + } catch { + return fallback; + } +} diff --git a/blackbox/src/openai-compatible-client.mjs b/blackbox/src/openai-compatible-client.mjs new file mode 100644 index 0000000..4ae6bdc --- /dev/null +++ b/blackbox/src/openai-compatible-client.mjs @@ -0,0 +1,41 @@ +export class OpenAICompatibleClient { + constructor({ baseUrl, apiKey = "local", model, temperature = 0.2, timeoutMs = 120000 }) { + if (!baseUrl) throw new Error("OpenAICompatibleClient requires baseUrl"); + if (!model) throw new Error("OpenAICompatibleClient requires model"); + this.baseUrl = baseUrl.replace(/\/+$/, ""); + this.apiKey = apiKey; + this.model = model; + this.temperature = temperature; + this.timeoutMs = timeoutMs; + } + + async chat({ system, user, temperature = this.temperature }) { + const payload = { + model: this.model, + messages: [ + { role: "system", content: system }, + { role: "user", content: user }, + ], + temperature, + }; + const response = await fetch(`${this.baseUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey || "local"}`, + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(this.timeoutMs), + }); + + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error(`Local model HTTP ${response.status}: ${body.slice(0, 500)}`); + } + + const data = await response.json(); + const content = data.choices?.[0]?.message?.content; + if (!content) throw new Error("Local model returned empty content"); + return { content, usage: data.usage || null }; + } +} diff --git a/blackbox/src/promote-fixtures.mjs b/blackbox/src/promote-fixtures.mjs new file mode 100644 index 0000000..6539e53 --- /dev/null +++ b/blackbox/src/promote-fixtures.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node + +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { toFixtureCandidate } from "./fixtures.mjs"; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const input = args._[0] || (await readLatestRunPath("blackbox/runs")); + const out = args.out || "tests/fixtures/scoring-cases.generated.json"; + const limit = Number(args.limit || 50); + const records = await readJsonLines(input); + const candidates = records + .filter((record) => record.judge?.fixtureWorthy || record.judge?.verdict !== "pass" || record.error) + .slice(0, limit) + .map(toFixtureCandidate); + + await fs.mkdir(path.dirname(out), { recursive: true }); + await fs.writeFile(out, `${JSON.stringify(candidates, null, 2)}\n`); + console.log(`Wrote ${candidates.length} fixture candidate(s) to ${out}`); +} + +async function readJsonLines(filePath) { + const raw = await fs.readFile(filePath, "utf8"); + return raw + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +async function readLatestRunPath(outputDir) { + const latest = (await fs.readFile(path.join(outputDir, "latest.txt"), "utf8")).trim(); + return path.join(outputDir, latest); +} + +function parseArgs(argv) { + const args = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) { + args._.push(arg); + continue; + } + const key = arg.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) args[key] = true; + else { + args[key] = next; + i++; + } + } + return args; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/blackbox/src/prompts.mjs b/blackbox/src/prompts.mjs new file mode 100644 index 0000000..ee6fdda --- /dev/null +++ b/blackbox/src/prompts.mjs @@ -0,0 +1,101 @@ +export const GENERATOR_SYSTEM_PROMPT = `You generate realistic flawed English text for grammar correction testing. + +Return ONLY JSON: +{ + "original": "the flawed text", + "intendedMeaning": "what the writer meant", + "errorTags": ["tense", "punctuation", "spelling"], + "notes": "why this case is interesting" +} + +Rules: +- The original must contain 1-4 grammar, spelling, punctuation, or word-choice errors. +- Preserve a realistic user voice: email, chat, forms, product feedback, school/work notes. +- Include tricky cases sometimes: repeated words, standalone i, idioms, punctuation-only fixes, ambiguous there/their/they're. +- Do not include private or real-person data.`; + +export function generatorUserPrompt({ index, seed }) { + return `Generate case #${index}. Research focus: ${seed || "mixed grammar correction edge cases"}.`; +} + +export const JUDGE_SYSTEM_PROMPT = `You are a strict evaluator for a grammar correction system. + +Return ONLY JSON: +{ + "verdict": "pass" | "fail" | "interesting", + "risk": "none" | "false_accept" | "false_reject" | "semantic_change" | "bad_visibility" | "weak_correction" | "cascade_issue", + "shouldAccept": true, + "meaningPreserved": true, + "grammarImproved": true, + "visibleSuggestionsSafe": true, + "reason": "brief reason", + "fixtureWorthy": true +} + +Judge the system behavior, not the model personality. Prefer "interesting" for borderline cases worth regression testing.`; + +export function judgeUserPrompt({ generated, correctlyResult, scoring, error }) { + return JSON.stringify( + { + task: "Evaluate Correctly grammar correction behavior", + generated, + correctlyResult, + scoring, + error: error ? { message: error.message || String(error) } : null, + }, + null, + 2, + ); +} + +export const ANALYST_SYSTEM_PROMPT = `You are a senior engineer analyzing Correctly blackbox research results. + +Your job is to suggest concrete next engineering steps, not to rewrite code. + +Return ONLY JSON: +{ + "summary": "short overall assessment", + "metrics": { + "mainRisks": ["bad_visibility", "cascade_issue"], + "confidence": "low" | "medium" | "high" + }, + "recommendations": [ + { + "priority": "P0" | "P1" | "P2" | "P3", + "area": "extractDisplayChanges" | "scoreAcceptedCorrection" | "cascade_cache" | "prompts" | "fixture_quality" | "model_quality", + "title": "short action title", + "evidenceCaseIds": ["0001"], + "problem": "what went wrong", + "suggestedChange": "specific code or policy change to consider", + "suggestedTests": ["fixture or unit/e2e test to add"] + } + ], + "fixtureReview": { + "promoteAsIs": ["case-id"], + "promoteWithEdits": ["case-id"], + "discard": ["case-id"] + } +} + +Use these mappings: +- bad visible individual changes usually point to extractDisplayChanges being too permissive. +- good full correction but unsafe/missing changes usually points to display extraction, not acceptance. +- repeated fallback or wrong level usually points to cascade/cache policy. +- judge contradictions or noisy generated cases point to fixture_quality. +- weak fixer behavior without scoring bug points to model_quality or prompts. + +Be conservative. Prefer fixture/test suggestions before scoring-rule changes.`; + +export function analystUserPrompt({ runSummary, selectedRecords, evaluation }) { + return JSON.stringify( + { + task: "Analyze blackbox grammar scoring research and propose next engineering steps", + runSummary, + selectedRecords, + evaluationSummary: evaluation?.summary || null, + failedFixtureResults: evaluation?.results?.filter((result) => !result.passed).slice(0, 20) || [], + }, + null, + 2, + ); +} diff --git a/blackbox/src/run.mjs b/blackbox/src/run.mjs new file mode 100644 index 0000000..a73d7b2 --- /dev/null +++ b/blackbox/src/run.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node + +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { extractDisplayChanges, scoreAcceptedCorrection } from "../../lib/score.js"; +import { createProvider } from "../../providers/provider-registry.js"; +import { installChromeStub } from "./chrome-stub.mjs"; +import { safeJsonParseObject } from "./json.mjs"; +import { OpenAICompatibleClient } from "./openai-compatible-client.mjs"; +import { GENERATOR_SYSTEM_PROMPT, generatorUserPrompt, JUDGE_SYSTEM_PROMPT, judgeUserPrompt } from "./prompts.mjs"; + +const DEFAULT_CASES = 10; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const configPath = args.config || "blackbox/config.local.json"; + const config = await readJson(configPath); + const caseCount = Number(args.cases || config.caseCount || DEFAULT_CASES); + const outputDir = args.outDir || config.outputDir || "blackbox/runs"; + const runId = `${timestamp()}-${slug(config.runName || "blackbox")}`; + const outputPath = path.join(outputDir, `${runId}.jsonl`); + + installChromeStub(); + await fs.mkdir(outputDir, { recursive: true }); + + const generator = new OpenAICompatibleClient(config.generator); + const judge = new OpenAICompatibleClient(config.judge || config.generator); + const fixerConfig = config.fixer || {}; + const provider = createProvider( + fixerConfig.providerId || "ollama", + fixerConfig.apiKey || "", + fixerConfig.model, + fixerConfig.baseUrl, + ); + + const summary = { + total: 0, + pass: 0, + fail: 0, + interesting: 0, + errors: 0, + fixtureWorthy: 0, + outputPath, + }; + + for (let index = 1; index <= caseCount; index++) { + const record = await runCase({ index, config, generator, provider, judge }); + summary.total++; + if (record.error) summary.errors++; + const verdict = record.judge?.verdict; + if (verdict === "pass") summary.pass++; + else if (verdict === "fail") summary.fail++; + else if (verdict === "interesting") summary.interesting++; + if (record.judge?.fixtureWorthy) summary.fixtureWorthy++; + + await appendJsonLine(outputPath, record); + console.log(formatProgress(record)); + } + + await writeLatestPointer(outputDir, outputPath); + console.log(JSON.stringify(summary, null, 2)); +} + +async function runCase({ index, config, generator, provider, judge }) { + const startedAt = new Date().toISOString(); + const id = `${String(index).padStart(4, "0")}-${Date.now()}`; + let generated = null; + let correctlyResult = null; + let scoring = null; + let judgeResult = null; + let error = null; + + try { + generated = await generateCase(generator, { index, seed: config.seed }); + correctlyResult = await provider.correctGrammar(generated.original); + scoring = scoreCorrectlyResult(correctlyResult, generated.original); + judgeResult = await judgeCase(judge, { generated, correctlyResult, scoring }); + } catch (err) { + error = { + message: err.message || String(err), + stack: err.stack || null, + }; + if (generated) { + judgeResult = await judgeCase(judge, { generated, correctlyResult, scoring, error }).catch((judgeErr) => ({ + verdict: "interesting", + risk: "cascade_issue", + shouldAccept: false, + meaningPreserved: false, + grammarImproved: false, + visibleSuggestionsSafe: false, + reason: `System errored and judge failed: ${judgeErr.message}`, + fixtureWorthy: true, + })); + } + } + + return { + id, + startedAt, + provider: { + id: provider.providerId, + model: provider.model, + }, + generated, + correctlyResult, + scoring, + judge: judgeResult, + error, + }; +} + +async function generateCase(generator, { index, seed }) { + const { content } = await generator.chat({ + system: GENERATOR_SYSTEM_PROMPT, + user: generatorUserPrompt({ index, seed }), + temperature: generator.temperature, + responseFormatJson: true, + }); + const parsed = safeJsonParseObject(content); + if (!parsed?.original || typeof parsed.original !== "string") { + throw new Error(`Generator returned invalid case: ${content.slice(0, 500)}`); + } + return { + original: parsed.original, + intendedMeaning: parsed.intendedMeaning || "", + errorTags: Array.isArray(parsed.errorTags) ? parsed.errorTags : [], + notes: parsed.notes || "", + }; +} + +function scoreCorrectlyResult(result, original) { + const normalized = { + corrected: result.corrected, + changes: Array.isArray(result.changes) ? result.changes : [], + confidence: normalizeModelConfidence(result.confidence), + }; + const level = result.cascadeLevel || 1; + const acceptance = scoreAcceptedCorrection(normalized, original, level); + const extraction = extractDisplayChanges(normalized, original); + return { + accepted: acceptance.accepted, + acceptanceScore: acceptance.acceptanceScore, + reasons: acceptance.reasons, + displayChanges: extraction.displayChanges, + hiddenChanges: extraction.hiddenChanges, + cascadeLevel: result.cascadeLevel || null, + displayConfidence: result.confidence ?? null, + }; +} + +async function judgeCase(judge, payload) { + const { content } = await judge.chat({ + system: JUDGE_SYSTEM_PROMPT, + user: judgeUserPrompt(payload), + temperature: judge.temperature, + responseFormatJson: true, + }); + const parsed = safeJsonParseObject(content); + if (!parsed?.verdict) { + return { + verdict: "interesting", + risk: "cascade_issue", + shouldAccept: false, + meaningPreserved: false, + grammarImproved: false, + visibleSuggestionsSafe: false, + reason: `Judge returned invalid JSON: ${content.slice(0, 300)}`, + fixtureWorthy: true, + }; + } + return { + verdict: normalizeEnum(parsed.verdict, ["pass", "fail", "interesting"], "interesting"), + risk: normalizeEnum( + parsed.risk, + ["none", "false_accept", "false_reject", "semantic_change", "bad_visibility", "weak_correction", "cascade_issue"], + "none", + ), + shouldAccept: Boolean(parsed.shouldAccept), + meaningPreserved: Boolean(parsed.meaningPreserved), + grammarImproved: Boolean(parsed.grammarImproved), + visibleSuggestionsSafe: Boolean(parsed.visibleSuggestionsSafe), + reason: parsed.reason || "", + fixtureWorthy: Boolean(parsed.fixtureWorthy), + }; +} + +function normalizeModelConfidence(confidence) { + if (typeof confidence !== "number" || !Number.isFinite(confidence)) return 5; + if (confidence >= 1 && confidence <= 10) return confidence; + return Math.max(1, Math.min(10, Math.round(confidence / 10))); +} + +function normalizeEnum(value, allowed, fallback) { + return allowed.includes(value) ? value : fallback; +} + +async function readJson(filePath) { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw); +} + +async function appendJsonLine(filePath, value) { + await fs.appendFile(filePath, `${JSON.stringify(value)}\n`); +} + +async function writeLatestPointer(outputDir, outputPath) { + const relative = path.relative(outputDir, outputPath); + await fs.writeFile(path.join(outputDir, "latest.txt"), `${relative}\n`); +} + +function parseArgs(argv) { + const args = {}; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) continue; + const key = arg.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) args[key] = true; + else { + args[key] = next; + i++; + } + } + return args; +} + +function timestamp() { + return new Date().toISOString().replace(/[:.]/g, "-"); +} + +function slug(value) { + return String(value || "run") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 80); +} + +function formatProgress(record) { + const verdict = record.judge?.verdict || "error"; + const risk = record.judge?.risk || "unknown"; + const original = record.generated?.original || ""; + return `[${record.id}] ${verdict}/${risk}: ${original.slice(0, 100)}`; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/package.json b/package.json index 21aaa1d..a8e87fd 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,11 @@ "test:e2e:provider": "playwright test tests/e2e-playwright/provider-contract.spec.js", "test:e2e:popup": "playwright test tests/e2e-playwright/popup-model-loading.spec.js", "test:e2e:all": "playwright test tests/e2e-playwright", + "blackbox": "node blackbox/src/run.mjs", + "blackbox:promote": "node blackbox/src/promote-fixtures.mjs", + "blackbox:evaluate": "node blackbox/src/evaluate-fixtures.mjs", + "blackbox:analyze": "node blackbox/src/analyze-results.mjs", + "blackbox:autoresearch": "node blackbox/src/autoresearch.mjs", "lint": "biome check .", "lint:fix": "biome check . --fix", "build:clean": "rm -rf dist web-ext-artifacts correctly-chrome.zip correctly-firefox.xpi", diff --git a/tests/fixtures/scoring-cases.generated.json b/tests/fixtures/scoring-cases.generated.json new file mode 100644 index 0000000..92d463a --- /dev/null +++ b/tests/fixtures/scoring-cases.generated.json @@ -0,0 +1,1152 @@ +[ + { + "id": "0001-1780237092311", + "original": "i think its a good idea to have the meeting on tuesday. however, i dont know if everyone will be avaliable.", + "level": 1, + "rawResponse": { + "corrected": "I think it's a good idea to have the meeting on Tuesday. However, I don't know if everyone will be available.", + "changes": [ + { + "original": "its", + "replacement": "it's", + "explanation": "Contractions require an apostrophe." + }, + { + "original": "avaliable", + "replacement": "available", + "explanation": "Spelling correction." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I think it's a good idea to have the meeting on Tuesday. However, I don't know if everyone will be available.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "avaliable", + "replacement": "available" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified and corrected the spelling errors, contractions, and punctuation issues in the original text." + }, + { + "id": "0002-1780237152802", + "original": "I seen him yesterday at the store, and he said he was going to call me back. But I haven't hearred from him yet.", + "level": 1, + "rawResponse": { + "corrected": "I saw him yesterday at the store, and he said he was going to call me back. But I haven't heard from him yet.", + "changes": [ + { + "original": "seen", + "replacement": "saw", + "explanation": "Past participle needed for perfect tense; 'I have seen' vs. simple past 'I saw'." + }, + { + "original": "hearred", + "replacement": "heard", + "explanation": "Common misspelling; 'heard' is the correct past tense of 'hear'." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I saw him yesterday at the store, and he said he was going to call me back. But I haven't heard from him yet.", + "displayChanges": [ + { + "original": "seen", + "replacement": "saw" + }, + { + "original": "hearred", + "replacement": "heard" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "All corrections were accurate and targeted, with no duplicate or overlapping changes. The corrected text matches the usable changes, indicating a high level of confidence in the correction." + }, + { + "id": "0003-1780237181114", + "original": "I’d appreciate it if you could sent the file over as soon as possible. thanks!", + "level": 1, + "rawResponse": { + "corrected": "I'd appreciate it if you could send the file over as soon as possible. Thanks!", + "changes": [ + { + "original": "I’d appreciate it if you could sent the file over as soon as possible. thanks!", + "replacement": "I'd appreciate it if you could send the file over as soon as possible. Thanks!", + "explanation": "Changed single quote to apostrophe for contraction; added missing period and exclamation point." + } + ], + "confidence": 9 + }, + "expected": { + "accept": true, + "corrected": "I'd appreciate it if you could send the file over as soon as possible. Thanks!", + "displayChanges": [ + { + "original": "I’d appreciate it if you could sent the file over as soon as possible. thanks!", + "replacement": "I'd appreciate it if you could send the file over as soon as possible. Thanks!" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The model returned a whole-text edit instead of targeted changes, which may not accurately reflect the intended corrections." + }, + { + "id": "0004-1780237196353", + "original": "I think that its best to schedule the meeting for next tuesday, as i have a conflict on monday.", + "level": 3, + "rawResponse": { + "corrected": "I think that it's best to schedule the meeting for next Tuesday, as I have a conflict on Monday.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think that it's best to schedule the meeting for next Tuesday, as I have a conflict on Monday.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the possessive/contraction error and did not provide a full-text correction with structured changes." + }, + { + "id": "0005-1780237213417", + "original": "I was wondering if you could send me the report, its really important for my presentation tomorow.", + "level": 1, + "rawResponse": { + "corrected": "I was wondering if you could send me the report; it's really important for my presentation tomorrow.", + "changes": [ + { + "original": "report", + "replacement": "report;", + "explanation": "Added a semicolon to separate the independent clauses for better clarity." + }, + { + "original": "its", + "replacement": "it's", + "explanation": "Corrected the contraction for 'it is'." + }, + { + "original": "presentation tomorow", + "replacement": "presentation tomorrow", + "explanation": "Corrected the misspelled word 'tomorow'." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I was wondering if you could send me the report; it's really important for my presentation tomorrow.", + "displayChanges": [ + { + "original": "report", + "replacement": "report;" + }, + { + "original": "its", + "replacement": "it's" + }, + { + "original": "presentation tomorow", + "replacement": "presentation tomorrow" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "Correctly identified and corrected all errors, including punctuation, spelling, and tense issues." + }, + { + "id": "0006-1780237227143", + "original": "I'm really exited to here back from you soonest.", + "level": 1, + "rawResponse": { + "corrected": "I'm really excited to hear back from you soonest.", + "changes": [ + { + "original": "here", + "replacement": "hear", + "explanation": "Corrected the verb form to match the subject 'you'." + } + ], + "confidence": 9 + }, + "expected": { + "accept": true, + "corrected": "I'm really excited to hear back from you soonest.", + "displayChanges": [ + { + "original": "here", + "replacement": "hear" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the error and provided a clear explanation of the correction. However, it did not catch the slight inconsistency between the corrected text and the changes array." + }, + { + "id": "0007-1780237240710", + "original": "I’m really exited to here back from you soon, lets schedule a call next week!", + "level": 2, + "rawResponse": { + "corrected": "I'm really excited to hear back from you soon, let's schedule a call next week!", + "changes": [ + { + "original": "I’m", + "replacement": "I'm", + "explanation": "Missing apostrophe in contraction." + }, + { + "original": "really exited", + "replacement": "really excited", + "explanation": "Misspelled 'excited'." + }, + { + "original": "here back", + "replacement": "hear back", + "explanation": "Incorrect word order; should be 'hear' instead of 'here'." + }, + { + "original": "lets schedule", + "replacement": "let's schedule", + "explanation": "Missing apostrophe in contraction." + } + ], + "confidence": 9 + }, + "expected": { + "accept": true, + "corrected": "I'm really excited to hear back from you soon, let's schedule a call next week!", + "displayChanges": [ + { + "original": "I’m", + "replacement": "I'm" + }, + { + "original": "really exited", + "replacement": "really excited" + }, + { + "original": "here back", + "replacement": "hear back" + }, + { + "original": "lets schedule", + "replacement": "let's schedule" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified and corrected the errors in the input text, including missing apostrophes, misspelled words, and incorrect word order." + }, + { + "id": "0008-1780237255589", + "original": "I think the meeting should of been rescheduled to next tuesday.", + "level": 1, + "rawResponse": { + "corrected": "I think the meeting should have been rescheduled to next Tuesday.", + "changes": [ + { + "original": "should of", + "replacement": "should have", + "explanation": "Corrected the contraction 'should of' to the proper form 'should have'." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I think the meeting should have been rescheduled to next Tuesday.", + "displayChanges": [ + { + "original": "should of", + "replacement": "should have" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the error, provided a clear explanation of the correction, and preserved the original meaning." + }, + { + "id": "0009-1780237266345", + "original": "I think its really important that we finalise the report by friday.", + "level": 1, + "rawResponse": { + "corrected": "I think it's really important that we finalize the report by Friday.", + "changes": [ + { + "original": "its", + "replacement": "it's", + "explanation": "Contractions: 'it is' should be contracted as 'it's'." + }, + { + "original": "finalise", + "replacement": "finalize", + "explanation": "Spelling: American English prefers 'finalize' over British English 'finalise'." + }, + { + "original": "friday", + "replacement": "Friday", + "explanation": "Capitalization: Days of the week should be capitalized." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I think it's really important that we finalize the report by Friday.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "finalise", + "replacement": "finalize" + }, + { + "original": "friday", + "replacement": "Friday" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "All changes were targeted and corrected the original text without altering its tone." + }, + { + "id": "0010-1780237280008", + "original": "I need to confirm the meeting with John, hes not responded to my last email yet. Also can you check if the presentation is ready for tomorrow?", + "level": 3, + "rawResponse": { + "corrected": "I need to confirm the meeting with John; he hasn't responded to my last email yet. Also, can you check if the presentation is ready for tomorrow?", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I need to confirm the meeting with John; he hasn't responded to my last email yet. Also, can you check if the presentation is ready for tomorrow?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the comma usage after the introductory phrase and did not replace 'hes' with 'he'." + }, + { + "id": "0011-1780237295502", + "original": "I think its a really good idea, but could we maybe look at the budget first?", + "level": 2, + "rawResponse": { + "corrected": "I think it's a really good idea, but could we maybe look at the budget first?", + "changes": [ + { + "original": "its", + "replacement": "it's", + "explanation": "'It's' is the correct contraction for 'it is', while 'its' would imply possession." + } + ], + "confidence": 9 + }, + "expected": { + "accept": true, + "corrected": "I think it's a really good idea, but could we maybe look at the budget first?", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the common contraction mistake and provided a clear explanation. The corrected text matches the intended meaning, and the changes are targeted." + }, + { + "id": "0012-1780237307753", + "original": "I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?", + "level": 1, + "rawResponse": { + "corrected": "I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?", + "changes": [], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I was really excited to hear back from you regarding the project, but I still haven't received anything. Is there something going on?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The corrected text matches the source, and no changes were made." + }, + { + "id": "0013-1780237320955", + "original": "I'm really exited to hear back from you about the project, lets schedule a quick meeting sometime next week. it would be really good.", + "level": 2, + "rawResponse": { + "corrected": "I'm really excited to hear back from you about the project. Let's schedule a quick meeting sometime next week—it would be great.", + "changes": [ + { + "original": "I'm really exited", + "replacement": "I'm really excited", + "explanation": "Corrected the misspelling of 'excited'." + }, + { + "original": "lets schedule", + "replacement": "Let's schedule", + "explanation": "Added an apostrophe to form the contraction of 'let us'." + }, + { + "original": "it would be really good", + "replacement": "it would be great", + "explanation": "Changed 'really good' to 'great' for a more concise expression." + } + ], + "confidence": 8 + }, + "expected": { + "accept": true, + "corrected": "I'm really excited to hear back from you about the project. Let's schedule a quick meeting sometime next week—it would be great.", + "displayChanges": [ + { + "original": "I'm really exited", + "replacement": "I'm really excited" + }, + { + "original": "lets schedule", + "replacement": "Let's schedule" + }, + { + "original": "it would be really good", + "replacement": "it would be great" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The corrected text differs slightly from the changes (0.87), indicating a potential issue with consistency." + }, + { + "id": "0014-1780237340539", + "original": "I think its important to note that the report, which was submitted late yesterday, needs reviewed again.", + "level": 1, + "rawResponse": { + "corrected": "I think it's important to note that the report, which was submitted late yesterday, needs to be reviewed again.", + "changes": [ + { + "original": "its", + "replacement": "it's", + "explanation": "Contractions: 'it is' should be contracted as 'it's'." + }, + { + "original": "needs reviewed", + "replacement": "needs to be reviewed", + "explanation": "Verb tense: 'reviewed' should be followed by the infinitive 'to be' for correct usage." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I think it's important to note that the report, which was submitted late yesterday, needs to be reviewed again.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "needs reviewed", + "replacement": "needs to be reviewed" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified and corrected the errors in grammar, spelling, and verb tense. The changes were targeted and consistent with the intended meaning." + }, + { + "id": "0015-1780237354015", + "original": "I think it would be realy helpful if you could check my work by tuesday. Thanks!", + "level": 1, + "rawResponse": { + "corrected": "I think it would be really helpful if you could check my work by Tuesday.", + "changes": [ + { + "original": "realy", + "replacement": "really", + "explanation": "Corrected common misspelling of 'real'." + }, + { + "original": "tuesday", + "replacement": "Tuesday", + "explanation": "Capitalized proper noun 'Tuesday'." + } + ], + "confidence": 9 + }, + "expected": { + "accept": true, + "corrected": "I think it would be really helpful if you could check my work by Tuesday.", + "displayChanges": [ + { + "original": "realy", + "replacement": "really" + }, + { + "original": "tuesday", + "replacement": "Tuesday" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "Corrected common misspelling of 'real' and capitalized proper noun 'Tuesday', but the corrected text differs slightly from the changes (0.94)." + }, + { + "id": "0016-1780237365148", + "original": "I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are an updates.", + "level": 2, + "rawResponse": { + "corrected": "I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are any updates.", + "changes": [ + { + "original": "confirmation", + "replacement": "any updates", + "explanation": "The word 'confirmation' is incorrect in this context. The sentence should request an update or information about the status of the confirmation." + } + ], + "confidence": 8 + }, + "expected": { + "accept": true, + "corrected": "I appreciate your prompt response, however I am still awaiting confirmation from my supervisor. Please let me know if there are any updates.", + "displayChanges": [ + { + "original": "confirmation", + "replacement": "any updates" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the error in 'confirmation' and replaced it with 'any updates'. The corrected text is consistent with the changes made, and the cascade level is within a safe range." + }, + { + "id": "0017-1780237378095", + "original": "I think its really important to make shure the report is finalized by friday.", + "level": 1, + "rawResponse": { + "corrected": "I think it's really important to make sure the report is finalized by Friday.", + "changes": [ + { + "original": "its", + "replacement": "it's", + "explanation": "Contractions: 'it is' should be contracted as 'it's'." + }, + { + "original": "make shure", + "replacement": "make sure", + "explanation": "Spelling correction: 'shure' should be spelled as 'sure'." + }, + { + "original": "friday", + "replacement": "Friday", + "explanation": "Capitalization: Days of the week should be capitalized." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I think it's really important to make sure the report is finalized by Friday.", + "displayChanges": [ + { + "original": "its", + "replacement": "it's" + }, + { + "original": "make shure", + "replacement": "make sure" + }, + { + "original": "friday", + "replacement": "Friday" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "All changes were targeted and corrected the original text, with no duplicate or overlapping changes." + }, + { + "id": "0018-1780237390024", + "original": "I was really exited to hear about your proposal, but i think we need to discuss it further before finalizing anything. Its got some potential, but also a few kinks.", + "level": 3, + "rawResponse": { + "corrected": "I was really excited to hear about your proposal, but I think we need to discuss it further before finalizing anything. It's got some potential, but also a few kinks.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear about your proposal, but I think we need to discuss it further before finalizing anything. It's got some potential, but also a few kinks.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Empty response without structured changes" + }, + { + "id": "0019-1780237408838", + "original": "I hope to recieve your reply soonest. Let me know if you need anything else, thanks!", + "level": 1, + "rawResponse": { + "corrected": "I hope to receive your reply soonest. Let me know if you need anything else, thanks!", + "changes": [ + { + "original": "recieve", + "replacement": "receive", + "explanation": "Correct spelling of 'receive'." + } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "I hope to receive your reply soonest. Let me know if you need anything else, thanks!", + "displayChanges": [ + { + "original": "recieve", + "replacement": "receive" + } + ], + "hiddenChangeCount": 0 + }, + "notes": "The system correctly identified the spelling error and suggested a correction. The changes array contains only one change, which is correct and targeted." + }, + { + "id": "0020-1780237421158", + "original": "I was really exited to hear about the project, but i’m not sure if I have time currently.", + "level": 3, + "rawResponse": { + "corrected": "I was really excited to hear about the project, but I'm not sure if I have time currently.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear about the project, but I'm not sure if I have time currently.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the spelling error in 'exited' and did not provide any suggestions for improvement." + }, + { + "id": "0021-1780237437281", + "original": "I think its important that we circle back on this item next week, if possible. Let me know your thoughts.", + "level": 3, + "rawResponse": { + "corrected": "I think it's important that we circle back on this item next week, if possible. Let me know your thoughts.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's important that we circle back on this item next week, if possible. Let me know your thoughts.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a full-text correction with structured changes, but received an empty response." + }, + { + "id": "0022-1780237448458", + "original": "I'm really excited about the project, its gonna be a great success i think.", + "level": 3, + "rawResponse": { + "corrected": "I'm really excited about the project; it's going to be a great success, I think.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I'm really excited about the project; it's going to be a great success, I think.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the sentence and did not provide any visible suggestions." + }, + { + "id": "0023-1780237459409", + "original": "I was really exited to hear about the project, but I think we need too clarify some things before moving forward. Its important that everyones on the same page.", + "level": 3, + "rawResponse": { + "corrected": "I was really excited to hear about the project, but I think we need to clarify some things before moving forward. It's important that everyone is on the same page.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear about the project, but I think we need to clarify some things before moving forward. It's important that everyone is on the same page.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the subject-verb agreement error ('everyones' vs 'everyone is') and did not provide any suggestions for improvement." + }, + { + "id": "0024-1780237474126", + "original": "I was realy exited to hear back from you about the project, but i haven’t seen anything yet. Could you let me know when it might be ready?", + "level": 3, + "rawResponse": { + "corrected": "I was really excited to hear back from you about the project, but I haven't seen anything yet. Could you let me know when it might be ready?", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was really excited to hear back from you about the project, but I haven't seen anything yet. Could you let me know when it might be ready?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the spelling error 'realy' to 'really', resulting in an empty response." + }, + { + "id": "0025-1780237486400", + "original": "I think we should of schedule a meeting next week to discuss the new project, its really important.", + "level": 3, + "rawResponse": { + "corrected": "I think we should schedule a meeting next week to discuss the new project; it's really important.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think we should schedule a meeting next week to discuss the new project; it's really important.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the error and provided an empty response." + }, + { + "id": "0026-1780237496055", + "original": "I seen him yesterday and he said that the project is due next week. However, I haven't started it yet so i’m a little worried.", + "level": 3, + "rawResponse": { + "corrected": "I saw him yesterday and he said that the project is due next week. However, I haven't started it yet so I'm a little worried.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I saw him yesterday and he said that the project is due next week. However, I haven't started it yet so I'm a little worried.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a full-text correction with structured changes, but only the corrected string was provided." + }, + { + "id": "0027-1780237506793", + "original": "I think its important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possable.", + "level": 3, + "rawResponse": { + "corrected": "I think it's important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possible.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's important to note that the project will be delayed due to unforeseen circumstances. I'll update everyone with more details as soon as possible.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal language 'possable' vs 'possible', which may not require correction depending on the context." + }, + { + "id": "0028-1780237519603", + "original": "I wanted to reach out regarding the project, its going well so far but i'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?", + "level": 3, + "rawResponse": { + "corrected": "I wanted to reach out regarding the project; it's going well so far, but I'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I wanted to reach out regarding the project; it's going well so far, but I'm facing a slight challenge with integrating the new API. Can we schedule a quick call to discuss it?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The corrected text still contains a comma splice ('it's going well so far but...') and the response time is relatively high (664ms)." + }, + { + "id": "0029-1780237536187", + "original": "i think its a good idea but maybe we should double check the numbers before sending it out to all the team members.", + "level": 3, + "rawResponse": { + "corrected": "I think it's a good idea, but maybe we should double-check the numbers before sending it out to all the team members.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea, but maybe we should double-check the numbers before sending it out to all the team members.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a non-empty changes array, but it was empty." + }, + { + "id": "0030-1780237546885", + "original": "I think its a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please sent me the document?", + "level": 3, + "rawResponse": { + "corrected": "I think it's a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please send me the document?", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's a really great idea, but we need to make sure everyone is on board before moving forward. Also, could you please send me the document?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "empty response and high cascade level penalized the score" + }, + { + "id": "0031-1780237559359", + "original": "I think the report is good, but like, maybe a few more details would be helpfull?", + "level": 3, + "rawResponse": { + "corrected": "I think the report is good, but maybe a few more details would be helpful.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think the report is good, but maybe a few more details would be helpful.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The AI failed to correct the informal language ('like, maybe') and omitted the comma before 'but', despite detecting spelling errors. This suggests a lack of understanding of formal writing conventions." + }, + { + "id": "0032-1780237572268", + "original": "i think it would be grate if you could look into this issue. its been going on for a while now and is really impacting our workflow.", + "level": 3, + "rawResponse": { + "corrected": "I think it would be great if you could look into this issue. It's been going on for a while now and is really impacting our workflow.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it would be great if you could look into this issue. It's been going on for a while now and is really impacting our workflow.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the spelling error 'grate' and did not provide any suggestions." + }, + { + "id": "0033-1780237583786", + "original": "I was thinking about the project, and i feel like we should revisit the timeline. Its kinda tight right now.", + "level": 3, + "rawResponse": { + "corrected": "I was thinking about the project, and I feel like we should revisit the timeline. It's kind of tight right now.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was thinking about the project, and I feel like we should revisit the timeline. It's kind of tight right now.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the standalone 'i' error and did not provide any suggestions for improvement." + }, + { + "id": "0034-1780237596256", + "original": "I seen him leave earlier, and he said it was urgent. I'll call you back later.", + "level": 3, + "rawResponse": { + "corrected": "I saw him leave earlier, and he said it was urgent. I'll call you back later.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I saw him leave earlier, and he said it was urgent. I'll call you back later.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the 'seen' error in the original text, resulting in an incorrect corrected string." + }, + { + "id": "0035-1780237606258", + "original": "I was wondering if you could sent me the report before friday. Its really needed for my presentation.", + "level": 3, + "rawResponse": { + "corrected": "I was wondering if you could send me the report before Friday. It's really needed for my presentation.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was wondering if you could send me the report before Friday. It's really needed for my presentation.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal tone and missing period at the end of the sentence." + }, + { + "id": "0036-1780237616891", + "original": "i think we should of scheduled the meeting for tuesday, not monday. its more in line with everyone's availability.", + "level": 3, + "rawResponse": { + "corrected": "I think we should have scheduled the meeting for Tuesday, not Monday. It's more in line with everyone's availability.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think we should have scheduled the meeting for Tuesday, not Monday. It's more in line with everyone's availability.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The model failed to correct the verb tense error and did not provide any visible suggestions." + }, + { + "id": "0037-1780237627819", + "original": "I think its a good idea to finalize the report by friday.", + "level": 3, + "rawResponse": { + "corrected": "I think it's a good idea to finalize the report by Friday.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea to finalize the report by Friday.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Empty response without structured changes" + }, + { + "id": "0038-1780237635988", + "original": "I am writting to enquire about the possibility of a refund, as my order arrived damaged. Its quite upsetting.", + "level": 3, + "rawResponse": { + "corrected": "I am writing to inquire about the possibility of a refund, as my order arrived damaged. It's quite upsetting.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I am writing to inquire about the possibility of a refund, as my order arrived damaged. It's quite upsetting.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a full-text correction with structured changes, but received an empty response." + }, + { + "id": "0039-1780237647678", + "original": "I'm writting this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.", + "level": 3, + "rawResponse": { + "corrected": "I'm writing this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I'm writing this to inform you about the ongoing project, its progressing well so far. However, we do need your approval before moving to phase two.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the comma splice error and did not provide any suggestions for improvement." + }, + { + "id": "0040-1780237661808", + "original": "I think we need to go over the proposal again. Its not really clear on what actions are needed next, and I'm worried about the deadline.", + "level": 3, + "rawResponse": { + "corrected": "I think we need to go over the proposal again. It's not really clear on what actions are needed next, and I'm worried about the deadline.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think we need to go over the proposal again. It's not really clear on what actions are needed next, and I'm worried about the deadline.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a non-empty changes array, but it was empty." + }, + { + "id": "0041-1780237673036", + "original": "I feel like this report is not quite done yet, its missing a few key details and I need to go back over it. Please let me know if you have any questions.", + "level": 3, + "rawResponse": { + "corrected": "I feel like this report is not quite done yet; it's missing a few key details, and I need to go back over it. Please let me know if you have any questions.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I feel like this report is not quite done yet; it's missing a few key details, and I need to go back over it. Please let me know if you have any questions.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the sentence correctly, and also did not provide any visible suggestions." + }, + { + "id": "0042-1780237686366", + "original": "I think its a good idea, but are you sure about the timeline? We need to finalize it soon.", + "level": 3, + "rawResponse": { + "corrected": "I think it's a good idea, but are you sure about the timeline? We need to finalize it soon.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea, but are you sure about the timeline? We need to finalize it soon.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Penalty for empty response and cascade level is too high, but corrected consistency penalty is 0." + }, + { + "id": "0043-1780237696360", + "original": "I think their going to need more time to complete the project. Its looking like a difficult task.", + "level": 3, + "rawResponse": { + "corrected": "I think they're going to need more time to complete the project. It's looking like a difficult task.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think they're going to need more time to complete the project. It's looking like a difficult task.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal writing habit of using 'its' instead of 'it's', and also didn't provide any visible suggestions for improvement." + }, + { + "id": "0044-1780237709319", + "original": "I think its really important that we follow up on thier request ASAP. Let me know if you have any question's.", + "level": 3, + "rawResponse": { + "corrected": "I think it's really important that we follow up on their request ASAP. Let me know if you have any questions.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's really important that we follow up on their request ASAP. Let me know if you have any questions.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal communication errors, such as 'thier' vs. 'their', and did not provide any visible suggestions." + }, + { + "id": "0045-1780237722263", + "original": "I was realy looking forward to the meeting, but I didn’t got a chance to present my findings. Its really frustating.", + "level": 3, + "rawResponse": { + "corrected": "I was really looking forward to the meeting, but I didn't get a chance to present my findings. It's really frustrating.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was really looking forward to the meeting, but I didn't get a chance to present my findings. It's really frustrating.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the sentence structure and did not provide any meaningful suggestions." + }, + { + "id": "0046-1780237735921", + "original": "I think that we should of schedule a meeting for next week. Its crucial to finalize the project timeline.", + "level": 3, + "rawResponse": { + "corrected": "I think that we should schedule a meeting for next week. It's crucial to finalize the project timeline.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think that we should schedule a meeting for next week. It's crucial to finalize the project timeline.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the error and did not provide any meaningful suggestions." + }, + { + "id": "0047-1780237747329", + "original": "I think its really important that we finalize the report by friday, okay?", + "level": 3, + "rawResponse": { + "corrected": "I think it's really important that we finalize the report by Friday, okay?", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's really important that we finalize the report by Friday, okay?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the informal closing ('okay?') and did not provide any suggestions for improvement." + }, + { + "id": "0048-1780237757597", + "original": "I have been working on the report, and I think it’s mostly done. However their are a few sections that still need more work, specially the financial projections.", + "level": 3, + "rawResponse": { + "corrected": "I have been working on the report, and I think it's mostly done. However, there are a few sections that still need more work, especially the financial projections.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I have been working on the report, and I think it's mostly done. However, there are a few sections that still need more work, especially the financial projections.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a non-empty changes array or more detailed feedback on the correction process." + }, + { + "id": "0049-1780237771683", + "original": "I think its a good idea but are you shore?", + "level": 3, + "rawResponse": { + "corrected": "I think it's a good idea, but are you sure?", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I think it's a good idea, but are you sure?", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "The system failed to correct the grammar error and also returned an empty response." + }, + { + "id": "0050-1780237781746", + "original": "I was wondering if you could please look ovver the proposal i sent you yesterday. Let me know your thoughts as soon as possable.", + "level": 3, + "rawResponse": { + "corrected": "I was wondering if you could please look over the proposal I sent you yesterday. Let me know your thoughts as soon as possible.", + "changes": [], + "confidence": 6 + }, + "expected": { + "accept": false, + "corrected": "I was wondering if you could please look over the proposal I sent you yesterday. Let me know your thoughts as soon as possible.", + "displayChanges": [], + "hiddenChangeCount": 0 + }, + "notes": "Expected a more comprehensive correction, including changes to the text and a higher confidence score." + } +] diff --git a/tests/fixtures/scoring-cases.json b/tests/fixtures/scoring-cases.json new file mode 100644 index 0000000..b164b3a --- /dev/null +++ b/tests/fixtures/scoring-cases.json @@ -0,0 +1,27 @@ +[ + { + "id": "standalone-i-hidden-punctuation", + "original": "so i didnt had any time tolarend", + "level": 1, + "rawResponse": { + "corrected": "So I didn't have any time to learn.", + "changes": [ + { "original": "so", "replacement": "So", "explanation": "Capitalize first word of sentence." }, + { "original": "i", "replacement": "I", "explanation": "Pronoun I should be capitalized." }, + { "original": "didnt", "replacement": "didn't", "explanation": "Add apostrophe for contraction." }, + { "original": "had", "replacement": "have", "explanation": "Use base verb after didn't." }, + { "original": "tolarend", "replacement": "learn", "explanation": "Correct misspelled word." }, + { "original": "", "replacement": ".", "explanation": "Add terminal period." } + ], + "confidence": 10 + }, + "expected": { + "accept": true, + "corrected": "So I didn't have any time to learn.", + "displayChanges": [{ "original": "i", "replacement": "I" }, { "original": "tolarend", "replacement": "learn" }], + "hiddenChangeCount": 1, + "minAcceptanceScore": 60 + }, + "notes": "Standalone i must not match inside time; punctuation insertion is hidden but full correction accepted." + } +]