Make compare_pdfs coverage fail closed

docs/COMPARISON_EVALUATION.md

@@ -195,11 +195,19 @@ Candidate page-alignment records contain `before_page` or null, `after_page` or
 null, a `same`, `moved`, `inserted`, or `deleted` relation, stable content
 anchors, and an optional ambiguity group. V1 permits only one-to-one-plus-null
 alignments. Repeated or otherwise ambiguous pages retain the ambiguity rather
-than being guessed. Each before/after region is expressed separately in
-top-left PDF points and binds the page box and rotation used for conversion.
+than being guessed, and that ambiguity degrades the semantic, text, and
+structure channels to `partial` (typed `REPEATED_PAGE_AMBIGUITY`) rather than
+being silently skipped under supported coverage. Likewise, a compared page
+whose Extraction IR reports a failed or partial text layer or extraction drops
+the semantic and text channels to `unavailable` or `partial`: a scanned or
+image-only page is never scored as fully text-covered. Each before/after region
+is expressed separately in top-left PDF points and binds the page box and
+rotation used for conversion.
 
 Unknown and unavailable are first-class states. An empty change list is valid
-only when every channel required by the case completed successfully.
+only when every channel required by the case completed successfully; when a
+channel is degraded, `no_reported_changes` is fail-closed and does not report
+green.
 
 ## Deterministic scoring
 

docs/MCP_CONTRACT.md

@@ -141,13 +141,32 @@ parser, observation digest, page count, and pre/post immutability evidence for
 both inputs. It aligns pages without resolving repeated-page ambiguity, emits
 source-bound observations and typed changes across seven coverage channels,
 and keeps widgets under the form channel rather than ordinary annotations.
+Coverage is degraded, never inflated, for content the engine did not actually
+compare: a compared page whose Extraction IR reports a failed text layer or
+extraction drops the semantic and text channels to `unavailable`, a partial
+one drops them to `partial` (a scanned, image-only, or otherwise non-text
+page is therefore never reported as fully covered by the text channels), and
+any repeated/template page the aligner refuses to pair degrades the semantic,
+text, and structure channels to `partial` with a typed `REPEATED_PAGE_AMBIGUITY`
+reason rather than being silently skipped. A checkbox's displayed appearance
+state (`/AS`) is captured on the observation but intentionally not a separate
+compared property, because the parser folds it into the observed field value,
+so a change in it is already reported through that value rather than duplicated.
+This folding does not hold for radio groups: the parser reports the shared
+group value for every widget and does not expose per-widget `/AS`, so a change
+to an individual radio widget's displayed state while the group value is
+unchanged is **not currently detected** — a named coverage gap, not a claim of
+full form-appearance coverage.
 Evidence display regions preserve PDF.js viewport coordinates even when PDF
 content is clipped or lies partly outside the CropBox; consumers clip those
 regions for display rather than rewriting the source coordinates.
 Default-material suppressions are retained as reversible typed decisions;
-forensic mode reports them. A complete result means the requested channels
-were observed under this policy. It never sets an equivalence claim, and an
-empty reported set is not proof that the files are semantically identical.
+forensic mode reports them. A complete result means every requested channel was
+fully observed under this policy; a partial result names the channels and pages
+that were not. `no_reported_changes` is fail-closed: an empty change set is not
+reported as green when any requested channel is less than fully supported. It
+never sets an equivalence claim, and an empty reported set is not proof that the
+files are semantically identical.
 
 Comparison refuses page-cap prefixes, changed sources, malformed PDFs,
 encrypted inputs, output-cap truncation, unknown input fields, and invalid

pdf-toolkit-mcp-share/server/output-schemas.js

@@ -1551,10 +1551,10 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({
     limitations: stringArray,
   }),
   compare_pdfs: object({
-    schema_version: { const: "1.0" },
+    schema_version: { const: "1.1" },
     engine: object({
       name: { const: "pdf-tools.deterministic-comparison" },
-      version: { const: "0.1.0" },
+      version: { const: "0.2.0" },
       parser: object({ name: { const: "pdfjs-dist" }, version: { const: "5.4.624" } }),
       renderer: object({
         name: { const: "native-canvas" },

pdf-toolkit-mcp-share/server/pdf-comparison.js

@@ -4,10 +4,15 @@ import {
   validatePdfObservationSemantics,
 } from "./pdf-observations.js";
 
-export const PDF_COMPARISON_SCHEMA_VERSION = "1.0";
+// Bumped 0.1.0 -> 0.2.0 (schema 1.0 -> 1.1) when coverage began degrading on
+// unreadable IR page status (Bug 1) and repeated-page alignment ambiguity
+// (Bug 2). Coverage is wire-visible, so a change in what `supported` means is a
+// behavior change and earns a deliberate version bump. The schema const in
+// server/output-schemas.js mirrors both strings.
+export const PDF_COMPARISON_SCHEMA_VERSION = "1.1";
 export const PDF_COMPARISON_ENGINE = Object.freeze({
   name: "pdf-tools.deterministic-comparison",
-  version: "0.1.0",
+  version: "0.2.0",
   parser: Object.freeze({ name: "pdfjs-dist", version: "5.4.624" }),
   renderer: Object.freeze({
     name: "native-canvas",
@@ -37,15 +42,22 @@ export const PDF_COMPARISON_CHANNELS = Object.freeze([
 export const PDF_COMPARISON_SIDES = Object.freeze(["before", "after"]);
 
 // Reasons the comparison raises itself, rather than inheriting from a document
-// observation. `TEXT_EXTRACTION_TRUNCATED` is per side because it describes one
-// document's layout extraction; the visual reasons describe the run.
+// observation. The visual reasons and `REPEATED_PAGE_AMBIGUITY` describe the
+// run as a whole (repeated-page ambiguity is a property of the pair, not of one
+// side), so they are not side-prefixed. The sided reasons below each describe
+// one document's own layout extraction.
 const PDF_COMPARISON_OWN_COVERAGE_REASONS = Object.freeze([
   "VISUAL_NOT_REQUESTED",
   "VISUAL_RENDERER_UNAVAILABLE",
   "VISUAL_ALIGNED_PAGE_COMPARISON_SKIPPED",
+  "REPEATED_PAGE_AMBIGUITY",
 ]);
 const PDF_COMPARISON_OWN_SIDED_COVERAGE_REASONS = Object.freeze([
   "TEXT_EXTRACTION_TRUNCATED",
+  "TEXT_LAYER_FAILED",
+  "TEXT_LAYER_PARTIAL",
+  "EXTRACTION_FAILED",
+  "EXTRACTION_PARTIAL",
 ]);
 
 /*
@@ -271,7 +283,16 @@ function coverage(status = "supported", reasonCodes = []) {
   return { status, reason_codes: [...new Set(reasonCodes)].sort(compareCodePoints) };
 }
 
-export function derivePdfComparisonCoverage(documents, includeVisual) {
+// Never let a channel's status improve: unavailable outranks partial, which
+// outranks supported. Every degradation in this function goes through here so
+// the ordering is applied once.
+const COVERAGE_STATUS_RANK = Object.freeze({ supported: 0, partial: 1, unavailable: 2 });
+function raiseCoverage(entry, status, reasonCode) {
+  if (COVERAGE_STATUS_RANK[status] > COVERAGE_STATUS_RANK[entry.status]) entry.status = status;
+  entry.reason_codes.push(reasonCode);
+}
+
+export function derivePdfComparisonCoverage(documents, includeVisual, alignments = []) {
   const result = Object.fromEntries(PDF_COMPARISON_CHANNELS.map(channel => [channel, coverage()]));
   const mapping = {
     metadata: "metadata",
@@ -297,6 +318,41 @@ export function derivePdfComparisonCoverage(documents, includeVisual) {
       }
     }
   }
+  // Bug 1: the truncation check above only catches a text layer the extractor
+  // deliberately cut short. It ignores the IR page's own `text_layer_status`
+  // and `extraction_status`, so a page whose text layer or extraction *failed*
+  // (or was only partial) — a scanned/image-only page, a page the parser could
+  // not read — still reported `supported` semantic and text coverage. The
+  // guiding rule is that we must never claim `supported` for a page the text
+  // channel could not read. A `failed` status means the page produced no
+  // observable text at all, so it contributes `unavailable` (matching how an
+  // unavailable observation channel already propagates here); a `partial`
+  // status contributes `partial`. Structure is left to the observation-derived
+  // `pages` channel above; only the two text-bearing channels degrade here.
+  for (const document of documents) {
+    const side = document.side.toUpperCase();
+    const conditions = [
+      ["unavailable", page => page.text_layer_status === "failed", `${side}_TEXT_LAYER_FAILED`],
+      ["unavailable", page => page.extraction_status === "failed", `${side}_EXTRACTION_FAILED`],
+      ["partial", page => page.text_layer_status === "partial", `${side}_TEXT_LAYER_PARTIAL`],
+      ["partial", page => page.extraction_status === "partial", `${side}_EXTRACTION_PARTIAL`],
+    ];
+    for (const [status, predicate, reasonCode] of conditions) {
+      if (!document.layout.pages.some(predicate)) continue;
+      for (const channel of ["semantic", "text"]) raiseCoverage(result[channel], status, reasonCode);
+    }
+  }
+  // Bug 2: a `repeated_ambiguous` alignment is a repeated/template page the
+  // engine refused to guess a partner for. Such pages are excluded from aligned
+  // text comparison and from inserted/deleted structure detection, so their
+  // content was never compared. Surfacing that as partial coverage on the three
+  // channels whose per-page comparison was skipped keeps every consumer honest;
+  // the ambiguity is a property of the pair, so the reason is not side-prefixed.
+  if (alignments.some(alignment => alignment.match_basis === "repeated_ambiguous")) {
+    for (const channel of ["semantic", "text", "structure"]) {
+      raiseCoverage(result[channel], "partial", "REPEATED_PAGE_AMBIGUITY");
+    }
+  }
   if (!includeVisual) {
     result.visual = coverage("unavailable", ["VISUAL_NOT_REQUESTED"]);
   } else if (documents.some(document => document.renders.some(render => !render?.binary))) {
@@ -765,6 +821,24 @@ function detectFormChanges(state, includeVisual, alignments) {
   const beforeByKey = grouped(beforeItems);
   const afterByKey = grouped(afterItems);
   let changes = 0;
+  // `appearance_state` (the widget `/AS`) is captured on every observation
+  // (output-schemas.js:263) but is NOT compared here, and adding it to this
+  // list would not help. For CHECKBOX widgets the pinned pdfjs 5.4.624 resolves
+  // `fieldValue` from `/AS`, so the observed `value` already reflects the
+  // displayed state (even when a file's `/V` and `/AS` disagree, `value`
+  // follows `/AS`); comparing `appearance_state` too would only duplicate that
+  // change. That redundancy is pinned by test/compare-pdfs-coverage.test.js.
+  //
+  // KNOWN GAP (not redundancy): for RADIO groups pdfjs does not expose
+  // per-widget `appearanceState`, and `fieldValue` is the shared parent `/V`
+  // for every widget. So `appearance_state` falls back to that same shared
+  // `fieldValue`, and a per-widget `/AS` change with an unchanged group `/V`
+  // changes neither `value` nor the fallback `appearance_state` — it is NOT
+  // detected. Adding `appearance_state` to `properties` cannot close this,
+  // because the observed value is the same fallback; detecting it requires
+  // capturing the real per-widget `/AS` in the observation layer. Tracked as a
+  // separate follow-up; named honestly in docs/MCP_CONTRACT.md rather than
+  // claimed as covered.
   const properties = ["type", "value", "default_value", "options", "flags", "widget_page",
     "widget_native_region", "widget_display_region", "rotation"];
   for (const key of new Set([...beforeByKey.keys(), ...afterByKey.keys()])) {
@@ -978,11 +1052,13 @@ export function buildPdfComparison({
 }) {
   validateDocumentInput(before, "before");
   validateDocumentInput(after, "after");
-  const coverageByChannel = derivePdfComparisonCoverage([before, after], includeVisual);
   const alignments = alignComparisonPages(before.layout.pages, after.layout.pages, {
     beforeCompositeAnchors: pageCompositeAnchors(before),
     afterCompositeAnchors: pageCompositeAnchors(after),
   });
+  // Coverage derivation needs the alignments so repeated-page ambiguity (Bug 2)
+  // degrades the affected channels in the engine, before any consumer sees it.
+  const coverageByChannel = derivePdfComparisonCoverage([before, after], includeVisual, alignments);
   const pairs = alignedPagePairs(before, after, alignments);
   const state = createState(mode, before, after);
   detectStructure(state, alignments);
@@ -1261,6 +1337,18 @@ export function validatePdfComparisonSemantics(payload) {
     .filter((value, index, values) => values.indexOf(value) === index).sort(compareCodePoints);
   if (canonical(payload.limitations) !== canonical(expectedLimitations)) semanticError("limitations do not match coverage reasons");
 
+  // Bug 2 invariant: a repeated-ambiguous alignment means the engine did not
+  // compare those pages' semantic, text, or structure content, so none of those
+  // channels may still claim `supported`, and each must carry the typed reason.
+  if (payload.page_alignments.some(alignment => alignment.match_basis === "repeated_ambiguous")) {
+    for (const channel of ["semantic", "text", "structure"]) {
+      const entry = payload.coverage[channel];
+      if (entry.status === "supported" || !entry.reason_codes.includes("REPEATED_PAGE_AMBIGUITY")) {
+        semanticError(`${channel} coverage ignores repeated-page ambiguity`);
+      }
+    }
+  }
+
   const beforePages = new Set();
   const afterPages = new Set();
   for (const alignment of payload.page_alignments) {

scripts/eval-generate-comparison-coverage-fixtures.mjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+
+/*
+ * Deterministic fixtures for the compare_pdfs coverage-honesty bugs. Same
+ * discipline as scripts/eval-generate-comparison-fixtures.mjs: frozen dates,
+ * fixed geometry, no personal data, byte-reproducible output. These feed
+ * test/compare-pdfs-coverage.test.js.
+ *
+ *   coverage-text-before.pdf / coverage-text-after.pdf
+ *     Two clean single-page text-layer documents (a pure text change between
+ *     them). Their IR extraction_status is "complete", so semantic and text
+ *     coverage must stay "supported" — the control for Bug 1.
+ *   coverage-nontext.pdf
+ *     A single page with no text at all, only a filled rectangle. Its IR
+ *     text_layer_status is "empty" and extraction_status is "partial"
+ *     (vector-only, not a text-layer candidate), the scanned/image-only shape
+ *     Bug 1 must degrade semantic/text coverage for.
+ *   coverage-repeated-before.pdf / coverage-repeated-after.pdf
+ *     Two pages of identical text, in identical documents. Every page is a
+ *     repeated/template page the aligner refuses to pair, so the comparison
+ *     compares nothing — Bug 2 must surface that as partial coverage rather
+ *     than trivially-green "no reported changes".
+ *   coverage-appearance-before.pdf / coverage-appearance-after.pdf
+ *     A checkbox whose logical value (/V = Yes) is identical in both, but whose
+ *     displayed appearance state (/AS) is "Yes" before and "Off" after. The two
+ *     differ only in appearance_state — Bug 3 must report a form_field change.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { PDFDocument, PDFName, StandardFonts, rgb } from "pdf-lib";
+
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const DEFAULT_OUTPUT_DIR = path.join(
+  REPO_ROOT,
+  "test",
+  "fixtures",
+  "eval",
+  "comparison",
+  "coverage",
+);
+const FIXED_DATE = new Date("2026-07-21T00:00:00.000Z");
+const PAGE_SIZE = [612, 792];
+
+function freezeMetadata(pdf) {
+  pdf.setTitle("Synthetic comparison coverage fixture");
+  pdf.setAuthor("Open Document Alliance PDF Tools maintainers");
+  pdf.setSubject("Synthetic PDF comparison coverage fixture; contains no personal data");
+  pdf.setKeywords(["synthetic", "comparison", "coverage"]);
+  pdf.setCreator("scripts/eval-generate-comparison-coverage-fixtures.mjs");
+  pdf.setProducer("pdf-lib 1.17.1");
+  pdf.setCreationDate(FIXED_DATE);
+  pdf.setModificationDate(FIXED_DATE);
+}
+
+const SAVE_OPTIONS = Object.freeze({
+  useObjectStreams: false,
+  addDefaultPage: false,
+  updateFieldAppearances: false,
+  objectsPerTick: Number.POSITIVE_INFINITY,
+});
+
+async function buildTextDocument(sentence) {
+  const pdf = await PDFDocument.create();
+  freezeMetadata(pdf);
+  const font = await pdf.embedFont(StandardFonts.Helvetica);
+  const page = pdf.addPage(PAGE_SIZE);
+  page.drawText(sentence, { x: 72, y: 700, size: 16, font, color: rgb(0.08, 0.08, 0.08) });
+  return pdf.save(SAVE_OPTIONS);
+}
+
+async function buildNonTextDocument() {
+  const pdf = await PDFDocument.create();
+  freezeMetadata(pdf);
+  // No text is drawn: only a filled rectangle, so the page has a vector paint
+  // operation and no text items at all.
+  const page = pdf.addPage(PAGE_SIZE);
+  page.drawRectangle({ x: 120, y: 320, width: 372, height: 160, color: rgb(0.2, 0.2, 0.2) });
+  return pdf.save(SAVE_OPTIONS);
+}
+
+async function buildRepeatedDocument() {
+  const pdf = await PDFDocument.create();
+  freezeMetadata(pdf);
+  const font = await pdf.embedFont(StandardFonts.Helvetica);
+  for (let index = 0; index < 2; index += 1) {
+    const page = pdf.addPage(PAGE_SIZE);
+    page.drawText("Repeated template page", { x: 72, y: 700, size: 16, font, color: rgb(0.08, 0.08, 0.08) });
+  }
+  return pdf.save(SAVE_OPTIONS);
+}
+
+async function buildAppearanceDocument(displayedState) {
+  const pdf = await PDFDocument.create();
+  freezeMetadata(pdf);
+  const page = pdf.addPage(PAGE_SIZE);
+  const form = pdf.getForm();
+  const checkbox = form.createCheckBox("Agree");
+  checkbox.addToPage(page, { x: 72, y: 680, width: 24, height: 24 });
+  // Logical value is checked (/V = Yes) in both documents.
+  checkbox.check();
+  form.updateFieldAppearances();
+  // Force only the displayed appearance state (/AS) to differ between the two
+  // documents, leaving /V = Yes unchanged.
+  for (const widget of checkbox.acroField.getWidgets()) {
+    widget.dict.set(PDFName.of("AS"), PDFName.of(displayedState));
+  }
+  return pdf.save({ ...SAVE_OPTIONS, updateFieldAppearances: false });
+}
+
+const VARIANTS = Object.freeze([
+  ["coverage-text-before.pdf", () => buildTextDocument("Coverage sentinel alpha remains text.")],
+  ["coverage-text-after.pdf", () => buildTextDocument("Coverage sentinel bravo remains text.")],
+  ["coverage-nontext.pdf", () => buildNonTextDocument()],
+  ["coverage-repeated-before.pdf", () => buildRepeatedDocument()],
+  ["coverage-repeated-after.pdf", () => buildRepeatedDocument()],
+  ["coverage-appearance-before.pdf", () => buildAppearanceDocument("Yes")],
+  ["coverage-appearance-after.pdf", () => buildAppearanceDocument("Off")],
+]);
+
+export async function generateComparisonCoverageFixtures(outputDir = DEFAULT_OUTPUT_DIR) {
+  await fs.mkdir(outputDir, { recursive: true });
+  const generated = [];
+  for (const [filename, build] of VARIANTS) {
+    const bytes = await build();
+    await fs.writeFile(path.join(outputDir, filename), bytes);
+    generated.push(filename);
+  }
+  return generated;
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const outputIndex = process.argv.indexOf("--output-dir");
+  const outputDir = outputIndex >= 0
+    ? path.resolve(process.argv[outputIndex + 1])
+    : DEFAULT_OUTPUT_DIR;
+  const generated = await generateComparisonCoverageFixtures(outputDir);
+  process.stdout.write(`${generated.map(name => path.join(outputDir, name)).join("\n")}\n`);
+}

server/output-schemas.js

@@ -1551,10 +1551,10 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({
     limitations: stringArray,
   }),
   compare_pdfs: object({
-    schema_version: { const: "1.0" },
+    schema_version: { const: "1.1" },
     engine: object({
       name: { const: "pdf-tools.deterministic-comparison" },
-      version: { const: "0.1.0" },
+      version: { const: "0.2.0" },
       parser: object({ name: { const: "pdfjs-dist" }, version: { const: "5.4.624" } }),
       renderer: object({
         name: { const: "native-canvas" },