From 6f170d8e54f09e1760ff3f756f58df1796f05a62 Mon Sep 17 00:00:00 2001
From: JP <jp@JPs-Pursuit-MacBook-Air.local>
Date: Tue, 19 May 2026 11:26:14 -0400
Subject: [PATCH] feat(pebble): polish export markdown + pin format with tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Tightens the prospect-research export so partial runs, long claims, and
heterogeneous confidence values render cleanly, and pins the document
shape with structural tests so it can't regress silently.

What changed:
- pebble/formatting.py (new): escape_md_table_cell, truncate_for_cell,
  normalize_confidence — pure helpers, no I/O.
- pebble/export.py: empty-state section when claims+summary are blank;
  confidence rendered as "Medium (0.62)" / "High" / "Unknown" instead of
  raw values; claim text > 240 chars truncates with a footnote pointing
  to a "Full claim text" detail section; sources ordered by first claim
  appearance with "(N claims)" suffix and source_title links when present.
- pebble/tests/test_export.py (new): 19 structural assertions on header,
  empty state, summary, claims table, truncation, confidence, sources.
- pebble/tests/test_formatting.py (new): 26 helper unit tests.

Signature of render_profile_markdown is unchanged. PDF path is untouched
beyond a smoke check.

Test counts: pebble 290 -> 335 (+45 new), no regressions.
financial_forecasting 827 passed/22 skipped, unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 pebble/export.py                | 111 +++++++++----
 pebble/formatting.py            |  52 ++++++
 pebble/tests/test_export.py     | 285 ++++++++++++++++++++++++++++++++
 pebble/tests/test_formatting.py |  72 ++++++++
 4 files changed, 492 insertions(+), 28 deletions(-)
 create mode 100644 pebble/formatting.py
 create mode 100644 pebble/tests/test_export.py
 create mode 100644 pebble/tests/test_formatting.py

diff --git a/pebble/export.py b/pebble/export.py
index 084aa7e3..31333df9 100644
--- a/pebble/export.py
+++ b/pebble/export.py
@@ -2,23 +2,28 @@
 
 from datetime import datetime, timezone
 
+from .formatting import escape_md_table_cell, normalize_confidence, truncate_for_cell
+
+_CLAIM_TEXT_LIMIT = 240
+
 
 def render_profile_markdown(profile: dict, prospect_name: str, prospect_org: str) -> str:
     """Render a research profile as a Markdown document.
 
     Args:
-        profile: Profile dict with claims, summary, confidence_score.
+        profile: Profile dict with claims, summary, confidence_score, and
+            optionally `partial` (bool) and `partial_reason` (str).
         prospect_name: Display name of the prospect.
-        prospect_org: Organization name.
+        prospect_org: Organization name (may be blank).
 
     Returns:
-        Markdown string.
+        Markdown string. Stable structure across runs — pinned by tests in
+        `pebble/tests/test_export.py`.
     """
     lines: list[str] = []
     generated = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
-    confidence = profile.get("confidence_score", "unknown")
+    confidence = normalize_confidence(profile.get("confidence_score"))
 
-    # Header
     lines.append(f"# Prospect Research: {prospect_name}")
     if prospect_org:
         lines.append(f"**Organization:** {prospect_org}")
@@ -28,46 +33,99 @@ def render_profile_markdown(profile: dict, prospect_name: str, prospect_org: str
         lines.append("**Status:** Partial (some agents failed)")
     lines.append("")
 
-    # Summary
-    summary = profile.get("summary", "")
+    summary = (profile.get("summary") or "").strip()
+    claims = profile.get("claims") or []
+
+    if not summary and not claims:
+        lines.append("## No information gathered")
+        lines.append("")
+        reason = (profile.get("partial_reason") or "").strip()
+        if reason:
+            lines.append(f"This research run did not produce any claims. Reason: {reason}")
+        else:
+            lines.append(
+                "This research run did not produce any claims. The agents may have "
+                "timed out or found no public information for this prospect."
+            )
+        lines.append("")
+        return "\n".join(lines)
+
     if summary:
         lines.append("## Summary")
         lines.append("")
         lines.append(summary)
         lines.append("")
 
-    # Claims table
-    claims = profile.get("claims", [])
+    truncated_claims: list[tuple[int, str]] = []
     if claims:
         lines.append(f"## Claims ({len(claims)})")
         lines.append("")
         lines.append("| # | Claim | Source | Confidence | Status |")
         lines.append("|---|-------|--------|------------|--------|")
         for i, claim in enumerate(claims, 1):
-            text = claim.get("text", "").replace("|", "\\|").replace("\n", " ")
+            raw_text = claim.get("text", "")
+            short, was_truncated = truncate_for_cell(raw_text, _CLAIM_TEXT_LIMIT)
+            cell_text = escape_md_table_cell(short)
+            if was_truncated:
+                cell_text = f"{cell_text} [^c{i}]"
+                truncated_claims.append((i, raw_text))
             url = claim.get("source_url", "")
             source_link = f"[Link]({url})" if url else "-"
-            conf = claim.get("confidence", "medium")
-            temporal = claim.get("temporal_status", "")
-            if temporal and temporal != "unknown":
-                status = temporal
-            else:
-                status = "-"
-            lines.append(f"| {i} | {text} | {source_link} | {conf} | {status} |")
+            conf_cell = normalize_confidence(claim.get("confidence", "medium"))
+            temporal = (claim.get("temporal_status") or "").strip()
+            status = temporal if temporal and temporal != "unknown" else "-"
+            lines.append(f"| {i} | {cell_text} | {source_link} | {conf_cell} | {status} |")
         lines.append("")
 
-    # Sources list (unique URLs)
-    urls = sorted({c.get("source_url", "") for c in claims if c.get("source_url")})
-    if urls:
-        lines.append("## Sources")
-        lines.append("")
-        for url in urls:
-            lines.append(f"- {url}")
-        lines.append("")
+        if truncated_claims:
+            lines.append("### Full claim text")
+            lines.append("")
+            for idx, full_text in truncated_claims:
+                normalized = full_text.replace("\r\n", "\n").strip()
+                lines.append(f"[^c{idx}]: {normalized}")
+            lines.append("")
+
+    source_lines = _render_sources_section(claims)
+    if source_lines:
+        lines.extend(source_lines)
 
     return "\n".join(lines)
 
 
+def _render_sources_section(claims: list[dict]) -> list[str]:
+    """Build the `## Sources` section ordered by first-appearance in claims.
+
+    Each URL appears once; multi-claim URLs get a `(N claims)` suffix.
+    Uses `source_title` when present, falling back to the bare URL.
+    """
+    counts: dict[str, int] = {}
+    titles: dict[str, str] = {}
+    order: list[str] = []
+    for claim in claims:
+        url = (claim.get("source_url") or "").strip()
+        if not url:
+            continue
+        if url not in counts:
+            counts[url] = 0
+            order.append(url)
+        counts[url] += 1
+        title = (claim.get("source_title") or "").strip()
+        if title and url not in titles:
+            titles[url] = title
+
+    if not order:
+        return []
+
+    out: list[str] = ["## Sources", ""]
+    for url in order:
+        title = titles.get(url)
+        label = f"[{title}]({url})" if title else url
+        suffix = f" ({counts[url]} claims)" if counts[url] > 1 else ""
+        out.append(f"- {label}{suffix}")
+    out.append("")
+    return out
+
+
 def render_profile_pdf(markdown_text: str) -> bytes:
     """Best-effort PDF rendering from markdown text.
 
@@ -78,7 +136,6 @@ def render_profile_pdf(markdown_text: str) -> bytes:
         import markdown as md
 
         html = md.markdown(markdown_text, extensions=["tables"])
-        # Wrap in minimal HTML for readability
         full_html = f"""<!DOCTYPE html>
 <html><head><meta charset="utf-8">
 <style>
@@ -92,8 +149,6 @@ def render_profile_pdf(markdown_text: str) -> bytes:
             from weasyprint import HTML
             return HTML(string=full_html).write_pdf()
         except ImportError:
-            # weasyprint not installed -- return markdown as bytes
             return markdown_text.encode("utf-8")
     except ImportError:
-        # markdown library not installed -- return raw text
         return markdown_text.encode("utf-8")
diff --git a/pebble/formatting.py b/pebble/formatting.py
new file mode 100644
index 00000000..7e8bd0de
--- /dev/null
+++ b/pebble/formatting.py
@@ -0,0 +1,52 @@
+"""Shared formatting helpers for Pebble output renderers.
+
+Pure functions, no I/O. Used by `pebble.export` (legacy markdown profile) and
+intended for adoption by `pebble.orchestrator.renderer` in a follow-up.
+"""
+
+from __future__ import annotations
+
+
+def escape_md_table_cell(text: str) -> str:
+    """Escape a string for safe embedding in a Markdown table cell.
+
+    Markdown tables split rows on unescaped `|`, and any newline ends the row.
+    """
+    return text.replace("\\", "\\\\").replace("|", "\\|").replace("\n", " ").replace("\r", " ")
+
+
+def truncate_for_cell(text: str, limit: int = 240) -> tuple[str, bool]:
+    """Truncate `text` to `limit` characters, appending an ellipsis if shortened.
+
+    Returns `(maybe_truncated, was_truncated)`. The ellipsis counts toward the
+    limit so the returned string is never longer than `limit`.
+    """
+    if len(text) <= limit:
+        return text, False
+    return text[: max(0, limit - 1)] + "…", True
+
+
+def normalize_confidence(value: object) -> str:
+    """Render a heterogeneous confidence value as a human-readable band.
+
+    Accepts:
+      - float-like in [0.0, 1.0] -> `"High (0.84)"`, `"Medium (0.62)"`, `"Low (0.21)"`
+      - string `"high"` / `"medium"` / `"low"` (case-insensitive) -> `"High"` etc.
+      - anything else (None, empty, unrecognized) -> `"Unknown"`
+    """
+    if isinstance(value, bool):
+        return "Unknown"
+    if isinstance(value, (int, float)):
+        try:
+            score = float(value)
+        except (TypeError, ValueError):
+            return "Unknown"
+        if 0.0 <= score <= 1.0:
+            band = "High" if score >= 0.7 else "Medium" if score >= 0.4 else "Low"
+            return f"{band} ({score:.2f})"
+        return "Unknown"
+    if isinstance(value, str):
+        key = value.strip().lower()
+        if key in {"high", "medium", "low"}:
+            return key.capitalize()
+    return "Unknown"
diff --git a/pebble/tests/test_export.py b/pebble/tests/test_export.py
new file mode 100644
index 00000000..7ff83f96
--- /dev/null
+++ b/pebble/tests/test_export.py
@@ -0,0 +1,285 @@
+"""Structural tests for pebble.export.render_profile_markdown.
+
+These tests pin the output **format** so regressions are caught loudly.
+They assert on document structure (sections, ordering, table shape) rather
+than substring presence, so reordering or rewording a section will fail.
+"""
+
+import re
+
+from pebble.export import render_profile_markdown, render_profile_pdf
+
+
+def _claim(text: str, **overrides) -> dict:
+    """Helper: build a claim dict with sensible defaults."""
+    base = {
+        "text": text,
+        "source_url": "",
+        "confidence": "medium",
+        "temporal_status": "",
+    }
+    base.update(overrides)
+    return base
+
+
+def _section_indices(md: str) -> dict[str, int]:
+    """Return a map of section heading text -> line index for `##` headings."""
+    out: dict[str, int] = {}
+    for i, line in enumerate(md.splitlines()):
+        if line.startswith("## "):
+            out[line[3:].strip()] = i
+    return out
+
+
+# --- Header --------------------------------------------------------------
+
+
+def test_header_section_always_present():
+    md = render_profile_markdown(
+        {"summary": "x", "claims": [], "confidence_score": "high"},
+        "Jane Doe",
+        "Acme Corp",
+    )
+    lines = md.splitlines()
+    assert lines[0] == "# Prospect Research: Jane Doe"
+    assert lines[1] == "**Organization:** Acme Corp"
+    assert lines[2].startswith("**Generated:** ")
+    # Timestamp shape: YYYY-MM-DD HH:MM UTC
+    assert re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2} UTC", lines[2])
+    assert lines[3] == "**Confidence:** High"
+
+
+def test_header_omits_organization_when_blank():
+    md = render_profile_markdown(
+        {"summary": "x", "claims": [], "confidence_score": "high"},
+        "Jane Doe",
+        "",
+    )
+    lines = md.splitlines()
+    assert lines[0] == "# Prospect Research: Jane Doe"
+    assert lines[1].startswith("**Generated:** ")
+
+
+def test_partial_status_line_only_when_partial():
+    md_partial = render_profile_markdown(
+        {"summary": "x", "claims": [], "partial": True}, "J", "A"
+    )
+    md_complete = render_profile_markdown({"summary": "x", "claims": []}, "J", "A")
+    assert "**Status:** Partial (some agents failed)" in md_partial
+    assert "**Status:**" not in md_complete
+
+
+# --- Empty-state --------------------------------------------------------
+
+
+def test_empty_state_when_no_data():
+    md = render_profile_markdown(
+        {"summary": "", "claims": [], "partial": True}, "Jane", "Acme"
+    )
+    sections = _section_indices(md)
+    assert "No information gathered" in sections
+    assert "Summary" not in sections
+    assert "Claims (0)" not in sections
+    assert "Sources" not in sections
+
+
+def test_empty_state_uses_partial_reason_when_provided():
+    md = render_profile_markdown(
+        {"summary": "", "claims": [], "partial_reason": "All agents timed out"},
+        "J",
+        "A",
+    )
+    assert "All agents timed out" in md
+
+
+def test_empty_state_has_generic_fallback_without_reason():
+    md = render_profile_markdown({"summary": "", "claims": []}, "J", "A")
+    assert "did not produce any claims" in md
+
+
+def test_summary_alone_skips_empty_state():
+    md = render_profile_markdown(
+        {"summary": "Some prose.", "claims": []}, "J", "A"
+    )
+    sections = _section_indices(md)
+    assert "Summary" in sections
+    assert "No information gathered" not in sections
+
+
+# --- Summary ------------------------------------------------------------
+
+
+def test_summary_block_emitted_when_present():
+    md = render_profile_markdown(
+        {"summary": "Founder of Acme.", "claims": [_claim("c1")]}, "J", "A"
+    )
+    lines = md.splitlines()
+    summary_idx = lines.index("## Summary")
+    assert lines[summary_idx + 1] == ""
+    assert lines[summary_idx + 2] == "Founder of Acme."
+
+
+def test_summary_block_omitted_when_blank():
+    md = render_profile_markdown(
+        {"summary": "   ", "claims": [_claim("c1")]}, "J", "A"
+    )
+    assert "## Summary" not in md
+
+
+# --- Claims table -------------------------------------------------------
+
+
+def test_claims_table_structure():
+    md = render_profile_markdown(
+        {
+            "summary": "",
+            "claims": [
+                _claim("First claim", source_url="https://a.example"),
+                _claim("Second claim", source_url="https://b.example"),
+            ],
+        },
+        "J",
+        "A",
+    )
+    lines = md.splitlines()
+    header_idx = lines.index("## Claims (2)")
+    # Section: heading, blank, table-header, separator, 2 data rows, blank.
+    assert lines[header_idx + 1] == ""
+    assert lines[header_idx + 2] == "| # | Claim | Source | Confidence | Status |"
+    assert lines[header_idx + 3] == "|---|-------|--------|------------|--------|"
+    assert lines[header_idx + 4].startswith("| 1 | First claim |")
+    assert lines[header_idx + 5].startswith("| 2 | Second claim |")
+    # Each data row has exactly the right column count (6 pipes for 5 cols).
+    for row_idx in (header_idx + 4, header_idx + 5):
+        assert lines[row_idx].count("|") - lines[row_idx].count("\\|") == 6
+
+
+def test_claim_text_escapes_pipes_and_newlines():
+    md = render_profile_markdown(
+        {"summary": "", "claims": [_claim("a|b\nc")]},
+        "J",
+        "A",
+    )
+    # In the table cell the pipe is escaped and the newline becomes a space.
+    assert "| a\\|b c |" in md
+
+
+def test_claim_truncation_above_limit():
+    long_text = "x" * 300
+    md = render_profile_markdown(
+        {"summary": "", "claims": [_claim(long_text)]},
+        "J",
+        "A",
+    )
+    lines = md.splitlines()
+    # Truncated text in table cell ends with ellipsis + footnote anchor.
+    row = next(ln for ln in lines if ln.startswith("| 1 |"))
+    assert "…" in row
+    assert "[^c1]" in row
+    # Detail section follows the table.
+    detail_idx = lines.index("### Full claim text")
+    footnote = lines[detail_idx + 2]
+    assert footnote.startswith("[^c1]: ")
+    assert footnote.endswith("x" * 50)  # full text restored
+
+
+def test_claim_at_limit_not_truncated():
+    text = "y" * 240
+    md = render_profile_markdown(
+        {"summary": "", "claims": [_claim(text)]},
+        "J",
+        "A",
+    )
+    assert "### Full claim text" not in md
+    assert "[^c1]" not in md
+
+
+def test_claim_confidence_normalized():
+    md = render_profile_markdown(
+        {
+            "summary": "",
+            "claims": [
+                _claim("c1", confidence=0.84),
+                _claim("c2", confidence="low"),
+                _claim("c3", confidence=None),
+            ],
+        },
+        "J",
+        "A",
+    )
+    lines = md.splitlines()
+    row1 = next(ln for ln in lines if ln.startswith("| 1 |"))
+    row2 = next(ln for ln in lines if ln.startswith("| 2 |"))
+    row3 = next(ln for ln in lines if ln.startswith("| 3 |"))
+    assert "High (0.84)" in row1
+    assert "Low" in row2 and "(0." not in row2
+    assert "Unknown" in row3
+
+
+def test_top_level_confidence_normalized():
+    md_float = render_profile_markdown(
+        {"summary": "x", "claims": [], "confidence_score": 0.62}, "J", "A"
+    )
+    md_missing = render_profile_markdown({"summary": "x", "claims": []}, "J", "A")
+    assert "**Confidence:** Medium (0.62)" in md_float
+    assert "**Confidence:** Unknown" in md_missing
+
+
+# --- Sources ------------------------------------------------------------
+
+
+def test_sources_section_ordering_by_first_appearance():
+    md = render_profile_markdown(
+        {
+            "summary": "",
+            "claims": [
+                _claim("c1", source_url="https://b.example"),
+                _claim("c2", source_url="https://a.example"),
+                _claim("c3", source_url="https://b.example"),
+            ],
+        },
+        "J",
+        "A",
+    )
+    lines = md.splitlines()
+    sources_idx = lines.index("## Sources")
+    # First-appearance order: b before a, with b carrying a multi-claim suffix.
+    assert lines[sources_idx + 2] == "- https://b.example (2 claims)"
+    assert lines[sources_idx + 3] == "- https://a.example"
+
+
+def test_sources_section_uses_titles_when_present():
+    md = render_profile_markdown(
+        {
+            "summary": "",
+            "claims": [
+                _claim(
+                    "c1",
+                    source_url="https://a.example",
+                    source_title="A Example Page",
+                ),
+                _claim("c2", source_url="https://b.example"),
+            ],
+        },
+        "J",
+        "A",
+    )
+    assert "- [A Example Page](https://a.example)" in md
+    assert "- https://b.example" in md
+
+
+def test_sources_section_omitted_when_no_urls():
+    md = render_profile_markdown(
+        {"summary": "x", "claims": [_claim("c1")]}, "J", "A"
+    )
+    assert "## Sources" not in md
+
+
+# --- PDF smoke ----------------------------------------------------------
+
+
+def test_render_profile_pdf_smoke_returns_bytes():
+    md = "# Hello"
+    out = render_profile_pdf(md)
+    assert isinstance(out, bytes)
+    assert len(out) > 0
diff --git a/pebble/tests/test_formatting.py b/pebble/tests/test_formatting.py
new file mode 100644
index 00000000..2703cb20
--- /dev/null
+++ b/pebble/tests/test_formatting.py
@@ -0,0 +1,72 @@
+"""Unit tests for pebble.formatting helpers."""
+
+import pytest
+
+from pebble.formatting import escape_md_table_cell, normalize_confidence, truncate_for_cell
+
+
+class TestEscapeMdTableCell:
+    def test_pipe_is_escaped(self):
+        assert escape_md_table_cell("a|b") == "a\\|b"
+
+    def test_newlines_become_spaces(self):
+        assert escape_md_table_cell("line1\nline2\r\nline3") == "line1 line2  line3"
+
+    def test_backslash_is_doubled_before_pipe_escape(self):
+        # Without doubling, a literal backslash before a pipe could be
+        # interpreted as the table-cell escape itself.
+        assert escape_md_table_cell("a\\b|c") == "a\\\\b\\|c"
+
+    def test_empty_string_passthrough(self):
+        assert escape_md_table_cell("") == ""
+
+
+class TestTruncateForCell:
+    def test_returns_unchanged_below_limit(self):
+        text, truncated = truncate_for_cell("short", 240)
+        assert text == "short"
+        assert truncated is False
+
+    def test_truncates_above_limit_with_ellipsis(self):
+        text, truncated = truncate_for_cell("a" * 300, 240)
+        assert truncated is True
+        assert text.endswith("…")
+        assert len(text) == 240
+
+    def test_at_limit_not_truncated(self):
+        text, truncated = truncate_for_cell("x" * 240, 240)
+        assert truncated is False
+        assert text == "x" * 240
+
+    def test_one_over_limit_truncated(self):
+        text, truncated = truncate_for_cell("x" * 241, 240)
+        assert truncated is True
+        assert len(text) == 240
+
+
+class TestNormalizeConfidence:
+    @pytest.mark.parametrize(
+        "value,expected",
+        [
+            (0.84, "High (0.84)"),
+            (0.70, "High (0.70)"),
+            (0.62, "Medium (0.62)"),
+            (0.40, "Medium (0.40)"),
+            (0.21, "Low (0.21)"),
+            (0.0, "Low (0.00)"),
+            (1.0, "High (1.00)"),
+            ("high", "High"),
+            ("MEDIUM", "Medium"),
+            (" low ", "Low"),
+            (None, "Unknown"),
+            ("", "Unknown"),
+            ("unknown", "Unknown"),
+            ("garbage", "Unknown"),
+            (1.5, "Unknown"),
+            (-0.1, "Unknown"),
+            (True, "Unknown"),
+            (False, "Unknown"),
+        ],
+    )
+    def test_normalization(self, value, expected):
+        assert normalize_confidence(value) == expected