pdrbot.py

#!/./.venv/bin/python
"""
PDRBot - Daily Criminal Opinions Scraper

Downloads criminal law opinions from all 14 Texas Courts of Appeals
for the previous business day and stores them in data/ with date-based organization.
Runs Tuesday-Saturday at 12:01 AM to collect opinions from the previous day.
"""

import requests
from bs4 import BeautifulSoup
import os
import shutil
import re
import sys
import json
import time
import logging
import sqlite3
import smtplib
import imaplib
import email
import hashlib
from contextlib import contextmanager
from datetime import datetime, timedelta
from urllib.parse import urljoin, quote
from pathlib import Path
from PyPDF2 import PdfReader, PdfWriter
from dotenv import load_dotenv
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
from email.utils import parsedate_to_datetime
from reportlab.lib.pagesizes import letter
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak, Table, TableStyle
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.lib.units import inch
from reportlab.lib import colors
from reportlab.lib.enums import TA_LEFT, TA_CENTER, TA_JUSTIFY

# Add mwb_common to path
sys.path.insert(0, os.path.expanduser('~/github/mwb_common'))
from mwb_claude import call_claude_with_retry, get_current_model
import case_styles  # noqa: E402  -- local sibling module
import defense_wins  # noqa: E402  -- local sibling module
import brief_harvest  # noqa: E402  -- local sibling module

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)



ANALYSIS_JSON_SCHEMA = {
    "type": "object",
    "properties": {
        "appellant_name": {
            "type": "string",
            "description": "Full name of the appellant as identified on page 1 of the opinion."
        },
        "case_numbers": {
            "type": "array",
            "items": {"type": "string"},
            "description": "All COA cause numbers for the opinion(s) analyzed."
        },
        "issues": {
            "type": "array",
            "items": {
                "type": "object",
                "properties": {
                    "headline": {"type": "string", "description": "One sentence, max 15 words, why the issue is PDR-worthy."},
                    "issue_description": {"type": "string", "description": "The novel or controversial legal question."},
                    "discussion": {"type": "string", "description": "Quotes or specific references from the opinion showing novelty or controversy."},
                    "authority_conflicts": {"type": "string", "description": "Which courts take each side; whether the split is acknowledged; recency; CCA status. Empty string if none."},
                    "relevant_precedent": {"type": "string", "description": "Other cases, statutes, or principles cited in the opinion that frame the controversy. Empty string if none."},
                    "pdr_score": {"type": "integer", "minimum": 1, "maximum": 10, "description": "1-3 routine; 4-5 arguable; 6-7 unsettled; 8-10 split/unresolved-constitutional/logical-flaw."},
                    "matched_open_questions": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": {
                                "id": {"type": "integer", "description": "Catalog open-question id from cca-judges catalog."},
                                "explanation": {"type": "string", "description": "One-sentence explanation of how the COA opinion implicates the cataloged question."}
                            },
                            "required": ["id", "explanation"]
                        },
                        "description": "Optional. Cataloged CCA-flagged open questions whose resolution the opinion implicates."
                    }
                },
                "required": ["headline", "issue_description", "discussion", "pdr_score"]
            }
        },
        "issue_count": {"type": "integer", "minimum": 0},
        "state_is_appellant": {
            "type": "boolean",
            "description": "True when the State of Texas is the appellant — i.e., the State, not the defendant, brought the appeal. False for ordinary defense appeals and for original proceedings styled \"In re ...\"."
        },
        "disposition": {
            "type": "string",
            "enum": [
                "affirmed",
                "reversed",
                "reversed_in_part",
                "modified_and_affirmed",
                "vacated",
                "remanded",
                "dismissed",
                "petition_granted",
                "petition_denied",
                "abated",
                "other"
            ],
            "description": "Outcome the court of appeals ordered for the appellant. Use \"reversed\" for any judgment fully set aside (reversed-and-remanded, reversed-and-rendered, reversed-and-acquitted). Use \"reversed_in_part\" only when the court reversed some counts/issues and affirmed others. Use \"modified_and_affirmed\" when the court modified the judgment (e.g., struck a fee or court cost) but affirmed the conviction. Use \"vacated\" for a vacatur not styled as a reversal. Use \"dismissed\" for jurisdictional dismissals (untimely notice, Anders that becomes a dismissal, plea-bargain waiver dismissal). For mandamus or habeas original proceedings use \"petition_granted\" or \"petition_denied\". \"abated\" for abatements. \"other\" only when no listed value fits."
        }
    },
    "required": ["appellant_name", "case_numbers", "issues", "issue_count"]
}


def _is_defense_win(disposition, state_is_appellant):
    """Defense wins when the State appealed and lost (affirmance), or
    when the defense appealed and won (any flavor of reversal/vacatur).
    Returns False for unknown or mixed dispositions; the stamp is
    intentionally conservative."""
    if disposition is None or state_is_appellant is None:
        return False
    if state_is_appellant:
        return disposition == "affirmed"
    return disposition in ("reversed", "reversed_in_part", "vacated")


def render_analysis_prose(text):
    """Render analysis_text for display. Accepts either JSON (new) or prose
    (legacy/stub). Returns a TERSE REPORT-formatted string."""
    if not text:
        return text
    stripped = text.strip()
    if not (stripped.startswith("{") and stripped.endswith("}")):
        return text
    try:
        data = json.loads(stripped)
    except ValueError:
        return text
    lines = []
    count = data.get("issue_count", 0)
    header = "TERSE REPORT: INTERESTING LEGAL ISSUES" if count else "TERSE REPORT: NO INTERESTING ISSUES"
    lines.append(header)
    lines.append("")
    name = data.get("appellant_name", "")
    nums = ", ".join(data.get("case_numbers", []) or [])
    if name or nums:
        lines.append(f"Appellant: {name}" + (f"  Case No.: {nums}" if nums else ""))
        lines.append("")
    disp = data.get("disposition")
    state_app = data.get("state_is_appellant")
    if disp or state_app is not None:
        bits = []
        if disp:
            bits.append(f"Disposition: {disp}")
        if state_app is True:
            bits.append("Appellant: State")
        elif state_app is False:
            bits.append("Appellant: defendant")
        if _is_defense_win(disp, state_app):
            bits.append("DEFENSE WIN")
        lines.append("  ·  ".join(bits))
        lines.append("")
    for i, issue in enumerate(data.get("issues", []) or [], 1):
        lines.append(f"Issue {i}:")
        lines.append(f"  Headline: {issue.get('headline', '')}")
        lines.append(f"  Description: {issue.get('issue_description', '')}")
        lines.append(f"  Discussion: {issue.get('discussion', '')}")
        if issue.get("authority_conflicts"):
            lines.append(f"  Authority Conflicts: {issue['authority_conflicts']}")
        if issue.get("relevant_precedent"):
            lines.append(f"  Relevant Precedent: {issue['relevant_precedent']}")
        matches = issue.get("matched_open_questions") or []
        if matches:
            lines.append("  Matched Open Questions:")
            for mq in matches:
                qid = mq.get("id", "?")
                expl = mq.get("explanation", "")
                lines.append(f"    [{qid}] {expl}")
        lines.append(f"  PDR Score: {issue.get('pdr_score', 0)}")
        lines.append("")
    lines.append(f"Issue Count: {count}")
    return "\n".join(lines)


def parse_analysis_json(text):
    """Return the structured dict if text is JSON; None if it is prose or invalid."""
    if not text:
        return None
    stripped = text.strip()
    if not (stripped.startswith("{") and stripped.endswith("}")):
        return None
    try:
        return json.loads(stripped)
    except ValueError:
        return None

class PDRBot:
    def __init__(self, data_dir="data"):
        # Load environment variables
        load_dotenv()
        
        self.base_url = os.getenv('BASE_URL', "https://search.txcourts.gov/")
        self.data_dir = data_dir
        self.db_path = os.path.join(data_dir, os.getenv('DB_NAME', "pdrbot.db"))
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': os.getenv('USER_AGENT', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')
        })
        
        # Claude CLI configuration
        self.analysis_enabled = os.getenv('ANALYSIS_ENABLED', 'true').lower() == 'true'
        self.claude_model = get_current_model()
        logger.info(f"Using claude CLI with Max subscription for analysis (model: {self.claude_model})")
        
        # Load analysis prompt
        self.analysis_prompt = self.load_analysis_prompt()
        
        # Configuration from environment
        self.request_timeout = int(os.getenv('REQUEST_TIMEOUT', '30'))
        self.max_retries = int(os.getenv('MAX_RETRIES', '3'))
        self.download_delay = int(os.getenv('DOWNLOAD_DELAY', '1'))
        
        # Email configuration
        self.email_enabled = os.getenv('EMAIL_ENABLED', 'false').lower() == 'true'
        self.email_smtp_host = os.getenv('EMAIL_SMTP_HOST', 'smtp.gmail.com')
        self.email_smtp_port = int(os.getenv('EMAIL_SMTP_PORT', '587'))
        self.email_from = os.getenv('EMAIL_FROM')
        self.email_auth_user = os.getenv('EMAIL_AUTH_USER', self.email_from)  # Default to FROM if not specified
        self.email_password = os.getenv('EMAIL_PASSWORD')
        # Support multiple email recipients (comma-separated)
        email_to_raw = os.getenv('EMAIL_TO')
        if email_to_raw:
            self.email_to = [email.strip() for email in email_to_raw.split(',')]
        else:
            self.email_to = []
        self.email_subject_prefix = os.getenv('EMAIL_SUBJECT_PREFIX', 'PDRBot Daily Report')
        
        # Subscription management configuration
        self.subscription_email = os.getenv('SUBSCRIPTION_EMAIL')
        self.subscription_auth_user = os.getenv('SUBSCRIPTION_AUTH_USER', self.subscription_email)
        self.subscription_password = os.getenv('SUBSCRIPTION_PASSWORD')
        self.subscription_imap_host = os.getenv('SUBSCRIPTION_IMAP_HOST', 'imap.fastmail.com')
        self.subscription_imap_port = int(os.getenv('SUBSCRIPTION_IMAP_PORT', '993'))
        self.members_file = os.getenv('MEMBERS_FILE', 'data/members.json')
        self.last_check_file = os.getenv('LAST_CHECK_FILE', 'data/last_subscription_check.txt')
        
        # Ensure data directory exists
        os.makedirs(data_dir, exist_ok=True)
        
        # Initialize database
        self.init_database()
    
    def init_database(self):
        """Initialize SQLite database with required tables"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        # Create opinions table
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS opinions (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                case_number TEXT NOT NULL,
                court TEXT NOT NULL,
                opinion_date DATE NOT NULL,
                opinion_type TEXT NOT NULL,
                justice_name TEXT,
                filename TEXT NOT NULL,
                file_path TEXT NOT NULL,
                case_url TEXT NOT NULL,
                pdf_url TEXT,
                download_timestamp TIMESTAMP DEFAULT (datetime('now', 'localtime')),
                UNIQUE(case_number, opinion_type, justice_name)
            )
        ''')
        
        # Create daily_runs table to track execution
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS daily_runs (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                run_date DATE NOT NULL,
                target_date DATE NOT NULL,
                total_courts_checked INTEGER DEFAULT 0,
                total_cases_found INTEGER DEFAULT 0,
                total_files_downloaded INTEGER DEFAULT 0,
                run_timestamp TIMESTAMP DEFAULT (datetime('now', 'localtime')),
                status TEXT DEFAULT 'running',
                error_message TEXT
            )
        ''')
        
        # Create court_rollover table to track courts with no opinions for next day checking
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS court_rollover (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                court_number INTEGER NOT NULL,
                original_date DATE NOT NULL,
                created_timestamp TIMESTAMP DEFAULT (datetime('now', 'localtime')),
                UNIQUE(court_number, original_date)
            )
        ''')
        
        # Create analysis table
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS analysis (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                opinion_id INTEGER NOT NULL,
                case_number TEXT NOT NULL,
                court TEXT NOT NULL,
                opinion_date DATE NOT NULL,
                analysis_text TEXT NOT NULL,
                has_interesting_issues BOOLEAN NOT NULL DEFAULT 0,
                issue_count INTEGER DEFAULT 0,
                analysis_timestamp TIMESTAMP DEFAULT (datetime('now', 'localtime')),
                claude_model TEXT NOT NULL,
                FOREIGN KEY (opinion_id) REFERENCES opinions (id),
                UNIQUE(opinion_id)
            )
        ''')
        
        # Create representatives table
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS representatives (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                case_number TEXT NOT NULL,
                court TEXT NOT NULL,
                opinion_date DATE NOT NULL,
                party_name TEXT NOT NULL,
                party_type TEXT NOT NULL,
                representative_names TEXT NOT NULL,
                scrape_timestamp TIMESTAMP DEFAULT (datetime('now', 'localtime')),
                UNIQUE(case_number, court, party_name)
            )
        ''')

        # Add columns if they don't exist (for existing databases)
        for col, table in [('pdf_url', 'opinions'), ('pdr_score', 'analysis')]:
            try:
                cursor.execute(f'ALTER TABLE {table} ADD COLUMN {col} {"TEXT" if col == "pdf_url" else "INTEGER"}')
                logger.info(f"Added {col} column to existing {table} table")
            except sqlite3.OperationalError:
                pass  # Column already exists
        
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS imap_state (
                mailbox TEXT PRIMARY KEY,
                last_uid INTEGER DEFAULT 0,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')

        conn.commit()
        conn.close()
    
    def _get_last_imap_uid(self, mailbox):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('SELECT last_uid FROM imap_state WHERE mailbox = ?', (mailbox,))
        row = cursor.fetchone()
        conn.close()
        return row[0] if row else 0

    def _set_last_imap_uid(self, mailbox, uid):
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            INSERT INTO imap_state (mailbox, last_uid, updated_at)
            VALUES (?, ?, CURRENT_TIMESTAMP)
            ON CONFLICT(mailbox) DO UPDATE SET
                last_uid = excluded.last_uid,
                updated_at = CURRENT_TIMESTAMP
        ''', (mailbox, uid))
        conn.commit()
        conn.close()

    def get_current_business_day(self):
        """Get the current business day (skip weekends)"""
        today = datetime.now().date()
        
        # If today is weekend, go back to Friday
        while today.weekday() >= 5:  # Saturday=5, Sunday=6
            today -= timedelta(days=1)
        
        return today
    
    def add_court_to_rollover(self, court_number, original_date):
        """Add a court to rollover list for checking tomorrow"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        try:
            cursor.execute('''
                INSERT OR IGNORE INTO court_rollover (court_number, original_date)
                VALUES (?, ?)
            ''', (court_number, original_date))
            conn.commit()
        except Exception as e:
            logger.error(f"Failed to add court {court_number} to rollover: {e}")
        finally:
            conn.close()
    
    def get_rollover_courts(self, original_date):
        """Get courts that need to be checked from previous day"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            SELECT court_number FROM court_rollover 
            WHERE original_date = ?
            ORDER BY court_number
        ''', (original_date,))
        results = [row[0] for row in cursor.fetchall()]
        conn.close()
        return results
    
    def clear_rollover_courts(self, original_date):
        """Clear rollover courts for a specific date after processing"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            DELETE FROM court_rollover WHERE original_date = ?
        ''', (original_date,))
        conn.commit()
        conn.close()
    
    def load_analysis_prompt(self):
        """Load the analysis prompt from the pdrbot-prompt file, then append
        the current CCA open-questions catalog so the analyzer can flag
        opinions that implicate questions current CCA judges have written
        separately about."""
        prompt_file = Path("pdrbot-prompt")
        if prompt_file.exists():
            base = prompt_file.read_text().strip()
        else:
            logger.warning("pdrbot-prompt file not found, using default prompt")
            base = "Analyze this legal opinion for interesting legal issues."

        catalog_path = Path(
            "/home/ubuntu/github/cca-opinions/reports/special-interests/catalog.json"
        )
        if not catalog_path.exists():
            logger.warning("open-questions catalog not found at %s — skipping",
                           catalog_path)
            return base

        try:
            catalog = json.loads(catalog_path.read_text())
        except Exception as e:
            logger.warning("could not parse catalog (%s) — skipping", e)
            return base

        questions = catalog.get("questions", [])
        if not questions:
            return base

        lines = [
            base,
            "",
            "## Known PDR-Worthy Open Questions (from CCA side opinions)",
            "",
            f"Texas Court of Criminal Appeals judges have flagged the "
            f"following {len(questions)} open questions in concurrences and "
            f"dissents (2015–present). These are issues a future PDR could "
            f"pick up.",
            "",
            "When an issue you identify in the COA opinion implicates one or "
            "more of these questions — the opinion's reasoning or holding "
            "turns on the question's resolution, applies contested doctrine "
            "the question targets, or sits in tension with how a CCA judge "
            "has framed the question — populate the issue's "
            "`matched_open_questions` array with the catalog id(s) and a "
            "one-sentence explanation of how. A match is a strong "
            "PDR-worthiness signal; multiple matches usually warrant a "
            "higher PDR score. Catalog: " + catalog.get("source", "") + ".",
            "",
            "Catalog:",
        ]
        for q in questions:
            examples = q.get("examples", []) or []
            ex_bits = []
            for ex in examples[:2]:
                style = ex.get("case_style") or ""
                cn = ex.get("case_number") or ""
                ex_bits.append(f"{style}, {cn}" if style else cn)
            ex_str = "; ".join(ex_bits) if ex_bits else ""
            line = (
                f"{q['id']}. [{q['judge']}] {q['question']}"
                + (f" — example: {ex_str}" if ex_str else "")
            )
            lines.append(line)

        return "\n".join(lines)

    @contextmanager
    def smtp_connection(self):
        """Context manager for SMTP connections"""
        if self.email_smtp_port == 465:
            server = smtplib.SMTP_SSL(self.email_smtp_host, self.email_smtp_port)
        else:
            server = smtplib.SMTP(self.email_smtp_host, self.email_smtp_port)
            server.starttls()
        server.login(self.email_auth_user, self.email_password)
        try:
            yield server
        finally:
            server.quit()

    def extract_pdr_score(self, analysis_text):
        """Extract PDR Score (1-10) from analysis text.

        Returns an integer score, or None if not found/parseable.
        """
        _d = parse_analysis_json(analysis_text)
        if _d is not None:
            scores = [i.get("pdr_score") for i in _d.get("issues", []) or [] if i.get("pdr_score")]
            return max(scores) if scores else None
        patterns = [
            r'▪\s*PDR Score[^:]*:\s*(\d+)',
            r'\*\*PDR Score[^:]*:\*\*\s*(\d+)',
            r'(?m)^PDR Score[^:]*:\s*(\d+)',
        ]
        for pattern in patterns:
            match = re.search(pattern, analysis_text)
            if match:
                score = int(match.group(1))
                if 1 <= score <= 10:
                    return score
        return None

    def extract_text_from_pdf(self, file_path):
        """Extract text content from a PDF file"""
        try:
            with open(file_path, 'rb') as file:
                pdf_reader = PdfReader(file)
                text = ""
                for page in pdf_reader.pages:
                    text += page.extract_text() + "\n"
                return text.strip()
        except Exception as e:
            logger.error(f"Failed to extract text from {file_path}: {e}")
            return None
    
    def analyze_opinion_with_claude(self, text_content, case_number):
        """Two-pass analysis: Haiku triage → Opus on hits.

        Haiku 4.5 classifies the opinion as INTERESTING or ROUTINE. ROUTINE saves
        a stub and skips the expensive pass; INTERESTING gets the full Opus pass.
        Haiku failures fall through to Opus (safety — never drop a case silently).
        """
        if not self.analysis_enabled:
            logger.warning("Analysis not enabled - skipping")
            return None

        try:
            logger.info(f"Analyzing {case_number}: {len(text_content):,} chars")

            triage_line = self._triage_with_haiku(text_content, case_number)
            if triage_line and triage_line.upper().startswith("ROUTINE"):
                logger.info(f"Triage {case_number}: {triage_line[:120]}")
                return (
                    "TERSE REPORT: NO INTERESTING ISSUES\n\n"
                    f"[Triage: Haiku classified as ROUTINE. {triage_line}]"
                )

            if triage_line:
                logger.info(f"Triage {case_number}: {triage_line[:120]}")
            else:
                logger.info(f"Triage {case_number}: unavailable — running full analysis")

            full_prompt = f"{self.analysis_prompt}\n\n--- OPINION TEXT ---\n{text_content}"
            analysis_text = call_claude_with_retry(
                prompt=full_prompt,
                timeout=300,
                max_retries=3,
                base_delay=5,
                json_schema=ANALYSIS_JSON_SCHEMA,
            )
            logger.info(f"Completed analysis for {case_number}")
            return analysis_text

        except Exception as e:
            logger.error(f"Failed to analyze {case_number} with Claude: {e}")
            return None

    def _triage_with_haiku(self, text_content, case_number):
        """Fast Haiku pass. Returns first line of the response, or None on
        failure (caller falls through to full Opus pass)."""
        triage_prompt = (
            "You are the first pass of a two-stage triage for a Texas "
            "criminal-defense appellate practice. Cases you mark INTERESTING "
            "go to a more capable model (Opus) for full analysis; cases you "
            "mark ROUTINE are not analyzed further. Your only job is to "
            "filter out truly cookie-cutter dispositions — the deeper model "
            "is the authoritative judge of what is PDR-worthy.\n\n"
            "Reply with exactly one line.\n\n"
            "Default to INTERESTING. Mark ROUTINE only when the opinion is "
            "truly cookie-cutter — a boilerplate disposition that could have "
            "been written from a template, with no contested legal question, "
            "no debatable application of law to fact, no concurrence or "
            "dissent, and no discussion beyond reciting settled doctrine and "
            "applying it to overwhelming or uncontested facts.\n\n"
            "Cookie-cutter patterns (non-exhaustive):\n"
            "- Anders affirmance with no arguable grounds and no "
            "fine/cost/restitution issue\n"
            "- Jurisdictional dismissal for untimely notice of appeal, no "
            "tolling argument\n"
            "- Habeas dismissed for failure to comply with statutory "
            "prerequisites, no merits discussion\n"
            "- Mandamus denied solely on want of presentment or want of "
            "clear right, no contested record\n"
            "- Frivolous-appeal dismissal with no preserved issues briefed\n"
            "- Plea-bargain waiver dismissal under Rule 25.2(a)(2) with no "
            "certification dispute\n\n"
            "Anything else is INTERESTING — including any concurrence or "
            "dissent, any sufficiency or burden-of-proof challenge against "
            "non-overwhelming facts, any case where the court engages with a "
            "doctrinal question even briefly, any case where the court "
            "relies on cited authority to defeat a non-frivolous argument, "
            "and any case decided on a close or contested record.\n\n"
            "Always escalate when you see explicit textual signals like:\n"
            "- the opinion itself flags an unresolved or unsettled question "
            "('has not determined,' 'has not addressed,' 'we have not "
            "found,' 'no controlling authority,' 'unsettled,' 'open "
            "question,' or a Pattern Jury Charges committee note flagging "
            "a gap)\n"
            "- the court relies on pre-2000 Court of Criminal Appeals "
            "authority for a contested point\n"
            "- the court acknowledges or describes a split among Texas "
            "courts of appeals (even where it picks a side)\n"
            "- a concurrence or dissent rejects the majority's framework\n\n"
            "A false ROUTINE loses a PDR-worthy case; a false INTERESTING "
            "costs only one extra Opus call. When in doubt, escalate.\n\n"
            "Your entire reply must be a single line, exactly one of:\n"
            "- 'INTERESTING.'  (no justification — Opus will analyze)\n"
            "- 'ROUTINE: <one-sentence reason>'"
        )
        try:
            result = call_claude_with_retry(
                prompt=f"{triage_prompt}\n\n--- OPINION TEXT ---\n{text_content}",
                timeout=60,
                max_retries=2,
                base_delay=3,
                model="claude-haiku-4-5",
            )
            if not result:
                return None
            return result.strip().splitlines()[0]
        except Exception as e:
            logger.warning(f"Haiku triage failed for {case_number}: {e} — "
                           f"falling through to full analysis")
            return None
    
    def clean_analysis_text(self, analysis_text):
        """Clean analysis text by removing introductory commentary, meta-commentary, and first-person statements"""
        import re

        # Remove common intro phrases and first-person commentary
        # These patterns capture various ways Claude might introduce the analysis
        intro_patterns = [
            # "I'll/I will/Let me analyze..." patterns
            r"^I'll analyze this.*?(?:\n|\.)\s*\n*",
            r"^Let me analyze this.*?(?:\n|\.)\s*\n*",
            r"^I will analyze this.*?(?:\n|\.)\s*\n*",
            r"^I must analyze this.*?(?:\n|\.)\s*\n*",
            r"^Analyzing this.*?(?:\n|\.)\s*\n*",

            # "Looking at this..." patterns (single and multi-line)
            r"^Looking at this.*?(?:\n\n|\*\*)",

            # "I need to/I must examine/check..." patterns
            r"^I need to (?:examine|check|analyze).*?(?:\n|\.)\s*\n*",
            r"^I must (?:examine|check|analyze).*?(?:\n|\.)\s*\n*",

            # "I find..." patterns at the beginning
            r"^I find (?:no interesting issues|that this).*?(?:\n|\.)\s*\n*",
            r"^I do not find.*?(?:\n|\.)\s*\n*",

            # Meta-commentary about language requirements/compliance
            r"^CRITICAL LANGUAGE REQUIREMENT.*?(?=TERSE REPORT|Appellant Name|\Z)",
            r"^I have reviewed the forbidden words.*?(?:\n|\.)\s*\n*",
            r"^.*compliance checklist.*?(?:\n\n|\Z)",
            r"^.*COMPLIANCE CHECKLIST.*?(?:\n\n|\Z)",
            r"^Every sentence will use only approved.*?(?:\n|\.)\s*\n*",
        ]

        cleaned = analysis_text
        for pattern in intro_patterns:
            cleaned = re.sub(pattern, '', cleaned, flags=re.IGNORECASE | re.DOTALL)

        # Remove leading whitespace/newlines
        cleaned = cleaned.lstrip()

        return cleaned

    def extract_headlines_from_analysis(self, analysis_text):
        """Extract Headline field values from analysis text.

        Handles multiple formatting variants:
          ▪ Headline: ...
          **Headline:** ...
          Headline: ...

        Returns a list of headline strings. Returns empty list for older
        analyses that pre-date the Headline field (backward compatible).
        """
        _d = parse_analysis_json(analysis_text)
        if _d is not None:
            return [i.get("headline", "") for i in _d.get("issues", []) or [] if i.get("headline")]
        patterns = [
            r'▪\s*Headline:\s*(.+)',
            r'\*\*Headline:\*\*\s*(.+)',
            r'(?m)^Headline:\s*(.+)',
        ]
        headlines = []
        for pattern in patterns:
            matches = re.findall(pattern, analysis_text)
            if matches:
                headlines = [m.strip() for m in matches]
                break
        return headlines

    def extract_appellant_name(self, analysis_text):
        """Extract the appellant name / case style from the structured header.

        Looks for the 'Appellant Name and Case Number' field in various
        formatting variants and returns the value, or None if not found.
        """
        _d = parse_analysis_json(analysis_text)
        if _d is not None:
            return _d.get("appellant_name") or None
        patterns = [
            r'▪\s*Appellant Name[^:]*:\s*(.+)',
            r'\*\*Appellant Name[^:]*:\*\*\s*(.+)',
            r'(?m)^Appellant Name[^:]*:\s*(.+)',
        ]
        for pattern in patterns:
            match = re.search(pattern, analysis_text)
            if match:
                return match.group(1).strip()
        return None

    def save_analysis_to_db(self, opinion_id, case_number, court, opinion_date, analysis_text):
        """Save analysis results to database

        For consolidated cases (multiple case numbers in same PDF), saves the
        analysis to all opinion records that share the same file_path.
        """
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()

        try:
            # Clean the analysis text
            cleaned_text = self.clean_analysis_text(analysis_text)

            # Check for execution errors
            if "execution error" in cleaned_text.lower():
                logger.warning(f"Analysis for {case_number} contains execution error - marking as failed")
                has_interesting_issues = False
                issue_count = 0
            else:
                # Prefer the structured-JSON envelope emitted by the current
                # triage-plus-full-analysis pipeline. If the analysis is not
                # JSON, fall back to the legacy "no interesting issues" phrase
                # check and markdown issue markers.
                parsed_issues = None
                try:
                    parsed = json.loads(cleaned_text)
                    if isinstance(parsed, dict) and isinstance(parsed.get('issues'), list):
                        parsed_issues = parsed['issues']
                except (json.JSONDecodeError, ValueError):
                    pass

                if parsed_issues is not None:
                    issue_count = len(parsed_issues)
                    has_interesting_issues = issue_count > 0
                elif "no interesting issues" in cleaned_text.lower():
                    has_interesting_issues = False
                    issue_count = 0
                else:
                    issue_patterns = [
                        r'▪\s*Issue Description:',
                        r'\*\*Issue Description:\*\*',
                        r'\*\*Issue \d+:',
                        r'Issue \d+:',
                        r'▪\s*Headline:',
                        r'\*\*Headline:\*\*',
                    ]
                    issue_count = 0
                    for pattern in issue_patterns:
                        count = len(re.findall(pattern, cleaned_text))
                        issue_count = max(issue_count, count)
                    has_interesting_issues = issue_count > 0

            # Extract PDR score
            pdr_score = self.extract_pdr_score(cleaned_text)

            # Extract cached disposition / state_is_appellant fields from
            # JSON-form analyses so the slip-opinions and triage renderers
            # can stamp defense wins without re-parsing analysis_text.
            disposition_val = None
            state_is_appellant_val = None
            _parsed_meta = parse_analysis_json(cleaned_text)
            if _parsed_meta is not None:
                if isinstance(_parsed_meta.get("disposition"), str):
                    disposition_val = _parsed_meta["disposition"]
                if isinstance(_parsed_meta.get("state_is_appellant"), bool):
                    state_is_appellant_val = 1 if _parsed_meta["state_is_appellant"] else 0

            # Get the file_path for this opinion
            cursor.execute('SELECT file_path FROM opinions WHERE id = ?', (opinion_id,))
            result = cursor.fetchone()
            if not result:
                logger.error(f"Opinion ID {opinion_id} not found")
                return False

            file_path = result[0]

            # Find all opinion_ids that share this file_path (consolidated cases)
            cursor.execute('SELECT id, case_number, court, opinion_date FROM opinions WHERE file_path = ?', (file_path,))
            all_opinions = cursor.fetchall()

            # Save analysis for all opinions that share this file
            for oid, cnum, crt, odate in all_opinions:
                cursor.execute('''
                    INSERT OR REPLACE INTO analysis
                    (opinion_id, case_number, court, opinion_date, analysis_text,
                     has_interesting_issues, issue_count, claude_model, pdr_score,
                     disposition, state_is_appellant)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                ''', (oid, cnum, crt, odate, cleaned_text,
                      has_interesting_issues, issue_count, self.claude_model, pdr_score,
                      disposition_val, state_is_appellant_val))

            if len(all_opinions) > 1:
                logger.info(f"Saved analysis to {len(all_opinions)} consolidated cases sharing {file_path}")

            conn.commit()
            return True
        except Exception as e:
            logger.error(f"Error saving analysis to database: {e}")
            return False
        finally:
            conn.close()
    
    def get_unanalyzed_opinions(self):
        """Get opinions that haven't been analyzed yet

        Returns only one opinion per unique file_path to avoid analyzing
        the same PDF multiple times for consolidated cases.
        """
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()

        try:
            cursor.execute('''
                SELECT MIN(o.id) as id, o.case_number, o.court, o.opinion_date, o.file_path
                FROM opinions o
                LEFT JOIN analysis a ON o.id = a.opinion_id
                WHERE a.opinion_id IS NULL
                GROUP BY o.file_path
                ORDER BY o.opinion_date DESC, o.case_number
            ''')
            return cursor.fetchall()
        except Exception as e:
            logger.error(f"Error fetching unanalyzed opinions: {e}")
            return []
        finally:
            conn.close()
    
    def process_opinion_analysis(self, opinion_id, case_number, court, opinion_date, file_path):
        """Process a single opinion for analysis"""
        logger.info(f"Analyzing opinion: {case_number}")
        
        # Extract text from PDF
        text_content = self.extract_text_from_pdf(file_path)
        if not text_content:
            logger.error(f"Could not extract text from {file_path}")
            return False
        
        # Analyze with Claude
        analysis_result = self.analyze_opinion_with_claude(text_content, case_number)
        if not analysis_result:
            logger.error(f"Could not analyze {case_number}")
            return False
        
        # Save analysis to database
        success = self.save_analysis_to_db(opinion_id, case_number, court, opinion_date, analysis_result)
        if success:
            logger.info(f"Saved analysis for {case_number}")

            # If case has interesting issues, scrape representative information
            # Use the cleaned/parsed result from DB to stay consistent
            cleaned = self.clean_analysis_text(analysis_result)
            issue_count = 0
            for pattern in [r'▪\s*Headline:', r'\*\*Headline:\*\*']:
                issue_count = max(issue_count, len(re.findall(pattern, cleaned)))
            if issue_count > 0 and "no interesting issues" not in cleaned.lower():
                case_url = self.generate_case_url(case_number, court)
                self.scrape_case_representatives(case_url, case_number, court, opinion_date)
                time.sleep(1)
        
        return success
    
    def run_analysis_batch(self, limit=None):
        """Process unanalyzed opinions in batches"""
        if not self.analysis_enabled:
            logger.info("Analysis is disabled")
            return
        
        unanalyzed = self.get_unanalyzed_opinions()
        if limit:
            unanalyzed = unanalyzed[:limit]
        
        if not unanalyzed:
            logger.info("No unanalyzed opinions found")
            return
        
        logger.info(f"Processing {len(unanalyzed)} unanalyzed opinions")
        processed = 0
        failed = 0
        
        for opinion_id, case_number, court, opinion_date, file_path in unanalyzed:
            try:
                if self.process_opinion_analysis(opinion_id, case_number, court, opinion_date, file_path):
                    processed += 1
                else:
                    failed += 1
                
                # Rate limiting - be respectful to Claude API
                time.sleep(2)
                
            except Exception as e:
                logger.error(f"Error processing {case_number}: {e}")
                failed += 1
        
        logger.info(f"Analysis batch complete: {processed} processed, {failed} failed")
        
        # Don't generate intermediate reports - only generate after all analysis is complete
    
    def analyze_directory_pdfs(self, directory_path):
        """Analyze all PDF files in a specific directory"""
        if not os.path.exists(directory_path):
            logger.error(f"Directory not found: {directory_path}")
            return
        
        # Find all PDF files in the directory
        pdf_files = []
        for filename in os.listdir(directory_path):
            if filename.endswith('.pdf') and not filename.startswith('analysis_report'):
                pdf_path = os.path.join(directory_path, filename)
                # Extract case number from filename
                case_number = filename.replace('.pdf', '')
                pdf_files.append((case_number, pdf_path))
        
        if not pdf_files:
            logger.info(f"No PDF files found in {directory_path}")
            return
        
        logger.info(f"Found {len(pdf_files)} PDF files in {directory_path}")
        processed = 0
        failed = 0
        
        for case_number, pdf_path in pdf_files:
            try:
                # Check if this case is already analyzed
                conn = sqlite3.connect(self.db_path)
                cursor = conn.cursor()
                cursor.execute('''
                    SELECT COUNT(*) FROM opinions o
                    JOIN analysis a ON o.id = a.opinion_id
                    WHERE o.case_number = ?
                ''', (case_number,))
                already_analyzed = cursor.fetchone()[0] > 0
                conn.close()
                
                if already_analyzed:
                    logger.info(f"Skipping {case_number} - already analyzed")
                    continue
                
                # Find or create opinion record
                conn = sqlite3.connect(self.db_path)
                cursor = conn.cursor()
                cursor.execute('SELECT id FROM opinions WHERE case_number = ?', (case_number,))
                result = cursor.fetchone()
                
                if result:
                    opinion_id = result[0]
                else:
                    # Create a basic opinion record for this PDF
                    # Extract court and date from case number (e.g., "01-23-00771-CR")
                    parts = case_number.split('-')
                    if len(parts) >= 3:
                        court = f"COA{parts[0]}"
                        year = f"20{parts[1]}"
                        # Use a default date based on directory name
                        dir_name = os.path.basename(directory_path)
                        if len(dir_name) == 8 and dir_name.isdigit():
                            opinion_date = f"{dir_name[:4]}-{dir_name[4:6]}-{dir_name[6:8]}"
                        else:
                            opinion_date = f"{year}-01-01"  # Default date
                        
                        cursor.execute('''
                            INSERT INTO opinions 
                            (case_number, court, opinion_date, opinion_type, filename, file_path, case_url)
                            VALUES (?, ?, ?, ?, ?, ?, ?)
                        ''', (case_number, court, opinion_date, "combined", os.path.basename(pdf_path), pdf_path, ""))
                        opinion_id = cursor.lastrowid
                        conn.commit()
                    else:
                        logger.warning(f"Could not parse case number format: {case_number}")
                        conn.close()
                        continue
                
                conn.close()
                
                # Process the analysis
                if self.process_opinion_analysis(opinion_id, case_number, court, opinion_date, pdf_path):
                    processed += 1
                else:
                    failed += 1
                
                # Rate limiting
                time.sleep(2)
                
            except Exception as e:
                logger.error(f"Error processing {case_number}: {e}")
                failed += 1
        
        logger.info(f"Directory analysis complete: {processed} processed, {failed} failed")
        
        # Don't generate intermediate reports - only generate after all analysis is complete
    
    def backfill_pdf_urls(self):
        """Backfill PDF URLs for existing records that don't have them"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        try:
            # Find records without PDF URLs
            cursor.execute('''
                SELECT id, case_number, court, opinion_date 
                FROM opinions 
                WHERE pdf_url IS NULL OR pdf_url = ''
            ''')
            records = cursor.fetchall()
            
            if not records:
                logger.info("No records need PDF URL backfill")
                return
            
            logger.info(f"Backfilling PDF URLs for {len(records)} records")
            
            for opinion_id, case_number, court, opinion_date in records:
                # Try to reconstruct the PDF URLs by re-scraping the case
                try:
                    coa_num = int(court.replace("COA", ""))
                    date_obj = datetime.strptime(str(opinion_date), '%Y-%m-%d').date()
                    
                    # Get the docket page for this case
                    url = self.get_docket_url(coa_num, date_obj)
                    response = self.get_with_retry(url)
                    
                    if response:
                        soup = BeautifulSoup(response.content, 'html.parser')
                        criminal_cases = self.parse_criminal_causes(soup)
                        
                        # Find the matching case
                        for case in criminal_cases:
                            if case['case_number'] == case_number:
                                pdf_urls = [link['url'] for link in case['pdf_links']]
                                pdf_urls_string = ';'.join(pdf_urls) if pdf_urls else None
                                
                                # Update the record
                                cursor.execute('''
                                    UPDATE opinions 
                                    SET pdf_url = ?
                                    WHERE id = ?
                                ''', (pdf_urls_string, opinion_id))
                                
                                logger.info(f"Updated PDF URLs for {case_number}")
                                break
                    
                    # Small delay to be respectful
                    time.sleep(1)
                    
                except Exception as e:
                    logger.warning(f"Could not backfill PDF URL for {case_number}: {e}")
            
            conn.commit()
            logger.info("PDF URL backfill complete")
            
        except Exception as e:
            logger.error(f"Error during PDF URL backfill: {e}")
        finally:
            conn.close()
    
    def get_analysis_results(self, date_filter=None, interesting_only=True, date_range=None):
        """Get analysis results for report generation

        Deduplicates by PDF content to ensure each unique PDF only appears once,
        even if downloaded multiple times with different filenames (consolidated cases).

        Args:
            date_filter: Exact date match (str or datetime)
            interesting_only: Only return cases with interesting issues
            date_range: Tuple of (start_date, end_date) for range queries
        """
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()

        try:
            # First get all matching results
            query = '''
                SELECT a.case_number, a.court, a.opinion_date, a.analysis_text,
                       a.has_interesting_issues, a.issue_count, a.analysis_timestamp,
                       o.file_path, o.case_url, o.pdf_url, o.opinion_type,
                       a.pdr_score
                FROM analysis a
                JOIN opinions o ON a.opinion_id = o.id
            '''
            params = []
            conditions = []

            # Exclude very short analyses (likely failed/incomplete)
            conditions.append("LENGTH(a.analysis_text) > 100")

            # Filter to only interesting cases if requested
            if interesting_only:
                conditions.append("a.has_interesting_issues = 1")

            if date_filter:
                conditions.append("a.opinion_date = ?")
                params.append(date_filter)
            elif date_range:
                start_date, end_date = date_range
                conditions.append("a.opinion_date >= ? AND a.opinion_date <= ?")
                params.extend([start_date, end_date])

            if conditions:
                query += " WHERE " + " AND ".join(conditions)

            query += " ORDER BY a.has_interesting_issues DESC, COALESCE(a.pdr_score, 0) DESC, a.opinion_date DESC, a.case_number"

            cursor.execute(query, params)
            all_results = cursor.fetchall()

            # Deduplicate by PDF content hash AND by consolidated case numbers
            # But preserve different opinion types (lead vs concurring vs dissenting)
            seen_hashes = {}
            seen_case_sets = {}  # Track (case_numbers, opinion_type) to allow different opinion types
            unique_results = []

            for result in all_results:
                file_path = result[7]  # file_path is at index 7
                case_number = result[0]  # case_number is at index 0
                opinion_type = result[10]  # opinion_type is at index 10

                # Determine if this is a lead opinion or a side opinion (concurring/dissenting)
                # Lead opinions: op, mem, combined, op+con, op+dis, mem+con, mem+dis, etc.
                # We want to deduplicate lead opinions separately from side opinions
                is_side_opinion = opinion_type in ('con', 'dis')

                # Compute hash of PDF file
                try:
                    if os.path.exists(file_path):
                        with open(file_path, 'rb') as f:
                            pdf_content = f.read()
                            pdf_hash = hashlib.md5(pdf_content).hexdigest()

                        # Check if we've seen this exact PDF before
                        if pdf_hash in seen_hashes:
                            logger.debug(f"Skipping duplicate PDF: {file_path} (same as {seen_hashes[pdf_hash]})")
                            continue

                        # Extract case numbers from the first page to detect consolidated cases
                        # that may have different PDF hashes but same content
                        try:
                            import subprocess
                            text_result = subprocess.run(
                                ['pdftotext', '-l', '1', file_path, '-'],
                                capture_output=True, text=True, timeout=10
                            )
                            if text_result.returncode == 0:
                                first_page = text_result.stdout
                                case_nums = set(re.findall(r'\d{2}-\d{2}-\d{5}-CR', first_page))
                                if case_nums:
                                    # Create a key combining case numbers AND whether it's a side opinion
                                    # This allows lead + concurring + dissenting to all appear
                                    case_set_key = (frozenset(case_nums), is_side_opinion, opinion_type if is_side_opinion else 'lead')
                                    if case_set_key in seen_case_sets:
                                        logger.debug(f"Skipping consolidated duplicate: {case_number} [{opinion_type}] (same as {seen_case_sets[case_set_key]})")
                                        continue
                                    seen_case_sets[case_set_key] = case_number
                        except Exception as e:
                            logger.debug(f"Could not extract text from {file_path} for dedup: {e}")

                        seen_hashes[pdf_hash] = file_path
                        unique_results.append(result)
                    else:
                        # If file doesn't exist, include it anyway (will show error in report)
                        unique_results.append(result)
                except Exception as e:
                    logger.warning(f"Error hashing {file_path}: {e}, including anyway")
                    unique_results.append(result)

            return unique_results
        except Exception as e:
            logger.error(f"Error fetching analysis results: {e}")
            return []
        finally:
            conn.close()
    
    def generate_case_url(self, case_number, court):
        """Generate the online case URL"""
        base_url = "https://search.txcourts.gov/Case.aspx?cn="
        url = f"{base_url}{case_number}"
        if court:
            m = re.search(r"(\d+)", str(court))
            if m:
                url += f"&coa=coa{int(m.group(1)):02d}"
        return url

    def _harvest_target_from_result(self, result):
        """Build a brief_harvest case dict from a get_analysis_results row."""
        case_number = result[0]
        court = result[1]
        opinion_date = result[2] if len(result) > 2 else ""
        analysis_text = result[3] if len(result) > 3 else ""
        stored_url = result[8] if len(result) > 8 else None

        style = ""
        data = parse_analysis_json(analysis_text)
        if data:
            state_app = data.get("state_is_appellant")
            name = (data.get("appellant_name") or "").strip()
            if state_app is True:
                style = (
                    f"State of Texas v. {name}" if name
                    else "State of Texas v. Defendant"
                )
            else:
                # False or unknown: ordinary defense appeal (the common case).
                style = (
                    f"{name} v. State of Texas" if name
                    else "Defendant v. State of Texas"
                )
        else:
            name = self.extract_appellant_name(analysis_text) or ""
            if name:
                style = f"{name} v. State of Texas"

        case_url = (stored_url or "").strip() or self.generate_case_url(
            case_number, court)
        return {
            "date": str(opinion_date) if opinion_date is not None else "",
            "court": court or "",
            "case_number": case_number,
            "style": style,
            "disposition": "",
            "case_url": case_url,
        }

    @staticmethod
    def _counsel_html(counsel, color="#2874a6"):
        """Render defense-counsel lines for an HTML case card."""
        if not counsel:
            return ""
        blocks = []
        for c in counsel:
            parts = [c["name"]]
            if c.get("bar_number"):
                parts.append(f"SBOT {c['bar_number']}")
            if c.get("email"):
                parts.append(
                    f'<a href="mailto:{c["email"]}" '
                    f'style="color:{color};">{c["email"]}</a>')
            blocks.append(
                f'<div style="margin-top:2px;font-size:12px;color:#333;">'
                f'&#9878;&#65039; {" &middot; ".join(parts)}</div>')
        return "".join(blocks)

    @staticmethod
    def _counsel_plain_lines(counsel, label="Defense counsel"):
        """Plain-text counsel lines for the text/plain email body."""
        if not counsel:
            return ""
        lines = []
        for c in counsel:
            bits = [c["name"]]
            if c.get("bar_number"):
                bits.append(f"SBOT {c['bar_number']}")
            if c.get("email"):
                bits.append(c["email"])
            lines.append(f"    {label}: {' -- '.join(bits)}\n")
        return "".join(lines)

    def scrape_case_representatives(self, case_url, case_number, court, opinion_date):
        """Scrape representative information from case URL"""
        try:
            logger.info(f"Scraping representatives for case {case_number}")
            response = self.get_with_retry(case_url)
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Find the parties panel - search for panel-heading containing "Parties"
            parties_panel = soup.find('div', class_=['panel-heading', 'panel-heading-content'], string=lambda text: text and 'Parties' in text)
            if not parties_panel:
                logger.warning(f"No parties panel found for case {case_number}")
                return []
            
            # Navigate to the table containing party information
            panel_content = parties_panel.find_next_sibling('div', class_='panel-content')
            if not panel_content:
                logger.warning(f"No panel content found for case {case_number}")
                return []
            
            # Find the grid table
            party_table = panel_content.find('table', class_='rgMasterTable')
            if not party_table:
                logger.warning(f"No party table found for case {case_number}")
                return []
            
            representatives = []
            tbody = party_table.find('tbody')
            if tbody:
                for row in tbody.find_all('tr'):
                    cells = row.find_all('td')
                    if len(cells) >= 3:
                        party_name = cells[0].get_text(strip=True)
                        party_type = cells[1].get_text(strip=True)
                        representative_cell = cells[2]
                        
                        # Skip "State of Texas" entries
                        if "State of Texas" in party_name or "Criminal - State of Texas" in party_type:
                            continue
                        
                        # Extract representative names (they may be separated by <br> tags)
                        rep_names = []
                        for br in representative_cell.find_all('br'):
                            br.replace_with('\n')
                        rep_text = representative_cell.get_text()
                        
                        # Clean and split representative names
                        for name in rep_text.split('\n'):
                            name = name.strip()
                            if name and name not in rep_names:
                                rep_names.append(name)
                        
                        if rep_names:
                            representatives.append({
                                'party_name': party_name,
                                'party_type': party_type,
                                'representative_names': ', '.join(rep_names)
                            })
            
            # Save to database
            self.save_representatives_to_db(case_number, court, opinion_date, representatives)
            
            return representatives
            
        except Exception as e:
            logger.error(f"Error scraping representatives for case {case_number}: {e}")
            return []
    
    def save_representatives_to_db(self, case_number, court, opinion_date, representatives):
        """Save representative information to database"""
        if not representatives:
            return
        
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        try:
            for rep in representatives:
                cursor.execute('''
                    INSERT OR REPLACE INTO representatives 
                    (case_number, court, opinion_date, party_name, party_type, representative_names)
                    VALUES (?, ?, ?, ?, ?, ?)
                ''', (case_number, court, opinion_date, rep['party_name'], 
                      rep['party_type'], rep['representative_names']))
            
            conn.commit()
            logger.info(f"Saved {len(representatives)} representative entries for case {case_number}")
            
        except Exception as e:
            logger.error(f"Error saving representatives to database: {e}")
        finally:
            conn.close()
    
    def get_case_representatives(self, case_number, court):
        """Get representative information for a specific case"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        try:
            cursor.execute('''
                SELECT party_name, party_type, representative_names
                FROM representatives 
                WHERE case_number = ? AND court = ?
                ORDER BY party_type
            ''', (case_number, court))
            
            results = cursor.fetchall()
            return [{'party_name': row[0], 'party_type': row[1], 'representative_names': row[2]} 
                    for row in results]
        except Exception as e:
            logger.error(f"Error getting representatives for case {case_number}: {e}")
            return []
        finally:
            conn.close()

    def get_opinion_pdf_urls(self, case_number, court):
        """Get the original PDF URLs for a case by reconstructing from case page structure"""
        # This is a simplified approach - ideally we'd store these during download
        # For now, we'll link to the case page where users can access the PDFs
        court_num = court.replace("COA", "").zfill(2)
        case_encoded = quote(case_number)
        
        # The actual PDF URLs follow a pattern but are complex to reconstruct
        # So we'll provide the case page URL where PDFs can be accessed
        return [f"https://search.txcourts.gov/Case.aspx?cn={case_encoded}&coa={court_num}"]
    
    def generate_pdf_path(self, file_path):
        """Generate a relative path to the PDF file for links"""
        # Convert absolute path to relative from report location
        return os.path.relpath(file_path, self.data_dir)
    
    def create_pdf_styles(self):
        """Create custom styles for the PDF report"""
        styles = getSampleStyleSheet()
        
        # Custom styles
        styles.add(ParagraphStyle(
            name='CustomTitle',
            parent=styles['Heading1'],
            fontSize=18,
            spaceAfter=30,
            alignment=TA_CENTER,
            textColor=colors.darkblue
        ))
        
        styles.add(ParagraphStyle(
            name='CaseTitle',
            parent=styles['Heading2'],
            fontSize=14,
            spaceAfter=12,
            spaceBefore=20,
            textColor=colors.darkred
        ))
        
        styles.add(ParagraphStyle(
            name='CaseInfo',
            parent=styles['Normal'],
            fontSize=10,
            spaceAfter=8,
            textColor=colors.grey
        ))
        
        styles.add(ParagraphStyle(
            name='Analysis',
            parent=styles['Normal'],
            fontSize=11,
            spaceAfter=12,
            alignment=TA_JUSTIFY
        ))
        
        return styles
    
    def get_unique_filename(self, base_filename):
        """Generate a unique filename by appending -1, -2, etc. if file exists"""
        output_path = os.path.join(self.data_dir, base_filename)
        if not os.path.exists(output_path):
            return base_filename
        
        name, ext = os.path.splitext(base_filename)
        counter = 1
        while True:
            new_filename = f"{name}-{counter}{ext}"
            new_path = os.path.join(self.data_dir, new_filename)
            if not os.path.exists(new_path):
                return new_filename
            counter += 1
    
    def generate_analysis_report(self, date_filter=None, output_filename=None, date_range=None, custom_title=None):
        """Generate a comprehensive PDF report of analysis results

        Args:
            date_filter: Exact date for single-day reports
            output_filename: Custom output filename
            date_range: Tuple of (start_date, end_date) for multi-day reports
            custom_title: Custom title for the report
        """
        logger.info("Generating analysis PDF report...")

        # Get analysis results (only interesting cases)
        results = self.get_analysis_results(date_filter=date_filter, interesting_only=True, date_range=date_range)

        if not results:
            logger.info("No analyzed cases found for report generation")
            return None

        # Determine the opinion date for the report title
        opinion_dates = [row[2] for row in results]  # opinion_date column
        if custom_title:
            report_title = custom_title
            report_date = "combined"
        elif date_range:
            start_date, end_date = date_range
            report_title = f"{start_date} through {end_date} Handdowns"
            report_date = f"{start_date.replace('-', '')}_to_{end_date.replace('-', '')}"
        elif date_filter:
            # Handle both datetime objects and strings
            if hasattr(date_filter, 'strftime'):
                report_date = date_filter.strftime("%Y-%m-%d")
            else:
                report_date = str(date_filter)
            report_title = f"{report_date} Handdowns"
        elif opinion_dates:
            # Use the most common date or latest date
            from collections import Counter  # rare path, keep inline
            date_counter = Counter(opinion_dates)
            most_common_date = date_counter.most_common(1)[0][0]
            report_date = str(most_common_date)
            report_title = f"{report_date} Handdowns"
        else:
            report_date = datetime.now().strftime("%Y-%m-%d")
            report_title = f"{report_date} Handdowns"

        # Create output filename if not provided
        if not output_filename:
            base_filename = f"pdrbot_report_{report_date.replace('-', '')}.pdf"
            output_filename = self.get_unique_filename(base_filename)

        output_path = os.path.join(self.data_dir, output_filename)
        
        # Create PDF document
        doc = SimpleDocTemplate(
            output_path,
            pagesize=letter,
            rightMargin=72,
            leftMargin=72,
            topMargin=72,
            bottomMargin=18
        )
        
        # Get styles
        styles = self.create_pdf_styles()
        story = []
        
        # Title with opinion date
        title = Paragraph(f"PDRBot Report<br/>Interesting Legal Issues<br/>{report_title}", styles['CustomTitle'])
        story.append(title)
        story.append(Spacer(1, 20))
        
        # AI Disclaimer
        disclaimer_text = f"This report is AI-generated by {self.claude_model} for the sole purpose of assisting experts in finding cases that might be PDR worthy. Do not depend on it."
        disclaimer = Paragraph(disclaimer_text, styles['CaseInfo'])
        story.append(disclaimer)
        story.append(Spacer(1, 20))
        
        # Summary statistics
        total_cases = len(results)
        total_issues = sum(row[5] for row in results)  # issue_count column

        summary_data = [
            ['Cases with Interesting Issues:', str(total_cases)],
            ['Total Issues Found:', str(total_issues)],
            ['Report Generated:', datetime.now().strftime("%B %d, %Y at %I:%M %p")],
        ]
        
        summary_table = Table(summary_data, colWidths=[3*inch, 2*inch])
        summary_table.setStyle(TableStyle([
            ('ALIGN', (0, 0), (-1, -1), 'LEFT'),
            ('FONTNAME', (0, 0), (0, -1), 'Helvetica-Bold'),
            ('FONTSIZE', (0, 0), (-1, -1), 10),
            ('BOTTOMPADDING', (0, 0), (-1, -1), 6),
        ]))
        
        story.append(summary_table)
        story.append(Spacer(1, 30))

        # Process each case (all cases here are interesting since we filtered above)
        for i, (case_number, court, opinion_date, analysis_text, has_interesting,
                issue_count, analysis_timestamp, file_path, case_url, pdf_url, opinion_type,
                pdr_score) in enumerate(results):
            
            # Case title with PDR score
            score_label = f" [PDR {pdr_score}/10]" if pdr_score else ""
            case_title = f"Case {i+1}: {case_number} ({court}){score_label}"
            story.append(Paragraph(case_title, styles['CaseTitle']))
            
            # Case information with clickable links
            case_url_link = self.generate_case_url(case_number, court)
            case_info_parts = [
                f"<b>Date:</b> {opinion_date}",
                f"<b>Issues Found:</b> {issue_count}",
                f"<b>Case Page:</b> <link href='{case_url_link}' color='blue'>View Case Details</link>",
            ]
            
            # Add direct links to PDF opinions if available
            if pdf_url and pdf_url.strip():
                pdf_urls = pdf_url.split(';')
                if len(pdf_urls) == 1:
                    case_info_parts.append(f"<b>Opinion PDF:</b> <link href='{pdf_urls[0]}' color='blue'>Direct PDF Link</link>")
                else:
                    # Multiple PDFs - list them
                    for i, url in enumerate(pdf_urls, 1):
                        case_info_parts.append(f"<b>Opinion {i} PDF:</b> <link href='{url}' color='blue'>Direct PDF Link {i}</link>")
            else:
                # Fallback to case page
                case_info_parts.append(f"<b>Online Opinions:</b> <link href='{case_url_link}' color='blue'>View on Court Website</link>")
            
            # Optionally include local file link as well
            if file_path and os.path.exists(file_path):
                local_link = f"file://{os.path.abspath(file_path)}"
                case_info_parts.append(f"<b>Local PDF:</b> <link href='{local_link}' color='grey'>{os.path.basename(file_path)}</link>")
            
            case_info = "<br/>".join(case_info_parts)
            story.append(Paragraph(case_info, styles['CaseInfo']))
            story.append(Spacer(1, 10))
            
            # Add representative information for interesting cases
            if has_interesting:
                representatives = self.get_case_representatives(case_number, court)
                if representatives:
                    rep_info_parts = ["<b>Defense Representatives:</b>"]
                    for rep in representatives:
                        party_info = f"• <i>{rep['party_name']}</i> ({rep['party_type']}): {rep['representative_names']}"
                        rep_info_parts.append(party_info)
                    
                    rep_info = "<br/>".join(rep_info_parts)
                    story.append(Paragraph(rep_info, styles['CaseInfo']))
                    story.append(Spacer(1, 10))
            
            # Analysis text (clean up formatting for PDF)
            analysis_clean = render_analysis_prose(analysis_text).replace('▪', '•').replace('◦', '○')

            # Convert markdown formatting to HTML for PDF
            # Remove markdown headers (# and ##)
            analysis_clean = re.sub(r'^#+\s*', '', analysis_clean, flags=re.MULTILINE)
            # Remove Priority Level sections completely
            analysis_clean = re.sub(r'\*\*▪ Priority Level:\*\*[^\n]*\n?', '', analysis_clean)
            analysis_clean = re.sub(r'\*\*Priority Level:\*\*[^\n]*\n?', '', analysis_clean)

            # Strip any existing HTML tags that Claude may have included
            analysis_clean = re.sub(r'<[^>]+>', '', analysis_clean)

            # Escape special HTML characters
            analysis_clean = analysis_clean.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')

            # Now safely convert markdown to HTML
            # Convert **bold** to <b>bold</b>
            analysis_clean = re.sub(r'\*\*(.*?)\*\*', r'<b>\1</b>', analysis_clean)
            # Convert *italic* to <i>italic</i> (but not when part of **)
            analysis_clean = re.sub(r'(?<!\*)\*([^*]+?)\*(?!\*)', r'<i>\1</i>', analysis_clean)
            
            # Split analysis into paragraphs for better formatting
            analysis_paragraphs = analysis_clean.split('\n\n')
            
            # Filter out empty paragraphs and very short ones that could cause blank pages
            valid_paragraphs = []
            for para in analysis_paragraphs:
                para_content = para.strip()
                # Skip empty paragraphs and very short lines that are just formatting artifacts
                if len(para_content) > 5 and not para_content.isspace():
                    valid_paragraphs.append(para_content)
            
            # Only add paragraphs if we have substantial content
            if valid_paragraphs:
                for para_content in valid_paragraphs:
                    try:
                        story.append(Paragraph(para_content, styles['Analysis']))
                        story.append(Spacer(1, 6))  # Small spacing between paragraphs
                    except Exception as e:
                        # If paragraph fails due to HTML parsing, try with plain text
                        logger.warning(f"Paragraph HTML parsing failed for {case_number}, using plain text: {str(e)[:100]}")
                        # Strip all markup and use plain text
                        plain_content = re.sub(r'<[^>]+>', '', para_content)
                        plain_content = plain_content.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')
                        story.append(Paragraph(plain_content, styles['Analysis']))
                        story.append(Spacer(1, 6))
            
            # Add page break except for last case, and only if we added content
            if i < len(results) - 1:
                # Add some spacing before page break to avoid orphaned content
                story.append(Spacer(1, 12))
                story.append(PageBreak())
        
        # Build PDF
        try:
            doc.build(story)
            logger.info(f"Analysis report generated: {output_path}")
            return output_path
        except Exception as e:
            logger.error(f"Error generating PDF report: {e}")
            return None
    
    def generate_daily_report(self, target_date=None):
        """Generate a report for a specific date"""
        if target_date is None:
            target_date = self.get_current_business_day()
        
        # Convert datetime to string format for database comparison
        if hasattr(target_date, 'strftime'):
            date_filter = target_date.strftime('%Y-%m-%d')
        else:
            date_filter = target_date
        
        return self.generate_analysis_report(date_filter=date_filter)
    
    def get_docket_url(self, coa_num, date):
        """Generate the docket URL for a specific court and date"""
        date_str = date.strftime("%m/%d/%Y")
        return f"{self.base_url}Docket.aspx?coa=coa{coa_num:02d}&FullDate={date_str}"
    
    def get_abbreviation_and_justice(self, description, disposition=""):
        """Get abbreviation for opinion type and justice name based on description and disposition"""
        description_lower = description.lower()
        disposition_lower = disposition.lower()
        
        # Check disposition ONLY if it contains concurring or dissenting
        if "concurring" in disposition_lower:
            # Look for justice name in description
            justice_match = re.search(r'(?:concurring )?opinion by (?:chief )?justice (\w+)', description_lower)
            justice_name = justice_match.group(1) if justice_match else None
            return "con", justice_name
        elif "dissenting" in disposition_lower:
            # Look for justice name in description
            justice_match = re.search(r'(?:dissenting )?opinion by (?:chief )?justice (\w+)', description_lower)
            justice_name = justice_match.group(1) if justice_match else None
            return "dis", justice_name
        
        # Otherwise, use description to determine type
        if "memorandum" in description_lower:
            return "mem", None
        elif "dissenting" in description_lower:
            justice_match = re.search(r'dissenting opinion by (?:chief )?justice (\w+)', description_lower)
            justice_name = justice_match.group(1) if justice_match else None
            return "dis", justice_name
        elif "concurring" in description_lower:
            justice_match = re.search(r'concurring opinion by (?:chief )?justice (\w+)', description_lower)
            justice_name = justice_match.group(1) if justice_match else None
            return "con", justice_name
        elif "opinion" in description_lower:
            return "op", None
        
        return "", None
    
    def extract_case_number(self, case_link_text):
        """Extract case number from link text"""
        match = re.search(r'\d{2}-\d{2}-\d{5}-CR', case_link_text)
        return match.group(0) if match else None
    
    def download_pdf(self, pdf_url, filepath, max_retries=None):
        """Download PDF file to specified path with retry logic"""
        if max_retries is None:
            max_retries = self.max_retries
        last_exception = None
        
        for attempt in range(max_retries):
            try:
                response = self.session.get(pdf_url, timeout=self.request_timeout)
                response.raise_for_status()
                
                # Validate response is actually a PDF
                if response.headers.get('content-type', '').lower() != 'application/pdf':
                    logger.warning(f"Response is not a PDF (attempt {attempt + 1}): {response.headers.get('content-type', 'unknown')}")
                    if attempt == max_retries - 1:
                        logger.error(f"Not a PDF after {max_retries} attempts: {os.path.basename(filepath)}")
                        return False
                    time.sleep(2 ** attempt)  # Exponential backoff
                    continue
                
                # Ensure directory exists
                os.makedirs(os.path.dirname(filepath), exist_ok=True)
                
                with open(filepath, 'wb') as f:
                    f.write(response.content)
                
                # Verify file was written and has content
                if os.path.exists(filepath) and os.path.getsize(filepath) > 0:
                    # Basic PDF validation - check for PDF magic bytes
                    with open(filepath, 'rb') as f:
                        header = f.read(4)
                        if not header.startswith(b'%PDF'):
                            raise Exception("Downloaded file is not a valid PDF")
                    
                    logger.info(f"Downloaded: {os.path.basename(filepath)}")
                    return True
                else:
                    raise Exception("File was not written or is empty")
                    
            except (requests.exceptions.RequestException, requests.exceptions.Timeout, 
                    requests.exceptions.ConnectionError, Exception) as e:
                last_exception = e
                if attempt < max_retries - 1:
                    wait_time = 2 ** attempt  # Exponential backoff: 1, 2, 4 seconds
                    logger.warning(f"Download failed (attempt {attempt + 1}/{max_retries}): {os.path.basename(filepath)} - {e}. Retrying in {wait_time}s...")
                    time.sleep(wait_time)
                else:
                    logger.error(f"Failed to download {os.path.basename(filepath)} after {max_retries} attempts: {e}")
        
        return False
    
    def get_with_retry(self, url, max_retries=None):
        """Make HTTP request with retry logic"""
        if max_retries is None:
            max_retries = self.max_retries
        last_exception = None
        
        for attempt in range(max_retries):
            try:
                response = self.session.get(url, timeout=self.request_timeout)
                response.raise_for_status()
                return response
                
            except (requests.exceptions.RequestException, requests.exceptions.Timeout, 
                    requests.exceptions.ConnectionError) as e:
                last_exception = e
                if attempt < max_retries - 1:
                    wait_time = 2 ** attempt  # Exponential backoff: 1, 2, 4 seconds
                    logger.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {url} - {e}. Retrying in {wait_time}s...")
                    time.sleep(wait_time)
                else:
                    logger.error(f"Failed to fetch {url} after {max_retries} attempts: {e}")
                    raise last_exception
    
    def parse_criminal_causes(self, soup):
        """Parse the Criminal Causes Decided section"""
        criminal_cases = []
        
        # Look for the "Criminal Causes Decided" heading
        criminal_heading = None
        h3_elements = soup.find_all('h3')
        for h3 in h3_elements:
            if 'Criminal Causes Decided' in h3.get_text():
                criminal_heading = h3
                break
        
        if not criminal_heading:
            return criminal_cases
        
        # Find the table that follows the heading
        table = criminal_heading.find_next('table', class_='rgMasterTable')
        if not table:
            return criminal_cases
        
        # Find all rows in the table body
        tbody = table.find('tbody')
        if not tbody:
            return criminal_cases
        
        rows = tbody.find_all('tr', class_=['rgRow', 'rgAltRow'])
        
        for row in rows:
            case_data = self.parse_case_row(row)
            if case_data:
                criminal_cases.append(case_data)
        
        return criminal_cases
    
    def parse_case_row(self, row):
        """Parse a single case row"""
        try:
            # Find the case number link
            case_link = row.find('a', href=re.compile(r'Case\.aspx\?cn=.*-CR'))
            if not case_link:
                return None
            
            case_number = self.extract_case_number(case_link.text.strip())
            if not case_number:
                return None
            
            # Find the disposition column - look for td with class "caseDisp"
            disposition = ""
            disposition_cell = row.find('td', class_='caseDisp')
            if disposition_cell:
                disposition = disposition_cell.text.strip()
            else:
                # Fallback: try 3rd column if caseDisp class not found
                cells = row.find_all('td')
                if len(cells) >= 3:
                    disposition = cells[2].text.strip()
            
            # Find all PDF links in this row
            pdf_links = []
            doc_tables = row.find_all('table', class_='docGrid')
            
            for doc_table in doc_tables:
                pdf_link = doc_table.find('a', href=re.compile(r'SearchMedia\.aspx'))
                if pdf_link:
                    # Get the description (opinion type)
                    desc_cell = pdf_link.find_parent('td').find_previous_sibling('td')
                    description = desc_cell.text.strip() if desc_cell else ""
                    
                    # Clean up the PDF URL
                    href = pdf_link['href']
                    href = href.replace('" + this.CurrentWebState.CurrentCourt + @"', 'coa01')
                    pdf_url = urljoin(self.base_url, href)
                    pdf_links.append({
                        'url': pdf_url,
                        'description': description
                    })
            
            if pdf_links:
                return {
                    'case_number': case_number,
                    'pdf_links': pdf_links,
                    'disposition': disposition
                }
        
        except Exception as e:
            logger.error(f"Error parsing case row: {e}")
        
        return None
    
    def get_opinion_sort_order(self, opinion_type):
        """Get sort order for opinion types: main opinions first, then concurring, then dissenting"""
        order_map = {
            'mem': 1,  # Memorandum opinions first
            'op': 1,   # Regular opinions first
            'con': 2,  # Concurring opinions second
            'dis': 3,  # Dissenting opinions last
            'opinion': 1  # Default opinions first
        }
        return order_map.get(opinion_type, 4)  # Unknown types go last
    
    def concatenate_pdfs(self, pdf_paths, output_path):
        """Concatenate multiple PDF files into one"""
        writer = PdfWriter()
        
        for pdf_path in pdf_paths:
            if os.path.exists(pdf_path):
                try:
                    reader = PdfReader(pdf_path)
                    for page in reader.pages:
                        writer.add_page(page)
                except Exception as e:
                    logger.error(f"Error reading PDF {pdf_path}: {e}")
        
        with open(output_path, 'wb') as output_file:
            writer.write(output_file)
    
    def process_case_opinions(self, case_number, case_info, court_name, date, date_folder):
        """Download and concatenate all opinions for a single case"""
        opinions = case_info['opinions']
        temp_files = []
        downloaded_count = 0
        
        # Final combined filename
        final_filename = f"{case_number}.pdf"
        final_filepath = os.path.join(date_folder, final_filename)
        
        # Check if final file already exists
        if os.path.exists(final_filepath):
            logger.info(f"Combined file already exists: {final_filename}")
            return 0
        
        # Sort opinions: main (mem/op) first, then concurring, then dissenting
        sorted_opinions = sorted(opinions, key=lambda x: (x['sort_order'], x['justice_name'] or ''))
        
        # Download individual opinion PDFs to temp files
        failed_opinions = []
        for opinion in sorted_opinions:
            temp_path = os.path.join(date_folder, opinion['temp_filename'])
            
            if self.download_pdf(opinion['url'], temp_path):
                temp_files.append(temp_path)
                downloaded_count += 1
            else:
                failed_opinions.append(opinion)
            
            time.sleep(self.download_delay)  # Delay between downloads
        
        # Log failed downloads for visibility
        if failed_opinions:
            failed_descriptions = [op['description'] for op in failed_opinions]
            logger.warning(f"Failed to download {len(failed_opinions)} opinions for case {case_number}: {', '.join(failed_descriptions)}")
        
        # Concatenate PDFs if we have multiple opinions, otherwise just rename
        if len(temp_files) > 1:
            logger.info(f"Concatenating {len(temp_files)} opinions for case {case_number}")
            self.concatenate_pdfs(temp_files, final_filepath)
        elif len(temp_files) == 1:
            src_path = temp_files[0]
            if os.path.exists(final_filepath):
                logger.info(f"Combined file already exists, dropping stale temp: {src_path}")
                try:
                    os.remove(src_path)
                except OSError:
                    pass
                return 0
            if not os.path.exists(src_path):
                logger.error(f"Temp file vanished before rename: {src_path}")
                return 0
            shutil.move(src_path, final_filepath)
            temp_files = []  # Don't delete the file we just renamed
        else:
            logger.warning(f"No PDFs downloaded for case {case_number}")
            return 0
        
        # Clean up temp files
        for temp_file in temp_files:
            try:
                os.remove(temp_file)
            except Exception as e:
                logger.warning(f"Could not delete temp file {temp_file}: {e}")
        
        # Generate case URL and save to database
        case_url = f"https://search.txcourts.gov/Case.aspx?cn={case_number}"
        
        # Create summary of opinion types for database
        opinion_types = []
        justices = []
        for opinion in sorted_opinions:
            opinion_types.append(opinion['abbrev'])
            if opinion['justice_name']:
                justices.append(f"{opinion['abbrev']}_{opinion['justice_name']}")
        
        opinion_type_summary = '+'.join(opinion_types)
        justice_summary = ';'.join(justices) if justices else None
        
        # Collect all PDF URLs for this case
        pdf_urls = [opinion['url'] for opinion in sorted_opinions]
        pdf_urls_string = ';'.join(pdf_urls) if pdf_urls else None
        
        # Save to database
        self.save_opinion_to_db(
            case_number=case_number,
            court=court_name,
            opinion_date=date,
            opinion_type=opinion_type_summary,
            justice_name=justice_summary,
            filename=final_filename,
            file_path=final_filepath,
            case_url=case_url,
            pdf_url=pdf_urls_string
        )
        
        logger.info(f"Created combined opinion file: {final_filename}")
        return 1  # Return 1 for the combined file
    
    def save_opinion_to_db(self, case_number, court, opinion_date, opinion_type, justice_name, filename, file_path, case_url, pdf_url=None):
        """Save opinion information to database"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        try:
            cursor.execute('''
                INSERT OR IGNORE INTO opinions
                (case_number, court, opinion_date, opinion_type, justice_name, filename, file_path, case_url, pdf_url)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
            ''', (case_number, court, opinion_date, opinion_type, justice_name, filename, file_path, case_url, pdf_url))

            conn.commit()
            inserted = cursor.rowcount > 0
            # When a new opinion row appears for a case we haven't seen
            # before, scrape its style off search.txcourts.gov so the
            # slip-opinions page can show the caption. Failures are
            # non-fatal; the row is cached either way for later retry.
            try:
                case_styles.ensure_table(conn)
                case_styles.get_or_fetch_style(conn, case_number)
            except Exception as e:
                logger.warning(f"case_styles fetch failed for {case_number}: {e}")
            return inserted
        except Exception as e:
            logger.error(f"Error saving to database: {e}")
            return False
        finally:
            conn.close()
    
    def scrape_court_date(self, coa_num, date, date_folder):
        """Scrape opinions for a specific court and date"""
        url = self.get_docket_url(coa_num, date)
        date_str = date.strftime("%Y-%m-%d")
        court_name = f"COA{coa_num:02d}"
        
        logger.info(f"Scraping {court_name} for {date_str}")
        
        try:
            response = self.get_with_retry(url)
            soup = BeautifulSoup(response.content, 'html.parser')
            criminal_cases = self.parse_criminal_causes(soup)
            
            if not criminal_cases:
                logger.debug(f"No criminal cases found for {court_name} on {date_str}")
                return 0, 0
            
            downloaded_count = 0
            total_cases = len(criminal_cases)
            total_opinions_attempted = 0
            
            # Group opinions by case for concatenation
            case_opinions = {}
            for case in criminal_cases:
                case_number = case['case_number']
                disposition = case.get('disposition', '')
                
                if case_number not in case_opinions:
                    case_opinions[case_number] = {
                        'case_data': case,
                        'opinions': []
                    }
                
                for i, pdf_link in enumerate(case['pdf_links']):
                    description = pdf_link['description']
                    pdf_url = pdf_link['url']
                    
                    # Generate filename
                    abbrev, justice_name = self.get_abbreviation_and_justice(description, disposition)
                    if abbrev:
                        if justice_name and abbrev in ['con', 'dis']:
                            temp_filename = f"{case_number}_{abbrev}_{justice_name}_temp.pdf"
                        else:
                            temp_filename = f"{case_number}_{abbrev}_temp.pdf"
                    else:
                        suffix = f"_{i+1}" if len(case['pdf_links']) > 1 else ""
                        temp_filename = f"{case_number}{suffix}_temp.pdf"
                    
                    case_opinions[case_number]['opinions'].append({
                        'url': pdf_url,
                        'description': description,
                        'disposition': disposition,
                        'abbrev': abbrev or "opinion",
                        'justice_name': justice_name,
                        'temp_filename': temp_filename,
                        'sort_order': self.get_opinion_sort_order(abbrev or "opinion")
                    })
                    total_opinions_attempted += 1
            
            # Process each case: download individual PDFs and concatenate
            for case_number, case_info in case_opinions.items():
                try:
                    downloaded_count += self.process_case_opinions(
                        case_number, case_info, court_name, date, date_folder
                    )
                except Exception as e:
                    logger.error(f"Error processing case {case_number}: {e}")
                
                # Be respectful with delays
                time.sleep(self.download_delay)
            
            logger.info(f"Downloaded {downloaded_count} files from {court_name} on {date_str}")
            if total_opinions_attempted > downloaded_count:
                failed_count = total_opinions_attempted - downloaded_count
                logger.warning(f"Failed to download {failed_count} out of {total_opinions_attempted} opinions for {court_name} on {date_str}")
            return total_cases, downloaded_count
            
        except Exception as e:
            logger.error(f"Error scraping {court_name} on {date_str}: {e}")
            return 0, 0
    
    def run_daily_scrape(self):
        """Run daily scrape for today's opinions and rollover courts from yesterday
        
        Checks today's opinions for all courts. Courts with no opinions are added
        to rollover list for checking tomorrow. Also checks any rollover courts
        from yesterday that had no opinions.
        """
        today = self.get_current_business_day()
        yesterday = today - timedelta(days=1)
        run_date = datetime.now().date()
        
        logger.info(f"Starting daily scrape for today ({today.strftime('%Y-%m-%d')}) with rollover from yesterday")
        
        # Create date-based folder for today
        date_folder = os.path.join(self.data_dir, today.strftime('%Y%m%d'))
        os.makedirs(date_folder, exist_ok=True)
        
        # Initialize run record
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            INSERT INTO daily_runs (run_date, target_date, status)
            VALUES (?, ?, 'running')
        ''', (run_date, today))
        run_id = cursor.lastrowid
        conn.commit()
        conn.close()
        
        total_cases = 0
        total_downloaded = 0
        courts_checked = 0
        courts_with_no_opinions = []
        
        try:
            # First, check rollover courts from yesterday
            rollover_courts = self.get_rollover_courts(yesterday)
            if rollover_courts:
                logger.info(f"Checking {len(rollover_courts)} rollover courts from yesterday: {rollover_courts}")
                for coa_num in rollover_courts:
                    cases_found, files_downloaded = self.scrape_court_date(coa_num, yesterday, date_folder)
                    total_cases += cases_found
                    total_downloaded += files_downloaded
                    courts_checked += 1
                    
                    # Small delay between courts
                    time.sleep(int(os.getenv('COURT_DELAY', '2')))
                
                # Clear processed rollover courts
                self.clear_rollover_courts(yesterday)
            
            # Now scrape all 14 courts for today
            logger.info(f"Checking all courts for today's opinions ({today.strftime('%Y-%m-%d')})")
            for coa_num in range(1, 15):
                cases_found, files_downloaded = self.scrape_court_date(coa_num, today, date_folder)
                total_cases += cases_found
                total_downloaded += files_downloaded
                courts_checked += 1
                
                # Track courts with no opinions for rollover
                if cases_found == 0:
                    courts_with_no_opinions.append(coa_num)
                
                # Small delay between courts
                time.sleep(int(os.getenv('COURT_DELAY', '2')))
            
            # Add courts with no opinions to rollover for tomorrow
            if courts_with_no_opinions:
                logger.info(f"Adding {len(courts_with_no_opinions)} courts to rollover for tomorrow: {courts_with_no_opinions}")
                for coa_num in courts_with_no_opinions:
                    self.add_court_to_rollover(coa_num, today)
            
            # Update run record as completed
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            cursor.execute('''
                UPDATE daily_runs 
                SET total_courts_checked = ?, total_cases_found = ?, 
                    total_files_downloaded = ?, status = 'completed'
                WHERE id = ?
            ''', (courts_checked, total_cases, total_downloaded, run_id))
            conn.commit()
            conn.close()
            
            logger.info(f"Daily scrape completed!")
            logger.info(f"Courts checked: {courts_checked}")
            logger.info(f"Total cases found: {total_cases}")
            logger.info(f"Total files downloaded: {total_downloaded}")
            logger.info(f"Files saved to: {date_folder}")
            
            # Run analysis if enabled and new files were downloaded
            if self.analysis_enabled and total_downloaded > 0:
                logger.info("Starting analysis of newly downloaded opinions...")
                self.run_analysis_batch(limit=total_downloaded)
            
        except Exception as e:
            # Update run record with error
            conn = sqlite3.connect(self.db_path)
            cursor = conn.cursor()
            cursor.execute('''
                UPDATE daily_runs 
                SET total_courts_checked = ?, total_cases_found = ?, 
                    total_files_downloaded = ?, status = 'error', error_message = ?
                WHERE id = ?
            ''', (courts_checked, total_cases, total_downloaded, str(e), run_id))
            conn.commit()
            conn.close()
            
            logger.error(f"Daily scrape failed: {e}")
            raise

    def generate_prompt_pdf(self, target_date):
        """Generate a PDF containing the analysis prompt"""
        if not self.analysis_prompt:
            logger.error("No analysis prompt available")
            return None
        
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        output_path = os.path.join(self.data_dir, f"pdrbot_prompt_{timestamp}.pdf")
        
        try:
            # Create PDF document
            doc = SimpleDocTemplate(output_path, pagesize=letter,
                                  rightMargin=72, leftMargin=72, topMargin=72, bottomMargin=18)
            
            # Create custom styles
            styles = getSampleStyleSheet()
            styles.add(ParagraphStyle(
                'CustomTitle',
                parent=styles['Heading1'],
                fontSize=16,
                spaceAfter=30,
                alignment=TA_CENTER,
                textColor=colors.black
            ))
            styles.add(ParagraphStyle(
                'PromptText',
                parent=styles['Normal'],
                fontSize=10,
                spaceAfter=12,
                alignment=TA_JUSTIFY,
                textColor=colors.black,
                fontName='Courier',
                leading=12
            ))
            
            story = []
            
            # Title
            title = f"PDRBot Analysis Prompt"
            story.append(Paragraph(title, styles['CustomTitle']))
            story.append(Spacer(1, 20))
            
            # Date information
            date_info = f"Report Date: {target_date}<br/>Generated: {datetime.now().strftime('%B %d, %Y at %I:%M %p')}"
            story.append(Paragraph(date_info, styles['Normal']))
            story.append(Spacer(1, 30))
            
            # Prompt content - split into paragraphs for better formatting
            prompt_lines = self.analysis_prompt.split('\n')
            current_paragraph = []
            
            for line in prompt_lines:
                line = line.strip()
                if not line:
                    # Empty line - end current paragraph if it has content
                    if current_paragraph:
                        para_text = '<br/>'.join(current_paragraph)
                        story.append(Paragraph(para_text, styles['PromptText']))
                        story.append(Spacer(1, 6))
                        current_paragraph = []
                else:
                    # Escape HTML characters and add to current paragraph
                    escaped_line = line.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
                    current_paragraph.append(escaped_line)
            
            # Add any remaining paragraph
            if current_paragraph:
                para_text = '<br/>'.join(current_paragraph)
                story.append(Paragraph(para_text, styles['PromptText']))
            
            # Build PDF
            doc.build(story)
            logger.info(f"Prompt PDF generated: {output_path}")
            return output_path
            
        except Exception as e:
            logger.error(f"Error generating prompt PDF: {e}")
            return None

    def _build_email_html(self, target_date, results, interesting_count,
                          wins=None, counsel_by_case=None):
        """Build HTML email body with case cards and direct links.

        wins: list of defense-win dicts from defense_wins.collect_defense_wins(),
        rendered as a green section at the top of the email.
        counsel_by_case: case_number -> list of counsel dicts for interesting cases.
        """
        wins = wins or []
        counsel_by_case = counsel_by_case or {}
        # Build case cards
        case_cards_html = ""
        if interesting_count > 0:
            for result in results:
                case_number = result[0]
                court = result[1]
                analysis_text = result[3]
                issue_count = result[5]
                pdf_url = result[9]
                pdr_score = result[11] if len(result) > 11 else None

                appellant_name = self.extract_appellant_name(analysis_text)
                case_label = appellant_name if appellant_name else case_number
                headlines = self.extract_headlines_from_analysis(analysis_text)
                case_url = self.generate_case_url(case_number, court)

                # Score badge color
                if pdr_score and pdr_score >= 8:
                    badge_color = "#c0392b"  # red
                elif pdr_score and pdr_score >= 6:
                    badge_color = "#d68910"  # amber
                elif pdr_score:
                    badge_color = "#2874a6"  # blue
                else:
                    badge_color = "#7f8c8d"  # gray

                score_badge = ""
                if pdr_score:
                    score_badge = (
                        f'<span style="display:inline-block;background:{badge_color};'
                        f'color:#fff;font-weight:bold;padding:2px 8px;border-radius:3px;'
                        f'font-size:13px;margin-left:8px;">{pdr_score}/10</span>'
                    )

                headlines_html = ""
                if headlines:
                    items = "".join(
                        f"<li style='margin:3px 0;color:#333;'>{hl}</li>"
                        for hl in headlines
                    )
                    headlines_html = f"<ul style='margin:6px 0 0 0;padding-left:20px;'>{items}</ul>"
                else:
                    issue_word = "issue" if issue_count == 1 else "issues"
                    headlines_html = f"<p style='margin:6px 0 0 0;color:#555;'>{issue_count} {issue_word}</p>"

                counsel_html = self._counsel_html(
                    counsel_by_case.get(case_number, []), color="#2874a6")

                # Direct PDF link if available
                link_html = f'<a href="{case_url}" style="color:#2874a6;text-decoration:none;">Case page</a>'
                if pdf_url and pdf_url.strip():
                    first_pdf = pdf_url.split(';')[0]
                    link_html += f' &middot; <a href="{first_pdf}" style="color:#2874a6;text-decoration:none;">Opinion PDF</a>'

                case_cards_html += f"""
                <tr><td style="padding:10px 16px;border-bottom:1px solid #eee;">
                    <div style="font-size:15px;font-weight:bold;color:#1a1a1a;">
                        {case_label} <span style="font-weight:normal;color:#666;">({court})</span>{score_badge}
                    </div>
                    {headlines_html}
                    {counsel_html}
                    <div style="margin-top:4px;font-size:12px;">{link_html}</div>
                </td></tr>"""

        # Defense-wins section (top of the email).
        wins_html = ""
        if wins:
            win_cards = ""
            for w in wins:
                links = (
                    f'<a href="{w["case_url"]}" style="color:#1e8449;text-decoration:none;">Case page</a>'
                )
                if w.get("pdf_url"):
                    links += (
                        f' &middot; <a href="{w["pdf_url"]}" '
                        f'style="color:#1e8449;text-decoration:none;">Opinion PDF</a>'
                    )
                counsel_html = self._counsel_html(
                    w.get("counsel", []), color="#1e8449")
                win_cards += f'''
                <tr><td style="padding:10px 16px;border-bottom:1px solid #d5e8d4;background:#f4faf4;">
                    <div style="font-size:15px;font-weight:bold;color:#145a32;">{w['style']}</div>
                    <div style="margin-top:2px;font-size:13px;color:#333;">{w['case_number']}
                        <span style="color:#666;">({w['court']}, {w['date']})</span>
                        <span style="display:inline-block;background:#1e8449;color:#fff;font-weight:bold;padding:2px 8px;border-radius:3px;font-size:12px;margin-left:8px;">{w['disposition']}</span>
                    </div>
                    {counsel_html}
                    <div style="margin-top:4px;font-size:12px;">{links}</div>
                </td></tr>'''
            wins_html = f'''
    <tr><td style="padding:12px 24px;background:#eaf7ea;border-bottom:1px solid #d5e8d4;">
        <div style="font-size:16px;font-weight:bold;color:#145a32;">&#127942; Defense Wins</div>
    </td></tr>
    <tr><td style="padding:0;">
        <table width="100%" cellpadding="0" cellspacing="0">
            {win_cards}
        </table>
    </td></tr>'''

        # Assemble full HTML
        if interesting_count > 0:
            summary_line = f"{interesting_count} case{'s' if interesting_count != 1 else ''} with interesting issues"
        else:
            summary_line = "No interesting issues identified"
        if not wins:
            summary_line += ' <span style="color:#888;">&middot; no new defense wins</span>'

        html = f"""<!DOCTYPE html>
<html><head><meta charset="utf-8"></head>
<body style="margin:0;padding:0;font-family:-apple-system,Helvetica,Arial,sans-serif;background:#f5f5f5;">
<table width="100%" cellpadding="0" cellspacing="0" style="max-width:600px;margin:0 auto;background:#ffffff;">
    <tr><td style="background:#1a1a2e;padding:20px 24px;">
        <div style="color:#ffffff;font-size:20px;font-weight:bold;">PDRBot Daily Report</div>
        <div style="color:#aaa;font-size:14px;margin-top:4px;">{target_date}</div>
    </td></tr>{wins_html}
    <tr><td style="padding:16px 24px;background:#f8f8f8;border-bottom:1px solid #ddd;">
        <div style="font-size:15px;color:#333;">{summary_line}</div>
    </td></tr>
    <tr><td style="padding:0;">
        <table width="100%" cellpadding="0" cellspacing="0">
            {case_cards_html}
        </table>
    </td></tr>
    <tr><td style="padding:16px 24px;background:#f8f8f8;border-top:1px solid #ddd;font-size:12px;color:#888;">
        Full analysis is in the attached PDF report.<br>
        PDRBot runs at 9:10 a.m. Opinions released after that will appear in the next day's report.<br><br>
        See an error? Email <a href="mailto:mb@ivi3.com" style="color:#2874a6;">mb@ivi3.com</a>.
        &middot; <a href="https://github.com/markwbennett/PDRbot" style="color:#2874a6;">Source code</a><br><br>
        To unsubscribe, reply with "unsubscribe" in the subject or body.
    </td></tr>
</table>
</body></html>"""
        return html

    def _build_email_plain(self, target_date, results, interesting_count,
                           wins=None, counsel_by_case=None):
        """Build plain-text fallback email body."""
        wins = wins or []
        counsel_by_case = counsel_by_case or {}
        if wins:
            wins_section = "DEFENSE WINS\n\n"
            for w in wins:
                wins_section += (
                    f"  {w['style']}\n"
                    f"    {w['case_number']} ({w['court']}, {w['date']}) -- {w['disposition']}\n"
                )
                wins_section += self._counsel_plain_lines(
                    w.get("counsel", []), label="Defense counsel")
                wins_section += f"    {w['case_url']}\n\n"
        else:
            wins_section = "No new defense wins on the COA dockets.\n\n"
        if interesting_count > 0:
            cases_section = f"{interesting_count} case(s) with interesting issues:\n\n"
            for result in results:
                case_number = result[0]
                court = result[1]
                analysis_text = result[3]
                issue_count = result[5]
                pdr_score = result[11] if len(result) > 11 else None

                appellant_name = self.extract_appellant_name(analysis_text)
                case_label = appellant_name if appellant_name else case_number
                if court:
                    case_label += f" ({court})"

                score_str = f" [PDR {pdr_score}/10]" if pdr_score else ""
                headlines = self.extract_headlines_from_analysis(analysis_text)
                if headlines:
                    cases_section += f"  {case_label}{score_str}\n"
                    for hl in headlines:
                        cases_section += f"    - {hl}\n"
                else:
                    issue_word = "issue" if issue_count == 1 else "issues"
                    cases_section += f"  {case_label}{score_str} -- {issue_count} {issue_word}\n"
                cases_section += self._counsel_plain_lines(
                    counsel_by_case.get(case_number, []),
                    label="Defense counsel")
                cases_section += "\n"
        else:
            cases_section = "No interesting issues were identified."

        return f"""PDRBot Daily Report for {target_date}

{wins_section}{cases_section}
Full analysis is in the attached PDF report.

PDRBot runs at 9:10 a.m. Opinions released after that will appear in the next day's report.

See an error? Email mb@ivi3.com.
Source: https://github.com/markwbennett/PDRbot

To unsubscribe, reply with 'unsubscribe' in the subject or body.
"""

    def send_email_report(self, report_path, target_date, date_range=None,
                          wins=None, wins_state=None):
        """Send email with PDF report attachment

        Args:
            report_path: Path to the PDF report
            target_date: Date string for subject line
            date_range: Optional tuple of (start_date, end_date) for querying results
        """
        if not self.email_enabled:
            logger.info("Email sending is disabled")
            return False

        all_recipients = self.get_all_recipients()

        if not all([self.email_from, self.email_auth_user, self.email_password]) or not all_recipients:
            logger.error("Email configuration incomplete or no recipients")
            return False

        # report_path may be None when PDF generation failed; we still send the
        # inline HTML/plain body so subscribers see the analyses.
        if report_path and not os.path.exists(report_path):
            logger.error(f"Report file not found: {report_path}; sending without attachment")
            report_path = None

        # Get interesting issues for the date(s)
        if date_range:
            results = self.get_analysis_results(date_range=date_range, interesting_only=True)
        else:
            results = self.get_analysis_results(date_filter=target_date, interesting_only=True)
        interesting_count = len(results)

        # Defense wins from the COA dockets (ported from tx-judicial-scraper).
        # run_daily_automation scrapes wins once and passes them in so the
        # docket is not fetched twice; fall back to scraping here when called
        # directly (e.g. the resume path) without pre-scraped wins.
        if wins is None:
            try:
                wins, wins_state = defense_wins.collect_defense_wins()
            except Exception as exc:
                logger.warning(f"Defense-wins docket check failed: {exc}")
                wins, wins_state = [], None

        # Harvest defense counsel (name / SBOT / email) from briefs for every
        # case on this report — defense wins and interesting opinions alike —
        # so the listing is outreach-ready. Never raises.
        win_case_numbers = {w["case_number"] for w in wins}
        harvest_targets = list(wins)
        for result in results:
            target = self._harvest_target_from_result(result)
            if target["case_number"] not in win_case_numbers:
                harvest_targets.append(target)
        # Re-enrich wins that arrived already enriched from run_daily_automation
        # only when counsel is missing; otherwise still fill interesting cases.
        needs_enrich = [
            t for t in harvest_targets
            if not t.get("counsel")
        ]
        if needs_enrich:
            brief_harvest.enrich_cases(needs_enrich, self.db_path)
        counsel_by_case = {
            t["case_number"]: t.get("counsel", []) for t in harvest_targets
        }
        for w in wins:
            if not w.get("counsel"):
                w["counsel"] = counsel_by_case.get(w["case_number"], [])

        # Subject line: show count + up to three top headlines.
        if interesting_count == 0:
            interesting_text = "no interesting issues"
        elif interesting_count == 1:
            interesting_text = "1 case"
        else:
            interesting_text = f"{interesting_count} cases"
        subject = f"{self.email_subject_prefix}\u2014{target_date}\u2014{interesting_text}"

        # Append up to three short headlines so the inbox preview carries signal.
        if interesting_count > 0:
            headlines = []
            for r in results[:3]:
                text = r[3] if len(r) > 3 else ""
                extracted = self.extract_headlines_from_analysis(text) or []
                if extracted:
                    headlines.append(extracted[0])
            if headlines:
                trimmed = [h[:50].strip().rstrip(".") for h in headlines]
                suffix = "; ".join(trimmed)
                combined = f"{subject}: {suffix}"
                subject = combined if len(combined) <= 180 else combined[:177] + "..."

        if wins:
            subject = f"{len(wins)} defense win{'s' if len(wins) != 1 else ''}\u2014{subject}"

        # Build email content
        html_body = self._build_email_html(
            target_date, results, interesting_count,
            wins=wins, counsel_by_case=counsel_by_case)
        plain_body = self._build_email_plain(
            target_date, results, interesting_count,
            wins=wins, counsel_by_case=counsel_by_case)

        # Read report attachment once (may be absent if PDF generation failed).
        report_data = None
        report_filename = None
        if report_path:
            with open(report_path, "rb") as f:
                report_data = f.read()
            report_filename = os.path.basename(report_path)

        try:
            successful_sends = 0
            with self.smtp_connection() as server:
                for recipient in all_recipients:
                    try:
                        msg = MIMEMultipart('alternative')
                        msg['From'] = self.email_from
                        msg['To'] = recipient
                        msg['Subject'] = subject

                        # RFC 8058 one-click unsubscribe — required for Gmail/Yahoo deliverability.
                        unsub_address = os.getenv('UNSUBSCRIBE_EMAIL', self.subscription_email or self.email_from)
                        msg['List-Unsubscribe'] = f'<mailto:{unsub_address}?subject=unsubscribe>'
                        msg['List-Unsubscribe-Post'] = 'List-Unsubscribe=One-Click'

                        # Plain-text first, HTML second (clients prefer last)
                        msg.attach(MIMEText(plain_body, 'plain'))
                        msg.attach(MIMEText(html_body, 'html'))

                        if report_data is not None:
                            # Attach report PDF.
                            part = MIMEBase('application', 'octet-stream')
                            part.set_payload(report_data)
                            encoders.encode_base64(part)
                            part.add_header(
                                'Content-Disposition',
                                f'attachment; filename="{report_filename}"'
                            )
                            outer = MIMEMultipart('mixed')
                            outer['From'] = msg['From']
                            outer['To'] = msg['To']
                            outer['Subject'] = msg['Subject']
                            outer['List-Unsubscribe'] = msg['List-Unsubscribe']
                            outer['List-Unsubscribe-Post'] = msg['List-Unsubscribe-Post']
                            outer.attach(msg)
                            outer.attach(part)
                            envelope = outer
                        else:
                            envelope = msg

                        server.sendmail(self.email_from, [recipient], envelope.as_string())
                        logger.info(f"Email sent to {recipient}")
                        successful_sends += 1

                    except Exception as e:
                        logger.error(f"Failed to send email to {recipient}: {e}")
                        continue

            if successful_sends > 0:
                logger.info(f"Sent emails to {successful_sends}/{len(all_recipients)} recipients")
                if wins and wins_state is not None:
                    try:
                        defense_wins.mark_reported(wins_state, wins)
                    except Exception as exc:
                        logger.warning(f"Failed to persist defense-wins state: {exc}")
                return True
            else:
                logger.error("Failed to send email to any recipients")
                return False

        except Exception as e:
            logger.error(f"Failed to send emails: {e}")
            return False

    def send_test_email(self, recipient_email):
        """Send a test email to a specific recipient"""
        if not self.email_enabled:
            logger.info("Email sending is disabled")
            return False

        if not all([self.email_from, self.email_auth_user, self.email_password]):
            logger.error("Email configuration incomplete")
            return False

        target_date = datetime.now().strftime('%Y-%m-%d')
        results = self.get_analysis_results(date_filter=target_date, interesting_only=True)
        interesting_count = len(results)

        if interesting_count == 0:
            interesting_text = "no interesting issues"
        elif interesting_count == 1:
            interesting_text = "1 case"
        else:
            interesting_text = f"{interesting_count} cases"

        try:
            msg = MIMEMultipart('alternative')
            msg['From'] = self.email_from
            msg['To'] = recipient_email
            msg['Subject'] = f"{self.email_subject_prefix}\u2014{target_date}\u2014{interesting_text} (TEST)"

            plain_body = f"TEST EMAIL - PDRBot Daily Report for {target_date}\n\nThis is a test email to verify email functionality.\n\nTo unsubscribe, reply with 'unsubscribe' in the subject or body.\n"
            html_body = self._build_email_html(target_date, results, interesting_count)

            msg.attach(MIMEText(plain_body, 'plain'))
            msg.attach(MIMEText(html_body, 'html'))

            with self.smtp_connection() as server:
                server.sendmail(self.email_from, [recipient_email], msg.as_string())

            logger.info(f"Test email sent to {recipient_email}")
            return True

        except Exception as e:
            logger.error(f"Failed to send test email to {recipient_email}: {e}")
            return False

    def check_execution_errors(self, date_str):
        """Check for execution errors on a given date and return count"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()

        cursor.execute('''
            SELECT COUNT(*)
            FROM analysis a
            WHERE a.opinion_date = ? AND LOWER(a.analysis_text) LIKE '%execution error%'
        ''', (date_str,))

        count = cursor.fetchone()[0]
        conn.close()
        return count

    def retry_execution_errors(self, date_str):
        """Retry all cases with execution errors for a given date"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()

        cursor.execute('''
            SELECT o.id, o.case_number, o.court, o.opinion_date, o.file_path
            FROM opinions o
            JOIN analysis a ON o.id = a.opinion_id
            WHERE a.opinion_date = ? AND LOWER(a.analysis_text) LIKE '%execution error%'
        ''', (date_str,))

        error_cases = cursor.fetchall()
        conn.close()

        if not error_cases:
            return 0

        logger.info(f"Retrying {len(error_cases)} cases with execution errors")
        successful = 0

        for opinion_id, case_number, court, opinion_date, file_path in error_cases:
            logger.info(f"Retrying {case_number}...")

            # Extract text
            text_content = self.extract_text_from_pdf(file_path)
            if not text_content:
                logger.error(f"Failed to extract text from {file_path}")
                continue

            # Analyze
            analysis_result = self.analyze_opinion_with_claude(text_content, case_number)

            if not analysis_result or "execution error" in analysis_result.lower():
                logger.warning(f"Still getting execution error for {case_number}")
                continue

            # Success! Save the new analysis
            success = self.save_analysis_to_db(opinion_id, case_number, court, opinion_date, analysis_result)
            if success:
                logger.info(f"Successfully re-analyzed {case_number}")
                successful += 1

            # Rate limiting
            time.sleep(2)

        logger.info(f"Retry complete: {successful}/{len(error_cases)} successful")
        return successful

    def check_court_staleness(self, threshold_days=21):
        """Return per-court list of (court, days_since_last_interesting). Any
        court past threshold_days is flagged stale — usually a scraper regression.
        Returns a list of dicts; empty if all courts are current or have no history."""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute("""
            SELECT o.court,
                   MAX(o.opinion_date) AS last_interesting,
                   JULIANDAY('now') - JULIANDAY(MAX(o.opinion_date)) AS days_ago
            FROM opinions o
            JOIN analysis a ON a.opinion_id = o.id
            WHERE a.is_interesting = 1
            GROUP BY o.court
        """)
        rows = cursor.fetchall()
        conn.close()
        stale = []
        for court, last, days in rows:
            if days is None:
                continue
            days_int = int(days)
            if days_int > threshold_days:
                stale.append({"court": court, "last_interesting": last, "days_ago": days_int})
        return stale

    def run_triage_audit(self, sample_size=10):
        """Sample N recent Haiku-ROUTINE analyses, re-run each through the full
        analysis pass (Opus via the main prompt), and report the disagreement
        rate. A disagreement is when Opus identifies any interesting issue on
        a case Haiku classified ROUTINE — i.e., a false negative.
        """
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute("""
            SELECT a.id, a.case_number, a.analysis_text, o.file_path, o.court
            FROM analysis a
            JOIN opinions o ON a.opinion_id = o.id
            WHERE a.analysis_text LIKE '%Haiku classified as ROUTINE%'
            ORDER BY a.analysis_timestamp DESC
            LIMIT ?
        """, (sample_size,))
        samples = cursor.fetchall()
        conn.close()

        if not samples:
            print("No Haiku-ROUTINE analyses on record yet.")
            return

        print(f"Auditing {len(samples)} Haiku-ROUTINE analyses against the full prompt...\n")
        disagreements = []
        errors = 0
        for aid, case_num, haiku_text, file_path, court in samples:
            if not file_path or not os.path.exists(file_path):
                errors += 1
                print(f"  [SKIP] {case_num}: PDF missing ({file_path})")
                continue
            text = self.extract_text_from_pdf(file_path)
            if not text:
                errors += 1
                print(f"  [SKIP] {case_num}: PDF extraction failed")
                continue
            full_prompt = f"{self.analysis_prompt}\n\n--- OPINION TEXT ---\n{text}"
            try:
                opus_text = call_claude_with_retry(
                    prompt=full_prompt,
                    timeout=300,
                    max_retries=2,
                    base_delay=5,
                )
            except Exception as exc:
                errors += 1
                print(f"  [ERROR] {case_num}: {exc}")
                continue
            opus_found_interesting = bool(opus_text) and not opus_text.strip().upper().startswith(
                "TERSE REPORT: NO INTERESTING"
            )
            verdict = "DISAGREE" if opus_found_interesting else "AGREE"
            if opus_found_interesting:
                disagreements.append((case_num, court, opus_text))
            print(f"  [{verdict}] {case_num} ({court})")

        evaluated = len(samples) - errors
        rate = (len(disagreements) / evaluated * 100.0) if evaluated else 0.0
        print(f"\nSampled: {len(samples)}  Evaluated: {evaluated}  "
              f"Disagreements: {len(disagreements)}  Rate: {rate:.1f}%")
        if rate > 5.0:
            print("WARNING: triage disagreement rate >5%. Consider loosening the Haiku prompt.")
        if disagreements:
            print("\nCases where Opus identified issues Haiku missed:")
            for case_num, court, text in disagreements:
                first_line = text.strip().splitlines()[0] if text else ""
                print(f"  - {case_num} ({court}): {first_line[:120]}")

    def run_daily_automation(self, resume_run_id=None):
        """Run complete daily automation with resumption support"""
        target_date = self.get_current_business_day()
        date_str = target_date.strftime('%Y-%m-%d')
        
        # Check for incomplete runs first
        if resume_run_id is None:
            incomplete_runs = self.find_incomplete_runs(date_str)
            if incomplete_runs:
                latest_run = incomplete_runs[0]
                run_id = latest_run[0]
                logger.info(f"Found incomplete run {run_id} for {date_str}. Resuming...")
                return self.resume_incomplete_run(run_id)
        else:
            logger.info(f"Resuming specific run {resume_run_id}")
            return self.resume_incomplete_run(resume_run_id)
        
        # Start new run
        logger.info(f"Starting new daily automation for {date_str}")
        
        # Create run record
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            INSERT INTO daily_runs (run_date, target_date, status)
            VALUES (?, ?, ?)
        ''', (datetime.now().date(), target_date, 'running'))
        run_id = cursor.lastrowid
        conn.commit()
        conn.close()
        
        try:
            # Step 1: Scrape opinions
            logger.info("Step 1: Scraping opinions...")
            self.update_run_state(run_id, status='scraping')
            self.resume_daily_scrape(run_id, target_date)
            
            # Step 2: Run analysis
            # Use the date-agnostic queue: rolled-over opinions from prior
            # business days share today's run, so a date-filtered count
            # would skip them whenever today's courts have not yet posted.
            if self.analysis_enabled:
                unanalyzed_count = len(self.get_unanalyzed_opinions())
                if unanalyzed_count > 0:
                    logger.info(f"Step 2: Running analysis on {unanalyzed_count} cases...")
                    self.update_run_state(run_id, status='analyzing')
                    self.run_analysis_batch()
                else:
                    logger.info("Step 2: No cases need analysis")
            else:
                logger.warning("Analysis is disabled or Claude client not available")

            # Step 2.5: Check for and retry execution errors
            logger.info("Step 2.5: Checking for execution errors...")
            execution_errors = self.check_execution_errors(date_str)
            if execution_errors > 0:
                logger.warning(f"Found {execution_errors} execution errors - retrying...")
                self.retry_execution_errors(date_str)

            # Step 3: Generate report
            # First try a single-day report for the target date.  If that
            # finds nothing (e.g. after a backlog catch-up where today's
            # date has no interesting cases but earlier dates do), widen the
            # window to cover all unreported dates since the last completed
            # run.
            logger.info("Step 3: Generating report...")
            self.update_run_state(run_id, status='reporting')
            report_generation_error = None
            report_path = None
            report_date_label = date_str
            report_date_range = None
            try:
                report_path = self.generate_daily_report(target_date)
            except Exception as exc:
                report_generation_error = exc
                logger.error(f"generate_daily_report raised: {exc}")

            if not report_path and report_generation_error is None:
                # No interesting cases for today — widen the window.
                last_completed = self._get_last_completed_date()
                if last_completed:
                    range_start = (datetime.strptime(last_completed, '%Y-%m-%d').date()
                                   + timedelta(days=1)).strftime('%Y-%m-%d')
                else:
                    range_start = date_str
                range_end = date_str
                if range_start < range_end:
                    logger.info(f"No interesting cases for {date_str}; "
                                f"trying range {range_start} to {range_end}")
                    report_date_range = (range_start, range_end)
                    try:
                        report_path = self.generate_analysis_report(
                            date_range=report_date_range)
                    except Exception as exc:
                        report_generation_error = exc
                        logger.error(f"generate_analysis_report raised: {exc}")
                    report_date_label = f"{range_start} to {range_end}"

            # Defense-wins docket scrape runs on EVERY execution, independent
            # of whether there is an interesting opinion to email. A win that
            # posts after a morning run, or lands on a day with no interesting
            # opinions, must still go out. Scrape once here and hand the result
            # to send_email_report so the docket is not fetched twice.
            try:
                wins, wins_state = defense_wins.collect_defense_wins()
            except Exception as exc:
                logger.warning(f"Defense-wins docket check failed: {exc}")
                wins, wins_state = [], None
            # Defense-counsel harvest for wins + interesting cases runs inside
            # send_email_report (where both sets of cases are available).

            if not report_path and report_generation_error is None and not wins:
                # Genuine no-cases state — no interesting opinions AND no new
                # defense wins. Not a failure: the automation ran cleanly, there
                # was just nothing worth emailing. Return True so the wrapper
                # script does not send a spurious failure alert.
                self.update_run_state(run_id, status='no_cases')
                logger.info("No report generated (no interesting cases or defense wins)")
                return True

            if not report_path and report_generation_error is None:
                # No interesting opinions, but defense wins are pending: send a
                # wins-only email (no PDF attachment).
                logger.info(
                    f"No interesting cases for {date_str}, but "
                    f"{len(wins)} new defense win(s); sending wins-only email")

            if report_generation_error is not None:
                # PDF generation crashed but analyses succeeded — send email
                # without the PDF attachment so the results still reach subscribers.
                logger.warning("PDF generation failed; sending attachment-less email")
                self.update_run_state(run_id, status='reporting',
                                    error_message=f"PDF failed: {report_generation_error}")

            # Step 4: Check subscription emails
            logger.info("Step 4: Checking subscription emails...")
            self.check_subscription_emails()

            # Step 5: Send email
            if self.email_enabled:
                logger.info("Step 5: Sending email...")
                self.update_run_state(run_id, status='emailing')
                email_success = self.send_email_report(
                    report_path, report_date_label, date_range=report_date_range,
                    wins=wins, wins_state=wins_state)
                
                if email_success:
                    self.update_run_state(run_id, status='completed')
                    logger.info("Daily automation completed successfully")
                    return True
                else:
                    self.update_run_state(run_id, status='email_failed', 
                                        error_message="Email sending failed")
                    logger.error("Daily automation completed but email failed")
                    return False
            else:
                self.update_run_state(run_id, status='completed')
                logger.info("Daily automation completed (email disabled)")
                return True
                
        except Exception as e:
            error_msg = f"Daily automation failed: {str(e)}"
            self.update_run_state(run_id, status='failed', error_message=error_msg)
            logger.error(error_msg)
            return False

    def get_run_state(self, run_id):
        """Get the current state of a run"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            SELECT status, total_courts_checked, total_cases_found, 
                   total_files_downloaded, target_date, error_message
            FROM daily_runs 
            WHERE id = ?
        ''', (run_id,))
        
        result = cursor.fetchone()
        conn.close()
        
        if result:
            return {
                'status': result[0],
                'courts_checked': result[1],
                'cases_found': result[2],
                'files_downloaded': result[3],
                'target_date': result[4],
                'error_message': result[5]
            }
        return None
    
    def update_run_state(self, run_id, status=None, courts_checked=None, 
                        cases_found=None, files_downloaded=None, error_message=None):
        """Update the state of a run"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        updates = []
        params = []
        
        if status is not None:
            updates.append("status = ?")
            params.append(status)
        if courts_checked is not None:
            updates.append("total_courts_checked = ?")
            params.append(courts_checked)
        if cases_found is not None:
            updates.append("total_cases_found = ?")
            params.append(cases_found)
        if files_downloaded is not None:
            updates.append("total_files_downloaded = ?")
            params.append(files_downloaded)
        if error_message is not None:
            updates.append("error_message = ?")
            params.append(error_message)
        
        if updates:
            params.append(run_id)
            cursor.execute(f'''
                UPDATE daily_runs 
                SET {", ".join(updates)}
                WHERE id = ?
            ''', params)
            conn.commit()
        
        conn.close()
    
    def _get_last_completed_date(self):
        """Return the target_date of the most recent completed daily run, or None."""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        cursor.execute('''
            SELECT target_date FROM daily_runs
            WHERE status = 'completed'
            ORDER BY target_date DESC
            LIMIT 1
        ''')
        row = cursor.fetchone()
        conn.close()
        return str(row[0]) if row else None

    def find_incomplete_runs(self, target_date=None):
        """Find runs that were interrupted and can be resumed"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        if target_date:
            cursor.execute('''
                SELECT id, run_date, target_date, status, total_courts_checked,
                       total_cases_found, total_files_downloaded
                FROM daily_runs 
                WHERE target_date = ? AND status IN ('running', 'scraping', 'analyzing', 'reporting')
                ORDER BY run_timestamp DESC
            ''', (target_date,))
        else:
            cursor.execute('''
                SELECT id, run_date, target_date, status, total_courts_checked,
                       total_cases_found, total_files_downloaded
                FROM daily_runs 
                WHERE status IN ('running', 'scraping', 'analyzing', 'reporting')
                ORDER BY run_timestamp DESC
            ''')
        
        results = cursor.fetchall()
        conn.close()
        
        return results
    
    def resume_incomplete_run(self, run_id):
        """Resume an incomplete run from where it left off"""
        state = self.get_run_state(run_id)
        if not state:
            logger.error(f"Run {run_id} not found")
            return False
        
        target_date_str = state['target_date']
        target_date = datetime.strptime(target_date_str, '%Y-%m-%d').date()
        
        logger.info(f"Resuming run {run_id} for {target_date_str} from status: {state['status']}")
        
        try:
            # Update status to indicate we're resuming
            self.update_run_state(run_id, status='resuming')
            
            # Determine what steps need to be completed
            if state['status'] in ['running', 'scraping']:
                # Resume scraping
                logger.info("Resuming scraping phase...")
                self.update_run_state(run_id, status='scraping')
                self.resume_daily_scrape(run_id, target_date)
            
            # Check if analysis is needed
            unanalyzed_count = len(self.get_unanalyzed_opinions())
            if unanalyzed_count > 0 and self.analysis_enabled:
                logger.info(f"Resuming analysis phase... {unanalyzed_count} cases to analyze")
                self.update_run_state(run_id, status='analyzing')
                self.run_analysis_batch()
            
            # Generate report if needed
            if state['status'] in ['running', 'scraping', 'analyzing', 'reporting']:
                logger.info("Generating report...")
                self.update_run_state(run_id, status='reporting')
                report_path = self.generate_daily_report(target_date)
                
                if report_path and self.email_enabled:
                    logger.info("Sending email...")
                    self.update_run_state(run_id, status='emailing')
                    email_success = self.send_email_report(report_path, target_date_str)
                    
                    if email_success:
                        self.update_run_state(run_id, status='completed')
                        logger.info(f"Run {run_id} resumed and completed successfully")
                        return True
                    else:
                        self.update_run_state(run_id, status='email_failed', 
                                            error_message="Email sending failed")
                        logger.error(f"Run {run_id} completed but email failed")
                        return False
                else:
                    self.update_run_state(run_id, status='completed')
                    logger.info(f"Run {run_id} resumed and completed successfully (no email)")
                    return True
            
        except Exception as e:
            error_msg = f"Resume failed: {str(e)}"
            self.update_run_state(run_id, status='failed', error_message=error_msg)
            logger.error(f"Failed to resume run {run_id}: {e}")
            return False
    
    def get_unanalyzed_opinions_for_date(self, target_date):
        """Get unanalyzed opinions for a specific date

        Returns only one opinion per unique file_path to avoid analyzing
        the same PDF multiple times for consolidated cases.
        """
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()

        cursor.execute('''
            SELECT MIN(o.id) as id, o.case_number, o.court, o.opinion_date, o.file_path
            FROM opinions o
            LEFT JOIN analysis a ON o.id = a.opinion_id
            WHERE o.opinion_date = ? AND a.opinion_id IS NULL
            GROUP BY o.file_path
            ORDER BY o.case_number
        ''', (target_date,))

        results = cursor.fetchall()
        conn.close()
        return results
    
    def resume_daily_scrape(self, run_id, target_date):
        """Resume daily scrape with progress tracking
        
        Handles rollover courts from yesterday if resuming today's run,
        then continues with regular court checking and rollover tracking.
        """
        today = self.get_current_business_day()
        yesterday = today - timedelta(days=1)
        date_folder = os.path.join(self.data_dir, target_date.strftime('%Y%m%d'))
        os.makedirs(date_folder, exist_ok=True)
        
        courts_checked = 0
        total_cases = 0
        total_downloaded = 0
        courts_with_no_opinions = []
        
        try:
            # If resuming today's run, check rollover courts first
            if target_date == today:
                rollover_courts = self.get_rollover_courts(yesterday)
                if rollover_courts:
                    logger.info(f"Resuming: checking {len(rollover_courts)} rollover courts from yesterday")
                    for coa_num in rollover_courts:
                        cases_found, files_downloaded = self.scrape_court_date(coa_num, yesterday, date_folder)
                        total_cases += cases_found
                        total_downloaded += files_downloaded
                        courts_checked += 1
                        time.sleep(int(os.getenv('COURT_DELAY', '2')))
                    self.clear_rollover_courts(yesterday)
            
            for coa_num in range(1, 15):  # Courts 1-14
                try:
                    courts_checked += 1
                    logger.info(f"Scraping COA{coa_num:02d} for {target_date.strftime('%Y-%m-%d')}")
                    
                    # Update progress
                    self.update_run_state(run_id, courts_checked=courts_checked)
                    
                    # Scrape this court
                    cases_found, files_downloaded = self.scrape_court_date(coa_num, target_date, date_folder)
                    
                    total_cases += cases_found
                    total_downloaded += files_downloaded
                    
                    # Track courts with no opinions for rollover (only for today's date)
                    if target_date == today and cases_found == 0:
                        courts_with_no_opinions.append(coa_num)
                    
                    # Update progress
                    self.update_run_state(run_id, cases_found=total_cases, 
                                        files_downloaded=total_downloaded)
                    
                    if files_downloaded > 0:
                        logger.info(f"Downloaded {files_downloaded} files from COA{coa_num:02d} on {target_date.strftime('%Y-%m-%d')}")
                    elif cases_found > 0:
                        logger.warning(f"Failed to download {cases_found} out of {cases_found} opinions for COA{coa_num:02d} on {target_date.strftime('%Y-%m-%d')}")
                    
                    # Small delay between courts
                    time.sleep(self.download_delay)
                    
                except Exception as e:
                    logger.error(f"Error scraping COA{coa_num:02d}: {e}")
                    continue
            
            # Add courts with no opinions to rollover for tomorrow (only for today's date)
            if target_date == today and courts_with_no_opinions:
                logger.info(f"Adding {len(courts_with_no_opinions)} courts to rollover for tomorrow: {courts_with_no_opinions}")
                for coa_num in courts_with_no_opinions:
                    self.add_court_to_rollover(coa_num, target_date)
            
            # Final update
            self.update_run_state(run_id, status='scrape_completed', 
                                courts_checked=courts_checked,
                                cases_found=total_cases,
                                files_downloaded=total_downloaded)
            
            logger.info(f"Resume scrape completed!")
            logger.info(f"Courts checked: {courts_checked}")
            logger.info(f"Total cases found: {total_cases}")
            logger.info(f"Total files downloaded: {total_downloaded}")
            
        except Exception as e:
            self.update_run_state(run_id, status='scrape_failed', 
                                error_message=f"Scraping failed: {str(e)}")
            raise

    def load_members(self):
        """Load members from the members file"""
        try:
            if os.path.exists(self.members_file):
                with open(self.members_file, 'r') as f:
                    data = json.load(f)
                    return data.get('members', [])
            return []
        except Exception as e:
            logger.error(f"Error loading members file: {e}")
            return []
    
    def save_members(self, members):
        """Save members to the members file"""
        try:
            # Ensure data directory exists
            os.makedirs(os.path.dirname(self.members_file), exist_ok=True)
            
            data = {
                'members': members,
                'last_updated': datetime.now().isoformat(),
                'total_members': len(members)
            }
            
            with open(self.members_file, 'w') as f:
                json.dump(data, f, indent=2)
            
            logger.info(f"Saved {len(members)} members to {self.members_file}")
            return True
        except Exception as e:
            logger.error(f"Error saving members file: {e}")
            return False
    
    def add_member(self, email_address):
        """Add a new member to the subscription list"""
        members = self.load_members()
        email_address = email_address.lower().strip()
        
        if email_address not in members:
            members.append(email_address)
            if self.save_members(members):
                logger.info(f"Added new member: {email_address}")
                return True
        else:
            logger.info(f"Member already exists: {email_address}")
            return True
        
        return False
    
    def remove_member(self, email_address):
        """Remove a member from the subscription list"""
        members = self.load_members()
        email_address = email_address.lower().strip()
        
        if email_address in members:
            members.remove(email_address)
            if self.save_members(members):
                logger.info(f"Removed member: {email_address}")
                return True
        else:
            logger.info(f"Member not found: {email_address}")
            return True
        
        return False
    
    def get_all_recipients(self):
        """Get all email recipients (static + members)"""
        # Start with configured static recipients
        static_recipients = self.email_to.copy() if self.email_to else []
        
        # Add dynamic members
        members = self.load_members()
        
        # Combine and deduplicate
        all_recipients = list(set(static_recipients + members))
        
        return all_recipients
    
    def get_last_subscription_check(self):
        """Get the timestamp of the last subscription check"""
        try:
            if os.path.exists(self.last_check_file):
                with open(self.last_check_file, 'r') as f:
                    timestamp_str = f.read().strip()
                    return datetime.fromisoformat(timestamp_str)
            else:
                # If no previous check, default to 7 days ago
                return datetime.now() - timedelta(days=7)
        except Exception as e:
            logger.error(f"Error reading last check time: {e}")
            # Fallback to 7 days ago
            return datetime.now() - timedelta(days=7)
    
    def save_last_subscription_check(self, timestamp):
        """Save the timestamp of the last subscription check"""
        try:
            # Ensure data directory exists
            os.makedirs(os.path.dirname(self.last_check_file), exist_ok=True)
            
            with open(self.last_check_file, 'w') as f:
                f.write(timestamp.isoformat())
            
            logger.info(f"Updated last subscription check time: {timestamp.isoformat()}")
            return True
        except Exception as e:
            logger.error(f"Error saving last check time: {e}")
            return False
    
    def check_subscription_emails(self):
        """Check the subscription mailbox for subscribe/unsubscribe requests"""
        if not all([self.subscription_email, self.subscription_auth_user, self.subscription_password]):
            logger.warning("Subscription email configuration incomplete")
            return False
        
        try:
            # Connect to IMAP server
            mail = imaplib.IMAP4_SSL(self.subscription_imap_host, self.subscription_imap_port)
            mail.login(self.subscription_auth_user, self.subscription_password)
            
            # Try PDRbot folder first, then fallback to main INBOX
            try:
                status, data = mail.select('INBOX/PDRbot')
                if status != 'OK':
                    mail.select('INBOX')
                    logger.info("Using main INBOX folder for subscription emails")
                else:
                    logger.info("Using INBOX/PDRbot folder for subscription emails")
            except:
                mail.select('INBOX')
                logger.info("Fallback to main INBOX folder for subscription emails")
            
            # UID-based incremental search: idempotent, not spoofable via Date header.
            mailbox_key = 'subscription'
            last_uid = self._get_last_imap_uid(mailbox_key)
            search_criteria = f'{last_uid + 1}:*' if last_uid else 'ALL'
            status, message_ids = mail.uid('search', None, search_criteria)

            if status != 'OK':
                logger.error("Failed to search for emails by UID")
                return False

            message_ids = message_ids[0].split()
            # UID SEARCH 'N:*' returns the last message even when there are none newer.
            if last_uid:
                message_ids = [mid for mid in message_ids if int(mid) > last_uid]
            logger.info(f"Found {len(message_ids)} new emails since UID {last_uid}")
            max_uid_seen = last_uid
            processed_count = 0
            
            for msg_id in message_ids:
                try:
                    uid_int = int(msg_id)
                    if uid_int > max_uid_seen:
                        max_uid_seen = uid_int
                    # Fetch email by UID
                    status, msg_data = mail.uid('fetch', msg_id, '(RFC822)')
                    if status != 'OK':
                        continue
                    
                    # Parse email
                    email_message = email.message_from_bytes(msg_data[0][1])
                    
                    # Check if email date is actually after last check
                    email_date_str = email_message.get('Date')
                    if email_date_str:
                        try:
                            email_date = parsedate_to_datetime(email_date_str)
                            # Only process emails that arrived after our last check
                            if email_date <= last_check:
                                continue
                        except Exception:
                            pass  # If we can't parse the date, process the email anyway
                    
                    sender = email_message.get('From')
                    subject = email_message.get('Subject', '')
                    
                    # Extract sender email address
                    if '<' in sender and '>' in sender:
                        sender_email = sender.split('<')[1].split('>')[0]
                    else:
                        sender_email = sender
                    
                    # Get email body and subject
                    body = self.get_email_body(email_message)
                    subject = email_message.get('Subject', '').lower().strip()
                    
                    # Check both body and subject for subscription keywords
                    body_lower = body.lower().strip() if body else ""
                    is_subscribe = body_lower.startswith('subscribe') or subject == 'subscribe'
                    is_unsubscribe = body_lower.startswith('unsubscribe') or subject == 'unsubscribe'
                    
                    if is_subscribe:
                        if self.add_member(sender_email):
                            self.send_confirmation_email(sender_email, 'subscribed')
                            source = 'subject' if subject == 'subscribe' else 'body'
                            logger.info(f"Processed subscription request from {sender_email} (via {source})")
                            processed_count += 1
                    
                    elif is_unsubscribe:
                        if self.remove_member(sender_email):
                            self.send_confirmation_email(sender_email, 'unsubscribed')
                            source = 'subject' if subject == 'unsubscribe' else 'body'
                            logger.info(f"Processed unsubscription request from {sender_email} (via {source})")
                            processed_count += 1
                    
                    # Mark as read to avoid reprocessing
                    mail.store(msg_id, '+FLAGS', '\\Seen')
                    
                except Exception as e:
                    logger.error(f"Error processing email {msg_id}: {e}")
                    continue
            
            mail.close()
            mail.logout()
            
            # Save the current time as the last check time
            if max_uid_seen > last_uid:
                self._set_last_imap_uid(mailbox_key, max_uid_seen)
            self.save_last_subscription_check(datetime.now())
            
            if processed_count > 0:
                logger.info(f"Processed {processed_count} subscription requests")
            
            return True
            
        except Exception as e:
            logger.error(f"Error checking subscription emails: {e}")
            return False
    
    def get_email_body(self, email_message):
        """Extract text body from email message"""
        try:
            if email_message.is_multipart():
                for part in email_message.walk():
                    if part.get_content_type() == "text/plain":
                        return part.get_payload(decode=True).decode('utf-8', errors='ignore')
            else:
                return email_message.get_payload(decode=True).decode('utf-8', errors='ignore')
        except Exception as e:
            logger.error(f"Error extracting email body: {e}")
            return None
    
    def send_confirmation_email(self, recipient, action):
        """Send confirmation email for subscription changes"""
        try:
            msg = MIMEMultipart()
            msg['From'] = self.email_from
            msg['To'] = recipient
            msg['Subject'] = f"PDRBot Subscription {action.title()}"

            if action == 'subscribed':
                body = f"""You are now subscribed to PDRBot daily reports.

You will receive daily criminal law opinion analysis reports from the Texas Courts of Appeals.

To unsubscribe at any time, send an email to {self.subscription_email} with "unsubscribe" in the subject or body.
"""
            else:
                body = f"""You have been unsubscribed from PDRBot daily reports.

To resubscribe at any time, send an email to {self.subscription_email} with "subscribe" in the subject or body.
"""

            msg.attach(MIMEText(body, 'plain'))

            with self.smtp_connection() as server:
                server.sendmail(self.email_from, [recipient], msg.as_string())

            logger.info(f"Sent {action} confirmation to {recipient}")
            return True

        except Exception as e:
            logger.error(f"Error sending confirmation email to {recipient}: {e}")
            return False

def main():
    """Main entry point for PDRBot"""
    bot = PDRBot()
    
    if len(sys.argv) > 1:
        command = sys.argv[1].lower()
        
        if command == "analyze":
            # Run analysis on unanalyzed opinions
            limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
            bot.run_analysis_batch(limit=limit)
        elif command == "scrape":
            # Run daily scrape only
            bot.run_daily_scrape()
        elif command == "both":
            # Run scrape then analysis
            bot.run_daily_scrape()
            if bot.analysis_enabled:
                bot.run_analysis_batch()
        elif command == "report":
            # Generate PDF report from existing analyses
            if len(sys.argv) > 2:
                try:
                    date_str = sys.argv[2]
                    target_date = datetime.strptime(date_str, '%Y-%m-%d').date()
                    report_path = bot.generate_daily_report(target_date)
                except ValueError:
                    print("Invalid date format. Use YYYY-MM-DD")
                    sys.exit(1)
            else:
                report_path = bot.generate_analysis_report()
            
            if report_path:
                print(f"Report generated: {report_path}")
            else:
                print("No report generated (no interesting cases found)")
        elif command == "daily-report":
            # Generate report for today's analyses or specified date
            target_date = None
            if len(sys.argv) > 2:
                date_str = sys.argv[2]
                try:
                    target_date = datetime.strptime(date_str, "%Y-%m-%d")
                except ValueError:
                    print(f"Error: Invalid date format '{date_str}'. Use YYYY-MM-DD format.")
                    sys.exit(1)
            
            report_path = bot.generate_daily_report(target_date)
            if report_path:
                print(f"Daily report generated: {report_path}")
            else:
                print("No daily report generated (no interesting cases found)")
        elif command == "backfill-urls":
            # Backfill PDF URLs for existing records
            bot.backfill_pdf_urls()
        elif command == "analyze-dir":
            # Analyze all PDFs in a specific directory
            if len(sys.argv) > 2:
                directory_path = sys.argv[2]
                bot.analyze_directory_pdfs(directory_path)
            else:
                print("Usage: python pdrbot.py analyze-dir <directory_path>")
                sys.exit(1)
        elif command == "auto" or command == "automation":
            # Run full daily automation: scrape, analyze, report, and email
            success = bot.run_daily_automation()
            if success:
                print("Daily automation completed successfully")
            else:
                print("Daily automation failed")
                sys.exit(1)
        elif command == "resume":
            # Resume an incomplete run
            if len(sys.argv) > 2:
                try:
                    run_id = int(sys.argv[2])
                    success = bot.resume_incomplete_run(run_id)
                    if success:
                        print(f"Run {run_id} resumed and completed successfully")
                    else:
                        print(f"Failed to resume run {run_id}")
                        sys.exit(1)
                except ValueError:
                    print("Invalid run ID. Must be a number.")
                    sys.exit(1)
            else:
                # Show incomplete runs
                incomplete = bot.find_incomplete_runs()
                if incomplete:
                    print("Incomplete runs found:")
                    for run_id, run_date, target_date, status, courts, cases, files in incomplete:
                        print(f"  Run {run_id}: {target_date} (status: {status}, courts: {courts}, cases: {cases}, files: {files})")
                    print("\nUse 'python pdrbot.py resume <run_id>' to resume a specific run")
                else:
                    print("No incomplete runs found")
        elif command == "status":
            # Show recent run status
            incomplete = bot.find_incomplete_runs()
            if incomplete:
                print("Incomplete runs:")
                for run_id, run_date, target_date, status, courts, cases, files in incomplete:
                    print(f"  Run {run_id}: {target_date} (status: {status})")
            else:
                print("No incomplete runs")
            
            # Show recent completed runs
            conn = sqlite3.connect(bot.db_path)
            cursor = conn.cursor()
            cursor.execute('''
                SELECT id, target_date, status, total_files_downloaded, run_timestamp
                FROM daily_runs 
                WHERE status IN ('completed', 'failed', 'email_failed')
                ORDER BY run_timestamp DESC 
                LIMIT 5
            ''')
            recent = cursor.fetchall()
            conn.close()
            
            if recent:
                print("\nRecent completed runs:")
                for run_id, target_date, status, files, timestamp in recent:
                    print(f"  Run {run_id}: {target_date} ({status}, {files} files) - {timestamp}")
            else:
                print("No recent completed runs")
        elif command == "members":
            # Show subscription members
            members = bot.load_members()
            static_recipients = bot.email_to if bot.email_to else []
            
            print(f"Static recipients ({len(static_recipients)}):")
            for email in static_recipients:
                print(f"  - {email}")
            
            print(f"\nDynamic members ({len(members)}):")
            for email in members:
                print(f"  - {email}")
            
            all_recipients = bot.get_all_recipients()
            print(f"\nTotal unique recipients: {len(all_recipients)}")
        elif command == "check-subscriptions":
            # Manually check subscription emails
            print("Checking subscription emails...")
            success = bot.check_subscription_emails()
            if success:
                print("✅ Subscription check completed")
            else:
                print("❌ Subscription check failed")
        elif command == "triage-audit":
            n = int(sys.argv[2]) if len(sys.argv) > 2 else 10
            bot.run_triage_audit(sample_size=n)
        elif command == "court-staleness":
            threshold = int(sys.argv[2]) if len(sys.argv) > 2 else 21
            stale = bot.check_court_staleness(threshold_days=threshold)
            if not stale:
                print(f"All courts current (threshold: {threshold} days).")
            else:
                print(f"Stale courts (no interesting issues in >{threshold} days):")
                for entry in stale:
                    print(f"  {entry['court']}: last {entry['last_interesting']} "
                          f"({entry['days_ago']} days ago)")
        elif command == "test-email":
            # Send test email to specified recipient
            if len(sys.argv) > 2:
                recipient = sys.argv[2]
                print(f"Sending test email to {recipient}...")
                success = bot.send_test_email(recipient)
                if success:
                    print("✅ Test email sent successfully")
                else:
                    print("❌ Test email failed")
            else:
                print("Usage: python pdrbot.py test-email <email@address.com>")
                sys.exit(1)
        else:
            print("Usage: python pdrbot.py [scrape|analyze|both|report|daily-report|auto|resume|status|members|check-subscriptions|test-email|backfill-urls|analyze-dir] [options]")
            print("  scrape       - Download new opinions only")
            print("  analyze      - Analyze unanalyzed opinions only")
            print("  both         - Download and analyze (default)")
            print("  report       - Generate PDF report from all analyses")
            print("  report YYYY-MM-DD - Generate report for specific date")
            print("  daily-report [YYYY-MM-DD] - Generate report for specified date or today's analyses")
            print("  auto         - Full automation: scrape, analyze, report, and email")
            print("  resume [run_id] - Resume incomplete run or list incomplete runs")
            print("  status       - Show status of recent and incomplete runs")
            print("  members      - Show subscription members and recipients")
            print("  check-subscriptions - Manually check for subscription emails")
            print("  test-email <email> - Send test email to specified recipient")
            print("  triage-audit [N]   - Re-run Opus on N recent Haiku-ROUTINE cases; report disagreement rate")
            print("  court-staleness [days] - List courts with no interesting issue in the last N days (default 21)")
            print("  backfill-urls - Update existing records with direct PDF URLs")
            print("  analyze-dir <path> - Analyze all PDFs in specific directory")
            print("")
            print("Options:")
            print("  limit        - Maximum number of opinions to analyze (analyze mode only)")
            sys.exit(1)
    else:
        # Default: run daily scrape with analysis
        bot.run_daily_scrape()

if __name__ == "__main__":
    main()