cleaner.py

#!/usr/bin/env python3
"""
doc-cleaner — Convert PDF, DOCX, XLSX, and text files to clean, structured Markdown.

CJK-friendly. Table-friendly. Privacy-first.

Part of the notoriouslab open-source toolkit.
"""
import os

# protobuf descriptor-pool guard (D10): force the pure-Python protobuf
# implementation before anything can import the C/upb one. numbers-parser and
# keynote-parser vendor the same Apple .proto names and the upb pool aborts when
# both load in one process. Set here at the CLI/GUI entry root as defense in
# depth — parsers/numbers.py and parsers/iwork.py set it too, but those load
# lazily, so guarding the root protects against a future eager protobuf import.
# Idempotent; respects an explicit user override.
os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python")

import re
import sys
import json
import time
import argparse
import logging
import tempfile
from pathlib import Path


def _extract_version_regex(text):
    """Scan pyproject.toml text for the first `version = "..."` literal.

    Fallback for Python 3.9/3.10, which have no tomllib (the project supports
    3.9+). `[tool.briefcase]` is the first table in the file, so its version is
    the first match. Returns None when no version literal is present.
    """
    m = re.search(r'^\s*version\s*=\s*"([^"]+)"', text, re.MULTILINE)
    return m.group(1) if m else None


def _extract_version(text):
    """Return the briefcase version from pyproject.toml text, or None."""
    try:
        import tomllib
        return tomllib.loads(text)["tool"]["briefcase"]["version"]
    except Exception:
        # No tomllib (< 3.11), or the table was renamed/malformed. Reporting a
        # wrong version is worse than a cheap scan, so fall through.
        return _extract_version_regex(text)


def _read_version():
    """Version single source of truth: pyproject.toml `[tool.briefcase]`.

    Mirrors macapp._read_version() so the CLI and the App report the same
    number; the version was previously hardcoded here and silently went five
    releases stale. 'unknown' when pyproject.toml is unreachable (e.g. this
    file copied out on its own) — never a hardcoded number, which would just
    recreate the second source of truth.
    """
    try:
        text = (Path(__file__).parent / "pyproject.toml").read_text(encoding="utf-8")
    except OSError:
        return "unknown"
    return _extract_version(text) or "unknown"


__version__ = _read_version()

logger = logging.getLogger("doc-cleaner")

# Exit codes
EXIT_OK = 0             # all files processed successfully
EXIT_PARTIAL = 1        # some files failed
EXIT_NO_INPUT = 2       # no processable files found or config error

# Supported file extensions
SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".doc", ".xlsx", ".xls", ".csv", ".txt", ".md", ".pptx", ".ppt", ".dxf", ".jsonl", ".numbers", ".pages", ".key", ".epub"}

# Upper bound on files collected from a single recursive directory scan (GUI
# folder-drop, D3). Generous for real personal folders, but bounds a pathological
# tree. When hit, collection stops and a warning is logged (no silent truncation).
MAX_RECURSIVE_FILES = 1000

SCRIPT_DIR = Path(__file__).resolve().parent


def load_config(config_path):
    """Load JSON config, return empty dict if not found."""
    if config_path and os.path.exists(config_path):
        with open(config_path, "r", encoding="utf-8") as f:
            return json.load(f)
    return {}


def load_prompt(config, config_path=None):
    """Load the AI prompt template from config or default."""
    prompt_path = config.get("ai", {}).get("prompt_template")
    if prompt_path and not os.path.isabs(prompt_path):
        # Try relative to config dir first, then script dir
        candidates = []
        if config_path:
            candidates.append(os.path.join(os.path.dirname(config_path), prompt_path))
        candidates.append(os.path.join(SCRIPT_DIR, prompt_path))

        resolved = None
        for c in candidates:
            if os.path.exists(c):
                resolved = c
                break

        if resolved:
            with open(resolved, "r", encoding="utf-8") as f:
                return f.read()
        logger.warning(f"Prompt template not found: {prompt_path}, using default")
    elif prompt_path and os.path.isabs(prompt_path):
        if os.path.exists(prompt_path):
            with open(prompt_path, "r", encoding="utf-8") as f:
                return f.read()
        logger.warning(f"Prompt template not found: {prompt_path}, using default")

    # Default prompt
    default_path = os.path.join(SCRIPT_DIR, "prompts", "default.txt")
    if os.path.exists(default_path):
        with open(default_path, "r", encoding="utf-8") as f:
            return f.read()
    return "Analyze this document and output JSON with keys: title, summary, refined_markdown, tags."


def warn_config_secrets(config):
    """Warn if config.json contains fields that should be in .env instead."""
    secret_paths = [
        (["ai", "gemini", "api_key"], "GEMINI_API_KEY"),
        (["ai", "groq", "api_key"], "GROQ_API_KEY"),
        (["ai", "nvidia", "api_key"], "NVIDIA_API_KEY"),
        (["ai", "ollama", "api_key"], "OLLAMA_API_KEY"),
        (["ai", "openai", "api_key"], "OPENAI_API_KEY"),
        (["pdf", "password"], "PDF_PASSWORD"),
    ]
    for keys, env_name in secret_paths:
        obj = config
        for k in keys:
            obj = obj.get(k, {}) if isinstance(obj, dict) else {}
        if obj and isinstance(obj, str):
            logger.warning(
                f"⚠️  Secret found in config.json ({'.'.join(keys)}). "
                f"Move it to .env as {env_name} and remove from config.json. "
                f"config.json may be accidentally committed to git."
            )


def validate_patterns(config):
    """Pre-validate ad_truncation_patterns and ad_strip_patterns regex at startup."""
    for key in ("ad_truncation_patterns", "ad_strip_patterns"):
        for i, pat in enumerate(config.get(key, [])):
            try:
                re.compile(pat)
            except re.error as e:
                logger.error(f"Invalid regex in {key}[{i}]: {pat!r} — {e}")
                sys.exit(EXIT_NO_INPUT)


def create_ai_backend(ai_mode, config):
    """Create the appropriate AI backend based on config."""
    if ai_mode == "none":
        return None

    ai_config = config.get("ai", {})

    if ai_mode == "gemini":
        try:
            from ai.gemini import GeminiBackend
        except ImportError:
            logger.error(
                "Gemini backend requires google-genai. "
                "Install with: pip install google-genai python-dotenv"
            )
            sys.exit(EXIT_NO_INPUT)

        api_key = os.getenv("GEMINI_API_KEY")
        if not api_key:
            logger.error(
                "GEMINI_API_KEY not set. Add it to your .env file:\n"
                "  echo 'GEMINI_API_KEY=your-key-here' >> .env\n"
                "Do NOT put API keys in config.json — it may be committed to git."
            )
            sys.exit(EXIT_NO_INPUT)
        model = ai_config.get("gemini", {}).get("model", "gemini-2.5-pro")
        return GeminiBackend(api_key=api_key, model=model)

    if ai_mode == "groq":
        from ai.groq import GroqBackend

        api_key = os.getenv("GROQ_API_KEY")
        if not api_key:
            logger.error(
                "GROQ_API_KEY not set. Add it to your .env file:\n"
                "  echo 'GROQ_API_KEY=your-key-here' >> .env\n"
                "Do NOT put API keys in config.json — it may be committed to git."
            )
            sys.exit(EXIT_NO_INPUT)

        groq_config = ai_config.get("groq", {})
        model = groq_config.get("model", "meta-llama/llama-4-scout-17b-16e-instruct")
        base_url = groq_config.get("base_url", "https://api.groq.com/openai/v1")
        timeout = groq_config.get("timeout", 120)
        return GroqBackend(
            api_key=api_key,
            model=model,
            base_url=base_url,
            timeout=timeout,
        )

    if ai_mode == "nvidia":
        from ai.nvidia import NvidiaBackend

        api_key = os.getenv("NVIDIA_API_KEY")
        if not api_key:
            logger.error(
                "NVIDIA_API_KEY not set. Add it to your .env file:\n"
                "  echo 'NVIDIA_API_KEY=nvapi-your-key-here' >> .env\n"
                "Get a free key at https://build.nvidia.com"
            )
            sys.exit(EXIT_NO_INPUT)

        nvidia_config = ai_config.get("nvidia", {})
        model = nvidia_config.get("model", "meta/llama-3.2-90b-vision-instruct")
        base_url = nvidia_config.get("base_url", "https://integrate.api.nvidia.com/v1")
        timeout = nvidia_config.get("timeout", 180)
        return NvidiaBackend(
            api_key=api_key,
            model=model,
            base_url=base_url,
            timeout=timeout,
        )

    if ai_mode == "ollama":
        try:
            from ai.ollama import OllamaBackend
        except ImportError:
            logger.error(
                "Ollama backend requires the ollama package. "
                "Install with: pip install ollama\n"
                "Also ensure Ollama is running: https://ollama.com"
            )
            sys.exit(EXIT_NO_INPUT)

        ollama_config = ai_config.get("ollama", {})
        model = ollama_config.get("model", "qwen3.5:9b")
        host = ollama_config.get("host", "http://localhost:11434")
        vision_models = ollama_config.get("vision_models")
        return OllamaBackend(model=model, host=host, vision_models=vision_models)

    if ai_mode == "mlx":
        try:
            from ai.mlx import MLXBackend
        except ImportError:
            logger.error(
                "MLX backend requires mlx-lm. "
                "Install with: pip install mlx-lm\n"
                "Only available on Apple Silicon Macs."
            )
            sys.exit(EXIT_NO_INPUT)

        mlx_config = ai_config.get("mlx", {})
        model = mlx_config.get("model", "mlx-community/Qwen3-4B-4bit")
        max_tokens = mlx_config.get("max_tokens", 4096)
        return MLXBackend(model=model, max_tokens=max_tokens)

    if ai_mode == "openai":
        from ai.openai import OpenAIBackend

        openai_config = ai_config.get("openai", {})
        model = openai_config.get("model", "gpt-4o")
        base_url = openai_config.get("base_url", "http://localhost:8000/v1")
        api_key = os.getenv("OPENAI_API_KEY") or openai_config.get("api_key", "")
        timeout = openai_config.get("timeout", 120)
        return OpenAIBackend(
            api_key=api_key,
            model=model,
            base_url=base_url,
            timeout=timeout,
        )

    logger.error(f"Unknown AI backend: {ai_mode}")
    sys.exit(EXIT_NO_INPUT)


def parse_file(filepath, config):
    """
    Parse a document file and return extracted text + optional images.

    Returns: (text, images)
        - text: extracted text string (may be empty for scanned PDFs)
        - images: list of PIL.Image objects (for PDF vision mode), or None
    """
    ext = os.path.splitext(filepath)[1].lower()
    pdf_config = config.get("pdf", {})
    images = None
    text = ""

    if ext == ".pdf":
        try:
            from parsers import pdf
            from classifiers.pdf_classifier import classify, PdfType
            from classifiers.noise import clean_text
        except ImportError as e:
            raise ImportError(
                f"PDF processing requires PyMuPDF. Install with: pip install -r requirements.txt\n"
                f"  Missing: {e.name or e}"
            )

        # Decrypt if needed (before ODL or PyMuPDF extraction)
        password = pdf_config.get("password") or os.getenv("PDF_PASSWORD")
        target = filepath
        if password:
            decrypt_dir = pdf_config.get("decrypt_dir")
            decrypted = pdf.decrypt_pdf(filepath, password=password, output_dir=decrypt_dir)
            if decrypted:
                target = decrypted

        # Deterministic routing: table-bearing PDFs take the native PyMuPDF
        # table path (measured equal-or-better than ODL on tables, and ODL
        # collapses complex statement tables); prose PDFs keep ODL's layout
        # reconstruction. First-party find_tables scan, fail-open to ODL.
        if pdf.has_tables(target):
            odl_text = None
        else:
            # Try ODL extraction (high-quality prose layout)
            odl_text = pdf.extract_text_odl(target)

        # Classify (ODL text informs the decision if available)
        pdf_type, raw_text, metadata = classify(target, odl_text=odl_text)

        # For NATIVE and LAYOUT_BROKEN PDFs without ODL: re-extract with table
        # detection so tables are preserved as Markdown pipe tables.
        # LAYOUT_BROKEN is triggered when short_line_ratio > 70% — a PDF
        # that is mostly tables fires this heuristic because table cells create
        # many short lines, but find_tables() can still recover the structure.
        if pdf_type in (PdfType.NATIVE, PdfType.LAYOUT_BROKEN) and odl_text is None:
            table_text = pdf.extract_text_with_tables(target)
            if table_text:
                raw_text = table_text

        cutoff_patterns = config.get("ad_truncation_patterns")
        strip_patterns = config.get("ad_strip_patterns")
        strip_urls = config.get("strip_urls", True)
        text = clean_text(raw_text, cutoff_patterns=cutoff_patterns,
                          strip_patterns=strip_patterns, strip_urls=strip_urls)

        if pdf_type in (PdfType.SCANNED, PdfType.LAYOUT_BROKEN):
            dpi = pdf_config.get("dpi", 200)
            max_pages = pdf_config.get("max_pages", 15)
            images = pdf.extract_images(target, dpi=dpi, max_pages=max_pages)
            if not images and not text:
                logger.warning(f"No text or images extracted from {os.path.basename(filepath)}")

    elif ext == ".docx":
        try:
            from parsers.docx import parse
        except ImportError as e:
            raise ImportError(
                f"DOCX processing requires python-docx. Install with: pip install python-docx\n"
                f"  Missing: {e.name or e}"
            )
        text = parse(filepath)

    elif ext in (".xlsx", ".xls", ".csv"):
        try:
            from parsers.xlsx import parse
        except ImportError as e:
            raise ImportError(
                f"Spreadsheet processing requires pandas + openpyxl. "
                f"Install with: pip install pandas openpyxl\n"
                f"  Missing: {e.name or e}"
            )
        text = parse(filepath)

    elif ext == ".doc":
        try:
            from parsers.docx import parse_doc
        except ImportError as e:
            raise ImportError(
                f"DOC processing requires macOS textutil.\n"
                f"  Missing: {e.name or e}"
            )
        text = parse_doc(filepath)

    elif ext in (".pptx", ".ppt"):
        try:
            from parsers.pptx import parse
            text = parse(filepath)
        except ImportError as e:
            raise ImportError(
                f"Presentation processing requires python-pptx. Install with: pip install python-pptx\n"
                f"  Missing: {e.name or e}"
            )

    elif ext == ".dxf":
        try:
            from parsers.dxf import parse
            text = parse(filepath)
        except ImportError as e:
            raise ImportError(
                f"DXF processing requires ezdxf. Install with: pip install ezdxf\n"
                f"  Missing: {e.name or e}"
            )

    elif ext in (".txt", ".md"):
        from parsers.text import parse
        text = parse(filepath)

    elif ext == ".jsonl":
        from parsers.jsonl import parse
        text = parse(filepath)

    elif ext == ".numbers":
        from parsers.numbers import parse
        text = parse(filepath)

    elif ext in (".pages", ".key"):
        from parsers.iwork import parse
        text = parse(filepath)

    elif ext == ".epub":
        from parsers.epub import parse
        text = parse(filepath)

    else:
        logger.warning(f"Unsupported file type: {ext}")

    return text, images


def process_file(filepath, ai_backend, prompt, config, output_dir, output_format="md", dry_run=False):
    """
    Process a single file: parse → (optional AI) → Markdown/EPUB output.

    Returns: (status, output_path)
        - status: "ok" | "dry_run" | "no_content" | "write_error" | "error"
        - output_path: path to output file (primary format), or None on failure
    """
    filename = os.path.basename(filepath)
    stem = os.path.splitext(filename)[0]

    # Resolve the collision suffix (if any) first on the primary format
    primary_ext = ".md" if output_format in ("md", "both") else ".epub"
    primary_path = os.path.join(output_dir, f"{stem}{primary_ext}")
    if os.path.exists(primary_path):
        counter = 1
        while os.path.exists(os.path.join(output_dir, f"{stem}_{counter}{primary_ext}")):
            counter += 1
        stem = f"{stem}_{counter}"
        logger.info(f"  Output collision resolved to stem: {stem}")

    logger.info(f"Processing: {filename}")

    if dry_run:
        ext = os.path.splitext(filepath)[1].lower()
        if output_format == "both":
            logger.info(f"  [dry-run] Would process {filename} ({ext}) → {os.path.join(output_dir, stem)}.md and .epub")
        elif output_format == "epub":
            logger.info(f"  [dry-run] Would process {filename} ({ext}) → {os.path.join(output_dir, stem)}.epub")
        else:
            logger.info(f"  [dry-run] Would process {filename} ({ext}) → {os.path.join(output_dir, stem)}.md")
        return "dry_run", os.path.join(output_dir, f"{stem}.md" if output_format != "epub" else f"{stem}.epub")

    try:
        text, images = parse_file(filepath, config)

        if not text and not images:
            logger.warning(f"  No content extracted from {filename}")
            return "no_content", None

        # JSONL transcripts are pre-formatted Markdown — skip AI and write directly
        ext = os.path.splitext(filepath)[1].lower()
        if ext == ".jsonl":
            # JSONL transcripts are pre-formatted Markdown — bypass AI and PII redaction intentionally.
            # Output contains full conversation content; callers should treat the result as sensitive.
            logger.info("  JSONL transcript: AI and PII redaction bypassed — output contains full conversation")
            from output.markdown import render_raw_output
            frontmatter = config.get("output", {}).get("frontmatter", True)
            final_text = render_raw_output(text, filename=filename, source_path=filepath, frontmatter=frontmatter)
            md_path = os.path.join(output_dir, f"{stem}.md")
            if output_format in ("md", "both"):
                try:
                    fd, tmp_path = tempfile.mkstemp(dir=output_dir, suffix=".tmp")
                    with os.fdopen(fd, "w", encoding="utf-8") as f:
                        f.write(final_text)
                    os.replace(tmp_path, md_path)
                except OSError as e:
                    logger.error(f"  Write error for {filename}: {e}")
                    return "write_error", None
                logger.info(f"  → {md_path}")
            epub_path = os.path.join(output_dir, f"{stem}.epub")
            if output_format in ("epub", "both"):
                from output.epub import render_raw_epub
                # Pass the transcript markdown WITHOUT the YAML frontmatter —
                # title/source already go into the EPUB's own metadata, and
                # frontmatter passed as body text would render as visible YAML.
                content_epub = render_raw_epub(text, filename=filename, source_path=filepath)
                try:
                    fd, tmp_path = tempfile.mkstemp(dir=output_dir, suffix=".tmp")
                    with os.fdopen(fd, "wb") as f:
                        f.write(content_epub)
                    os.replace(tmp_path, epub_path)
                except OSError as e:
                    logger.error(f"  Write error for {filename}: {e}")
                    return "write_error", None
                logger.info(f"  → {epub_path}")
            return "ok", epub_path if output_format == "epub" else md_path

        # PII redaction (opt-in via config)
        pii_config = config.get("pii", {})
        pii_enabled = pii_config.get("enabled", False)
        pii_patterns = pii_config.get("patterns", None)  # None = all patterns

        if pii_enabled and text:
            from classifiers.pii import redact as redact_pii
            text, pii_count = redact_pii(text, enabled_patterns=pii_patterns)
            if pii_count:
                logger.info(f"  PII: {pii_count} item(s) redacted before processing")

        frontmatter = config.get("output", {}).get("frontmatter", True)

        raw_response = None
        if ai_backend:
            # AI mode: send to LLM for structuring
            # Retry once on transient errors (429/503/timeout) before fallback
            max_retries = config.get("ai", {}).get("max_retries", 1)
            last_err = None
            for attempt in range(1 + max_retries):
                try:
                    raw_response = ai_backend.call(prompt=prompt, images=images, text=text)
                    break
                except Exception as ai_err:
                    last_err = ai_err
                    if attempt < max_retries:
                        wait = 2 ** attempt  # 1s, 2s, ...
                        logger.warning(
                            f"  AI call failed ({ai_err}), retrying in {wait}s "
                            f"(attempt {attempt + 1}/{1 + max_retries})"
                        )
                        time.sleep(wait)

            if raw_response is None and not text:
                raise last_err  # no text to fall back on, propagate error

        # Parse AI JSON if we have it
        data = None
        if ai_backend and raw_response is not None:
            from ai.base import clean_json_response
            data = clean_json_response(raw_response)

            # Graceful degradation: if JSON repair failed badly, fall back to raw mode
            if data.get("status") == "partial_recovery" and text:
                logger.warning("  AI JSON output corrupted — falling back to raw mode")
                data = None
            elif pii_enabled:
                from classifiers.pii import redact as redact_pii
                if "refined_markdown" in data and isinstance(data["refined_markdown"], str):
                    data["refined_markdown"], _ = redact_pii(data["refined_markdown"], enabled_patterns=pii_patterns)
                if "summary" in data and isinstance(data["summary"], str):
                    data["summary"], _ = redact_pii(data["summary"], enabled_patterns=pii_patterns)

        # 1. Render Markdown if needed
        content_md = None
        if output_format in ("md", "both"):
            from output.markdown import render_ai_output, render_raw_output
            if data is not None:
                content_md = render_ai_output(
                    data, filename, source_path=filename,
                    frontmatter=frontmatter,
                )
            else:
                content_md = render_raw_output(
                    text, filename, source_path=filename,
                    frontmatter=frontmatter,
                )

            # Final PII sweep on rendered markdown output (catches AI-echoed PII)
            if pii_enabled:
                from classifiers.pii import redact as redact_pii
                content_md, pii_output_count = redact_pii(content_md, enabled_patterns=pii_patterns)
                if pii_output_count:
                    logger.info(f"  PII: {pii_output_count} item(s) redacted from Markdown output")

        # 2. Render EPUB if needed
        content_epub = None
        if output_format in ("epub", "both"):
            from output.epub import render_ai_epub, render_raw_epub
            epub_lang = config.get("output", {}).get("epub_language", "zh-TW")
            if data is not None:
                content_epub = render_ai_epub(
                    data, filename, source_path=filename, language=epub_lang
                )
            else:
                content_epub = render_raw_epub(
                    text, filename, source_path=filename, language=epub_lang
                )

        # Define safe write helper
        def safe_write(path, data_to_write, is_binary=False):
            os.makedirs(output_dir, exist_ok=True)
            mode = "wb" if is_binary else "w"
            encoding = None if is_binary else "utf-8"
            fd, tmp_path = tempfile.mkstemp(dir=output_dir, suffix=".tmp")
            try:
                if is_binary:
                    with os.fdopen(fd, mode) as f:
                        f.write(data_to_write)
                else:
                    with os.fdopen(fd, mode, encoding=encoding) as f:
                        f.write(data_to_write)
                os.replace(tmp_path, path)
            except BaseException:
                try:
                    os.unlink(tmp_path)
                except OSError:
                    pass
                raise

        # Write Markdown file
        md_path = os.path.join(output_dir, f"{stem}.md")
        if output_format in ("md", "both"):
            try:
                safe_write(md_path, content_md, is_binary=False)
                logger.info(f"  → {md_path}")
            except OSError as e:
                logger.error(f"  Write failed for {md_path}: {e}")
                return "write_error", None

        # Write EPUB file
        epub_path = os.path.join(output_dir, f"{stem}.epub")
        if output_format in ("epub", "both"):
            try:
                safe_write(epub_path, content_epub, is_binary=True)
                logger.info(f"  → {epub_path}")
            except OSError as e:
                logger.error(f"  Write failed for {epub_path}: {e}")
                return "write_error", None

        # Return status and primary output path
        primary_path = epub_path if output_format == "epub" else md_path
        return "ok", primary_path

    except Exception as e:
        logger.exception(f"  Failed: {filename}: {e}")
        return "error", None


def _collect_dir_recursive(input_path, real_root):
    """Recursively collect supported files under a directory (GUI folder-drop, D3).

    Preserves the symlink-escape guard (a file's resolved path must stay under
    the directory root), does not follow symlinked subdirectories
    (``followlinks=False``), traverses deterministically (sorted), and stops at
    ``MAX_RECURSIVE_FILES``. Returns ``(files, capped)`` where ``capped`` is True
    only when MORE than the cap existed (so exactly-cap files is not a false
    positive). Collects one past the cap to distinguish the two cases.
    """
    files = []
    capped = False
    for dirpath, dirnames, filenames in os.walk(input_path, followlinks=False):
        dirnames.sort()  # deterministic descent order
        for name in sorted(filenames):
            if os.path.splitext(name)[1].lower() not in SUPPORTED_EXTENSIONS:
                continue
            fp = os.path.realpath(os.path.join(dirpath, name))
            # P4 security: reject files whose resolved path escapes the root.
            if not fp.startswith(real_root + os.sep) and fp != real_root:
                logger.warning(f"Skipping symlink escape: {name}")
                continue
            files.append(fp)
            if len(files) > MAX_RECURSIVE_FILES:  # collected cap+1 → truly more exist
                capped = True
                break
        if capped:
            break
    if capped:
        files = files[:MAX_RECURSIVE_FILES]
        logger.warning(
            f"Recursive scan capped at {MAX_RECURSIVE_FILES} files; "
            "additional files were not collected"
        )
    return files, capped


def collect_files(input_path, recursive=False):
    """Collect processable files from a path (file or directory).

    With ``recursive=False`` (default, used by the CLI) a directory is scanned
    one level deep and subdirectories are skipped — behavior is byte-for-byte
    unchanged. With ``recursive=True`` (used by the GUI folder-drop) a directory
    is walked recursively via :func:`_collect_dir_recursive`.
    """
    if os.path.isfile(input_path):
        # Security: resolve symlinks to prevent directory traversal
        real_path = os.path.realpath(input_path)
        if os.path.islink(input_path):
            logger.info(f"Resolved symlink: {input_path} → {real_path}")
        ext = os.path.splitext(real_path)[1].lower()
        if ext in SUPPORTED_EXTENSIONS:
            return [real_path]
        else:
            logger.warning(f"Unsupported file type: {input_path}")
            return []

    if os.path.isdir(input_path):
        real_root = os.path.realpath(input_path)
        if recursive:
            # Drop the capped flag so the CLI return type stays a plain list.
            return _collect_dir_recursive(input_path, real_root)[0]
        files = []
        skipped_dirs = []
        for f in sorted(os.listdir(input_path)):
            fp = os.path.realpath(os.path.join(input_path, f))
            # P4 security: reject symlinks escaping the input directory
            if not fp.startswith(real_root + os.sep) and fp != real_root:
                logger.warning(f"Skipping symlink escape: {f}")
                continue
            if os.path.isdir(fp):
                skipped_dirs.append(f)
            elif os.path.isfile(fp) and os.path.splitext(f)[1].lower() in SUPPORTED_EXTENSIONS:
                files.append(fp)
        if skipped_dirs:
            logger.debug(
                f"Skipped {len(skipped_dirs)} subdirectory(ies) (non-recursive): "
                + ", ".join(skipped_dirs)
            )
        return files

    logger.error(f"Input not found: {input_path}")
    return []


def main():
    if sys.version_info < (3, 9):
        sys.exit("doc-cleaner requires Python 3.9+. Current: " + sys.version)

    parser = argparse.ArgumentParser(
        description="doc-cleaner — Convert documents to clean, structured Markdown.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=(
            "Examples:\n"
            "  python cleaner.py --input statement.pdf\n"
            "  python cleaner.py --input ./downloads/ --ai none\n"
            "  python cleaner.py --input report.xlsx --dry-run --verbose\n"
            "\n"
            "Part of the notoriouslab pipeline:\n"
            "  gmail-statement-fetcher → doc-cleaner → personal-cfo\n"
        ),
    )
    parser.add_argument("--input", "-i", required=True, help="file or directory to process")
    parser.add_argument("--output-dir", "-o", default="./output", help="output directory (default: ./output)")
    parser.add_argument("--config", default=None, help="path to config JSON (default: <script-dir>/config.json)")
    parser.add_argument("--ai", choices=["gemini", "groq", "nvidia", "ollama", "mlx", "openai", "none"], default=None, help="AI backend (default: from config or gemini)")
    parser.add_argument("--password", default=None, help="PDF decryption password (overrides .env and config)")
    parser.add_argument("--summary", action="store_true", help="print JSON summary to stdout after processing")
    parser.add_argument("--format", "-f", choices=["md", "epub", "both"], default="md", help="output format (default: md)")
    parser.add_argument("--dry-run", action="store_true", help="preview without writing files")
    parser.add_argument("--verbose", action="store_true", help="enable debug logging")
    parser.add_argument("--version", action="version", version=f"doc-cleaner {__version__}")

    args = parser.parse_args()

    # Logging
    log_level = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(
        level=log_level,
        format="%(asctime)s [%(levelname)s] %(message)s",
        datefmt="%H:%M:%S",
    )

    # Load .env if available
    try:
        from dotenv import load_dotenv
        load_dotenv()
    except ImportError:
        pass

    # Config
    config_path = args.config or os.path.join(SCRIPT_DIR, "config.json")
    config = load_config(config_path)

    # Security: warn if secrets leaked into config.json
    warn_config_secrets(config)

    # Validate regex patterns at startup
    validate_patterns(config)

    # PDF password priority: CLI > .env > config.json
    if args.password:
        if len(args.password) > 1024:
            logger.error("--password too long (max 1024 chars)")
            sys.exit(EXIT_NO_INPUT)
        config.setdefault("pdf", {})["password"] = args.password

    # AI mode priority: CLI --ai > config.json > default "gemini"
    ai_mode = args.ai or config.get("ai", {}).get("backend", "gemini")

    # Collect files
    files = collect_files(args.input)
    if not files:
        logger.error("No processable files found.")
        sys.exit(EXIT_NO_INPUT)

    logger.info(f"doc-cleaner v{__version__} — {len(files)} file(s) to process")
    if args.dry_run:
        logger.info("[DRY RUN] No files will be written.")

    # Process (delegates to core.py — config+backend+prompt built once and reused)
    from core import convert_files
    raw_results = convert_files(
        files,
        output_resolver=lambda _: args.output_dir,
        ai=ai_mode,
        output_format=args.format,
        config=config,
        config_path=config_path,
        dry_run=args.dry_run,
    )
    results = [
        {
            "file": r["file"],
            "output": os.path.relpath(r["output"]) if r["output"] else None,
            "status": r["status"],
        }
        for r in raw_results
    ]

    success = sum(1 for r in results if r["status"] in ("ok", "dry_run"))
    logger.info(f"Done: {success}/{len(files)} files processed.")

    # Machine-readable summary for AI agents and scripts
    if args.summary:
        summary = {
            "version": __version__,
            "total": len(files),
            "success": success,
            "failed": len(files) - success,
            "files": results,
        }
        print(json.dumps(summary, ensure_ascii=False))

    sys.exit(EXIT_OK if success == len(files) else EXIT_PARTIAL)


if __name__ == "__main__":
    main()