analyze_batch.py

"""
Summarizes a batch_results.json file into a readable eval report:
agreement scores per dish, and a breakdown of what each model said
when they disagreed.

Usage:
    python analyze_batch.py batch_results.json
"""
from __future__ import annotations

import json
import sys


def analyze(results_path: str) -> None:
    with open(results_path, encoding="utf-8") as f:
        data = json.load(f)

    print(f"{'Dish':<45} {'Score':>6}  {'Status'}")
    print("-" * 70)

    for item in data:
        filename = item["filename"]
        result = item["result"]

        if result is None:
            print(f"{filename:<45} {'—':>6}  FAILED: {item.get('error', 'unknown error')}")
            continue

        score = result["agreement_score"]
        status = "NEEDS REVIEW" if result["needs_review"] else "auto-approved"
        print(f"{filename:<45} {score:>6.2f}  {status}")

    # Now show details for anything that got flagged, so you can see WHY
    print("\n--- Disagreements worth a closer look ---\n")
    for item in data:
        result = item["result"]
        if result is None or not result["needs_review"]:
            continue

        gemini_entry = result["gemini_result"].get("dish_entry")
        groq_entry = result["groq_result"].get("dish_entry")

        print(f"{item['filename']}  (agreement: {result['agreement_score']:.2f})")
        if gemini_entry:
            print(f"  Gemini: {gemini_entry['name']}  {gemini_entry['price_band_inr']}  {gemini_entry['allergens']}")
        else:
            print(f"  Gemini: FAILED — {result['gemini_result'].get('error')}")

        if groq_entry:
            print(f"  Groq:   {groq_entry['name']}  {groq_entry['price_band_inr']}  {groq_entry['allergens']}")
        else:
            print(f"  Groq:   FAILED — {result['groq_result'].get('error')}")
        print()


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python analyze_batch.py batch_results.json")
        sys.exit(1)

    analyze(sys.argv[1])