03_llm_classifier.py

#!/usr/bin/env python3
"""03 — LLM classifier: content decides weak vs strong.

Offline heuristic stands in for a judge model. Tune base_threshold the same
way you would on a real Switchyard llm_classifier route.
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from lib.metrics import summarize_run
from lib.router import LlmClassifierRouter
from lib.scenarios import get_scenario, scenario_steps


def main() -> int:
    sc = get_scenario("simple-qa")
    steps = scenario_steps(sc)

    for threshold in (0.35, 0.5, 0.75):
        router = LlmClassifierRouter(base_threshold=threshold)
        decisions = [router.route(s) for s in steps]
        report = summarize_run(steps, decisions)
        print(f"\n=== threshold={threshold} ===")
        for s, d in zip(steps, decisions):
            score = f"{d.score:.2f}" if d.score is not None else "?"
            print(f"  [{score}] {d.target:8} ← {s.content[:60]}")
        print(
            f"  cost ${report['routed_cost_usd']:.4f} "
            f"(frontier ${report['frontier_only_usd']:.4f}, save {report['savings_pct']}%)"
        )

    print("\nTip: lower threshold → more strong traffic (quality); higher → more savings.")
    print("Real server config: configs/routes.llm_classifier.toml")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())