Skip to content

feat(eval): report quality-constrained localization cost #427

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fishmingyu merged 2 commits into from
Aug 5, 2026
Merged

feat(eval): report quality-constrained localization cost #427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
724b9b9
feat(eval): preserve auditable model usage
fishmingyu Aug 5, 2026
437e955
feat(eval): report quality-constrained localization cost
fishmingyu Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions codenib/eval/agent_runner/query_sweep.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,7 @@ def run_query_sweep(

for row, subset_id, skills, rep, cell_id, cell_path in plan:
cell_started = time.time()
out: Optional[Dict[str, Any]] = None
gt_blocks = collect_target_blocks(row)
target_files, target_symbols = query_targets(
row, simplified_symbols=cfg.gt_simplified_symbols
Expand Down Expand Up @@ -378,9 +379,13 @@ def run_query_sweep(
"trace_summary": out.get("trace_summary"),
"answer": out["answer"],
"total_turns": out["total_turns"],
"prompt_tokens": out["prompt_tokens"],
"completion_tokens": out["completion_tokens"],
"total_tokens": out["total_tokens"],
"cost_usd": out["cost_usd"],
"cost_provenance": out.get("cost_provenance"),
"cache_read_input_tokens": out["cache_read_input_tokens"],
"cache_creation_input_tokens": out["cache_creation_input_tokens"],
"elapsed_seconds": time.time() - cell_started,
"error": None,
}
Expand Down Expand Up @@ -414,9 +419,35 @@ def run_query_sweep(
"instance_id": instance_id,
"query_id": row["query_id"],
"category": row.get("category"),
"length_variant": row.get("length_variant"),
"source_config": row.get("source_config"),
"subset_id": subset_id,
"skills": skills,
"model": cfg.model,
"rep": rep,
"language": language_key_for_query_row(row),
"success": False,
"metrics": {},
"metrics_meaningful": bool(target_files or gt_blocks),
"query": row["query"],
"total_turns": out.get("total_turns") if out else None,
"prompt_tokens": out.get("prompt_tokens") if out else None,
"completion_tokens": (
out.get("completion_tokens") if out else None
),
"total_tokens": out.get("total_tokens") if out else None,
"cost_usd": out.get("cost_usd") if out else None,
"cost_provenance": (
out.get("cost_provenance")
if out
else {"kind": "unavailable", "model": cfg.model}
),
"cache_read_input_tokens": (
out.get("cache_read_input_tokens") if out else None
),
"cache_creation_input_tokens": (
out.get("cache_creation_input_tokens") if out else None
),
"error": str(exc),
"elapsed_seconds": time.time() - cell_started,
}
Expand Down
15 changes: 14 additions & 1 deletion codenib/eval/agent_runner/sweep.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Sequence

from codenib.llm.usage import runtime_cost_provenance

from .sweep_config import SweepConfig

# language_group (HF column) -> classify() language key
Expand Down Expand Up @@ -367,7 +369,6 @@ def _run_subagent(sys_prompt, q, mt):
# Sum token usage across every run so verify/scatter arms are charged for
# the closed loop, not just their final turn.
usage_totals = accounting.usage_totals()

return {
"nodes": observations["nodes"],
"tool_calls": observations["tool_calls"],
Expand All @@ -385,6 +386,7 @@ def _run_subagent(sys_prompt, q, mt):
"completion_tokens": usage_totals["completion_tokens"],
"total_tokens": usage_totals["total_tokens"],
"cost_usd": usage_totals["cost_usd"],
"cost_provenance": runtime_cost_provenance(cfg.model, usage_totals["cost_usd"]),
"cache_read_input_tokens": usage_totals["cache_read_input_tokens"],
"cache_creation_input_tokens": usage_totals["cache_creation_input_tokens"],
}
Expand Down Expand Up @@ -563,6 +565,7 @@ def run_sweep(cfg: SweepConfig, output_dir: Path, *, resume: bool = True) -> Dic
"completion_tokens": out["completion_tokens"],
"total_tokens": out["total_tokens"],
"cost_usd": out["cost_usd"],
"cost_provenance": out.get("cost_provenance"),
"cache_read_input_tokens": out["cache_read_input_tokens"],
"cache_creation_input_tokens": out["cache_creation_input_tokens"],
"elapsed_seconds": time.time() - t,
Expand Down Expand Up @@ -590,6 +593,16 @@ def run_sweep(cfg: SweepConfig, output_dir: Path, *, resume: bool = True) -> Dic
"metrics": {},
"metrics_meaningful": gt_meaningful,
"tool_calls": [],
"prompt_tokens": None,
"completion_tokens": None,
"total_tokens": None,
"cost_usd": None,
"cost_provenance": {
"kind": "unavailable",
"model": cfg.model,
},
"cache_read_input_tokens": None,
"cache_creation_input_tokens": None,
"error": str(exc),
"elapsed_seconds": time.time() - t,
}
Expand Down
26 changes: 26 additions & 0 deletions codenib/eval/pricing/anthropic-direct-2026-08-04.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"schema": "codenib.pricing-snapshot.v1",
"snapshot_id": "anthropic-direct-2026-08-04",
"currency": "USD",
"effective_date": "2026-08-04",
"retrieved_date": "2026-08-04",
"models": [
{
"model_id": "anthropic/claude-haiku-4-5-20251001",
"aliases": [
"anthropic/claude-haiku-4-5",
"claude-haiku-4-5-20251001"
],
"channel": "anthropic-api-direct",
"token_semantics": "prompt_excludes_cache",

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 5, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Fix Anthropic cache-hit repricing

When this bundled snapshot is used to reprice CodeNib cells produced through LiteLLM, Anthropic cache hits are overbilled: LiteLLM documents prompt_tokens as including cache-hit input tokens (https://docs.litellm.ai/docs/completion/prompt_caching), but prompt_excludes_cache makes project_cell_cost() leave those tokens in uncached_input and also add them again as cache_read_input whenever cache_read_input_tokens > 0. This inflates projected USD for cached runs and can change the constrained optimum, so the Anthropic/LiteLLM snapshot needs semantics that subtract cached reads (or a separate read/write-aware mode).

Useful? React with 👍 / 👎.

"cache_write_ttl": "5m",
"rates_per_million": {
"uncached_input": 1.0,
"output": 5.0,
"cache_read_input": 0.1,
"cache_write_input": 1.25
},
"source_url": "https://platform.claude.com/docs/en/about-claude/pricing"
}
]
}
Loading
Loading