braintrust.env.example

# =============================================================================
# Braintrust configuration — single source of truth for the AMFAM document
# classification experiments. Copy this file to braintrust.env and fill in
# real values. braintrust.env is gitignored (it holds API keys).
#
# Every experiment/report/dataset script in scripts/ reads these values via
# src/braintrust_config.py. Command-line flags still override them per-run.
# =============================================================================

# Braintrust organization this environment runs experiments in.
BRAINTRUST_ORG_ID=your-org-id-here

# Project used for experiments, reports, and datasets.
BRAINTRUST_PROJECT_NAME=AMFAM v2
BRAINTRUST_PROJECT_ID=your-project-id-here

# Project + dataset that holds the labeled source images (attachments + ground
# truth) that experiments classify. Usually the same project as above.
BRAINTRUST_DATASET_PROJECT=AMFAM v2
BRAINTRUST_DATASET=fixed_size_sampled

# Dataset that gets built from every qwen v1-v11 misclassification, with each
# row annotated by the prompt version it stemmed from.
BRAINTRUST_SMOKE_DATASET=qwen_misclassification_smoke_v1_v11

# Space-separated list of Qwen experiments (prompt versions v1-v11) whose
# misclassifications feed the smoke-test dataset. Entries may carry the
# suffix Braintrust appends to duplicate experiment names. Any experiment with
# zero misclassifications simply contributes no rows.
QWEN_EXPERIMENTS=\
qwen3.7-flash_v1_reasoning-5718f5da \
qwen3.7-flash_v2_reasoning \
qwen3.7-flash_v3_reasoning-14553e3a \
qwen3.7-flash_v4_reasoning \
qwen3.7-flash_v5_reasoning \
qwen3.7-flash_v6_reasoning \
qwen3.7-flash_v7_reasoning \
qwen3.7-flash_v8_reasoning \
qwen3.7-flash_v8.5_reasoning \
qwen3.7-flash_v9_reasoning \
qwen3.7-flash_v10_smoke \
qwen3.7-flash_v10_reasoning \
qwen3.7-flash_v11_smoke \
qwen3.7-flash_v11_reasoning

# Default model used to run experiments against the datasets.
BRAINTRUST_MODEL=qwen/qwen3.7-flash

# Stronger model used for confidence-gated escalation (routing) — the
# low-confidence tail is re-scored by this model. Empty disables escalation
# by default; --escalation-model on the eval runner still overrides it.
# BRAINTRUST_ESCALATION_MODEL=google/gemini-2.5-pro

# API base (only change if you use a Braintrust proxy).
BRAINTRUST_API_BASE=https://api.braintrust.dev

# API keys. Set BRAINTRUST_API_KEY to a key with write access to the project
# above so experiments/datasets can be created there. This is the "new" account
# key; a stale .env may still hold the previous account's key, so scripts must
# resolve via src/braintrust_config.py (braintrust.env loads first).
BRAINTRUST_API_KEY=your-braintrust-api-key-here
OPENROUTER_API_KEY=your-openrouter-api-key-here

# Optional: a separate OpenRouter key (research funding) reserved for large or
# significant runs only, used when a script explicitly invokes it or as 403
# quota failover. OPENROUTER_API_KEY remains the default for all runs.
# RESEARCH_FUNDING_API_KEY=your-research-funding-openrouter-key-here

# Optional: a separate read-only key for a different source account when the
# dataset lives somewhere else. Leave blank to use BRAINTRUST_API_KEY.
# DATA_BRAINTRUST_KEY=

# -----------------------------------------------------------------------------
# Agent account (optional) — a brand-new Braintrust account that can run
# scorers. The key normally lives in .env as AGENT_BRAINTRUST_API_KEY; these
# overrides let the agent run against its OWN org/project instead of the main
# account's. src/braintrust_config.load_agent_config() resolves them, falling
# back to the main values above for anything unset. The eval runner uses the
# agent profile with `--agent`, which registers only the exact_match scorer.
# -----------------------------------------------------------------------------
# AGENT_BRAINTRUST_ORG_ID=
# AGENT_BRAINTRUST_PROJECT_ID=
# AGENT_BRAINTRUST_PROJECT_NAME=
# AGENT_BRAINTRUST_DATASET_PROJECT=
# AGENT_BRAINTRUST_DATASET=
# AGENT_BRAINTRUST_API_BASE=