Extend eval --speech-model to all SDK-supported models

The SDK's SpeechModel enum stops at the legacy generation (best, nano,
slam-1, universal); the current models (universal-3-pro, universal-2)
are requested through its newer speech_models list parameter. Replace
the option's type with a CLI-level enum covering the union and route
each choice through the SDK parameter that accepts it.

https://claude.ai/code/session_01Ra3Q7AeGdx6uFxTtc7dL1K

Author: claude

aai_cli/commands/evaluate.py

@@ -9,6 +9,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from enum import StrEnum
 
 import assemblyai as aai
 import typer
@@ -21,6 +22,47 @@
 app = typer.Typer()
 
 
+class EvalSpeechModel(StrEnum):
+    """Every speech model the SDK can request, current generation first.
+
+    The SDK's own ``SpeechModel`` enum stops at the legacy generation; the current
+    models are requested through its newer ``speech_models`` list parameter, so the
+    CLI choices are the union of both.
+    """
+
+    universal_3_pro = "universal-3-pro"
+    universal_2 = "universal-2"
+    slam_1 = "slam-1"
+    universal = "universal"
+    best = "best"
+    nano = "nano"
+
+
+def _transcription_config(
+    speech_model: EvalSpeechModel | None, *, language_code: str | None, speaker_labels: bool
+) -> aai.TranscriptionConfig:
+    """Route the model choice through the SDK parameter that accepts it.
+
+    Legacy models go through ``speech_model`` (the SDK enum); current-generation
+    models only exist as ``speech_models`` list values.
+    """
+    legacy = {model.value for model in aai.SpeechModel}
+    return aai.TranscriptionConfig(
+        speech_model=(
+            aai.SpeechModel(speech_model.value)
+            if speech_model is not None and speech_model.value in legacy
+            else None
+        ),
+        speech_models=(
+            [speech_model.value]
+            if speech_model is not None and speech_model.value not in legacy
+            else None
+        ),
+        language_code=language_code,
+        speaker_labels=speaker_labels or None,
+    )
+
+
 def _pct(value: object) -> str:
     return f"{jsonshape.as_float(value):.2%}"
 
@@ -62,7 +104,7 @@ def _score_item(
 
 
 def _payload(
-    label: str, speech_model: aai.SpeechModel | None, results: list[_ItemResult]
+    label: str, speech_model: EvalSpeechModel | None, results: list[_ItemResult]
 ) -> dict[str, object]:
     payload: dict[str, object] = {
         "dataset": label,
@@ -123,7 +165,7 @@ def _render(payload: dict[str, object]) -> RenderableType:
             ("Score a model on 10 rows of an HF dataset", "assembly eval distil-whisper/meanwhile"),
             (
                 "Compare models on your own audio",
-                "assembly eval calls.csv --speech-model universal",
+                "assembly eval calls.csv --speech-model universal-3-pro",
             ),
             (
                 "Score diarization too (WER + DER)",
@@ -163,7 +205,7 @@ def evaluate(
     text_column: str | None = typer.Option(
         None, "--text-column", help="Reference text column name (default: auto-detect)."
     ),
-    speech_model: aai.SpeechModel | None = typer.Option(
+    speech_model: EvalSpeechModel | None = typer.Option(
         None, "--speech-model", help="Speech model to evaluate."
     ),
     language_code: str | None = typer.Option(
@@ -212,10 +254,8 @@ def body(state: AppState, json_mode: bool) -> None:
             with_speakers=speaker_labels,
         )
         api_key = config.resolve_api_key(profile=state.profile)
-        transcription_config = aai.TranscriptionConfig(
-            speech_model=speech_model,
-            language_code=language_code,
-            speaker_labels=speaker_labels or None,
+        transcription_config = _transcription_config(
+            speech_model, language_code=language_code, speaker_labels=speaker_labels
         )
         results: list[_ItemResult] = []
         for index, item in enumerate(data.items, start=1):

tests/__snapshots__/test_cli_output_snapshots.ambr

@@ -277,8 +277,9 @@
   │ --text-column             TEXT                      Reference text column    │
   │                                                     name (default:           │
   │                                                     auto-detect).            │
-  │ --speech-model            [best|nano|slam-1|univer  Speech model to          │
-  │                           sal]                      evaluate.                │
+  │ --speech-model            [universal-3-pro|univers  Speech model to          │
+  │                           al-2|slam-1|universal|be  evaluate.                │
+  │                           st|nano]                                           │
   │ --language-code           TEXT                      Force a language (e.g.   │
   │                                                     en_us).                  │
   │ --speaker-labels                                    Diarize and also score   │
@@ -302,7 +303,7 @@
    Score a model on 10 rows of an HF dataset
    $ assembly eval distil-whisper/meanwhile
    Compare models on your own audio
-   $ assembly eval calls.csv --speech-model universal
+   $ assembly eval calls.csv --speech-model universal-3-pro
    Score diarization too (WER + DER)
    $ assembly eval agent-calls.jsonl --speaker-labels
    Pick a subset/split and more rows

tests/test_eval_command.py

@@ -132,9 +132,33 @@ def test_speech_model_flag_reaches_config_and_output(tmp_path, mocker):
     assert _payload_of(result)["speech_model"] == "universal"
     tx_config = tx.call_args.kwargs["config"]
     assert tx_config.speech_model == aai.SpeechModel.universal
+    assert tx_config.speech_models is None  # legacy models ride the enum parameter only
     assert tx_config.speaker_labels is None  # not requested -> omitted, not False
 
 
+@pytest.mark.parametrize("model", ["universal-3-pro", "universal-2"])
+def test_current_models_ride_the_speech_models_list(tmp_path, mocker, model):
+    _auth()
+    _write_wer_manifest(tmp_path)
+    tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")])
+    result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", model, "--json"])
+    assert result.exit_code == 0
+    assert _payload_of(result)["speech_model"] == model
+    tx_config = tx.call_args.kwargs["config"]
+    assert tx_config.speech_models == [model]
+    assert tx_config.speech_model is None  # not in the SDK enum -> enum parameter omitted
+
+
+def test_no_speech_model_leaves_both_model_parameters_unset(tmp_path, mocker):
+    _auth()
+    _write_wer_manifest(tmp_path)
+    tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")])
+    assert runner.invoke(app, ["eval", "manifest.csv"]).exit_code == 0
+    tx_config = tx.call_args.kwargs["config"]
+    assert tx_config.speech_model is None
+    assert tx_config.speech_models is None
+
+
 def test_speech_model_named_in_human_header(tmp_path, mocker):
     _auth()
     _write_wer_manifest(tmp_path)