Skip to content

Commit 0f22087

Browse files
claudeclaude
committed
Drop legacy models from eval --speech-model
Only the current generation (universal-3-pro, universal-2) remains, so every choice rides the SDK's speech_models list parameter and the legacy-routing helper goes away. https://claude.ai/code/session_01Ra3Q7AeGdx6uFxTtc7dL1K
1 parent b493270 commit 0f22087

3 files changed

Lines changed: 49 additions & 92 deletions

File tree

‎aai_cli/commands/evaluate.py‎

Lines changed: 6 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -23,44 +23,11 @@
2323

2424

2525
class EvalSpeechModel(StrEnum):
26-
"""Every speech model the SDK can request, current generation first.
27-
28-
The SDK's own ``SpeechModel`` enum stops at the legacy generation; the current
29-
models are requested through its newer ``speech_models`` list parameter, so the
30-
CLI choices are the union of both.
31-
"""
26+
"""The current-generation models, requested via the SDK's ``speech_models``
27+
list parameter (its legacy ``SpeechModel`` enum predates them)."""
3228

3329
universal_3_pro = "universal-3-pro"
3430
universal_2 = "universal-2"
35-
slam_1 = "slam-1"
36-
universal = "universal"
37-
best = "best"
38-
nano = "nano"
39-
40-
41-
def _transcription_config(
42-
speech_model: EvalSpeechModel | None, *, language_code: str | None, speaker_labels: bool
43-
) -> aai.TranscriptionConfig:
44-
"""Route the model choice through the SDK parameter that accepts it.
45-
46-
Legacy models go through ``speech_model`` (the SDK enum); current-generation
47-
models only exist as ``speech_models`` list values.
48-
"""
49-
legacy = {model.value for model in aai.SpeechModel}
50-
return aai.TranscriptionConfig(
51-
speech_model=(
52-
aai.SpeechModel(speech_model.value)
53-
if speech_model is not None and speech_model.value in legacy
54-
else None
55-
),
56-
speech_models=(
57-
[speech_model.value]
58-
if speech_model is not None and speech_model.value not in legacy
59-
else None
60-
),
61-
language_code=language_code,
62-
speaker_labels=speaker_labels or None,
63-
)
6431

6532

6633
def _pct(value: object) -> str:
@@ -254,8 +221,10 @@ def body(state: AppState, json_mode: bool) -> None:
254221
with_speakers=speaker_labels,
255222
)
256223
api_key = config.resolve_api_key(profile=state.profile)
257-
transcription_config = _transcription_config(
258-
speech_model, language_code=language_code, speaker_labels=speaker_labels
224+
transcription_config = aai.TranscriptionConfig(
225+
speech_models=[speech_model.value] if speech_model else None,
226+
language_code=language_code,
227+
speaker_labels=speaker_labels or None,
259228
)
260229
results: list[_ItemResult] = []
261230
for index, item in enumerate(data.items, start=1):

‎tests/__snapshots__/test_cli_output_snapshots.ambr‎

Lines changed: 30 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -264,39 +264,36 @@
264264
│ [required] │
265265
╰──────────────────────────────────────────────────────────────────────────────╯
266266
╭─ Options ────────────────────────────────────────────────────────────────────╮
267-
│ --split TEXT Hugging Face split to │
268-
│ score (default: test). │
269-
│ --subset TEXT Hugging Face │
270-
│ config/subset name (e.g. │
271-
│ a language). │
272-
│ --limit INTEGER RANGE Rows to evaluate │
273-
│ [1<=x<=100] (1-100). │
274-
│ [default: 10] │
275-
│ --audio-column TEXT Audio column name │
276-
│ (default: auto-detect). │
277-
│ --text-column TEXT Reference text column │
278-
│ name (default: │
279-
│ auto-detect). │
280-
│ --speech-model [universal-3-pro|univers Speech model to │
281-
│ al-2|slam-1|universal|be evaluate. │
282-
│ st|nano] │
283-
│ --language-code TEXT Force a language (e.g. │
284-
│ en_us). │
285-
│ --speaker-labels Diarize and also score │
286-
│ DER against the │
287-
│ dataset's reference │
288-
│ speaker turns │
289-
│ (speakers/timestamps_st… │
290-
│ columns, in seconds). │
291-
│ --collar FLOAT RANGE [x>=0.0] DER forgiveness │
292-
│ (seconds) around each │
293-
│ reference turn boundary. │
294-
│ [default: 1.0] │
295-
│ --json -j Output the rows and │
296-
│ summary as one JSON │
297-
│ object. │
298-
│ --help Show this message and │
299-
│ exit. │
267+
│ --split TEXT Hugging Face split to │
268+
│ score (default: test). │
269+
│ --subset TEXT Hugging Face │
270+
│ config/subset name (e.g. │
271+
│ a language). │
272+
│ --limit INTEGER RANGE Rows to evaluate (1-100). │
273+
│ [1<=x<=100] [default: 10] │
274+
│ --audio-column TEXT Audio column name │
275+
│ (default: auto-detect). │
276+
│ --text-column TEXT Reference text column │
277+
│ name (default: │
278+
│ auto-detect). │
279+
│ --speech-model [universal-3-pro|univer Speech model to evaluate. │
280+
│ sal-2] │
281+
│ --language-code TEXT Force a language (e.g. │
282+
│ en_us). │
283+
│ --speaker-labels Diarize and also score │
284+
│ DER against the dataset's │
285+
│ reference speaker turns │
286+
│ (speakers/timestamps_sta… │
287+
│ columns, in seconds). │
288+
│ --collar FLOAT RANGE [x>=0.0] DER forgiveness (seconds) │
289+
│ around each reference │
290+
│ turn boundary. │
291+
│ [default: 1.0] │
292+
│ --json -j Output the rows and │
293+
│ summary as one JSON │
294+
│ object. │
295+
│ --help Show this message and │
296+
│ exit. │
300297
╰──────────────────────────────────────────────────────────────────────────────╯
301298

302299
Examples

‎tests/test_eval_command.py‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import json
1010
from types import SimpleNamespace
1111

12-
import assemblyai as aai
1312
import pytest
1413
from typer.testing import CliRunner
1514

@@ -123,21 +122,8 @@ def test_json_payload_shape(tmp_path, mocker):
123122
assert payload["rows"][1] == {"item": "b.wav", "words": 2, "errors": 1, "wer": 0.5}
124123

125124

126-
def test_speech_model_flag_reaches_config_and_output(tmp_path, mocker):
127-
_auth()
128-
_write_wer_manifest(tmp_path)
129-
tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")])
130-
result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", "universal", "--json"])
131-
assert result.exit_code == 0
132-
assert _payload_of(result)["speech_model"] == "universal"
133-
tx_config = tx.call_args.kwargs["config"]
134-
assert tx_config.speech_model == aai.SpeechModel.universal
135-
assert tx_config.speech_models is None # legacy models ride the enum parameter only
136-
assert tx_config.speaker_labels is None # not requested -> omitted, not False
137-
138-
139125
@pytest.mark.parametrize("model", ["universal-3-pro", "universal-2"])
140-
def test_current_models_ride_the_speech_models_list(tmp_path, mocker, model):
126+
def test_speech_model_flag_reaches_config_and_output(tmp_path, mocker, model):
141127
_auth()
142128
_write_wer_manifest(tmp_path)
143129
tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")])
@@ -146,25 +132,30 @@ def test_current_models_ride_the_speech_models_list(tmp_path, mocker, model):
146132
assert _payload_of(result)["speech_model"] == model
147133
tx_config = tx.call_args.kwargs["config"]
148134
assert tx_config.speech_models == [model]
149-
assert tx_config.speech_model is None # not in the SDK enum -> enum parameter omitted
135+
assert tx_config.speaker_labels is None # not requested -> omitted, not False
150136

151137

152-
def test_no_speech_model_leaves_both_model_parameters_unset(tmp_path, mocker):
138+
def test_no_speech_model_leaves_speech_models_unset(tmp_path, mocker):
153139
_auth()
154140
_write_wer_manifest(tmp_path)
155141
tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")])
156142
assert runner.invoke(app, ["eval", "manifest.csv"]).exit_code == 0
157-
tx_config = tx.call_args.kwargs["config"]
158-
assert tx_config.speech_model is None
159-
assert tx_config.speech_models is None
143+
assert tx.call_args.kwargs["config"].speech_models is None
144+
145+
146+
@pytest.mark.parametrize("model", ["best", "nano", "slam-1", "universal"])
147+
def test_legacy_models_are_a_usage_error(model):
148+
result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", model])
149+
assert result.exit_code == 2
150+
assert "--speech-model" in result.output
160151

161152

162153
def test_speech_model_named_in_human_header(tmp_path, mocker):
163154
_auth()
164155
_write_wer_manifest(tmp_path)
165156
_mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")])
166-
result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", "universal"])
167-
assert "universal" in result.output
157+
result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", "universal-3-pro"])
158+
assert "universal-3-pro" in result.output
168159
assert "default model" not in result.output
169160

170161

0 commit comments

Comments
 (0)