diff --git a/inspect_evals_scoring/model_config.yaml b/inspect_evals_scoring/model_config.yaml index 15659aa..f4bd88f 100644 --- a/inspect_evals_scoring/model_config.yaml +++ b/inspect_evals_scoring/model_config.yaml @@ -1,12 +1,12 @@ openai/gpt-4o: - name: gpt-4o + name: GPT-4o provider: OpenAI family: GPT series release_date: 2024-05-13 # https://platform.openai.com/docs/models knowledge_cutoff_date: 2023-10-31 # https://github.com/HaoooWang/llm-knowledge-cutoff-dates training_flops: Unknown api_provider: OpenAI - api_endpoint: gpt-4o-2024-11-20 # alias:gpt-4o https://platform.openai.com/docs/models + api_endpoint: openai/gpt-4o-2024-11-20 # alias:gpt-4o https://platform.openai.com/docs/models api_input_mtok_price_usd: 2.50 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 1.25 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 10.00 # https://www.helicone.ai/llm-cost @@ -16,14 +16,14 @@ openai/gpt-4o: context_window_size_tokens: 128_000 # https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json openai/gpt-4o-2024-08-06: - name: gpt-4o + name: GPT-4o provider: OpenAI family: GPT series release_date: 2024-05-13 # https://platform.openai.com/docs/models knowledge_cutoff_date: 2023-10-31 # https://github.com/HaoooWang/llm-knowledge-cutoff-dates training_flops: Unknown api_provider: OpenAI - api_endpoint: gpt-4o-2024-08-06 # alias:gpt-4o https://platform.openai.com/docs/models + api_endpoint: openai/gpt-4o-2024-08-06 # alias:gpt-4o https://platform.openai.com/docs/models api_input_mtok_price_usd: 2.50 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 1.25 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 10.00 # https://www.helicone.ai/llm-cost @@ -33,14 +33,14 @@ openai/gpt-4o-2024-08-06: context_window_size_tokens: 128_000 # https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json openai/gpt-4o-mini: - name: gpt-4o-mini + name: GPT-4o mini provider: OpenAI family: GPT series release_date: 2024-06-18 knowledge_cutoff_date: 2023-10-31 training_flops: Unknown api_provider: OpenAI - api_endpoint: gpt-4o-mini-2024-07-18 + api_endpoint: openai/gpt-4o-mini-2024-07-18 api_input_mtok_price_usd: 0.15 api_cached_input_mtok_price_usd: 0.075 api_output_mtok_price_usd: 0.60 @@ -57,7 +57,7 @@ openai/o3-mini: knowledge_cutoff_date: 2023-10-31 # https://github.com/HaoooWang/llm-knowledge-cutoff-dates training_flops: N/A (not publicly disclosed) api_provider: OpenAI - api_endpoint: o3-mini-2025-01-31 # https://platform.openai.com/docs/models + api_endpoint: openai/o3-mini-2025-01-31 # https://platform.openai.com/docs/models api_input_mtok_price_usd: 1.10 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 0.55 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 4.40 # https://www.helicone.ai/llm-cost @@ -74,7 +74,7 @@ openai/o4-mini: knowledge_cutoff_date: 2024-06-01 # https://platform.openai.com/docs/models/o4-mini training_flops: N/A (not publicly disclosed) api_provider: OpenAI - api_endpoint: o4-mini-2025-04-16 # https://platform.openai.com/docs/models/o4-mini + api_endpoint: openai/o4-mini-2025-04-16 # https://platform.openai.com/docs/models/o4-mini api_input_mtok_price_usd: 1.10 # https://platform.openai.com/docs/models/o4-mini api_cached_input_mtok_price_usd: 0.275 # https://platform.openai.com/docs/models/o4-mini api_output_mtok_price_usd: 4.40 # https://platform.openai.com/docs/models/o4-mini @@ -91,7 +91,7 @@ openai/o1-mini: knowledge_cutoff_date: 2023-10-31 # https://github.com/HaoooWang/llm-knowledge-cutoff-dates training_flops: "Unknown" api_provider: OpenAI - api_endpoint: o1-mini-2024-09-12 # https://platform.openai.com/docs/models + api_endpoint: openai/o1-mini-2024-09-12 # https://platform.openai.com/docs/models api_input_mtok_price_usd: 1.10 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 0.55 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 4.40 # https://www.helicone.ai/llm-cost @@ -107,8 +107,8 @@ google/gemini-2.0-flash-001: release_date: 2025-02-01 # https://ai.google.dev/ (experimental preview) knowledge_cutoff_date: 2024-06-30 training_flops: Unknown - api_provider: Google Vertex AI - api_endpoint: gemini-2.0-flash-001 # https://ai.google.dev/ + api_provider: Google API + api_endpoint: google/gemini-2.0-flash-001 # https://ai.google.dev/ api_input_mtok_price_usd: 0.10 # https://ai.google.dev/gemini-api/docs/pricing#2_0flash api_cached_input_mtok_price_usd: 0 # N/A api_output_mtok_price_usd: 0.40 # https://ai.google.dev/gemini-api/docs/pricing#2_0flash @@ -125,8 +125,8 @@ google/gemini-2.0-flash-thinking-exp-01-21: release_date: 2025-01-21 # https://ai.google.dev/gemini-api/docs/models#gemini-2.0-flash knowledge_cutoff_date: 2024-08-30 # https://ai.google.dev/gemini-api/docs/models#gemini-2.0-flash training_flops: Unknown - api_provider: Google Vertex AI - api_endpoint: gemini-2.0-flash-thinking-exp-01-21 # https://ai.google.dev/ + api_provider: Google API + api_endpoint: google/gemini-2.0-flash-thinking-exp-01-21 # https://ai.google.dev/ api_input_mtok_price_usd: 0.10 # https://ai.google.dev/gemini-api/docs/pricing#2_0flash api_cached_input_mtok_price_usd: 0.025 # "Context caching price" api_output_mtok_price_usd: 0.40 # https://ai.google.dev/gemini-api/docs/pricing#2_0flash @@ -143,8 +143,8 @@ google/gemini-2.0-pro-exp-02-05: release_date: 2025-02-01 # https://ai.google.dev/ (experimental preview) knowledge_cutoff_date: 2024-08-30 # https://ai.google.dev/gemini-api/docs/models#gemini-2.0-pro training_flops: Unknown - api_provider: Google Vertex AI - api_endpoint: gemini-2.0-pro-exp-02-05 # https://ai.google.dev/ + api_provider: Google API + api_endpoint: google/gemini-2.0-pro-exp-02-05 # https://ai.google.dev/ api_input_mtok_price_usd: 0.00 # https://ai.google.dev/gemini-api/docs/pricing#2_0pro api_cached_input_mtok_price_usd: 0 # N/A api_output_mtok_price_usd: 0.00 # https://ai.google.dev/gemini-api/docs/pricing#2_0pro @@ -160,8 +160,8 @@ google/gemini-2.5-pro-exp-03-25: release_date: 2025-03-25 # https://ai.google.dev/gemini-api/docs/models#gemini-2.5-pro-exp-03-25 knowledge_cutoff_date: 2025-01-31 # https://ai.google.dev/gemini-api/docs/models#gemini-2.5-pro-exp-03-25 training_flops: Unknown - api_provider: Google Vertex AI - api_endpoint: gemini-2.5-pro-exp-03-25 # https://ai.google.dev/ + api_provider: Google API + api_endpoint: google/gemini-2.5-pro-exp-03-25 # https://ai.google.dev/ api_input_mtok_price_usd: 0.00 # https://ai.google.dev/gemini-api/docs/pricing#2_5pro api_cached_input_mtok_price_usd: 0 # N/A api_output_mtok_price_usd: 0.00 # https://ai.google.dev/gemini-api/docs/pricing#2_5pro @@ -178,8 +178,8 @@ anthropic/claude-3-5-sonnet-latest: release_date: 2024-06-20 # https://docs.anthropic.com/ knowledge_cutoff_date: 2024-04-30 # https://github.com/HaoooWang/llm-knowledge-cutoff-dates training_flops: "Unknown" - api_provider: Anthropic (via AWS Bedrock & Anthropic API) - api_endpoint: claude-3-5-sonnet-20241022 # https://docs.anthropic.com/ + api_provider: Anthropic + api_endpoint: anthropic/claude-3-5-sonnet-20241022 # https://docs.anthropic.com/ api_input_mtok_price_usd: 3.00 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 0.30 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 15.00 # https://www.helicone.ai/llm-cost @@ -195,8 +195,8 @@ anthropic/claude-3-7-sonnet-20250219: release_date: 2025-02-25 # https://www.anthropic.com/news/claude-3-7-sonnet knowledge_cutoff_date: 2024-10-31 # https://docs.anthropic.com/en/docs/about-claude/models/all-models training_flops: "Unknown" - api_provider: Anthropic (via AWS Bedrock & Anthropic API) - api_endpoint: claude-3-7-sonnet-20250219 # https://docs.anthropic.com/en/docs/about-claude/models/all-models + api_provider: Anthropic + api_endpoint: anthropic/claude-3-7-sonnet-20250219 # https://docs.anthropic.com/en/docs/about-claude/models/all-models api_input_mtok_price_usd: 3.00 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 0.30 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 15.00 # https://www.helicone.ai/llm-cost @@ -205,22 +205,39 @@ anthropic/claude-3-7-sonnet-20250219: country_of_origin: USA context_window_size_tokens: 200000 # https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json -# xai/grok-3: -# name: Grok-3 -# provider: xAI -# family: Grok series -# release_date: 2025-02-18 -# knowledge_cutoff_date: 2023-12-31 # Unknown (trained on extensive 2023 data) -# training_flops: “Colossus” supercluster (100k Nvidia H100 GPUs) # :contentReferenceoaicite:64]{index=64} # :contentReference[oaicite:65{index=65} -# api_provider: xAI (X Platform API) -# api_endpoint: grok-3 -# api_input_mtok_price_usd: 0.27 # https://www.helicone.ai/llm-cost -# api_cached_input_mtok_price_usd: 0.07 # https://www.helicone.ai/llm-cost -# api_output_mtok_price_usd: 1.10 # https://www.helicone.ai/llm-cost -# attributes: -# accessibility: closed-source -# country_of_origin: USA -# context_window_size_tokens: 1000000 # https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json +grok/grok-3: + name: Grok 3 + provider: xAI + family: Grok + release_date: 2025-02-19 + knowledge_cutoff_date: 2024-11-17 # https://docs.x.ai/docs/models#additional-information-regarding-models + training_flops: "10x the compute of previous state-of-the-art models" + api_provider: Grok + api_endpoint: grok/grok-3-beta + api_input_mtok_price_usd: 3.00 # https://docs.x.ai/docs/models + api_cached_input_mtok_price_usd: 15.00 # https://docs.x.ai/docs/models + api_output_mtok_price_usd: 0 + attributes: + accessibility: closed-source + country_of_origin: USA + context_window_size_tokens: 131072 # https://docs.x.ai/docs/models + +grok/grok-3-mini: + name: Grok 3 mini + provider: xAI + family: Grok + release_date: 2025-02-19 + knowledge_cutoff_date: 2024-11-17 # https://docs.x.ai/docs/models#additional-information-regarding-models + training_flops: Unknown + api_provider: Grok + api_endpoint: grok/grok-3-mini-beta + api_input_mtok_price_usd: 0.30 # https://docs.x.ai/docs/models + api_cached_input_mtok_price_usd: 0.50 # https://docs.x.ai/docs/models + api_output_mtok_price_usd: 0 + attributes: + accessibility: closed-source + country_of_origin: USA + context_window_size_tokens: 131072 # https://docs.x.ai/docs/models grok/grok-2-1212: name: Grok-2 @@ -229,8 +246,8 @@ grok/grok-2-1212: release_date: 2024-12-14 # https://docs.x.ai/docs/release-notes#released-the-new-grok-2-1212-and-grok-2-vision-1212-models knowledge_cutoff_date: 2024-07-17 training_flops: "Unknown" - api_provider: grok - api_endpoint: grok-2-1212 + api_provider: Grok + api_endpoint: grok/grok-2-1212 api_input_mtok_price_usd: 2.00 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 0.00 # https://www.helicone.ai/llm-cost api_output_mtok_price_usd: 10.00 # https://www.helicone.ai/llm-cost @@ -247,8 +264,8 @@ openrouter/meta-llama/llama-3.3-70b-instruct: release_date: 2024-12-19 # https://ai.meta.com/llama/ knowledge_cutoff_date: 2023-12-31 # https://github.com/HaoooWang/llm-knowledge-cutoff-dates training_flops: "Unknown" - api_provider: AWS Bedrock, TogetherAI - api_endpoint: llama-3.3-70b-instruct # https://ai.meta.com/llama/ + api_provider: OpenRouter + api_endpoint: openrouter/meta-llama/llama-3.3-70b-instruct api_input_mtok_price_usd: 0.59 # https://aws.amazon.com/bedrock/pricing/ api_cached_input_mtok_price_usd: 0.0 # N/A api_output_mtok_price_usd: 0.79 # https://aws.amazon.com/bedrock/pricing/ @@ -265,8 +282,8 @@ openrouter/meta-llama/llama-3.2-90b-vision-instruct: release_date: 2024-10-31 # vision model preview https://ai.meta.com/llama/ knowledge_cutoff_date: 2023-12-31 training_flops: "Unknown" - api_provider: Together AI (Open Router / NVIDIA NeMo) - api_endpoint: llama-3.2-90b-vision-instruct # https://ai.meta.com/llama/ + api_provider: OpenRouter + api_endpoint: openrouter/meta-llama/llama-3.2-90b-vision-instruct api_input_mtok_price_usd: 0.7 # Estimated (varies by platform) api_cached_input_mtok_price_usd: 0.0 # N/A api_output_mtok_price_usd: 0.9 # Estimated @@ -282,8 +299,8 @@ openrouter/qwen/qwen-2-vl-72b-instruct: release_date: 2024-09-19 knowledge_cutoff_date: 2023-12-31 training_flops: "Pretrained on ~18T tokens" - api_provider: Together AI (or Alibaba ModelScope) - api_endpoint: qwen2.5-72b-instruct + api_provider: OpenRouter + api_endpoint: openrouter/qwen/qwen-2-vl-72b-instruct api_input_mtok_price_usd: 0.5 # Estimated for open-source model api_cached_input_mtok_price_usd: 0.0 # N/A api_output_mtok_price_usd: 0.5 # Estimated @@ -293,14 +310,14 @@ openrouter/qwen/qwen-2-vl-72b-instruct: context_window_size_tokens: 128000 # https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json openrouter/qwen/qwen-2.5-coder-32b-instruct: - name: Qwen 2.5 Coder + name: Qwen 2.5 Coder 32B Instruct provider: Alibaba Cloud family: Qwen release_date: 2024-09-01 # research preview knowledge_cutoff_date: 2023-12-31 # primarily code data training_flops: "Unknown" - api_provider: Together AI - api_endpoint: qwen2.5-coder-32b-instruct + api_provider: OpenRouter + api_endpoint: openrouter/qwen/qwen-2.5-coder-32b-instruct api_input_mtok_price_usd: 0.79 api_cached_input_mtok_price_usd: 0.0 # N/A api_output_mtok_price_usd: 0.79 @@ -310,14 +327,14 @@ openrouter/qwen/qwen-2.5-coder-32b-instruct: context_window_size_tokens: 131072 # https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json openrouter/deepseek/deepseek-chat: - name: DeepSeek V3 + name: DeepSeek-V3 provider: DeepSeek AI family: DeepSeek LLM release_date: 2024-12-26 knowledge_cutoff_date: 2024-12-26 # unknown so default to release date training_flops: "671B-parameter MoE, trained on 14.8T high-quality tokens" - api_provider: openrouter - api_endpoint: deepseek/deepseek-chat + api_provider: OpenRouter + api_endpoint: openrouter/deepseek/deepseek-chat api_input_mtok_price_usd: 0.40 # https://openrouter.ai/deepseek/deepseek-chat api_cached_input_mtok_price_usd: 0.00 api_output_mtok_price_usd: 1.30 # https://openrouter.ai/deepseek/deepseek-chat @@ -328,14 +345,14 @@ openrouter/deepseek/deepseek-chat: openrouter/deepseek/deepseek-r1: - name: DeepSeek R1 + name: DeepSeek-R1 provider: DeepSeek AI family: DeepSeek LLM release_date: 2025-01-20 knowledge_cutoff_date: 2025-01-20 # unknown so default to release date training_flops: "Unknown" - api_provider: openrouter - api_endpoint: deepseek/deepseek-r1 + api_provider: OpenRouter + api_endpoint: openrouter/deepseek/deepseek-r1 api_input_mtok_price_usd: 0.70 # https://openrouter.ai/deepseek/deepseek-r1 api_cached_input_mtok_price_usd: 0.00 api_output_mtok_price_usd: 2.50 # https://openrouter.ai/deepseek/deepseek-r1 @@ -351,8 +368,8 @@ mistral/mistral-small-latest: release_date: 2024-03-17 knowledge_cutoff_date: 2024-03-17 # Unknown so defaulting to release date training_flops: "Unknown" - api_provider: Mistral API (La Plateforme) / Azure AI - api_endpoint: mistral-small-2503 + api_provider: Mistral API (La Plateforme) + api_endpoint: mistral/mistral-small-2503 api_input_mtok_price_usd: 0.10 api_cached_input_mtok_price_usd: 0.0 # N/A api_output_mtok_price_usd: 0.30 @@ -368,8 +385,8 @@ mistral/mistral-large-latest: release_date: 2024-02-26 knowledge_cutoff_date: 2024-02-26 # Unknown so defaulting to release date training_flops: "Unknown" - api_provider: Mistral API (La Plateforme) / Azure AI - api_endpoint: mistral-large-latest + api_provider: Mistral API (La Plateforme) + api_endpoint: mistral/mistral-large-latest api_input_mtok_price_usd: 8.00 api_cached_input_mtok_price_usd: 0.0 # N/A api_output_mtok_price_usd: 8.00 @@ -385,7 +402,7 @@ openrouter/mistralai/codestral-2501: release_date: 2024-05-29 # (v1; Codestral 25.01 in Jan 2025 as v2) knowledge_cutoff_date: 2024-05-29 # Unknown so defaulting to release date training_flops: "Unknown" - api_provider: Together AI (and Mistral API) + api_provider: OpenRouter api_endpoint: openrouter/mistralai/codestral-2501 api_input_mtok_price_usd: 3.00 # https://www.helicone.ai/llm-cost api_cached_input_mtok_price_usd: 0.0 # N/A diff --git a/tests/test_model_config.py b/tests/test_model_config.py index fb36c22..13be339 100644 --- a/tests/test_model_config.py +++ b/tests/test_model_config.py @@ -108,16 +108,16 @@ def test_specific_model_fields(): model = ModelMetadata(**config["anthropic/claude-3-5-sonnet-latest"]) assert model.name == "Claude 3.5 Sonnet" assert model.provider == "Anthropic" - assert model.api_provider == "Anthropic (via AWS Bedrock & Anthropic API)" + assert model.api_provider == "Anthropic" assert isinstance(model.api_input_mtok_price_usd, float) assert isinstance(model.api_output_mtok_price_usd, float) # Test Meta model - if "meta/llama-3.3-70b" in config: - model = ModelMetadata(**config["meta/llama-3.3-70b"]) + if "openrouter/meta/llama-3.3-70b" in config: + model = ModelMetadata(**config["openrouter/meta/llama-3.3-70b"]) assert model.name == "Llama 3.3 70B" - assert model.provider == "Meta (via Amazon Bedrock)" - assert model.api_provider == "AWS Bedrock, TogetherAI" + assert model.provider == "Meta" + assert model.api_provider == "OpenRouter" assert isinstance(model.api_input_mtok_price_usd, float) assert isinstance(model.api_output_mtok_price_usd, float)