Add is_reasoning_model config flag for non-OpenAI thinking models

Extract OPENAI_REASONING_MODEL_PREFIXES to module-level constant and
is_reasoning_model() to a standalone function. Add is_reasoning_model
field to LLMModelConfig with 3-state logic: True (force reasoning),
False (force standard), None (auto-detect via OpenAI prefixes).

This allows users of non-OpenAI providers (Gemini, DeepSeek, etc.) to
explicitly mark models as reasoning models via config, without relying
on fragile prefix-based detection.

Closes #2

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: MateoTTR

openevolve/config.py

@@ -79,6 +79,11 @@ class LLMModelConfig:
     # Reasoning parameters
     reasoning_effort: Optional[str] = None
 
+    # Reasoning model override: True forces reasoning-model parameter conventions
+    # (max_completion_tokens, no temperature/top_p), False forces standard conventions,
+    # None (default) auto-detects based on known OpenAI reasoning model prefixes.
+    is_reasoning_model: Optional[bool] = None
+
     # Manual mode (human-in-the-loop)
     manual_mode: Optional[bool] = None
     _manual_queue_dir: Optional[str] = None

openevolve/llm/openai.py

@@ -20,6 +20,43 @@
 
 logger = logging.getLogger(__name__)
 
+# OpenAI reasoning models that require max_completion_tokens instead of max_tokens.
+# These models don't support temperature/top_p and use different parameters.
+OPENAI_REASONING_MODEL_PREFIXES = (
+    # O-series reasoning models
+    "o1-",
+    "o1",  # o1, o1-mini, o1-preview
+    "o3-",
+    "o3",  # o3, o3-mini, o3-pro
+    "o4-",  # o4-mini
+    # GPT-5 series are also reasoning models
+    "gpt-5-",
+    "gpt-5",  # gpt-5, gpt-5-mini, gpt-5-nano
+    # The GPT OSS series are also reasoning models
+    "gpt-oss-120b",
+    "gpt-oss-20b",
+)
+
+
+def is_reasoning_model(
+    model_name: str,
+    config_flag: Optional[bool] = None,
+) -> bool:
+    """Detect if a model should be treated as a reasoning model.
+
+    Args:
+        model_name: The model name/identifier.
+        config_flag: Explicit override from config. If True/False, returns that
+            value directly.  If None (default), auto-detects based on known
+            OpenAI reasoning model prefixes.
+
+    Returns:
+        True if the model should be treated as a reasoning model.
+    """
+    if config_flag is not None:
+        return config_flag
+    return model_name.lower().startswith(OPENAI_REASONING_MODEL_PREFIXES)
+
 
 def _iso_now() -> str:
     return datetime.now(tz=timezone.utc).isoformat()
@@ -63,9 +100,10 @@ def __init__(
         self.api_key = model_cfg.api_key
         self.random_seed = getattr(model_cfg, "random_seed", None)
         self.reasoning_effort = getattr(model_cfg, "reasoning_effort", None)
+        self.is_reasoning_model_flag = getattr(model_cfg, "is_reasoning_model", None)
 
         # Manual mode: enabled via llm.manual_mode in config.yaml
-        self.manual_mode = (getattr(model_cfg, "manual_mode", False) is True)
+        self.manual_mode = getattr(model_cfg, "manual_mode", False) is True
         self.manual_queue_dir: Optional[Path] = None
 
         if self.manual_mode:
@@ -114,29 +152,10 @@ async def generate_with_context(
         formatted_messages.extend(messages)
 
         # Set up generation parameters
-        # Define OpenAI reasoning models that require max_completion_tokens
-        # These models don't support temperature/top_p and use different parameters
-        OPENAI_REASONING_MODEL_PREFIXES = (
-            # O-series reasoning models
-            "o1-",
-            "o1",  # o1, o1-mini, o1-preview
-            "o3-",
-            "o3",  # o3, o3-mini, o3-pro
-            "o4-",  # o4-mini
-            # GPT-5 series are also reasoning models
-            "gpt-5-",
-            "gpt-5",  # gpt-5, gpt-5-mini, gpt-5-nano
-            # The GPT OSS series are also reasoning models
-            "gpt-oss-120b",
-            "gpt-oss-20b",
-        )
-
-        # Check if this is an OpenAI reasoning model based on model name pattern
-        # This works for all endpoints (OpenAI, Azure, OptiLLM, OpenRouter, etc.)
-        model_lower = str(self.model).lower()
-        is_openai_reasoning_model = model_lower.startswith(OPENAI_REASONING_MODEL_PREFIXES)
+        # Detect whether to use reasoning-model parameter conventions
+        is_reasoning = is_reasoning_model(self.model, self.is_reasoning_model_flag)
 
-        if is_openai_reasoning_model:
+        if is_reasoning:
             # For OpenAI reasoning models
             params = {
                 "model": self.model,

tests/test_reasoning_model_detection.py

@@ -0,0 +1,79 @@
+"""Tests for reasoning model detection logic (is_reasoning_model function)"""
+
+import unittest
+
+from openevolve.llm.openai import OPENAI_REASONING_MODEL_PREFIXES, is_reasoning_model
+
+
+class TestIsReasoningModel(unittest.TestCase):
+    """Test the is_reasoning_model() function"""
+
+    # Auto-detect (config_flag=None) -- OpenAI models
+    def test_openai_o_series_auto_detected(self):
+        for model in ["o1", "o1-mini", "o3", "o3-mini", "o3-pro", "o4-mini"]:
+            with self.subTest(model=model):
+                self.assertTrue(is_reasoning_model(model))
+
+    def test_openai_gpt5_auto_detected(self):
+        for model in ["gpt-5", "gpt-5-mini", "gpt-5-nano"]:
+            with self.subTest(model=model):
+                self.assertTrue(is_reasoning_model(model))
+
+    def test_openai_non_reasoning_not_detected(self):
+        for model in ["gpt-4o", "gpt-4o-mini", "gpt-4", "gpt-3.5-turbo"]:
+            with self.subTest(model=model):
+                self.assertFalse(is_reasoning_model(model))
+
+    # Auto-detect -- Non-OpenAI models should NOT be auto-detected
+    def test_gemini_not_auto_detected(self):
+        for model in ["gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"]:
+            with self.subTest(model=model):
+                self.assertFalse(is_reasoning_model(model))
+
+    def test_claude_not_auto_detected(self):
+        for model in ["claude-sonnet-4-5-20250929", "claude-opus-4-5-20251101"]:
+            with self.subTest(model=model):
+                self.assertFalse(is_reasoning_model(model))
+
+    def test_deepseek_not_auto_detected(self):
+        self.assertFalse(is_reasoning_model("deepseek-r1"))
+
+    # Explicit config_flag=True -- forces reasoning model
+    def test_explicit_true_overrides_auto_detect(self):
+        self.assertTrue(is_reasoning_model("gemini-2.5-flash", config_flag=True))
+        self.assertTrue(is_reasoning_model("deepseek-r1", config_flag=True))
+        self.assertTrue(is_reasoning_model("any-unknown-model", config_flag=True))
+
+    # Explicit config_flag=False -- forces non-reasoning model
+    def test_explicit_false_overrides_auto_detect(self):
+        # Even OpenAI reasoning models can be forced to non-reasoning
+        self.assertFalse(is_reasoning_model("o3-mini", config_flag=False))
+        self.assertFalse(is_reasoning_model("gpt-5", config_flag=False))
+
+    # Case insensitivity
+    def test_case_insensitive(self):
+        self.assertTrue(is_reasoning_model("O3-MINI"))
+        self.assertTrue(is_reasoning_model("GPT-5-MINI"))
+
+    # Backward compatibility
+    def test_none_config_flag_is_default(self):
+        """None config_flag should behave exactly like the old hardcoded logic"""
+        self.assertTrue(is_reasoning_model("o3-mini", config_flag=None))
+        self.assertFalse(is_reasoning_model("gpt-4o", config_flag=None))
+
+
+class TestReasoningModelPrefixes(unittest.TestCase):
+    """Test that the prefix constant is properly defined"""
+
+    def test_prefixes_is_tuple(self):
+        self.assertIsInstance(OPENAI_REASONING_MODEL_PREFIXES, tuple)
+
+    def test_prefixes_contains_o_series(self):
+        # At minimum, o1 and o3 should be in the prefixes
+        prefixes_str = " ".join(OPENAI_REASONING_MODEL_PREFIXES)
+        self.assertIn("o1", prefixes_str)
+        self.assertIn("o3", prefixes_str)
+
+
+if __name__ == "__main__":
+    unittest.main()