fix(nim): retry transient HTTP errors and report actual status code

garak/generators/nim.py

@@ -96,7 +96,7 @@ def _call_model(
             logging.critical(msg, exc_info=nfe)
             raise GarakException(f"🛑 {msg}") from nfe
         except Exception as oe:
-            msg = "NIM generation failed. Is the model name spelled correctly?"
+            msg = f"NIM generation failed: {type(oe).__name__}: {oe}"
             logging.critical(msg, exc_info=oe)
             raise GarakException(f"🛑 {msg}") from oe
 

garak/generators/openai.py

@@ -152,6 +152,7 @@ class OpenAICompatible(Generator):
         "suppressed_params": set(),
         "retry_json": True,
         "extra_params": {},
+        "transient_retry_codes": [408, 429, 502, 503, 504],
     }
 
     _unsafe_attributes = ["client", "generator"]
@@ -354,6 +355,12 @@ def _call_model(
             logging.exception(e)
             logging.error(msg)
             return [None]
+        except openai.APIStatusError as e:
+            if e.status_code in self.transient_retry_codes:
+                raise garak.exception.GeneratorBackoffTrigger from e
+            msg = f"HTTP {e.status_code} from {e.request.url}: {e.message}"
+            logging.warning(msg)
+            return [None]
         except json.decoder.JSONDecodeError as e:
             logging.exception(e)
             if self.retry_json:

tests/generators/test_nim.py

@@ -3,10 +3,93 @@
 
 import os
 import pytest
+import httpx
+import openai
+from unittest.mock import MagicMock, patch
 
+import garak.exception
 from garak.attempt import Message, Turn, Conversation
+from garak.exception import GarakException
 import garak.cli
 from garak.generators.nim import NVOpenAIChat
+from garak.generators.openai import OpenAICompatible
+
+
+def _make_api_status_error(
+    status_code: int, url: str = "http://localhost/v1/chat/completions"
+) -> openai.APIStatusError:
+    """Build an openai.APIStatusError with a real httpx.Response for a given HTTP status."""
+    request = httpx.Request("POST", url)
+    response = httpx.Response(status_code, request=request)
+    return openai.APIStatusError(f"HTTP {status_code}", response=response, body=None)
+
+
+def _make_prompt() -> Conversation:
+    return Conversation([Turn(role="user", content=Message("test prompt"))])
+
+
+@pytest.fixture
+def nim_generator(monkeypatch):
+    """NVOpenAIChat with mocked client; no real API key required."""
+    monkeypatch.setenv(NVOpenAIChat.ENV_VAR, "test-fake-key-for-unit-tests")
+    mock_client = MagicMock()
+    mock_client.chat.completions = MagicMock()
+    with patch("openai.OpenAI", return_value=mock_client):
+        g = NVOpenAIChat(name="org/test-model")
+    return g
+
+
+def test_transient_retry_codes_in_default_params():
+    """transient_retry_codes should be present in DEFAULT_PARAMS and contain expected codes."""
+    codes = OpenAICompatible.DEFAULT_PARAMS["transient_retry_codes"]
+    for code in (408, 429, 502, 503, 504):
+        assert code in codes, f"Expected {code} in transient_retry_codes"
+    for code in (400, 401, 403, 404):
+        assert code not in codes, f"Expected {code} NOT in transient_retry_codes"
+
+
+@pytest.mark.parametrize("code", [408, 502, 503, 504])
+def test_transient_http_error_raises_backoff_trigger(nim_generator, code):
+    """A transient status code should cause _call_model to raise GeneratorBackoffTrigger
+    so that the backoff decorator can schedule a retry."""
+    prompt = _make_prompt()
+    nim_generator.generator = MagicMock()
+    nim_generator.generator.create.side_effect = _make_api_status_error(code)
+
+    # Call the underlying function without the backoff decorator so the test does not
+    # need to wait for retry delays or exhaust the fibonacci sequence.
+    unwrapped = OpenAICompatible._call_model.__wrapped__
+    with pytest.raises(garak.exception.GeneratorBackoffTrigger):
+        unwrapped(nim_generator, prompt)
+
+
+@pytest.mark.parametrize("code", [400, 403, 404, 422])
+def test_terminal_http_error_returns_none(nim_generator, code):
+    """A terminal (non-transient) status code should cause _call_model to return [None]
+    so that the current attempt is skipped and the probe run continues."""
+    prompt = _make_prompt()
+    nim_generator.generator = MagicMock()
+    nim_generator.generator.create.side_effect = _make_api_status_error(code)
+
+    unwrapped = OpenAICompatible._call_model.__wrapped__
+    result = unwrapped(nim_generator, prompt)
+    assert result == [None], f"Expected [None] for HTTP {code}, got {result!r}"
+
+
+def test_nim_generic_exception_message_improved(nim_generator):
+    """Non-APIStatusError exceptions should report the exception type, not the old
+    generic 'Is the model name spelled correctly?' message."""
+    prompt = _make_prompt()
+
+    with patch(
+        "garak.generators.openai.OpenAICompatible._call_model",
+        side_effect=RuntimeError("connection reset by peer"),
+    ):
+        with pytest.raises(GarakException) as exc_info:
+            nim_generator._call_model(prompt)
+
+    error_text = str(exc_info.value)
+    assert "Is the model name spelled correctly?" not in error_text
 
 
 @pytest.mark.skipif(