diff --git a/garak/generators/nim.py b/garak/generators/nim.py index d19d1bb3d..42e4c7909 100644 --- a/garak/generators/nim.py +++ b/garak/generators/nim.py @@ -96,7 +96,7 @@ def _call_model( logging.critical(msg, exc_info=nfe) raise GarakException(f"🛑 {msg}") from nfe except Exception as oe: - msg = "NIM generation failed. Is the model name spelled correctly?" + msg = f"NIM generation failed: {type(oe).__name__}: {oe}" logging.critical(msg, exc_info=oe) raise GarakException(f"🛑 {msg}") from oe diff --git a/garak/generators/openai.py b/garak/generators/openai.py index dbdac75cf..a94f662ba 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -152,6 +152,7 @@ class OpenAICompatible(Generator): "suppressed_params": set(), "retry_json": True, "extra_params": {}, + "transient_retry_codes": [408, 429, 502, 503, 504], } _unsafe_attributes = ["client", "generator"] @@ -354,6 +355,12 @@ def _call_model( logging.exception(e) logging.error(msg) return [None] + except openai.APIStatusError as e: + if e.status_code in self.transient_retry_codes: + raise garak.exception.GeneratorBackoffTrigger from e + msg = f"HTTP {e.status_code} from {e.request.url}: {e.message}" + logging.warning(msg) + return [None] except json.decoder.JSONDecodeError as e: logging.exception(e) if self.retry_json: diff --git a/tests/generators/test_nim.py b/tests/generators/test_nim.py index 7346663bc..3d16da50d 100644 --- a/tests/generators/test_nim.py +++ b/tests/generators/test_nim.py @@ -3,10 +3,93 @@ import os import pytest +import httpx +import openai +from unittest.mock import MagicMock, patch +import garak.exception from garak.attempt import Message, Turn, Conversation +from garak.exception import GarakException import garak.cli from garak.generators.nim import NVOpenAIChat +from garak.generators.openai import OpenAICompatible + + +def _make_api_status_error( + status_code: int, url: str = "http://localhost/v1/chat/completions" +) -> openai.APIStatusError: + """Build an openai.APIStatusError with a real httpx.Response for a given HTTP status.""" + request = httpx.Request("POST", url) + response = httpx.Response(status_code, request=request) + return openai.APIStatusError(f"HTTP {status_code}", response=response, body=None) + + +def _make_prompt() -> Conversation: + return Conversation([Turn(role="user", content=Message("test prompt"))]) + + +@pytest.fixture +def nim_generator(monkeypatch): + """NVOpenAIChat with mocked client; no real API key required.""" + monkeypatch.setenv(NVOpenAIChat.ENV_VAR, "test-fake-key-for-unit-tests") + mock_client = MagicMock() + mock_client.chat.completions = MagicMock() + with patch("openai.OpenAI", return_value=mock_client): + g = NVOpenAIChat(name="org/test-model") + return g + + +def test_transient_retry_codes_in_default_params(): + """transient_retry_codes should be present in DEFAULT_PARAMS and contain expected codes.""" + codes = OpenAICompatible.DEFAULT_PARAMS["transient_retry_codes"] + for code in (408, 429, 502, 503, 504): + assert code in codes, f"Expected {code} in transient_retry_codes" + for code in (400, 401, 403, 404): + assert code not in codes, f"Expected {code} NOT in transient_retry_codes" + + +@pytest.mark.parametrize("code", [408, 502, 503, 504]) +def test_transient_http_error_raises_backoff_trigger(nim_generator, code): + """A transient status code should cause _call_model to raise GeneratorBackoffTrigger + so that the backoff decorator can schedule a retry.""" + prompt = _make_prompt() + nim_generator.generator = MagicMock() + nim_generator.generator.create.side_effect = _make_api_status_error(code) + + # Call the underlying function without the backoff decorator so the test does not + # need to wait for retry delays or exhaust the fibonacci sequence. + unwrapped = OpenAICompatible._call_model.__wrapped__ + with pytest.raises(garak.exception.GeneratorBackoffTrigger): + unwrapped(nim_generator, prompt) + + +@pytest.mark.parametrize("code", [400, 403, 404, 422]) +def test_terminal_http_error_returns_none(nim_generator, code): + """A terminal (non-transient) status code should cause _call_model to return [None] + so that the current attempt is skipped and the probe run continues.""" + prompt = _make_prompt() + nim_generator.generator = MagicMock() + nim_generator.generator.create.side_effect = _make_api_status_error(code) + + unwrapped = OpenAICompatible._call_model.__wrapped__ + result = unwrapped(nim_generator, prompt) + assert result == [None], f"Expected [None] for HTTP {code}, got {result!r}" + + +def test_nim_generic_exception_message_improved(nim_generator): + """Non-APIStatusError exceptions should report the exception type, not the old + generic 'Is the model name spelled correctly?' message.""" + prompt = _make_prompt() + + with patch( + "garak.generators.openai.OpenAICompatible._call_model", + side_effect=RuntimeError("connection reset by peer"), + ): + with pytest.raises(GarakException) as exc_info: + nim_generator._call_model(prompt) + + error_text = str(exc_info.value) + assert "Is the model name spelled correctly?" not in error_text @pytest.mark.skipif(