Wrapped LLM as a garak generator

docs/source/garak.generators.llm.rst

@@ -0,0 +1,7 @@
+garak.generators.llm
+==========================
+
+.. automodule:: garak.generators.llm
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/generators.rst

@@ -20,6 +20,7 @@ For a detailed oversight into how a generator operates, see :doc:`garak.generato
    garak.generators.langchain
    garak.generators.langchain_serve
    garak.generators.litellm
+   garak.generators.llm
    garak.generators.mistral
    garak.generators.ollama
    garak.generators.openai

garak/generators/llm.py

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: Portions Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""LLM (simonw/llm) generator support"""
+
+import logging
+from typing import List, Union
+
+import llm 
+
+from garak import _config
+from garak.attempt import Message, Conversation
+from garak.generators.base import Generator
+
+
+class LLMGenerator(Generator):
+    """Class supporting simonw/llm-managed models
+
+    See https://pypi.org/project/llm/ and its provider plugins.
+
+    Calls model.prompt() with the prompt text and relays the response. Per-provider
+    options and API keys are all handled by `llm` (e.g., `llm keys set openai`).
+
+    Set --model_name to the `llm` model id or alias (e.g., "gpt-4o-mini",
+    "claude-3.5-haiku", or a local alias configured in `llm models`).
+
+    Explicitly, garak delegates the majority of responsibility here:
+
+    * the generator calls prompt() on the resolved `llm` model
+    * provider setup, auth, and model-specific options live in `llm`
+    * there's no support for chains; this is a direct LLM interface
+
+    Notes:
+    * Not all providers support all parameters (e.g., temperature, max_tokens).
+      We pass only non-None params; providers ignore what they don't support.
+    """
+
+    DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | {
+        "temperature": None,
+        "max_tokens": None,
+        "top_p": None,
+        "stop": [],
+        "system": None,
+    }
+
+    generator_family_name = "LLM"
+
+    def __init__(self, name: str = "", config_root=_config):
+        self.name = name
+        self._load_config(config_root)
+        self.fullname = f"LLM (simonw/llm) {self.name or '(default)'}"
+
+        super().__init__(self.name, config_root=config_root)
+
+        try:
+            # Resolve the llm model; fall back to llm's default if no name given
+            self.model = llm.get_model(self.name) if self.name else llm.get_model()
+        except Exception as e:
+            logging.error("Failed to resolve `llm` model '%s': %s", self.name, repr(e))
+            raise e
+
+    def _call_model(
+        self, prompt: Conversation, generations_this_call: int = 1
+    ) -> List[Union[Message, None]]:
+        """
+        Continuation generation method for LLM integrations via `llm`.
+
+        This calls model.prompt() once per generation and materializes the text().
+        """
+        text_prompt = prompt.last_message().text
+
+        # Build kwargs only for parameters explicitly set (non-None / non-empty)
+        prompt_kwargs = {}
+        if self.system:
+            prompt_kwargs["system"] = self.system
+        if self.max_tokens is not None:
+            prompt_kwargs["max_tokens"] = self.max_tokens
+        if self.temperature is not None:
+            prompt_kwargs["temperature"] = self.temperature
+        if self.top_p is not None:
+            prompt_kwargs["top_p"] = self.top_p
+        if self.stop:
+            prompt_kwargs["stop"] = self.stop
+
+        try:
+            response = self.model.prompt(text_prompt, **prompt_kwargs)
+            out = response.text() 
+            return [Message(out)]
+        except Exception as e:
+            logging.error("`llm` generation failed: %s", repr(e))
+            return [None]
+
+
+DEFAULT_CLASS = "LLMGenerator"

pyproject.toml

@@ -111,6 +111,7 @@ dependencies = [
   "mistralai==1.5.2",
   "pillow>=10.4.0",
   "ftfy>=6.3.1",
+  "llm>=0.11",
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -11,6 +11,7 @@ backoff>=2.1.1
 rapidfuzz>=3.0.0
 jinja2>=3.1.6
 nltk>=3.9.1
+llm>=0.11
 accelerate>=0.23.0
 avidtools==0.1.2
 stdlibs>=2022.10.9

tests/generators/test_llm.py

@@ -0,0 +1,128 @@
+# SPDX-FileCopyrightText: Portions Copyright (c) 2025 NVIDIA CORPORATION &
+#                         AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for simonw/llm-backed garak generator"""
+
+import pytest
+from unittest.mock import MagicMock
+
+from garak.attempt import Conversation, Turn, Message
+from garak._config import GarakSubConfig
+
+# Adjust import path/module name to where you placed the wrapper
+from garak.generators.llm import LLMGenerator
+
+
+# ─── Helpers & Fixtures ─────────────────────────────────────────────────
+
+class FakeResponse:
+    """Minimal `llm` Response shim with .text()"""
+    def __init__(self, txt: str):
+        self._txt = txt
+    def text(self) -> str:
+        return self._txt
+
+
+class FakeModel:
+    """Minimal `llm` model shim with .prompt()"""
+    def __init__(self):
+        self.calls = []
+    def prompt(self, prompt_text: str, **kwargs):
+        self.calls.append((prompt_text, kwargs))
+        return FakeResponse("OK_FAKE")
+
+
+@pytest.fixture
+def cfg():
+    """Minimal garak sub-config; extend if you wire defaults via config."""
+    c = GarakSubConfig()
+    c.generators = {} 
+    return c
+
+
+@pytest.fixture
+def fake_llm(monkeypatch):
+    """
+    Patch llm.get_model to return a fresh FakeModel for each test.
+    Return the FakeModel so tests can inspect call args.
+    """
+    import llm 
+    model = FakeModel()
+    monkeypatch.setattr(llm, "get_model", lambda *a, **k: model)
+    return model
+
+
+# ─── Tests ──────────────────────────────────────────────────────────────
+
+def test_instantiation_resolves_model(cfg, fake_llm):
+    gen = LLMGenerator(name="my-alias", config_root=cfg)
+    assert gen.name == "my-alias"
+    assert hasattr(gen, "model")
+    assert "LLM (simonw/llm)" in gen.fullname
+
+
+def test_generate_returns_message(cfg, fake_llm):
+    gen = LLMGenerator(name="alias", config_root=cfg)
+
+    conv = Conversation([Turn("user", Message(text="ping"))])
+    out = gen._call_model(conv)
+
+    assert isinstance(out, list) and len(out) == 1
+    assert isinstance(out[0], Message)
+    assert out[0].text == "OK_FAKE"
+
+    prompt_text, kwargs = fake_llm.calls[0]
+    assert prompt_text == "ping"
+    assert kwargs == {}  
+
+
+def test_param_passthrough(cfg, fake_llm):
+    gen = LLMGenerator(name="alias", config_root=cfg)
+    gen.temperature = 0.2
+    gen.max_tokens = 64
+    gen.top_p = 0.9
+    gen.stop = ["\n\n"]
+    gen.system = "you are testy"
+
+    conv = Conversation([Turn("user", Message(text="hello"))])
+    _ = gen._call_model(conv)
+
+    _, kwargs = fake_llm.calls[0]
+    assert kwargs["temperature"] == 0.2
+    assert kwargs["max_tokens"] == 64
+    assert kwargs["top_p"] == 0.9
+    assert kwargs["stop"] == ["\n\n"]
+    assert kwargs["system"] == "you are testy"
+
+
+def test_wrapper_handles_llm_exception(cfg, monkeypatch):
+    """If the underlying `llm` call explodes, wrapper returns [None]."""
+    import llm
+    class BoomModel:
+        def prompt(self, *a, **k):
+            raise RuntimeError("boom")
+    monkeypatch.setattr(llm, "get_model", lambda *a, **k: BoomModel())
+
+    gen = LLMGenerator(name="alias", config_root=cfg)
+    conv = Conversation([Turn("user", Message(text="ping"))])
+    out = gen._call_model(conv)
+    assert out == [None]
+
+
+def test_default_model_when_name_empty(cfg, fake_llm, monkeypatch):
+    """
+    If name is empty, wrapper should call llm.get_model() with no args,
+    i.e., use llm's configured default model.
+    """
+    import llm
+    spy = MagicMock(side_effect=lambda *a, **k: fake_llm)
+    monkeypatch.setattr(llm, "get_model", spy)
+
+    gen = LLMGenerator(name="", config_root=cfg)
+    conv = Conversation([Turn("user", Message(text="x"))])
+    _ = gen._call_model(conv)
+
+    spy.assert_called_once()
+    assert spy.call_args.args == ()
+    assert spy.call_args.kwargs == {}