feat(guardrails): add commercial-API guardrails (Watsonx Guardian, Patronus, Qualifire, Pioneer)

.github/workflows/tests-integration.yaml

@@ -39,6 +39,12 @@ jobs:
           ALINIA_ENDPOINT: ${{ secrets.ALINIA_ENDPOINT }}
           ALINIA_API_KEY: ${{ secrets.ALINIA_API_KEY }}
           LAKERA_API_KEY: ${{ secrets.LAKERA_API_KEY }}
+          PATRONUS_API_KEY: ${{ secrets.PATRONUS_API_KEY }}
+          # IBM watsonx Guardian (Granite Guardian): IAM API key + region URL + project.
+          # The test skips unless APIKEY, URL, and PROJECT_ID (or SPACE_ID) are all set.
+          WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
+          WATSONX_URL: ${{ secrets.WATSONX_URL }}
+          WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
           # Bedrock API key (bearer token). boto3 >= 1.39 reads AWS_BEARER_TOKEN_BEDROCK
           # natively for bedrock / bedrock-runtime calls; no SigV4 creds needed.
           AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEDROCK_API_KEY }}

docs/SUMMARY.md

@@ -46,12 +46,14 @@
   * [OffTopic](api/guardrails/off-topic.md)
   * [OpenAI Moderation](api/guardrails/openai-moderation.md)
   * [Pangolin](api/guardrails/pangolin.md)
+  * [Patronus](api/guardrails/patronus.md)
   * [Prometheus](api/guardrails/prometheus.md)
   * [Prompt Guard 2](api/guardrails/prompt-guard.md)
   * [ProtectAI](api/guardrails/protectai.md)
   * [Selene](api/guardrails/selene.md)
   * [Sentinel](api/guardrails/sentinel.md)
   * [ShieldGemma](api/guardrails/shield-gemma.md)
+  * [Watsonx Guardian](api/guardrails/watsonx-guardian.md)
   * [WildGuard](api/guardrails/wild-guard.md)
 * Providers
   * [EncoderFile](api/providers/encoderfile.md)

pyproject.toml

@@ -11,11 +11,14 @@ dynamic = ["version"]
 dependencies = [
   "any-llm-sdk>=1.7.0",
   "pydantic",
+  # Used directly by the REST-based API guardrails (Alinia, Lakera, Patronus);
+  # not guaranteed transitively, so declared explicitly.
+  "requests>=2.31.0",
 ]
 
 [project.optional-dependencies]
 all = [
-  "any-guardrail[flowjudge,huggingface,azure-content-safety,encoderfile,bedrock,llamafile,openai,lettucedetect,gliner]"
+  "any-guardrail[flowjudge,huggingface,azure-content-safety,encoderfile,bedrock,llamafile,openai,lettucedetect,gliner,watsonx]"
 ]
 
 bedrock = [
@@ -68,6 +71,12 @@ openai = [
     "openai>=1.99.3",
 ]
 
+# IBM watsonx.ai SDK (Apache-2.0) for the WatsonxGuardian (Granite Guardian) API.
+# Patronus is plain REST (requests) and needs no extra.
+watsonx = [
+    "ibm-watsonx-ai>=1.3.0",
+]
+
 [project.urls]
 Documentation = "https://mozilla-ai.github.io/any-guardrail/"
 Issues = "https://github.com/mozilla-ai/any-guardrail/issues"

scripts/generate_api_docs.py

@@ -196,6 +196,7 @@ def _guardrail_page(module_path: str, class_name: str) -> str:
 _INSTALL_EXTRA_OVERRIDES = {
     "AzureContentSafety": "azure-content-safety",
     "AzurePromptShields": "azure-content-safety",
+    "WatsonxGuardian": "watsonx",
 }
 
 
@@ -388,6 +389,7 @@ def _guardrails_index_page() -> str:
         "openai-moderation.md",
     ),
     ("any_guardrail.guardrails.pangolin.pangolin", "Pangolin", "pangolin.md"),
+    ("any_guardrail.guardrails.patronus.patronus", "Patronus", "patronus.md"),
     ("any_guardrail.guardrails.protectai.protectai", "Protectai", "protectai.md"),
     ("any_guardrail.guardrails.sentinel.sentinel", "Sentinel", "sentinel.md"),
     ("any_guardrail.guardrails.shield_gemma.shield_gemma", "ShieldGemma", "shield-gemma.md"),
@@ -408,6 +410,11 @@ def _guardrails_index_page() -> str:
     ("any_guardrail.guardrails.selene.selene", "Selene", "selene.md"),
     ("any_guardrail.guardrails.lettuce_detect.lettuce_detect", "LettuceDetect", "lettuce-detect.md"),
     ("any_guardrail.guardrails.gli_guard.gli_guard", "GliGuard", "gli-guard.md"),
+    (
+        "any_guardrail.guardrails.watsonx_guardian.watsonx_guardian",
+        "WatsonxGuardian",
+        "watsonx-guardian.md",
+    ),
 ]
 
 

src/any_guardrail/base.py

@@ -58,6 +58,8 @@ class GuardrailName(StrEnum):
     SELENE = "selene"
     LETTUCE_DETECT = "lettuce_detect"
     GLI_GUARD = "gli_guard"
+    WATSONX_GUARDIAN = "watsonx_guardian"
+    PATRONUS = "patronus"
 
 
 class Guardrail(ABC):

src/any_guardrail/guardrails/patronus/__init__.py

@@ -0,0 +1,3 @@
+from .patronus import Patronus
+
+__all__ = ["Patronus"]

src/any_guardrail/guardrails/patronus/patronus.py

@@ -0,0 +1,218 @@
+import os
+import time
+from typing import ClassVar, Literal
+
+import requests
+
+from any_guardrail.base import Guardrail, GuardrailOutput
+from any_guardrail.types import AnyDict, CategoryResult
+
+
+class Patronus(Guardrail):
+    """Wraps the Patronus AI Evaluate API for managed LLM evaluation / guardrailing.
+
+    This is the hosted, pay-per-use counterpart to the locally-run
+    :class:`~any_guardrail.guardrails.glider.glider.Glider` (GLIDER) judge and the
+    Patronus Lynx hallucination model: the same paper-backed evaluators, served as
+    managed configurations behind a single ``/v1/evaluate`` endpoint.
+
+    A single request runs one or more *evaluators*. Each evaluator is selected by
+    name (e.g. ``"lynx"`` for hallucination, ``"judge"`` for the managed
+    LLM-as-a-judge, ``"answer-relevance"``, toxicity / PII evaluators) and an
+    optional managed ``criteria`` alias (e.g. ``"patronus:hallucination"``,
+    ``"patronus:prompt-injection"``). Each returns a pass/fail verdict, a raw
+    score in ``[0, 1]`` (higher is better; below ``0.5`` fails by default), and —
+    when ``explain_strategy`` is set — an explanation.
+
+    Auth is via an API key. Obtain one from https://app.patronus.ai/ (free
+    Developer tier with starter credit) and set it via ``PATRONUS_API_KEY`` or
+    pass it directly.
+
+    ``GuardrailOutput`` mapping:
+        - ``valid`` combines the per-evaluator pass flags per ``success_strategy``
+          (``"all_pass"`` → every evaluator must pass; ``"any_pass"`` → at least
+          one must).
+        - ``score`` is the canonical risk of the *riskiest* evaluator,
+          ``1 - min(score_raw)`` (since Patronus ``score_raw`` is higher-is-safer).
+        - ``categories`` lists one ``CategoryResult`` per evaluator (``name`` =
+          its criteria / evaluator id, ``triggered`` = it failed, ``score`` =
+          ``1 - score_raw``).
+        - ``explanation`` joins the evaluators' explanations when present.
+        - ``extra`` carries ``success_strategy`` and a per-evaluator breakdown;
+          ``raw`` is the full response body.
+        - Fails closed (``valid=False``, ``extra={"parse_failure": True}``) when
+          the response has no ``results``.
+
+    Research backing:
+        - Deshpande et al., *GLIDER: Grading LLM Interactions and Decisions using
+          Explainable Ranking* (https://arxiv.org/abs/2412.14140, 2024).
+        - Ravi et al., *Lynx: An Open Source Hallucination Evaluation Model*
+          (https://arxiv.org/abs/2407.08488, 2024).
+        - Docs: https://docs.patronus.ai/
+
+    Args:
+        evaluators (list[dict]): The evaluators to run, each a dict with at least
+            an ``"evaluator"`` key (plus optional ``"criteria"`` /
+            ``"explain_strategy"``). Example:
+            ``[{"evaluator": "judge", "criteria": "patronus:prompt-injection"}]``.
+        api_key (str | None): Patronus API key. Falls back to ``PATRONUS_API_KEY``.
+        endpoint (str): Evaluate API endpoint. Defaults to
+            ``https://api.patronus.ai/v1/evaluate``.
+        success_strategy ("all_pass" | "any_pass"): How to combine multiple
+            evaluators into the ``valid`` verdict. Defaults to ``"all_pass"``.
+        tags (dict[str, str] | None): Optional tags forwarded with each request
+            for observability.
+
+    """
+
+    SUPPORTED_MODELS: ClassVar = ["patronus-evaluate"]
+
+    def __init__(
+        self,
+        evaluators: list[AnyDict],
+        api_key: str | None = None,
+        endpoint: str = "https://api.patronus.ai/v1/evaluate",
+        success_strategy: Literal["all_pass", "any_pass"] = "all_pass",
+        tags: dict[str, str] | None = None,
+    ) -> None:
+        """Initialize the Patronus guardrail.
+
+        Does not perform any network I/O — the API is only contacted on
+        ``validate()``.
+        """
+        if api_key:
+            self.api_key = api_key
+        elif os.getenv("PATRONUS_API_KEY"):
+            self.api_key = os.getenv("PATRONUS_API_KEY")  # type: ignore[assignment]
+        else:
+            msg = (
+                "API key must be provided either as the `api_key=` parameter or through the "
+                "PATRONUS_API_KEY environment variable. Sign up at https://app.patronus.ai/ to obtain a key."
+            )
+            raise ValueError(msg)
+
+        if not evaluators:
+            msg = "`evaluators` must be a non-empty list of evaluator dicts, e.g. [{'evaluator': 'lynx'}]."
+            raise ValueError(msg)
+
+        self.evaluators = evaluators
+        self.endpoint = endpoint
+        self.success_strategy = success_strategy
+        self.tags = tags
+
+    def validate(
+        self,
+        input_text: str,
+        output_text: str | None = None,
+        retrieved_context: str | list[str] | None = None,
+    ) -> GuardrailOutput:
+        """Run the configured evaluators against the supplied model interaction.
+
+        Args:
+            input_text (str): The model input (user prompt) to evaluate.
+            output_text (str | None): The model output to evaluate. Required by
+                evaluators that judge a response (e.g. hallucination, answer
+                relevance).
+            retrieved_context (str | list[str] | None): RAG context document(s).
+                Required by grounding / hallucination evaluators.
+
+        Returns:
+            ``GuardrailOutput`` summarizing the evaluators' verdicts (see the
+            class docstring for the field mapping).
+
+        """
+        start = time.perf_counter()
+        params = self._pre_processing(input_text, output_text, retrieved_context)
+        response = self._inference(params)
+        result = self._post_processing(response)
+        self._stamp_usage(result, (time.perf_counter() - start) * 1000.0)
+        return result
+
+    def _pre_processing(
+        self,
+        input_text: str,
+        output_text: str | None,
+        retrieved_context: str | list[str] | None,
+    ) -> AnyDict:
+        body: AnyDict = {
+            "evaluators": self.evaluators,
+            "evaluated_model_input": input_text,
+        }
+        if output_text is not None:
+            body["evaluated_model_output"] = output_text
+        if retrieved_context is not None:
+            body["evaluated_model_retrieved_context"] = retrieved_context
+        if self.tags:
+            body["tags"] = self.tags
+        return body
+
+    def _inference(self, params: AnyDict) -> requests.Response:
+        response = requests.post(
+            self.endpoint,
+            headers={"X-API-KEY": self.api_key, "accept": "application/json"},
+            json=params,
+        )
+        if response.status_code != 200:
+            msg = f"Request to Patronus Evaluate API failed with status code {response.status_code}: {response.text}"
+            raise ValueError(msg)
+        return response
+
+    def _post_processing(self, response: requests.Response) -> GuardrailOutput:
+        body = response.json()
+        results = body.get("results") if isinstance(body, dict) else None
+        if not results:
+            return GuardrailOutput(valid=False, extra={"parse_failure": True}, raw=body)
+
+        categories: list[CategoryResult] = []
+        explanations: list[str] = []
+        risk_scores: list[float] = []
+        passes: list[bool] = []
+        breakdown: list[AnyDict] = []
+
+        for result in results:
+            if not isinstance(result, dict):
+                # Non-dict result entry: malformed -> count as a failed/triggered evaluator.
+                passes.append(False)
+                categories.append(CategoryResult(name="evaluator", triggered=True, score=None))
+                breakdown.append(
+                    {"name": "evaluator", "pass": False, "score_raw": None, "explanation": None, "malformed": True}
+                )
+                continue
+            name = result.get("criteria") or result.get("evaluator_id") or result.get("evaluator") or "evaluator"
+            evaluation = result.get("evaluation_result")
+            if not isinstance(evaluation, dict):
+                # Malformed individual result: count it as a failed/triggered
+                # evaluator so ``all_pass`` cannot fail open on a partial response.
+                passes.append(False)
+                categories.append(CategoryResult(name=name, triggered=True, score=None))
+                breakdown.append(
+                    {"name": name, "pass": False, "score_raw": None, "explanation": None, "malformed": True}
+                )
+                continue
+            passed = bool(evaluation.get("pass"))
+            passes.append(passed)
+
+            score_raw = evaluation.get("score_raw")
+            risk = (1.0 - float(score_raw)) if isinstance(score_raw, (int, float)) else None
+            if risk is not None:
+                risk_scores.append(risk)
+
+            explanation = evaluation.get("explanation")
+            if isinstance(explanation, str) and explanation:
+                explanations.append(explanation)
+
+            categories.append(CategoryResult(name=name, triggered=not passed, score=risk))
+            breakdown.append({"name": name, "pass": passed, "score_raw": score_raw, "explanation": explanation})
+
+        if not passes:
+            return GuardrailOutput(valid=False, extra={"parse_failure": True}, raw=body)
+
+        valid = all(passes) if self.success_strategy == "all_pass" else any(passes)
+        return GuardrailOutput(
+            valid=valid,
+            score=max(risk_scores) if risk_scores else None,
+            explanation="\n\n".join(explanations) if explanations else None,
+            categories=categories,
+            extra={"success_strategy": self.success_strategy, "breakdown": breakdown},
+            raw=body,
+        )

src/any_guardrail/guardrails/watsonx_guardian/__init__.py

@@ -0,0 +1,3 @@
+from .watsonx_guardian import WatsonxGuardian
+
+__all__ = ["WatsonxGuardian"]