Toxicity Detection validators

backend/app/api/API_USAGE.md

@@ -100,7 +100,7 @@ Endpoint:
 Optional filters:
 - `ids=<uuid>&ids=<uuid>`
 - `stage=input|output`
-- `type=uli_slur_match|pii_remover|gender_assumption_bias|ban_list|llm_critic|topic_relevance`
+- `type=uli_slur_match|pii_remover|gender_assumption_bias|ban_list|llm_critic|topic_relevance|llamaguard_7b|profanity_free`
 
 Example:
 
@@ -442,6 +442,8 @@ From `validators.json`:
 - `ban_list`
 - `llm_critic`
 - `topic_relevance`
+- `llamaguard_7b`
+- `profanity_free`
 
 Source of truth:
 - `backend/app/core/validators/validators.json`

backend/app/api/routes/guardrails.py

@@ -258,6 +258,9 @@ def add_validator_logs(
     for log in iteration.outputs.validator_logs:
         result = log.validation_result
 
+        if result is None:
+            continue
+
         if suppress_pass_logs and isinstance(result, PassResult):
             continue
 

backend/app/core/enum.py

@@ -32,3 +32,6 @@ class ValidatorType(Enum):
     GenderAssumptionBias = "gender_assumption_bias"
     BanList = "ban_list"
     TopicRelevance = "topic_relevance"
+    LLMCritic = "llm_critic"
+    LlamaGuard7B = "llamaguard_7b"
+    ProfanityFree = "profanity_free"

backend/app/core/validators/config/llamaguard_7b_safety_validator_config.py

@@ -0,0 +1,16 @@
+from typing import List, Literal, Optional
+
+from guardrails.hub import LlamaGuard7B
+
+from app.core.validators.config.base_validator_config import BaseValidatorConfig
+
+
+class LlamaGuard7BSafetyValidatorConfig(BaseValidatorConfig):
+    type: Literal["llamaguard_7b"]
+    policies: Optional[List[str]] = None
+
+    def build(self):
+        return LlamaGuard7B(
+            policies=self.policies,
+            on_fail=self.resolve_on_fail(),
+        )

backend/app/core/validators/config/profanity_free_safety_validator_config.py

@@ -0,0 +1,14 @@
+from typing import Literal
+
+from guardrails.hub import ProfanityFree
+
+from app.core.validators.config.base_validator_config import BaseValidatorConfig
+
+
+class ProfanityFreeSafetyValidatorConfig(BaseValidatorConfig):
+    type: Literal["profanity_free"]
+
+    def build(self):
+        return ProfanityFree(
+            on_fail=self.resolve_on_fail(),
+        )

backend/app/core/validators/validators.json

@@ -29,6 +29,16 @@
             "type": "topic_relevance",
             "version": "0.1.0",
             "source": "local"
+        },
+        {
+            "type": "llamaguard_7b",
+            "version": "0.1.0",
+            "source": "hub://guardrails/llamaguard_7b"
+        },
+        {
+            "type": "profanity_free",
+            "version": "0.1.0",
+            "source": "hub://guardrails/profanity_free"
         }
     ]
 }

backend/app/schemas/guardrail_config.py

@@ -24,6 +24,12 @@
 from app.core.validators.config.topic_relevance_safety_validator_config import (
     TopicRelevanceSafetyValidatorConfig,
 )
+from app.core.validators.config.llamaguard_7b_safety_validator_config import (
+    LlamaGuard7BSafetyValidatorConfig,
+)
+from app.core.validators.config.profanity_free_safety_validator_config import (
+    ProfanityFreeSafetyValidatorConfig,
+)
 
 ValidatorConfigItem = Annotated[
     Union[
@@ -32,6 +38,8 @@
         LexicalSlurSafetyValidatorConfig,
         LLMCriticSafetyValidatorConfig,
         PIIRemoverSafetyValidatorConfig,
+        LlamaGuard7BSafetyValidatorConfig,
+        ProfanityFreeSafetyValidatorConfig,
         TopicRelevanceSafetyValidatorConfig,
     ],
     Field(discriminator="type"),