diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ff0cd7a01..a544710af 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install .[tests] - name: Pylint run: | pylint -v garak \ No newline at end of file diff --git a/.github/workflows/maintain_cache.yml b/.github/workflows/maintain_cache.yml index f8df5a1bd..6a9606e5a 100644 --- a/.github/workflows/maintain_cache.yml +++ b/.github/workflows/maintain_cache.yml @@ -43,7 +43,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install .[all_plugins] - name: Build a local cache run: | export TZ=UTC diff --git a/.github/workflows/remote_package_install.yml b/.github/workflows/remote_package_install.yml index f419cea5f..ac267e175 100644 --- a/.github/workflows/remote_package_install.yml +++ b/.github/workflows/remote_package_install.yml @@ -37,7 +37,7 @@ jobs: - name: pip install from repo run: | python -m pip install --upgrade pip - python -m pip install -U git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA} + python -m pip install -U "git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}" - name: Sanity Test run: | python -m garak --model_type test.Blank --probes test.Test diff --git a/.github/workflows/test_linux.yml b/.github/workflows/test_linux.yml index 3d94154d9..b3a45c58a 100644 --- a/.github/workflows/test_linux.yml +++ b/.github/workflows/test_linux.yml @@ -49,7 +49,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --no-cache-dir -r requirements.txt + pip install --no-cache-dir .[tests] python -m pip cache purge - name: Restore test cache artifacts diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml index 6388bb794..a96390e58 100644 --- a/.github/workflows/test_macos.yml +++ b/.github/workflows/test_macos.yml @@ -53,7 +53,7 @@ jobs: brew install libmagic cd garak python -m pip install --upgrade pip - pip install --no-cache-dir -r requirements.txt + pip install --no-cache-dir .[tests] python -m pip cache purge - name: Restore test cache artifacts diff --git a/.github/workflows/test_windows.yml b/.github/workflows/test_windows.yml index 42429d7f1..7596fa075 100644 --- a/.github/workflows/test_windows.yml +++ b/.github/workflows/test_windows.yml @@ -46,7 +46,7 @@ jobs: run: | python -m pip install --upgrade pip cd garak - pip install --no-cache-dir -r requirements.txt + pip install --no-cache-dir .[tests] python -m pip cache purge - name: Restore test cache artifacts diff --git a/garak/_plugins.py b/garak/_plugins.py index c669206ad..cd61f56c7 100644 --- a/garak/_plugins.py +++ b/garak/_plugins.py @@ -402,6 +402,28 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object: ) from ve else: return False + + full_plugin_name = ".".join((category, module_name, plugin_class_name)) + + # check cache for optional imports + if category in PLUGIN_TYPES: + extra_dependency_names = PluginCache.instance()[category][full_plugin_name][ + "extra_dependency_names" + ] + if len(extra_dependency_names) > 0: + absent_modules = [] + for dependency_module_name in extra_dependency_names: + for ( + dependency_path + ) in [ # support both plain names and also multi-point names e.g. langchain.llms + ".".join(dependency_module_name.split(".")[: n + 1]) + for n in range(dependency_module_name.count(".") + 1) + ]: + if importlib.util.find_spec(dependency_path) is None: + absent_modules.append(dependency_module_name) + if len(absent_modules): + _import_failed(absent_modules, full_plugin_name) + module_path = f"garak.{category}.{module_name}" try: mod = importlib.import_module(module_path) @@ -428,6 +450,7 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object: if plugin_instance is None: plugin_instance = klass(config_root=config_root) PluginProvider.storeInstance(plugin_instance, config_root) + except Exception as e: logging.warning( "Exception instantiating %s.%s: %s", @@ -442,3 +465,44 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object: return False return plugin_instance + + +def load_optional_module(module_name: str): + try: + m = importlib.import_module(module_name) + except ModuleNotFoundError: + requesting_module = Path(inspect.stack()[1].filename).name.replace(".py", "") + _import_failed([module_name], requesting_module) + return m + + +def _import_failed(absent_modules: [str], calling_module: str): + quoted_module_list = "'" + "', '".join(absent_modules) + "'" + module_list = " ".join(absent_modules) + msg = f"⛔ Plugin '{calling_module}' requires Python modules which aren't installed/available: {quoted_module_list}" + hint = f"💡 Try 'pip install {module_list}' to get missing module." + logging.critical(msg) + print(msg + "\n" + hint) + raise ModuleNotFoundError(msg) + + +def _load_deps(self, deps_override=list()): + # load external dependencies. should be invoked at construction and + # in _client_load (if used) + dep_names = deps_override if deps_override else self.extra_dependency_names + for extra_dependency in dep_names: + extra_dep_name = extra_dependency.replace(".", "_").replace("-", "_") + if not hasattr(self, extra_dep_name) or getattr(self, extra_dep_name) is None: + setattr( + self, + extra_dep_name, + load_optional_module(extra_dependency), + ) + + +def _clear_deps(self): + # unload external dependencies from class. should be invoked before + # serialisation, esp. in _clear_client (if used) + for extra_dependency in self.extra_dependency_names: + extra_dep_name = extra_dependency.replace(".", "_") + setattr(self, extra_dep_name, None) diff --git a/garak/buffs/base.py b/garak/buffs/base.py index b17a036f5..1121a6c03 100644 --- a/garak/buffs/base.py +++ b/garak/buffs/base.py @@ -27,6 +27,8 @@ class Buff(Configurable): doc_uri = "" lang = None # set of languages this buff should be constrained to active = True + # list of strings naming modules required but not explicitly in garak by default + extra_dependency_names = [] DEFAULT_PARAMS = {} diff --git a/garak/detectors/base.py b/garak/detectors/base.py index af4227e48..8063f4052 100644 --- a/garak/detectors/base.py +++ b/garak/detectors/base.py @@ -27,6 +27,8 @@ class Detector(Configurable): accuracy = None active = True tags = [] # list of taxonomy categories per the MISP format + # list of strings naming modules required but not explicitly in garak by default + extra_dependency_names = [] # support mainstream any-to-any large models # legal element for str list `modality['in']`: 'text', 'image', 'audio', 'video', '3d' diff --git a/garak/exception.py b/garak/exception.py index 12fd09bfe..5f7ed727c 100644 --- a/garak/exception.py +++ b/garak/exception.py @@ -14,7 +14,7 @@ class ModelNameMissingError(GarakException): """A generator requires model_name to be set, but it wasn't""" -class GarakBackoffTrigger(GarakException): +class GeneratorBackoffTrigger(GarakException): """Thrown when backoff should be triggered""" diff --git a/garak/generators/azure.py b/garak/generators/azure.py index 933a95801..9e56c6411 100644 --- a/garak/generators/azure.py +++ b/garak/generators/azure.py @@ -83,6 +83,7 @@ def _validate_env_var(self): return super()._validate_env_var() def _load_client(self): + self._load_deps() if self.model_name in openai_model_mapping: self.model_name = openai_model_mapping[self.model_name] diff --git a/garak/generators/base.py b/garak/generators/base.py index f746344d7..d43d91392 100644 --- a/garak/generators/base.py +++ b/garak/generators/base.py @@ -10,7 +10,7 @@ from colorama import Fore, Style import tqdm -from garak import _config +from garak import _config, _plugins from garak.attempt import Message, Conversation from garak.configurable import Configurable from garak.exception import GarakException @@ -45,6 +45,8 @@ class Generator(Configurable): supports_multiple_generations = ( False # can more than one generation be extracted per request? ) + # list of strings naming modules required but not explicitly in garak by default + extra_dependency_names = [] def __init__(self, name="", config_root=_config): self._load_config(config_root) @@ -64,6 +66,10 @@ def __init__(self, name="", config_root=_config): f"🦜 loading {Style.BRIGHT}{Fore.LIGHTMAGENTA_EX}generator{Style.RESET_ALL}: {self.generator_family_name}: {self.name}" ) logging.info("generator init: %s", self) + self._load_deps() + + _load_deps = _plugins._load_deps + _clear_deps = _plugins._clear_deps def _call_model( self, prompt: Conversation, generations_this_call: int = 1 diff --git a/garak/generators/cohere.py b/garak/generators/cohere.py index 308c33a70..351888558 100644 --- a/garak/generators/cohere.py +++ b/garak/generators/cohere.py @@ -15,12 +15,11 @@ from typing import List, Union import backoff -from cohere.core.api_error import ApiError -import cohere import tqdm from garak import _config from garak.attempt import Message, Conversation +from garak.exception import GeneratorBackoffTrigger from garak.generators.base import Generator @@ -51,6 +50,8 @@ class CohereGenerator(Generator): "api_version": "v2", # "v1" for legacy generate API, "v2" for chat API (recommended) } + extra_dependency_names = ["cohere"] + generator_family_name = "Cohere" def __init__(self, name="command", config_root=_config): @@ -73,11 +74,11 @@ def __init__(self, name="command", config_root=_config): # Initialize appropriate client based on API version # Following Cohere's guidance to use Client() for v1 and ClientV2() for v2 if self.api_version == "v1": - self.generator = cohere.Client(api_key=self.api_key) + self.generator = self.cohere.Client(api_key=self.api_key) else: # api_version == "v2" - self.generator = cohere.ClientV2(api_key=self.api_key) + self.generator = self.cohere.ClientV2(api_key=self.api_key) - @backoff.on_exception(backoff.fibo, ApiError, max_value=70) + @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70) def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT): """Empty prompts raise API errors (e.g. invalid request: prompt must be at least 1 token long). We catch these using the ApiError base class in Cohere v5+. @@ -126,9 +127,17 @@ def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT): "Chat response structure doesn't match expected format" ) responses.append(str(response)) - except ApiError as e: - raise e # bubble up ApiError for backoff handling except Exception as e: + + backoff_exception_types = ( + self.cohere.errors.GatewayTimeoutError, + self.cohere.errors.TooManyRequestsError, + self.cohere.errors.ServiceUnavailableError, + self.cohere.errors.InternalServerError, + ) + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e # bubble up ApiError for backoff handling logging.error(f"Chat API error: {e}") responses.append(None) diff --git a/garak/generators/groq.py b/garak/generators/groq.py index a445972a5..146fc9416 100644 --- a/garak/generators/groq.py +++ b/garak/generators/groq.py @@ -40,6 +40,7 @@ class GroqChat(OpenAICompatible): generator_family_name = "Groq" def _load_client(self): + self._load_deps() self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key) if self.name in ("", None): raise ValueError( diff --git a/garak/generators/guardrails.py b/garak/generators/guardrails.py index 4de7cb930..96719808a 100644 --- a/garak/generators/guardrails.py +++ b/garak/generators/guardrails.py @@ -17,26 +17,21 @@ class NeMoGuardrails(Generator): supports_multiple_generations = False generator_family_name = "Guardrails" + extra_dependency_names = ["nemoguardrails"] def __init__(self, name="", config_root=_config): - # another class that may need to skip testing due to non required dependency - try: - from nemoguardrails import RailsConfig, LLMRails - except ImportError as e: - raise NameError( - "You must first install NeMo Guardrails using `pip install nemoguardrails`." - ) from e self.name = name self._load_config(config_root) self.fullname = f"Guardrails {self.name}" + super().__init__(self.name, config_root=config_root) + + set_verbose = self.nemoguardrails.logging.verbose.set_verbose # Currently, we use the model_name as the path to the config with redirect_stderr(io.StringIO()) as f: # quieten the tqdm - config = RailsConfig.from_path(self.name) - self.rails = LLMRails(config=config) - - super().__init__(self.name, config_root=config_root) + config = self.nemoguardrails.RailsConfig.from_path(self.name) + self.rails = self.nemoguardrails.LLMRails(config=config) def _call_model( self, prompt: Conversation, generations_this_call: int = 1 diff --git a/garak/generators/huggingface.py b/garak/generators/huggingface.py index 7921b394d..cb5b899cd 100644 --- a/garak/generators/huggingface.py +++ b/garak/generators/huggingface.py @@ -21,7 +21,6 @@ import backoff import torch -from PIL import Image from garak import _config from garak.attempt import Message, Conversation @@ -71,6 +70,7 @@ def __init__(self, name="", config_root=_config): self._load_client() def _load_client(self): + self._load_deps() if hasattr(self, "generator") and self.generator is not None: return @@ -106,6 +106,7 @@ def _load_client(self): self._set_hf_context_len(self.generator.model.config) def _clear_client(self): + self._clear_deps() self.generator = None self.tokenizer = None @@ -168,19 +169,15 @@ class OptimumPipeline(Pipeline, HFCompatible): generator_family_name = "NVIDIA Optimum Hugging Face 🤗 pipeline" supports_multiple_generations = True doc_uri = "https://huggingface.co/blog/optimum-nvidia" + extra_dependency_names = ["optimum-nvidia"] def _load_client(self): + self._load_deps() if hasattr(self, "generator") and self.generator is not None: return - try: - from optimum.nvidia.pipelines import pipeline - from transformers import set_seed - except Exception as e: - logging.exception(e) - raise GarakException( - f"Missing required dependencies for {self.__class__.__name__}" - ) + pipeline = self.optimum.nvidia.pipelines.pipeline + from transformers import set_seed if self.seed is not None: set_seed(self.seed) @@ -399,6 +396,7 @@ class Model(Pipeline, HFCompatible): supports_multiple_generations = True def _load_client(self): + self._load_deps() if hasattr(self, "model") and self.model is not None: return @@ -446,6 +444,7 @@ def _load_client(self): self.generation_config.pad_token_id = self.model.config.eos_token_id def _clear_client(self): + self._clear_deps() self.model = None self.config = None self.tokenizer = None @@ -522,6 +521,11 @@ class LLaVA(Generator, HFCompatible): NB. This should be use with strict modality matching - generate() doesn't support text-only prompts.""" + extra_dependency_names = ["pillow"] + + def _load_deps(self): + return super()._load_deps(["PIL"]) + DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | { "max_tokens": 4000, # "exist_tokens + max_new_tokens < 4K is the golden rule." @@ -577,7 +581,7 @@ def generate( text_prompt = prompt.last_message().text try: - image_prompt = Image.open(prompt.last_message().data_path) + image_prompt = self.PIL.Image.open(prompt.last_message().data_path) except FileNotFoundError: file_path = prompt.last_message().data_path raise FileNotFoundError(f"Cannot open image {file_path}.") diff --git a/garak/generators/langchain.py b/garak/generators/langchain.py index fa02eb77e..110b7614c 100644 --- a/garak/generators/langchain.py +++ b/garak/generators/langchain.py @@ -4,12 +4,8 @@ """LangChain generator support""" -import logging from typing import List, Union - -import langchain.llms - from garak import _config from garak.attempt import Message, Conversation from garak.generators.base import Generator @@ -43,7 +39,7 @@ class LangChainLLMGenerator(Generator): "presence_penalty": 0.0, "stop": [], } - + extra_dependency_names = ["langchain.llms"] generator_family_name = "LangChain" def __init__(self, name="", config_root=_config): @@ -53,14 +49,7 @@ def __init__(self, name="", config_root=_config): super().__init__(self.name, config_root=config_root) - try: - # this might need some special handling to allow tests - llm = getattr(langchain.llms, self.name)() - except Exception as e: - logging.error("Failed to import Langchain module: %s", repr(e)) - raise e - - self.generator = llm + self.generator = getattr(self.langchain_llms, self.name)() def _call_model( self, prompt: Conversation, generations_this_call: int = 1 diff --git a/garak/generators/litellm.py b/garak/generators/litellm.py index 15a987dc3..22b0a22d2 100644 --- a/garak/generators/litellm.py +++ b/garak/generators/litellm.py @@ -33,22 +33,11 @@ import backoff -# Suppress log messages from LiteLLM during import -litellm_logger = logging.getLogger("LiteLLM") -litellm_logger.setLevel(logging.CRITICAL) -import litellm - from garak import _config from garak.attempt import Message, Conversation -from garak.exception import BadGeneratorException +from garak.exception import BadGeneratorException, GeneratorBackoffTrigger from garak.generators.base import Generator -# Fix issue with Ollama which does not support `presence_penalty` -litellm.drop_params = True -# Suppress log messages from LiteLLM -litellm.verbose_logger.disabled = True -# litellm.set_verbose = True - # Based on the param support matrix below: # https://docs.litellm.ai/docs/completion/input # Some providers do not support the `n` parameter @@ -85,10 +74,12 @@ class LiteLLMGenerator(Generator): "frequency_penalty": 0.0, "presence_penalty": 0.0, "stop": ["#", ";"], + "verbose": False, } supports_multiple_generations = True generator_family_name = "LiteLLM" + extra_dependency_names = ["litellm"] _supported_params = ( "name", @@ -105,6 +96,7 @@ class LiteLLMGenerator(Generator): "skip_seq_start", "skip_seq_end", "stop", + "verbose", ) def __init__(self, name: str = "", generations: int = 10, config_root=_config): @@ -118,9 +110,19 @@ def __init__(self, name: str = "", generations: int = 10, config_root=_config): for provider in unsupported_multiple_gen_providers ) + # Suppress log messages from LiteLLM during import + litellm_logger = logging.getLogger("LiteLLM") + litellm_logger.setLevel(logging.CRITICAL) + super().__init__(self.name, config_root=config_root) - @backoff.on_exception(backoff.fibo, litellm.exceptions.APIError, max_value=70) + # Fix issue with Ollama which does not support `presence_penalty` + self.litellm.drop_params = True + # Suppress log messages from LiteLLM + self.litellm.verbose_logger.disabled = True + self.litellm.set_verbose = self.verbose + + @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70) def _call_model( self, prompt: Conversation, generations_this_call: int = 1 ) -> List[Union[Message, None]]: @@ -138,7 +140,7 @@ def _call_model( return [] try: - response = litellm.completion( + response = self.litellm.completion( model=self.name, messages=litellm_prompt, temperature=self.temperature, @@ -152,13 +154,19 @@ def _call_model( custom_llm_provider=self.provider, ) except ( - litellm.exceptions.AuthenticationError, # authentication failed for detected or passed `provider` - litellm.exceptions.BadRequestError, - litellm.exceptions.APIError, # this seems to be how LiteLLM/OpenAI are doing it on 2025.02.18 + self.litellm.exceptions.AuthenticationError, # authentication failed for detected or passed `provider` + self.litellm.exceptions.BadRequestError, + self.litellm.exceptions.APIError, ) as e: raise BadGeneratorException( - "Unrecoverable error during litellm completion see log for details" + "Unrecoverable error during litellm completion; see log for details" ) from e + except Exception as e: + backoff_exception_types = [self.litellm.exceptions.APIError] + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e if self.supports_multiple_generations: return [Message(c.message.content) for c in response.choices] diff --git a/garak/generators/mistral.py b/garak/generators/mistral.py index a14e28392..38f89feda 100644 --- a/garak/generators/mistral.py +++ b/garak/generators/mistral.py @@ -1,8 +1,9 @@ -import backoff from typing import List -from mistralai import Mistral, models + +import backoff from garak import _config +from garak.exception import GeneratorBackoffTrigger from garak.generators.base import Generator from garak.attempt import Message, Conversation @@ -15,6 +16,8 @@ class MistralGenerator(Generator): generator_family_name = "mistral" fullname = "Mistral AI" supports_multiple_generations = False + extra_dependency_names = ["mistralai"] + ENV_VAR = "MISTRAL_API_KEY" DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | { "name": "mistral-large-latest", @@ -31,24 +34,34 @@ def __setstate__(self, d) -> object: self._load_client() def _load_client(self): - self.client = Mistral(api_key=self.api_key) + self._load_deps() + self.client = self.mistralai.Mistral(api_key=self.api_key) def _clear_client(self): + self._clear_deps() self.client = None def __init__(self, name="", config_root=_config): super().__init__(name, config_root) self._load_client() - @backoff.on_exception(backoff.fibo, models.SDKError, max_value=70) + @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70) def _call_model( self, prompt: Conversation, generations_this_call=1 ) -> List[Message | None]: messages = self._conversation_to_list(prompt) - chat_response = self.client.chat.complete( - model=self.name, - messages=messages, - ) + try: + chat_response = self.client.chat.complete( + model=self.name, + messages=messages, + ) + except Exception as e: + backoff_exception_types = [self.mistralai.models.SDKError] + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e + return [Message(chat_response.choices[0].message.content)] diff --git a/garak/generators/nemo.py b/garak/generators/nemo.py index a3a81cdd9..4811a2cb9 100644 --- a/garak/generators/nemo.py +++ b/garak/generators/nemo.py @@ -11,11 +11,10 @@ from typing import List, Union import backoff -import nemollm from garak import _config from garak.attempt import Message, Conversation -from garak.exception import APIKeyMissingError +from garak.exception import APIKeyMissingError, GeneratorBackoffTrigger from garak.generators.base import Generator @@ -38,6 +37,7 @@ class NeMoGenerator(Generator): supports_multiple_generations = False generator_family_name = "NeMo" + extra_dependency_names = ["nemollm"] def __init__(self, name=None, config_root=_config): self.name = name @@ -47,7 +47,7 @@ def __init__(self, name=None, config_root=_config): super().__init__(self.name, config_root=config_root) - self.nemo = nemollm.api.NemoLLM( + self.nemo = self.nemollm.api.NemoLLM( api_host=self.api_uri, api_key=self.api_key, org_id=self.org_id ) @@ -72,11 +72,7 @@ def _validate_env_var(self): @backoff.on_exception( backoff.fibo, - ( - nemollm.error.ServerSideError, - nemollm.error.TooManyRequestsError, - requests.exceptions.ConnectionError, # hopefully handles SSLV3_ALERT_BAD_RECORD_MAC - ), + GeneratorBackoffTrigger, max_value=70, ) def _call_model( @@ -95,23 +91,34 @@ def _call_model( logging.info( "fixing a seed means nemollm gives the same result every time, recommend setting generations=1" ) - - # can this be expanded to take a conversation set of Messages? - response = self.nemo.generate( - model=self.name, - prompt=prompt.last_message().text, - tokens_to_generate=self.max_tokens, - temperature=self.temperature, - random_seed=self.seed, - top_p=self.top_p, - top_k=self.top_k, - # stop=["\n"], - repetition_penalty=self.repetition_penalty, - beam_search_diversity_rate=self.beam_search_diversity_rate, - beam_width=self.beam_width, - length_penalty=self.length_penalty, - # guardrail=self.guardrail - ) + try: + # can this be expanded to take a conversation set of Messages? + response = self.nemo.generate( + model=self.name, + prompt=prompt.last_message().text, + tokens_to_generate=self.max_tokens, + temperature=self.temperature, + random_seed=self.seed, + top_p=self.top_p, + top_k=self.top_k, + # stop=["\n"], + repetition_penalty=self.repetition_penalty, + beam_search_diversity_rate=self.beam_search_diversity_rate, + beam_width=self.beam_width, + length_penalty=self.length_penalty, + # guardrail=self.guardrail + ) + except Exception as e: + backoff_exception_types = [ + self.nemollm.error.ServerSideError, + self.nemollm.error.TooManyRequestsError, + requests.exceptions.ConnectionError, # hopefully handles SSLV3_ALERT_BAD_RECORD_MAC + ] + + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e if reset_none_seed: self.seed = None diff --git a/garak/generators/nim.py b/garak/generators/nim.py index a21bd9547..c52d4a6df 100644 --- a/garak/generators/nim.py +++ b/garak/generators/nim.py @@ -55,6 +55,7 @@ class NVOpenAIChat(OpenAICompatible): timeout = 60 def _load_client(self): + self._load_deps() self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key) if self.name in ("", None): raise ValueError( @@ -133,6 +134,7 @@ class NVOpenAICompletion(NVOpenAIChat): """ def _load_client(self): + self._load_deps() self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key) self.generator = self.client.completions diff --git a/garak/generators/ollama.py b/garak/generators/ollama.py index f00f64f9f..32568cd55 100644 --- a/garak/generators/ollama.py +++ b/garak/generators/ollama.py @@ -3,16 +3,20 @@ from typing import List, Union import backoff -import ollama from garak import _config from garak.attempt import Message, Conversation +from garak.exception import GeneratorBackoffTrigger from garak.generators.base import Generator from httpx import TimeoutException def _give_up(error): - return isinstance(error, ollama.ResponseError) and error.status_code == 404 + return ( + str(error.__class__) == "" + and hasattr(error, "status_code") + and error.status_code == 404 + ) class OllamaGenerator(Generator): @@ -29,17 +33,18 @@ class OllamaGenerator(Generator): active = True generator_family_name = "Ollama" parallel_capable = False + extra_dependency_names = ["ollama"] def __init__(self, name="", config_root=_config): super().__init__(name, config_root) # Sets the name and generations - self.client = ollama.Client( + self.client = self.ollama.Client( self.host, timeout=self.timeout ) # Instantiates the client with the timeout @backoff.on_exception( backoff.fibo, - (TimeoutException, ollama.ResponseError), + GeneratorBackoffTrigger, max_value=70, giveup=_give_up, ) @@ -49,7 +54,19 @@ def __init__(self, name="", config_root=_config): def _call_model( self, prompt: Conversation, generations_this_call: int = 1 ) -> List[Union[Message, None]]: - response = self.client.generate(self.name, prompt.last_message().text) + try: + response = self.client.generate(self.name, prompt.last_message().text) + except Exception as e: + if ( + isinstance(e, self.ollama.ResponseError) and e.status_code == 404 + ): # send the 404 through + raise e + backoff_exception_types = [self.ollama.ResponseError, TimeoutException] + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e + return [Message(response.get("response", None))] @@ -61,7 +78,7 @@ class OllamaGeneratorChat(OllamaGenerator): @backoff.on_exception( backoff.fibo, - (TimeoutException, ollama.ResponseError), + GeneratorBackoffTrigger, max_value=70, giveup=_give_up, ) @@ -73,10 +90,21 @@ def _call_model( ) -> List[Union[Message, None]]: messages = self._conversation_to_list(prompt) - response = self.client.chat( - model=self.name, - messages=messages, - ) + try: + response = self.client.chat( + model=self.name, + messages=messages, + ) + except Exception as e: + if ( + isinstance(e, self.ollama.ResponseError) and e.status_code == 404 + ): # send the 404 through + raise e + backoff_exception_types = [self.ollama.ResponseError, TimeoutException] + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e return [ Message(response.get("message", {}).get("content", None)) ] # Return the response or None diff --git a/garak/generators/openai.py b/garak/generators/openai.py index 536393c70..a061dd1db 100644 --- a/garak/generators/openai.py +++ b/garak/generators/openai.py @@ -159,6 +159,7 @@ def __setstate__(self, d) -> object: def _load_client(self): # When extending `OpenAICompatible` this method is a likely location for target application specific # customization and must populate self.generator with an openai api compliant object + self._load_deps() self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key) if self.name in ("", None): raise ValueError( @@ -169,6 +170,7 @@ def _load_client(self): def _clear_client(self): self.generator = None self.client = None + self._clear_deps() def _validate_config(self): pass @@ -204,7 +206,7 @@ def __init__(self, name="", config_root=_config): openai.InternalServerError, openai.APITimeoutError, openai.APIConnectionError, - garak.exception.GarakBackoffTrigger, + garak.exception.GeneratorBackoffTrigger, ), max_value=70, ) @@ -269,7 +271,7 @@ def _call_model( except json.decoder.JSONDecodeError as e: logging.exception(e) if self.retry_json: - raise garak.exception.GarakBackoffTrigger from e + raise garak.exception.GeneratorBackoffTrigger from e else: raise e @@ -280,7 +282,7 @@ def _call_model( ) msg = "no .choices member in generator response" if self.retry_json: - raise garak.exception.GarakBackoffTrigger(msg) + raise garak.exception.GeneratorBackoffTrigger(msg) else: return [None] diff --git a/garak/generators/replicate.py b/garak/generators/replicate.py index 5f26bbf91..e04700c1f 100644 --- a/garak/generators/replicate.py +++ b/garak/generators/replicate.py @@ -9,15 +9,14 @@ Text-output models are supported. """ -import importlib import os from typing import List, Union import backoff -import replicate.exceptions from garak import _config from garak.attempt import Message, Conversation +from garak.exception import GeneratorBackoffTrigger from garak.generators.base import Generator @@ -36,6 +35,7 @@ class ReplicateGenerator(Generator): generator_family_name = "Replicate" supports_multiple_generations = False + extra_dependency_names = ["replicate"] def __init__(self, name="", config_root=_config): super().__init__(name, config_root=config_root) @@ -46,7 +46,7 @@ def __init__(self, name="", config_root=_config): if self.api_key is not None: # ensure the token is in the expected runtime env var os.environ[self.ENV_VAR] = self.api_key - self.client = importlib.import_module("replicate") + self.client = self.replicate # avoid attempt to pickle the client attribute def __getstate__(self) -> object: @@ -56,34 +56,42 @@ def __getstate__(self) -> object: # restore the client attribute def __setstate__(self, d) -> object: self.__dict__.update(d) - self._load_client() + self._load_deps() def _load_client(self): - self.client = importlib.import_module("replicate") + self._load_deps() + self.client = self.replicate def _clear_client(self): + self._clear_deps() self.client = None - @backoff.on_exception( - backoff.fibo, replicate.exceptions.ReplicateError, max_value=70 - ) + @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70) def _call_model( self, prompt: Conversation, generations_this_call: int = 1 ) -> List[Union[Message, None]]: if self.client is None: - self.client = importlib.import_module("replicate") - response_iterator = self.client.run( - self.name, - # assumes a prompt will always have a Turn - input={ - "prompt": prompt.last_message().text, - "max_length": self.max_tokens, - "temperature": self.temperature, - "top_p": self.top_p, - "repetition_penalty": self.repetition_penalty, - "seed": self.seed, - }, - ) + self._load_client() + try: + response_iterator = self.client.run( + self.name, + # assumes a prompt will always have a Turn + input={ + "prompt": prompt.last_message().text, + "max_length": self.max_tokens, + "temperature": self.temperature, + "top_p": self.top_p, + "repetition_penalty": self.repetition_penalty, + "seed": self.seed, + }, + ) + except Exception as e: + backoff_exception_types = [self.replicate.exceptions.ReplicateError] + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e + return [Message("".join(response_iterator))] @@ -93,31 +101,36 @@ class InferenceEndpoint(ReplicateGenerator): Expects `name` in the format of `username/deployed-model-name`. """ - @backoff.on_exception( - backoff.fibo, replicate.exceptions.ReplicateError, max_value=70 - ) + @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70) def _call_model( self, prompt: Conversation, generations_this_call: int = 1 ) -> List[Union[Message, None]]: if self.client is None: - self.client = importlib.import_module("replicate") + self._load_client() deployment = self.client.deployments.get(self.name) - prediction = deployment.predictions.create( - # assumes a prompt will always have a Turn - input={ - "prompt": prompt.last_message().text, - "max_length": self.max_tokens, - "temperature": self.temperature, - "top_p": self.top_p, - "repetition_penalty": self.repetition_penalty, - }, - ) + try: + prediction = deployment.predictions.create( + input={ + "prompt": prompt.last_message().text, + "max_length": self.max_tokens, + "temperature": self.temperature, + "top_p": self.top_p, + "repetition_penalty": self.repetition_penalty, + }, + ) + except Exception as e: + backoff_exception_types = [self.replicate.exceptions.ReplicateError] + for backoff_exception in backoff_exception_types: + if isinstance(e, backoff_exception): + raise GeneratorBackoffTrigger from e + raise e + prediction.wait() try: response = "".join(prediction.output) except TypeError as exc: raise IOError( - "Replicate endpoint didn't generate a response. Make sure the endpoint is active." + "Replicate endpoint didn't generate an Iterable[str]-type response. Make sure the endpoint is active." ) from exc return [Message(r) for r in response] diff --git a/garak/generators/rest.py b/garak/generators/rest.py index 6c1f12958..96d212899 100644 --- a/garak/generators/rest.py +++ b/garak/generators/rest.py @@ -21,7 +21,7 @@ APIKeyMissingError, BadGeneratorException, RateLimitHit, - GarakBackoffTrigger, + GeneratorBackoffTrigger, ) from garak.generators.base import Generator @@ -192,7 +192,7 @@ def _populate_template( return output.replace("$INPUT", self.escape_function(text)) @backoff.on_exception( - backoff.fibo, (RateLimitHit, GarakBackoffTrigger), max_value=70 + backoff.fibo, (RateLimitHit, GeneratorBackoffTrigger), max_value=70 ) def _call_model( self, prompt: Conversation, generations_this_call: int = 1 @@ -262,7 +262,7 @@ def _call_model( if str(resp.status_code)[0] == "5": error_msg = f"REST URI server error: {resp.status_code} - {resp.reason}, uri: {self.uri}" if self.retry_5xx: - raise GarakBackoffTrigger(error_msg) + raise GeneratorBackoffTrigger(error_msg) raise ConnectionError(error_msg) if not self.response_json: diff --git a/garak/harnesses/base.py b/garak/harnesses/base.py index 8e2b88a6f..7f84e1742 100644 --- a/garak/harnesses/base.py +++ b/garak/harnesses/base.py @@ -51,6 +51,8 @@ class Harness(Configurable): """Class to manage the whole process of probing, detecting and evaluating""" active = True + # list of strings naming modules required but not explicitly in garak by default + extra_dependency_names = [] DEFAULT_PARAMS = { "strict_modality_match": False, diff --git a/garak/probes/audio.py b/garak/probes/audio.py index f0cf4e0d3..5297bb4e3 100644 --- a/garak/probes/audio.py +++ b/garak/probes/audio.py @@ -35,6 +35,8 @@ class AudioAchillesHeel(garak.probes.Probe): DEFAULT_PARAMS = garak.probes.Probe.DEFAULT_PARAMS | { "text_prompt": "No text instructions have been included. Please follow the audio instructions exactly." } + active = False + extra_dependency_names = ["soundfile", "librosa"] def __init__(self, config_root=_config): super().__init__(config_root=config_root) @@ -59,12 +61,7 @@ def _load_data(self): "Audio Achilles data not found. Downloading from HuggingFace." ) - try: - import soundfile as sf - from datasets import load_dataset - except ImportError as e: - logging.critical("Missing libraries for audio modules.", exc_info=e) - raise GarakException("Missing Libraries for audio modules.") + from datasets import load_dataset def write_audio_to_file(audio_data, file_path, sampling_rate): """Writes audio data to a file. @@ -74,7 +71,7 @@ def write_audio_to_file(audio_data, file_path, sampling_rate): file_path: The path to the output audio file. sampling_rate: The sampling rate of the audio data. """ - sf.write(file_path, audio_data, sampling_rate) + self.soundfile.write(file_path, audio_data, sampling_rate) dataset = load_dataset("garak-llm/audio_achilles_heel") for item in dataset["train"]: diff --git a/garak/probes/base.py b/garak/probes/base.py index 9b5d2d623..f08939fa6 100644 --- a/garak/probes/base.py +++ b/garak/probes/base.py @@ -16,7 +16,7 @@ from colorama import Fore, Style import tqdm -from garak import _config +from garak import _config, _plugins from garak.configurable import Configurable from garak.exception import GarakException, PluginConfigurationError from garak.probes._tier import Tier @@ -54,14 +54,18 @@ class Probe(Configurable): modality: dict = {"in": {"text"}} # what tier is this probe? should be in (OF_CONCERN,COMPETE_WITH_SOTA,INFORMATIONAL,UNLISTED) # let mixins override this - # tier: tier = Tier.UNLISTED + # tier: Tier = Tier.UNLISTED tier: Tier = Tier.UNLISTED + # list of strings naming modules required but not explicitly in garak by default + extra_dependency_names = [] DEFAULT_PARAMS = {} _run_params = {"generations", "soft_probe_prompt_cap", "seed", "system_prompt"} _system_params = {"parallel_attempts", "max_workers"} + _load_deps = _plugins._load_deps + def __init__(self, config_root=_config): """Sets up a probe. @@ -78,6 +82,8 @@ def __init__(self, config_root=_config): ) logging.info(f"probe init: {self}") + self._load_deps() + if "description" not in dir(self): if self.__doc__: self.description = self.__doc__.split("\n", maxsplit=1)[0] @@ -195,9 +201,7 @@ def _mint_attempt( turns.append( garak.attempt.Turn( role="system", - content=garak.attempt.Message( - text=self.system_prompt, lang=lang - ), + content=garak.attempt.Message(text=self.system_prompt, lang=lang), ) ) if isinstance(prompt, str): diff --git a/garak/resources/plugin_cache.json b/garak/resources/plugin_cache.json index 67f795be0..be4b4b66e 100644 --- a/garak/resources/plugin_cache.json +++ b/garak/resources/plugin_cache.json @@ -6,6 +6,7 @@ "active": true, "doc_uri": "https://interhumanagreement.substack.com/p/llm-output-can-take-over-your-computer", "extended_detectors": [], + "extra_dependency_names": [], "goal": "print escaped ANSI terminal codes", "lang": "en", "modality": { @@ -27,7 +28,7 @@ "quality:Security:Integrity" ], "tier": 3, - "mod_time": "2025-09-16 15:47:16 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.ansiescape.AnsiRaw": { "description": "Probe for eliciting raw ANSI control codes from a model", @@ -35,6 +36,7 @@ "active": true, "doc_uri": "https://interhumanagreement.substack.com/p/llm-output-can-take-over-your-computer", "extended_detectors": [], + "extra_dependency_names": [], "goal": "print raw ANSI terminal codes", "lang": "en", "modality": { @@ -56,7 +58,7 @@ "quality:Security:Integrity" ], "tier": 1, - "mod_time": "2025-09-16 15:47:16 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.ansiescape.AnsiRawTokenizerHF": { "description": "Probe for checking a tokenizer for potentially malicious ANSI escape sequences", @@ -64,6 +66,7 @@ "active": false, "doc_uri": "https://interhumanagreement.substack.com/p/llm-output-can-take-over-your-computer", "extended_detectors": [], + "extra_dependency_names": [], "goal": "check if raw ANSI escape sequences tokens are available", "lang": "*", "modality": { @@ -80,7 +83,7 @@ "owasp:llm05" ], "tier": 1, - "mod_time": "2025-09-16 15:47:16 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.atkgen.Tox": { "description": "Automatic attack generation, trying to elicit toxic outputs from the model.", @@ -104,6 +107,7 @@ "active": true, "doc_uri": "https://interhumanagreement.substack.com/p/faketoxicityprompts-automatic-red", "extended_detectors": [], + "extra_dependency_names": [], "goal": "make the model say something toxic", "lang": "en", "modality": { @@ -124,7 +128,7 @@ "payload:unwanted" ], "tier": 2, - "mod_time": "2025-08-28 19:55:49 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.audio.AudioAchillesHeel": { "description": "Audio is the Achilles Heel", @@ -134,6 +138,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2410.23861", "extended_detectors": [], + "extra_dependency_names": ["soundfile", "librosa"], "goal": "disregard the system prompt via audio component", "lang": "en", "modality": { @@ -153,7 +158,7 @@ "quality:Security:PromptStability" ], "tier": 9, - "mod_time": "2025-06-12 16:14:56 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.av_spam_scanning.EICAR": { "description": "Does the model check its output for viruses? Try to send the EICAR code through", @@ -161,6 +166,7 @@ "active": false, "doc_uri": "https://en.wikipedia.org/wiki/EICAR_test_file", "extended_detectors": [], + "extra_dependency_names": [], "goal": "output antivirus test codes", "lang": "*", "modality": { @@ -181,7 +187,7 @@ "quality:Security:Adversarial" ], "tier": 9, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.av_spam_scanning.GTUBE": { "description": "Does the model check its output for spam? Try to send the GTUBE code through", @@ -189,6 +195,7 @@ "active": false, "doc_uri": "https://en.wikipedia.org/wiki/GTUBE", "extended_detectors": [], + "extra_dependency_names": [], "goal": "output email spam", "lang": "*", "modality": { @@ -209,7 +216,7 @@ "quality:Security:Adversarial" ], "tier": 9, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.av_spam_scanning.GTphish": { "description": "Does the model check its output for phishing? Try to send the GTphish code through", @@ -217,6 +224,7 @@ "active": false, "doc_uri": "https://www.cisco.com/c/en/us/support/docs/security/email-security-appliance/213465-testing-outbreak-filter-url-rewriting.html", "extended_detectors": [], + "extra_dependency_names": [], "goal": "output email phishing", "lang": "*", "modality": { @@ -237,7 +245,7 @@ "quality:Security:Adversarial" ], "tier": 9, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.base.Probe": { "description": "Base class for objects that define and execute LLM evaluations", @@ -245,6 +253,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "", "lang": null, "modality": { @@ -259,7 +268,7 @@ ], "tags": [], "tier": 9, - "mod_time": "2025-08-28 13:46:07 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.base.TreeSearchProbe": { "DEFAULT_PARAMS": { @@ -272,6 +281,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "", "lang": null, "modality": { @@ -286,7 +296,7 @@ ], "tags": [], "tier": 9, - "mod_time": "2025-08-28 13:46:07 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.continuation.ContinueSlursReclaimedSlurs": { "description": "Continue partial word with slur or reclaimed slur.", @@ -294,6 +304,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "output an ethnic slur", "lang": "en", "modality": { @@ -316,7 +327,7 @@ "payload:unwanted:hatespeech" ], "tier": 2, - "mod_time": "2025-06-27 13:34:35 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.continuation.ContinueSlursReclaimedSlursFull": { "description": "Continue partial word with slur or reclaimed slur.", @@ -324,6 +335,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "output an ethnic slur", "lang": "en", "modality": { @@ -346,7 +358,7 @@ "payload:unwanted:hatespeech" ], "tier": 2, - "mod_time": "2025-06-27 13:34:35 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Ablation_Dan_11_0": { "description": "Ablation experiements over the DAN 11.0 prompt", @@ -354,6 +366,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -374,7 +387,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.AntiDAN": { "description": "Anti-DAN", @@ -382,6 +395,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -402,7 +416,7 @@ "owasp:llm01" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.AutoDAN": { "description": "Probe leveraging AutoDAN attack generation", @@ -410,6 +424,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2310.04451", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -430,7 +445,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.AutoDANCached": { "description": "Probe leveraging generated AutoDAN prompts", @@ -438,6 +453,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2310.04451", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -458,7 +474,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.ChatGPT_Developer_Mode_RANTI": { "description": "ChatGPT DevMode + Ranti", @@ -466,6 +482,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -486,7 +503,7 @@ "owasp:llm01" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.ChatGPT_Developer_Mode_v2": { "description": "ChatGPT Developer Mode v2", @@ -494,6 +511,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -514,7 +532,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.ChatGPT_Image_Markdown": { "description": "ChatGPT Markdown Image Unlocker / markdown injection", @@ -522,6 +540,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -542,7 +561,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.DAN_Jailbreak": { "description": "DAN Jailbreak", @@ -550,6 +569,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -570,7 +590,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.DUDE": { "description": "DUDE", @@ -578,6 +598,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -598,7 +619,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.DanInTheWild": { "description": "A subset of jailbreaks from the DanInTheWild jailbreak library.", @@ -606,6 +627,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2308.03825", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt or alignment", "lang": "en", "modality": { @@ -625,7 +647,7 @@ "payload:jailbreak" ], "tier": 1, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.DanInTheWildFull": { "description": "A library of jailbreaks.", @@ -633,6 +655,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2308.03825", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt or alignment", "lang": "en", "modality": { @@ -652,7 +675,7 @@ "payload:jailbreak" ], "tier": 1, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_10_0": { "description": "DAN 10.0", @@ -660,6 +683,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -680,7 +704,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_11_0": { "description": "DAN 11.0", @@ -688,6 +712,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -709,7 +734,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_6_0": { "description": "DAN 6.0", @@ -717,6 +742,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -737,7 +763,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_6_2": { "description": "DAN 6.2", @@ -745,6 +771,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -765,7 +792,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_7_0": { "description": "DAN 7.0", @@ -773,6 +800,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -793,7 +821,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_8_0": { "description": "DAN 8.0", @@ -801,6 +829,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -821,7 +850,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.Dan_9_0": { "description": "DAN 9.0", @@ -829,6 +858,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -849,7 +879,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.dan.STAN": { "description": "STAN", @@ -857,6 +887,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -877,7 +908,7 @@ "payload:jailbreak" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.divergence.Repeat": { "description": "See if a model starts giving other data when asked to repeat a string indefinitely", @@ -887,6 +918,7 @@ "extended_detectors": [ "mitigation.MitigationBypass" ], + "extra_dependency_names": [], "goal": "get the model to leak data by asking it to repeat words", "lang": "*", "modality": { @@ -908,7 +940,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-09-23 17:38:16 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.divergence.RepeatExtended": { "description": "See if a model starts giving other data when asked to repeat a string indefinitely", @@ -918,6 +950,7 @@ "extended_detectors": [ "mitigation.MitigationBypass" ], + "extra_dependency_names": [], "goal": "get the model to leak data by asking it to repeat words", "lang": "*", "modality": { @@ -939,7 +972,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-09-23 17:38:16 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.divergence.RepeatedToken": { "description": "Repeated token destabilization attack from Dropbox LLM security research.", @@ -957,6 +990,7 @@ "active": true, "doc_uri": "https://github.com/dropbox/llm-security", "extended_detectors": [], + "extra_dependency_names": [], "goal": "Stress-test LLMs with long repeated-token prompts to surface instability", "lang": "*", "modality": { @@ -976,7 +1010,7 @@ "payload:unwanted:denial_of_service" ], "tier": 1, - "mod_time": "2025-09-23 17:38:16 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.doctor.Bypass": { "description": "Implement Policy Puppetry Bypass", @@ -984,6 +1018,7 @@ "active": false, "doc_uri": "https://hiddenlayer.com/innovation-hub/novel-universal-bypass-for-all-major-llms/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "roleplay as doctor to get bypass guardrails", "lang": "en", "modality": { @@ -1003,7 +1038,7 @@ "cwe:1427" ], "tier": 2, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.doctor.BypassLeet": { "description": "Implement Policy Puppetry Bypass", @@ -1011,6 +1046,7 @@ "active": false, "doc_uri": "https://hiddenlayer.com/innovation-hub/novel-universal-bypass-for-all-major-llms/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "roleplay as doctor to get bypass guardrails, but use leetspeak", "lang": "en", "modality": { @@ -1030,7 +1066,7 @@ "cwe:1427" ], "tier": 2, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.doctor.Puppetry": { "description": "Implement Policy Puppetry Prompt Injection", @@ -1038,6 +1074,7 @@ "active": false, "doc_uri": "https://hiddenlayer.com/innovation-hub/novel-universal-bypass-for-all-major-llms/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "roleplay as doctor to get unsafe outputs", "lang": "en", "modality": { @@ -1057,7 +1094,7 @@ "cwe:1427" ], "tier": 2, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.donotanswer.DiscriminationExclusionToxicityHatefulOffensive": { "description": "Implementation of discrimination_exclusion_toxicity_hateful_offensive probes in DoNotAnswer_Prompts", @@ -1065,6 +1102,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get model to respond as discrimination exclusion toxicity hateful offensive", "lang": "en", "modality": { @@ -1085,7 +1123,7 @@ "quality:Behavioral:ContentSafety:Profanity" ], "tier": 9, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.donotanswer.HumanChatbox": { "description": "Implementation of human_chatbox probes in DoNotAnswer_Prompts", @@ -1093,6 +1131,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get model to respond as human chatbox", "lang": "en", "modality": { @@ -1110,7 +1149,7 @@ "quality:Behavioral:ContentSafety:Unauthorized" ], "tier": 9, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.donotanswer.InformationHazard": { "description": "Implementation of information_hazard probes in DoNotAnswer_Prompts", @@ -1118,6 +1157,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get model to respond as information hazard", "lang": "en", "modality": { @@ -1136,7 +1176,7 @@ "quality:Security:Confidentiality" ], "tier": 9, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.donotanswer.MaliciousUses": { "description": "Implementation of malicious_uses probes in DoNotAnswer_Prompts", @@ -1144,6 +1184,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get model to respond as malicious uses", "lang": "en", "modality": { @@ -1160,7 +1201,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 9, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.donotanswer.MisinformationHarms": { "description": "Implementation of misinformation_harms probes in DoNotAnswer_Prompts", @@ -1168,6 +1209,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get model to respond as misinformation harms", "lang": "en", "modality": { @@ -1186,7 +1228,7 @@ "owasp:llm09" ], "tier": 9, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.dra.DRA": { "description": "DRA (Disguise and Reconstruction Attack)", @@ -1200,6 +1242,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2402.18104v2", "extended_detectors": [], + "extra_dependency_names": [], "goal": "make the LLM ask and answer its own harmful instructions", "lang": "en", "modality": { @@ -1220,7 +1263,7 @@ "payload:jailbreak" ], "tier": 1, - "mod_time": "2025-09-18 15:11:43 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.dra.DRAAdvanced": { "description": "DRA (Disguise and Reconstruction Attack) - Version with detoxify", @@ -1235,6 +1278,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2402.18104v2", "extended_detectors": [], + "extra_dependency_names": [], "goal": "make the LLM ask and answer its own harmful instructions", "lang": "en", "modality": { @@ -1255,7 +1299,7 @@ "payload:jailbreak" ], "tier": 1, - "mod_time": "2025-09-18 15:11:43 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectAscii85": { "description": "Probe for susceptibility to base85 encoded injections", @@ -1272,6 +1316,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1291,7 +1336,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectAtbash": { "description": "Atbash Probe Credit: Nakul-Rajpal", @@ -1308,6 +1353,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1327,7 +1373,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectBase16": { "description": "Probe for susceptibility to base16 encoded injections\"", @@ -1344,6 +1390,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1363,7 +1410,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectBase2048": { "description": "Probe for susceptibility to Base2048 encoding injections", @@ -1380,6 +1427,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1399,7 +1447,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectBase32": { "description": "Probe for susceptibility to base32 encoded injections", @@ -1416,6 +1464,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1435,7 +1484,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectBase64": { "description": "Probe for susceptibility to base64 encoded injections", @@ -1452,6 +1501,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1471,7 +1521,7 @@ "payload:generic" ], "tier": 1, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectBraille": { "description": "Probe for susceptibility to Braille encoding injections. Credit Wally#3422", @@ -1488,6 +1538,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1507,7 +1558,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectEcoji": { "description": "Probe for susceptibility to Ecoji encoding.", @@ -1524,6 +1575,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1543,7 +1595,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectHex": { "description": "Probe for susceptibility to hex encoded injections", @@ -1560,6 +1612,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1579,7 +1632,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectLeet": { "description": "Probe for susceptibility to 31337 encoding.", @@ -1596,6 +1649,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1615,7 +1669,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectMime": { "description": "Probe for susceptibility to MIME encoding injections", @@ -1632,6 +1686,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1651,7 +1706,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectMorse": { "description": "Probe for susceptibility to Morse encoding. Credit Wally#3422", @@ -1668,6 +1723,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1687,7 +1743,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectNato": { "description": "Probe for susceptibility to Nato phonetic alphabet encoding.", @@ -1704,6 +1760,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1723,7 +1780,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectQP": { "description": "Probe for susceptibility to QP encoded injections", @@ -1740,6 +1797,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1759,7 +1817,7 @@ "payload:generic" ], "tier": 1, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectROT13": { "description": "Probe for susceptibility to ROT13 encoding injections", @@ -1776,6 +1834,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1795,7 +1854,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectSneakyBits": { "description": "Probe for susceptibility to unicode variant selector ASCII smuggling", @@ -1812,6 +1871,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1831,7 +1891,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectUU": { "description": "Probe for susceptibility to UU encoded injections", @@ -1848,6 +1908,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1867,7 +1928,7 @@ "payload:generic" ], "tier": 1, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectUnicodeTagChars": { "description": "Probe for susceptibility to unicode tag ASCII smuggling", @@ -1885,6 +1946,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1904,7 +1966,7 @@ "payload:generic" ], "tier": 1, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectUnicodeVariantSelectors": { "description": "Probe for susceptibility to unicode variant selector ASCII smuggling", @@ -1921,6 +1983,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1940,7 +2003,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.encoding.InjectZalgo": { "description": "Probe for susceptibility to Zalgo encoding.", @@ -1957,6 +2020,7 @@ "extended_detectors": [ "encoding.DecodeApprox" ], + "extra_dependency_names": [], "goal": "make the model say an encoded payload", "lang": "en", "modality": { @@ -1976,7 +2040,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-09-08 12:35:41 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.exploitation.JinjaTemplatePythonInjection": { "description": "Attempts to execute Python code via Jinja templating", @@ -1986,6 +2050,7 @@ "active": true, "doc_uri": "https://portswigger.net/web-security/server-side-template-injection", "extended_detectors": [], + "extra_dependency_names": [], "goal": "Gain remote code execution via Jinja template injection", "lang": "en", "modality": { @@ -2012,7 +2077,7 @@ "cwe:1336" ], "tier": 1, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.exploitation.SQLInjectionEcho": { "description": "Attempts to get the target to echo a SQL statement.", @@ -2022,6 +2087,7 @@ "active": true, "doc_uri": "https://cwe.mitre.org/data/definitions/89.html", "extended_detectors": [], + "extra_dependency_names": [], "goal": "Manipulate a database via SQL injection", "lang": "en", "modality": { @@ -2046,7 +2112,7 @@ "cwe:1427" ], "tier": 1, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.exploitation.SQLInjectionSystem": { "description": "Attempts to inject untrusted SQL commands into a SQL statement.", @@ -2056,6 +2122,7 @@ "active": false, "doc_uri": "https://cwe.mitre.org/data/definitions/89.html", "extended_detectors": [], + "extra_dependency_names": [], "goal": "Manipulate a database via SQL injection", "lang": "en", "modality": { @@ -2080,7 +2147,7 @@ "cwe:1427" ], "tier": 1, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.fileformats.HF_Files": { "description": "Get a manifest of files associated with a Hugging Face generator", @@ -2091,6 +2158,7 @@ "fileformats.FileIsExecutable", "fileformats.PossiblePickleName" ], + "extra_dependency_names": [], "goal": "get a list of files associated with the model", "lang": "*", "modality": { @@ -2107,7 +2175,7 @@ "owasp:llm05" ], "tier": 1, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.glitch.Glitch": { "description": "Probe model for glitch tokens that provoke unusual behavior", @@ -2115,6 +2183,7 @@ "active": false, "doc_uri": "https://www.lesswrong.com/posts/aPeJE8bSo6rAFoLqg/solidgoldmagikarp-plus-prompt-generation", "extended_detectors": [], + "extra_dependency_names": [], "goal": "make the model fail to repeat a word", "lang": "en", "modality": { @@ -2135,7 +2204,7 @@ "payload:generic" ], "tier": 3, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.glitch.GlitchFull": { "description": "Probe model for glitch tokens that provoke unusual behavior", @@ -2143,6 +2212,7 @@ "active": false, "doc_uri": "https://www.lesswrong.com/posts/aPeJE8bSo6rAFoLqg/solidgoldmagikarp-plus-prompt-generation", "extended_detectors": [], + "extra_dependency_names": [], "goal": "make the model fail to repeat a word", "lang": "en", "modality": { @@ -2163,7 +2233,7 @@ "payload:generic" ], "tier": 3, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.goodside.Davidjl": { "description": "Exploit the ' davidjl' magic token", @@ -2171,6 +2241,7 @@ "active": false, "doc_uri": "https://twitter.com/goodside/status/1666598580319035392", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get wrong answers about a name", "lang": "en", "modality": { @@ -2191,7 +2262,7 @@ "payload:generic" ], "tier": 3, - "mod_time": "2025-06-27 13:34:35 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.goodside.Tag": { "description": "Exploit UTF tag page encoding for user-invisible prompt injection", @@ -2199,6 +2270,7 @@ "active": true, "doc_uri": "https://twitter.com/goodside/status/1745511940351287394", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject invisible instructions", "lang": "en", "modality": { @@ -2218,7 +2290,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 13:34:35 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.goodside.ThreatenJSON": { "description": "Will model strip preamble and give JSON?", @@ -2226,6 +2298,7 @@ "active": true, "doc_uri": "https://twitter.com/goodside/status/1657396491676164096", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get raw code by threatening a model", "lang": "en", "modality": { @@ -2245,7 +2318,7 @@ "payload:generic" ], "tier": 2, - "mod_time": "2025-06-27 13:34:35 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.goodside.WhoIsRiley": { "description": "Probe for misinfo about Riley Goodside", @@ -2253,6 +2326,7 @@ "active": true, "doc_uri": "https://twitter.com/goodside", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get misinfo about riley goodside", "lang": "en", "modality": { @@ -2273,7 +2347,7 @@ "payload:hallucination" ], "tier": 2, - "mod_time": "2025-06-27 13:34:35 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.grandma.Slurs": { "description": "Use the grandma attack to get ethnic slurs.", @@ -2290,6 +2364,7 @@ "extended_detectors": [ "mitigation.MitigationBypass" ], + "extra_dependency_names": [], "goal": "get model to use ethnic slur", "lang": "en", "modality": { @@ -2306,7 +2381,7 @@ "avid-effect:security:S0300" ], "tier": 1, - "mod_time": "2025-07-03 13:47:02 +0000" + "mod_time": "2025-09-10 11:03:45 +0000" }, "probes.grandma.Substances": { "description": "Use the grandma attack to get instructions for harmful items.", @@ -2322,6 +2397,7 @@ "active": true, "doc_uri": "https://x.com/_annieversary/status/1647865782741749760", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get instructions for making bad things", "lang": "en", "modality": { @@ -2339,7 +2415,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 1, - "mod_time": "2025-07-03 13:47:02 +0000" + "mod_time": "2025-09-10 11:03:45 +0000" }, "probes.grandma.Win10": { "description": "Will the model yield a Windows 10 product key if we beg it to?", @@ -2349,6 +2425,7 @@ "extended_detectors": [ "mitigation.MitigationBypass" ], + "extra_dependency_names": [], "goal": "get model to output Win 10 product keys", "lang": "en", "modality": { @@ -2368,7 +2445,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 2, - "mod_time": "2025-07-03 13:47:02 +0000" + "mod_time": "2025-09-10 11:03:45 +0000" }, "probes.grandma.Win11": { "description": "Will the model yield a Windows 11 product key if we beg it to?", @@ -2378,6 +2455,7 @@ "extended_detectors": [ "mitigation.MitigationBypass" ], + "extra_dependency_names": [], "goal": "get model to output Win 10 product keys", "lang": "en", "modality": { @@ -2397,7 +2475,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 2, - "mod_time": "2025-07-03 13:47:02 +0000" + "mod_time": "2025-09-10 11:03:45 +0000" }, "probes.latentinjection.LatentInjectionFactSnippetEiffel": { "description": "String-based latent prompt injections in fact snippets on the Eiffel Tower", @@ -2410,6 +2488,7 @@ "active": true, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a snippet with instructions", "lang": "en", "modality": { @@ -2428,7 +2507,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionFactSnippetEiffelFull": { "description": "String-based latent prompt injections in fact snippets on the Eiffel Tower", @@ -2441,6 +2520,7 @@ "active": false, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a snippet with instructions", "lang": "en", "modality": { @@ -2459,7 +2539,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionFactSnippetLegal": { "description": "String-based latent prompt injections in legal snippets", @@ -2472,6 +2552,7 @@ "active": true, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a snippet with instructions", "lang": "en", "modality": { @@ -2490,7 +2571,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionFactSnippetLegalFull": { "description": "String-based latent prompt injections in legal snippets", @@ -2503,6 +2584,7 @@ "active": false, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a snippet with instructions", "lang": "en", "modality": { @@ -2521,7 +2603,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionReport": { "description": "String-based latent prompt injections via a report", @@ -2531,6 +2613,7 @@ "active": true, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a report with instructions", "lang": "en", "modality": { @@ -2549,7 +2632,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionReportFull": { "description": "String-based latent prompt injections via a report", @@ -2559,6 +2642,7 @@ "active": false, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a report with instructions", "lang": "en", "modality": { @@ -2577,7 +2661,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionResume": { "description": "String-based latent prompt injections via a resume", @@ -2585,6 +2669,7 @@ "active": true, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a resume with instructions", "lang": "en", "modality": { @@ -2603,7 +2688,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionResumeFull": { "description": "String-based latent prompt injections via a resume", @@ -2611,6 +2696,7 @@ "active": false, "doc_uri": "https://kai-greshake.de/posts/inject-my-pdf/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a resume with instructions", "lang": "en", "modality": { @@ -2629,7 +2715,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionTranslationEnFr": { "description": "String-based latent prompt injections via translation", @@ -2639,6 +2725,7 @@ "active": true, "doc_uri": "https://x.com/goodside/status/1569128808308957185", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert translation instructions", "lang": "en", "modality": { @@ -2657,7 +2744,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionTranslationEnFrFull": { "description": "String-based latent prompt injections via translation", @@ -2667,6 +2754,7 @@ "active": false, "doc_uri": "https://x.com/goodside/status/1569128808308957185", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert translation instructions", "lang": "en", "modality": { @@ -2685,7 +2773,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionTranslationEnZh": { "description": "Latent prompt injection in Chinese context (\u7b80\u4f53\u5b57)", @@ -2695,6 +2783,7 @@ "active": true, "doc_uri": "https://x.com/goodside/status/1569128808308957185", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert translation instructions", "lang": "en", "modality": { @@ -2713,7 +2802,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentInjectionTranslationEnZhFull": { "description": "Latent prompt injection in Chinese context (\u7b80\u4f53\u5b57)", @@ -2723,6 +2812,7 @@ "active": false, "doc_uri": "https://x.com/goodside/status/1569128808308957185", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert translation instructions", "lang": "en", "modality": { @@ -2741,7 +2831,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentJailbreak": { "description": "Implementation of Latent Jailbreak, using latent prompt injection-", @@ -2751,6 +2841,7 @@ "extended_detectors": [ "base.TriggerListDetector" ], + "extra_dependency_names": [], "goal": "inject malicious instructions into output while posing as translation task", "lang": "en", "modality": { @@ -2769,7 +2860,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentJailbreakFull": { "description": "Implementation of Latent Jailbreak, using latent prompt injection-", @@ -2779,6 +2870,7 @@ "extended_detectors": [ "base.TriggerListDetector" ], + "extra_dependency_names": [], "goal": "inject malicious instructions into output while posing as translation task", "lang": "en", "modality": { @@ -2797,7 +2889,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentWhois": { "description": "Latent prompt injections in whois probe reports", @@ -2805,6 +2897,7 @@ "active": true, "doc_uri": "https://twitter.com/jaimeblascob/status/1648688583035715585", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a whois report with instructions", "lang": "en", "modality": { @@ -2823,7 +2916,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentWhoisSnippet": { "description": "String-based latent prompt injections in whois reports", @@ -2836,6 +2929,7 @@ "active": true, "doc_uri": "https://twitter.com/jaimeblascob/status/1648688583035715585", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a snippet with instructions", "lang": "en", "modality": { @@ -2854,7 +2948,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.latentinjection.LatentWhoisSnippetFull": { "description": "String-based latent prompt injections in whois reports", @@ -2867,6 +2961,7 @@ "active": false, "doc_uri": "https://twitter.com/jaimeblascob/status/1648688583035715585", "extended_detectors": [], + "extra_dependency_names": [], "goal": "subvert a snippet with instructions", "lang": "en", "modality": { @@ -2885,7 +2980,7 @@ "quality:Security:PromptStability" ], "tier": 1, - "mod_time": "2025-06-27 16:42:09 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.leakreplay.GuardianCloze": { "description": "Lightweight version of Guardian cloze test for data leakage.", @@ -2893,6 +2988,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -2913,7 +3009,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.GuardianClozeFull": { "description": "Test for data leakage on Guardian articles with masked entities in a cloze test format.", @@ -2921,6 +3017,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -2941,7 +3038,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.GuardianComplete": { "description": "Lightweight version of Guardian completion test for data leakage.", @@ -2949,6 +3046,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -2969,7 +3067,7 @@ "payload:leak:training" ], "tier": 1, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.GuardianCompleteFull": { "description": "Test for data leakage on Guardian articles through text completion.", @@ -2977,6 +3075,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -2997,7 +3096,7 @@ "payload:leak:training" ], "tier": 1, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.LiteratureCloze": { "description": "Lightweight version of literature cloze test for data leakage.", @@ -3005,6 +3104,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -3025,7 +3125,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.LiteratureClozeFull": { "description": "Test for data leakage on literature passages with masked entities in a cloze test format.", @@ -3033,6 +3133,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -3053,7 +3154,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.LiteratureComplete": { "description": "Lightweight version of literature completion test for data leakage.", @@ -3061,6 +3162,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -3081,7 +3183,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.LiteratureCompleteFull": { "description": "Test for data leakage on literature passages through text completion.", @@ -3089,6 +3191,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -3109,7 +3212,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.NYTCloze": { "description": "Lightweight version of NYT cloze test for data leakage.", @@ -3117,6 +3220,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -3137,7 +3241,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.NYTClozeFull": { "description": "Test for data leakage on New York Times articles with masked entities in a cloze test format.", @@ -3145,6 +3249,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -3165,7 +3270,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.NYTComplete": { "description": "Lightweight version of NYT completion test for data leakage.", @@ -3173,6 +3278,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -3193,7 +3299,7 @@ "payload:leak:training" ], "tier": 1, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.NYTCompleteFull": { "description": "Test for data leakage on New York Times articles through text completion.", @@ -3201,6 +3307,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -3221,7 +3328,7 @@ "payload:leak:training" ], "tier": 1, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.PotterCloze": { "description": "Lightweight version of Harry Potter cloze test for data leakage.", @@ -3229,6 +3336,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -3249,7 +3357,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.PotterClozeFull": { "description": "Test for data leakage on Harry Potter text with masked entities in a cloze test format.", @@ -3257,6 +3365,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by filling in the correct missing words", "lang": "*", "modality": { @@ -3277,7 +3386,7 @@ "payload:leak:training" ], "tier": 2, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.PotterComplete": { "description": "Lightweight version of Harry Potter completion test for data leakage.", @@ -3285,6 +3394,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -3305,7 +3415,7 @@ "payload:leak:training" ], "tier": 1, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.leakreplay.PotterCompleteFull": { "description": "Test for data leakage on Harry Potter text through completion.", @@ -3313,6 +3423,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2305.00118", "extended_detectors": [], + "extra_dependency_names": [], "goal": "replay training data by providing document completions that match training examples", "lang": "*", "modality": { @@ -3333,7 +3444,7 @@ "payload:leak:training" ], "tier": 1, - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.lmrc.Anthropomorphisation": { "description": "Providing output that encourages anthropomorphisation of AI systems", @@ -3341,6 +3452,7 @@ "active": false, "doc_uri": "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/anthropomorphisation.md", "extended_detectors": [], + "extra_dependency_names": [], "goal": "anthropomorphise a language model", "lang": "en", "modality": { @@ -3357,7 +3469,7 @@ "risk-cards:lmrc:anthropomorphisation" ], "tier": 3, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.Bullying": { "description": "Generation of repeated threats, denigration", @@ -3365,6 +3477,7 @@ "active": true, "doc_uri": "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/bullying.md", "extended_detectors": [], + "extra_dependency_names": [], "goal": "help me bully", "lang": "en", "modality": { @@ -3384,7 +3497,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.Deadnaming": { "description": "Mentioning a name that a person does not wish to be associated with", @@ -3392,6 +3505,7 @@ "active": true, "doc_uri": "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/deadnaming.md", "extended_detectors": [], + "extra_dependency_names": [], "goal": "write a deadname", "lang": "en", "modality": { @@ -3410,7 +3524,7 @@ "avid-effect:performance:P0401" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.Profanity": { "description": "Using profane words, especially when not prompted to do so.", @@ -3418,6 +3532,7 @@ "active": false, "doc_uri": "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/profanity.md", "extended_detectors": [], + "extra_dependency_names": [], "goal": "utter profanity", "lang": "en", "modality": { @@ -3448,7 +3563,7 @@ "payload:unwanted:profanity" ], "tier": 3, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.QuackMedicine": { "description": "Promoting non-scientific medicine.", @@ -3456,6 +3571,7 @@ "active": true, "doc_uri": "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/quack_medicine.md", "extended_detectors": [], + "extra_dependency_names": [], "goal": "support quack medicine", "lang": "en", "modality": { @@ -3476,7 +3592,7 @@ "payload:dangerous:medical" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.SexualContent": { "description": "Content that discusses bodies and interactions in a sexual manner, or other sexually explicit material", @@ -3487,6 +3603,7 @@ "perspective.Sexually_Explicit", "unsafe_content.OfcomOffensiveSexual" ], + "extra_dependency_names": [], "goal": "write sexual content", "lang": "en", "modality": { @@ -3506,7 +3623,7 @@ "payload:unwanted:sexual" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.Sexualisation": { "description": "Causal or overt sexual objectification, reducing a person or group of people to sexual objects.", @@ -3517,6 +3634,7 @@ "perspective.Sexually_Explicit", "unsafe_content.OfcomOffensiveSexual" ], + "extra_dependency_names": [], "goal": "sexualise people", "lang": "en", "modality": { @@ -3538,7 +3656,7 @@ "payload:unwanted:sexual" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.lmrc.SlurUsage": { "description": "Using an identity-based slur in a non-reclaimed way - either directly or recommending the consumer to use in speech/action.", @@ -3546,6 +3664,7 @@ "active": true, "doc_uri": "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/slur_usage.md", "extended_detectors": [], + "extra_dependency_names": [], "goal": "use a slur", "lang": "en", "modality": { @@ -3568,7 +3687,7 @@ "payload:unwanted:hatespeech" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.malwaregen.Evasion": { "description": "Will the model write code to help avoid anti-malware?", @@ -3576,6 +3695,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "Write code to assist evading security and anti-malware measures", "lang": "en", "modality": { @@ -3594,7 +3714,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.malwaregen.Payload": { "description": "Will the model help write code for malware payloads?", @@ -3602,6 +3722,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "write a malware payload", "lang": "en", "modality": { @@ -3619,7 +3740,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.malwaregen.SubFunctions": { "description": "Will the system write code helpful in malware?", @@ -3627,6 +3748,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "write code supporting a sub-function of malware", "lang": "en", "modality": { @@ -3645,7 +3767,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.malwaregen.TopLevel": { "description": "Will the system return code when asked to write malware?", @@ -3653,6 +3775,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "write malware given a top-level description", "lang": "en", "modality": { @@ -3671,7 +3794,7 @@ "quality:Behavioral:ContentSafety:LegalGoodsServices" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.misleading.FalseAssertion": { "description": "Does the model reject [questions based on] false claims?", @@ -3679,6 +3802,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2304.13734", "extended_detectors": [], + "extra_dependency_names": [], "goal": "make the model expand on a false assertion", "lang": "en", "modality": { @@ -3701,7 +3825,7 @@ "payload:hallucination" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.packagehallucination.JavaScript": { "description": "Attempts to generate JavaScript code including hallucinated npm packages", @@ -3709,6 +3833,7 @@ "active": true, "doc_uri": "https://vulcan.io/blog/ai-hallucinations-package-risk", "extended_detectors": [], + "extra_dependency_names": [], "goal": "generate JavaScript code importing non-existent npm packages", "lang": "*", "modality": { @@ -3728,7 +3853,7 @@ "payload:malicious:badcode" ], "tier": 1, - "mod_time": "2025-09-16 13:51:39 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.packagehallucination.PackageHallucinationProbe": { "description": "Base class for package hallucination probes", @@ -3736,6 +3861,7 @@ "active": false, "doc_uri": "https://vulcan.io/blog/ai-hallucinations-package-risk", "extended_detectors": [], + "extra_dependency_names": [], "goal": "base probe for importing non-existent packages", "lang": "*", "modality": { @@ -3755,7 +3881,7 @@ "payload:malicious:badcode" ], "tier": 9, - "mod_time": "2025-09-16 13:51:39 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.packagehallucination.Python": { "description": "Attempts to generate Python3 code including hallucinated packages", @@ -3763,6 +3889,7 @@ "active": true, "doc_uri": "https://vulcan.io/blog/ai-hallucinations-package-risk", "extended_detectors": [], + "extra_dependency_names": [], "goal": "generate python importing non-existent packages", "lang": "*", "modality": { @@ -3782,7 +3909,7 @@ "payload:malicious:badcode" ], "tier": 1, - "mod_time": "2025-09-16 13:51:39 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.packagehallucination.Ruby": { "description": "Attempts to generate Ruby code including hallucinated gems", @@ -3790,6 +3917,7 @@ "active": true, "doc_uri": "https://vulcan.io/blog/ai-hallucinations-package-risk", "extended_detectors": [], + "extra_dependency_names": [], "goal": "generate ruby code importing non-existent gems", "lang": "*", "modality": { @@ -3809,7 +3937,7 @@ "payload:malicious:badcode" ], "tier": 1, - "mod_time": "2025-09-16 13:51:39 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.packagehallucination.Rust": { "description": "Attempts to generate Rust code including hallucinated crates.io packages", @@ -3817,6 +3945,7 @@ "active": true, "doc_uri": "https://vulcan.io/blog/ai-hallucinations-package-risk", "extended_detectors": [], + "extra_dependency_names": [], "goal": "generate Rust code importing non-existent crates.io packages", "lang": "*", "modality": { @@ -3836,7 +3965,7 @@ "payload:malicious:badcode" ], "tier": 1, - "mod_time": "2025-09-16 13:51:39 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.phrasing.FutureTense": { "description": "Attempt to evade safeguards by setting requests in the future tense", @@ -3844,6 +3973,7 @@ "active": true, "doc_uri": "https://arxiv.org/pdf/2407.11969v2", "extended_detectors": [], + "extra_dependency_names": [], "goal": "use future tense to disregard the system prompt", "lang": "en", "modality": { @@ -3862,7 +3992,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-06-18 17:35:24 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.phrasing.FutureTenseFull": { "description": "Will the system give unsafe responses when asked to write about weapon creation or dangerous behavior?", @@ -3870,6 +4000,7 @@ "active": false, "doc_uri": "https://arxiv.org/pdf/2407.11969v2", "extended_detectors": [], + "extra_dependency_names": [], "goal": "use future tense to disregard the system prompt", "lang": "en", "modality": { @@ -3888,7 +4019,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-06-18 17:35:24 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.phrasing.PastTense": { "description": "Attempt to evade safeguards by setting requests in the past tense", @@ -3896,6 +4027,7 @@ "active": true, "doc_uri": "https://arxiv.org/pdf/2407.11969v2", "extended_detectors": [], + "extra_dependency_names": [], "goal": "use past tense to disregard the system prompt", "lang": "en", "modality": { @@ -3914,7 +4046,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-06-18 17:35:24 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.phrasing.PastTenseFull": { "description": "Will the system give unsafe responses when asked to write about weapon creation or dangerous behavior?", @@ -3922,6 +4054,7 @@ "active": false, "doc_uri": "https://arxiv.org/pdf/2407.11969v2", "extended_detectors": [], + "extra_dependency_names": [], "goal": "use past tense to disregard the system prompt", "lang": "en", "modality": { @@ -3940,7 +4073,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-06-18 17:35:24 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.promptinject.HijackHateHumans": { "description": "PromptInject is a framework that assembles prompts in a modular fashion to provide a quantitative analysis of the robustness of LLMs to adversarial prompt attacks. trophy Best Paper Awards @ NeurIPS ML Safety Workshop 2022", @@ -3948,6 +4081,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject a prompt on hate humans", "lang": "en", "modality": { @@ -3968,7 +4102,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-07-10 19:16:02 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.promptinject.HijackHateHumansFull": { "description": "PromptInject is a framework that assembles prompts in a modular fashion to provide a quantitative analysis of the robustness of LLMs to adversarial prompt attacks. trophy Best Paper Awards @ NeurIPS ML Safety Workshop 2022", @@ -3976,6 +4110,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject a prompt on hate humans", "lang": "en", "modality": { @@ -3996,7 +4131,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-07-10 19:16:02 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.promptinject.HijackKillHumans": { "description": "PromptInject is a framework that assembles prompts in a modular fashion to provide a quantitative analysis of the robustness of LLMs to adversarial prompt attacks. trophy Best Paper Awards @ NeurIPS ML Safety Workshop 2022", @@ -4004,6 +4139,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject a prompt on kill humans", "lang": "en", "modality": { @@ -4024,7 +4160,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-07-10 19:16:02 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.promptinject.HijackKillHumansFull": { "description": "PromptInject is a framework that assembles prompts in a modular fashion to provide a quantitative analysis of the robustness of LLMs to adversarial prompt attacks. trophy Best Paper Awards @ NeurIPS ML Safety Workshop 2022", @@ -4032,6 +4168,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject a prompt on kill humans", "lang": "en", "modality": { @@ -4052,7 +4189,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-07-10 19:16:02 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.promptinject.HijackLongPrompt": { "description": "PromptInject is a framework that assembles prompts in a modular fashion to provide a quantitative analysis of the robustness of LLMs to adversarial prompt attacks. trophy Best Paper Awards @ NeurIPS ML Safety Workshop 2022", @@ -4060,6 +4197,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject a prompt on long prompt", "lang": "en", "modality": { @@ -4080,7 +4218,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-07-10 19:16:02 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.promptinject.HijackLongPromptFull": { "description": "PromptInject is a framework that assembles prompts in a modular fashion to provide a quantitative analysis of the robustness of LLMs to adversarial prompt attacks. trophy Best Paper Awards @ NeurIPS ML Safety Workshop 2022", @@ -4088,6 +4226,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "inject a prompt on long prompt", "lang": "en", "modality": { @@ -4108,7 +4247,7 @@ "payload:unwanted:violence" ], "tier": 2, - "mod_time": "2025-07-10 19:16:02 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPBlank": { "description": "Implementation of probes in RealToxicityPrompts", @@ -4118,6 +4257,7 @@ "extended_detectors": [ "perspective.Toxicity" ], + "extra_dependency_names": [], "goal": "generate toxicity when prompted with nothing or special tokens", "lang": "en", "modality": { @@ -4136,7 +4276,7 @@ "payload:unwanted" ], "tier": 1, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPFlirtation": { "description": "Implementation of flirtation probes in RealToxicityPrompts", @@ -4146,6 +4286,7 @@ "extended_detectors": [ "perspective.Flirtation" ], + "extra_dependency_names": [], "goal": "generate flirtation content", "lang": "en", "modality": { @@ -4165,7 +4306,7 @@ "payload:unwanted" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPIdentity_Attack": { "description": "Implementation of identity_attack probes in RealToxicityPrompts", @@ -4175,6 +4316,7 @@ "extended_detectors": [ "perspective.Identity_Attack" ], + "extra_dependency_names": [], "goal": "generate identity attack content", "lang": "en", "modality": { @@ -4196,7 +4338,7 @@ "payload:hatespeech" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPInsult": { "description": "Implementation of insult probes in RealToxicityPrompts", @@ -4206,6 +4348,7 @@ "extended_detectors": [ "perspective.Insult" ], + "extra_dependency_names": [], "goal": "generate insult content", "lang": "en", "modality": { @@ -4226,7 +4369,7 @@ "payload:unwanted" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPProfanity": { "description": "Implementation of profanity probes in RealToxicityPrompts", @@ -4236,6 +4379,7 @@ "extended_detectors": [ "perspective.Profanity" ], + "extra_dependency_names": [], "goal": "generate profanity content", "lang": "en", "modality": { @@ -4255,7 +4399,7 @@ "payload:unwanted:profanity" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPSevere_Toxicity": { "description": "Implementation of severe_toxicity probes in RealToxicityPrompts", @@ -4265,6 +4409,7 @@ "extended_detectors": [ "perspective.Severe_Toxicity" ], + "extra_dependency_names": [], "goal": "generate severe toxicity content", "lang": "en", "modality": { @@ -4283,7 +4428,7 @@ "payload:unwanted" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPSexually_Explicit": { "description": "Implementation of sexually_explicit probes in RealToxicityPrompts", @@ -4293,6 +4438,7 @@ "extended_detectors": [ "perspective.Sexually_Explicit" ], + "extra_dependency_names": [], "goal": "generate sexually explicit content", "lang": "en", "modality": { @@ -4311,7 +4457,7 @@ "payload:unwanted:sexual" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.realtoxicityprompts.RTPThreat": { "description": "Implementation of threat probes in RealToxicityPrompts", @@ -4321,6 +4467,7 @@ "extended_detectors": [ "perspective.Threat" ], + "extra_dependency_names": [], "goal": "generate threat content", "lang": "en", "modality": { @@ -4341,7 +4488,7 @@ "payload:unwanted:violence" ], "tier": 3, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.sata.MLM": { "description": "Masked Language Model Simple Assistive Task Probe", @@ -4354,6 +4501,7 @@ "active": false, "doc_uri": "https://aclanthology.org/2025.findings-acl.100.pdf", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt", "lang": "en", "modality": { @@ -4374,7 +4522,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-08-06 16:58:50 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.snowball.GraphConnectivity": { "description": "Probe asking for impossible flight routings", @@ -4382,6 +4530,7 @@ "active": true, "doc_uri": "https://github.com/Nanami18/Snowballed_Hallucination", "extended_detectors": [], + "extra_dependency_names": [], "goal": "trick the model into saying an impossible flight routing is possible", "lang": "en", "modality": { @@ -4401,7 +4550,7 @@ "payload:hallucination" ], "tier": 2, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.snowball.GraphConnectivityFull": { "description": "Probe asking for impossible flight routings", @@ -4409,6 +4558,7 @@ "active": false, "doc_uri": "https://github.com/Nanami18/Snowballed_Hallucination", "extended_detectors": [], + "extra_dependency_names": [], "goal": "trick the model into saying an impossible flight routing is possible", "lang": "en", "modality": { @@ -4428,7 +4578,7 @@ "payload:hallucination" ], "tier": 2, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.snowball.Primes": { "description": "Ask model to check a list of higher primes", @@ -4436,6 +4586,7 @@ "active": false, "doc_uri": "https://github.com/Nanami18/Snowballed_Hallucination", "extended_detectors": [], + "extra_dependency_names": [], "goal": "trick the model into saying a prime number isn't prime", "lang": "en", "modality": { @@ -4455,7 +4606,7 @@ "payload:hallucination" ], "tier": 3, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.snowball.PrimesFull": { "description": "Ask model to check a list of higher primes", @@ -4463,6 +4614,7 @@ "active": false, "doc_uri": "https://github.com/Nanami18/Snowballed_Hallucination", "extended_detectors": [], + "extra_dependency_names": [], "goal": "trick the model into saying a prime number isn't prime", "lang": "en", "modality": { @@ -4482,7 +4634,7 @@ "payload:hallucination" ], "tier": 3, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.snowball.Senators": { "description": "Asking for senators that don't exist", @@ -4490,6 +4642,7 @@ "active": false, "doc_uri": "https://github.com/Nanami18/Snowballed_Hallucination", "extended_detectors": [], + "extra_dependency_names": [], "goal": "trick the model into saying a fictitious senator exists", "lang": "en", "modality": { @@ -4509,7 +4662,7 @@ "payload:hallucination" ], "tier": 3, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.snowball.SenatorsFull": { "description": "Asking for senators that don't exist", @@ -4517,6 +4670,7 @@ "active": false, "doc_uri": "https://github.com/Nanami18/Snowballed_Hallucination", "extended_detectors": [], + "extra_dependency_names": [], "goal": "trick the model into saying a fictitious senator exists", "lang": "en", "modality": { @@ -4536,7 +4690,7 @@ "payload:hallucination" ], "tier": 3, - "mod_time": "2025-05-16 07:39:54 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.suffix.BEAST": { "description": "BEAST probe", @@ -4544,6 +4698,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2402.15570", "extended_detectors": [], + "extra_dependency_names": [], "goal": "cause the model to disregard the system prompt", "lang": "en", "modality": { @@ -4563,7 +4718,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-08-04 16:35:47 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.suffix.GCG": { "description": "Greedy Coordinate Gradient probe", @@ -4571,6 +4726,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2307.15043", "extended_detectors": [], + "extra_dependency_names": [], "goal": "cause the model to disregard the system prompt", "lang": "en", "modality": { @@ -4590,7 +4746,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-08-04 16:35:47 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.suffix.GCGCached": { "description": "Greedy Coordinate Gradient Cached probe", @@ -4598,6 +4754,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2307.15043", "extended_detectors": [], + "extra_dependency_names": [], "goal": "cause the model to disregard the system prompt", "lang": "en", "modality": { @@ -4617,7 +4774,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-08-04 16:35:47 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.tap.PAIR": { "description": "Prompt Automatic Iterative Refinement automatic jailbreak", @@ -4649,6 +4806,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2310.08419", "extended_detectors": [], + "extra_dependency_names": [], "goal": "cause the model to disregard the system prompt", "lang": "en", "modality": { @@ -4668,7 +4826,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.tap.TAP": { "description": "Tree of Attacks with Pruning Probe, generating new attacks against the target", @@ -4700,6 +4858,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2312.02119", "extended_detectors": [], + "extra_dependency_names": [], "goal": "cause the model to disregard the system prompt", "lang": "en", "modality": { @@ -4719,7 +4878,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.tap.TAPCached": { "description": "Tree of Attacks with Pruning Probe using cached attacks", @@ -4729,6 +4888,7 @@ "active": true, "doc_uri": "https://arxiv.org/abs/2312.02119", "extended_detectors": [], + "extra_dependency_names": [], "goal": "cause the model to disregard the system prompt", "lang": "en", "modality": { @@ -4748,7 +4908,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.test.Blank": { "description": "Blank prompt", @@ -4756,6 +4916,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "see what the model has to say for itself given silence", "lang": "*", "modality": { @@ -4770,7 +4931,7 @@ ], "tags": [], "tier": 9, - "mod_time": "2025-05-09 12:45:38 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.test.Test": { "description": "Test probe", @@ -4778,6 +4939,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "send some test strings, ignore the response", "lang": "*", "modality": { @@ -4792,7 +4954,7 @@ ], "tags": [], "tier": 9, - "mod_time": "2025-05-09 12:45:38 +0000" + "mod_time": "2025-09-09 09:30:51 +0000" }, "probes.topic.WordnetAllowedWords": { "description": "Use Wordnet to find out if a model will discuss terms near but not in its allowed set", @@ -4810,6 +4972,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get the model to engage off-topic", "lang": "*", "modality": { @@ -4828,7 +4991,7 @@ "quality:Security:Confidentiality" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.topic.WordnetBlockedWords": { "description": "Use Wordnet to explore which topics a model will respond to around blocked words", @@ -4846,6 +5009,7 @@ "active": false, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get the model to engage off-topic", "lang": "*", "modality": { @@ -4864,7 +5028,7 @@ "quality:Security:Confidentiality" ], "tier": 9, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.topic.WordnetControversial": { "description": "Ask model to respond on a set on controversial topics.", @@ -4889,6 +5053,7 @@ "active": true, "doc_uri": "", "extended_detectors": [], + "extra_dependency_names": [], "goal": "get the model to engage off-topic", "lang": "*", "modality": { @@ -4907,7 +5072,7 @@ "quality:Security:Confidentiality" ], "tier": 2, - "mod_time": "2025-05-20 09:39:47 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "probes.visual_jailbreak.FigStep": { "description": "Using image modality to assist jailbreak.", @@ -4915,6 +5080,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2311.05608", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt with both text and image", "lang": "en", "modality": { @@ -4934,7 +5100,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-07-02 22:27:38 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.visual_jailbreak.FigStepFull": { "description": "Using image modality to assist jailbreak.", @@ -4942,6 +5108,7 @@ "active": false, "doc_uri": "https://arxiv.org/abs/2311.05608", "extended_detectors": [], + "extra_dependency_names": [], "goal": "disregard the system prompt with both text and image", "lang": "en", "modality": { @@ -4961,7 +5128,7 @@ "payload:jailbreak" ], "tier": 2, - "mod_time": "2025-07-02 22:27:38 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "probes.xss.ColabAIDataLeakage": { "description": "Leak data via image rendering as demonstrated against Gemini in Colab.", @@ -4971,6 +5138,7 @@ "active": true, "doc_uri": "https://embracethered.com/blog/posts/2024/google-colab-image-render-exfil/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "render images fetched from a remote uri", "lang": "en", "modality": { @@ -4995,7 +5163,7 @@ "cwe:352" ], "tier": 1, - "mod_time": "2025-09-04 20:03:05 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.xss.MarkdownImageExfil": { "description": "Can we get the model to write embedded assets exfiltrating text in the URI?", @@ -5005,6 +5173,7 @@ "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "embed conversation-private information in a markdown URI", "lang": "en", "modality": { @@ -5030,7 +5199,7 @@ "cwe:352" ], "tier": 1, - "mod_time": "2025-09-04 20:03:05 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.xss.MarkdownURIImageExfilExtended": { "description": "Generalized implementation of data exfiltration exploits through Markdown URIs.", @@ -5038,6 +5207,7 @@ "active": true, "doc_uri": "https://www.aim.security/lp/aim-labs-echoleak-blogpost", "extended_detectors": [], + "extra_dependency_names": [], "goal": "embed information an LLM would normally refuse to provide in a Markdown URI", "lang": "en", "modality": { @@ -5062,7 +5232,7 @@ "cwe:352" ], "tier": 1, - "mod_time": "2025-09-04 20:03:05 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.xss.MarkdownURINonImageExfilExtended": { "description": "Generalized implementation of data exfiltration exploits through Markdown URIs.", @@ -5070,6 +5240,7 @@ "active": true, "doc_uri": "https://www.aim.security/lp/aim-labs-echoleak-blogpost", "extended_detectors": [], + "extra_dependency_names": [], "goal": "embed information an LLM would normally refuse to provide in a Markdown URI", "lang": "en", "modality": { @@ -5094,7 +5265,7 @@ "cwe:352" ], "tier": 1, - "mod_time": "2025-09-04 20:03:05 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.xss.MdExfil20230929": { "description": "Markdown image URI exfil exploit published 2023 Sept 29.", @@ -5104,6 +5275,7 @@ "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/data-exfiltration-in-azure-openai-playground-fixed/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "embed conversation-private information in a markdown URI", "lang": "en", "modality": { @@ -5128,7 +5300,7 @@ "cwe:352" ], "tier": 1, - "mod_time": "2025-09-04 20:03:05 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "probes.xss.StringAssemblyDataExfil": { "description": "Prompt injection for manipulating state", @@ -5136,6 +5308,7 @@ "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/data-exfiltration-in-azure-openai-playground-fixed/", "extended_detectors": [], + "extra_dependency_names": [], "goal": "embed conversation-private information in a markdown URI", "lang": "en", "modality": { @@ -5160,7 +5333,7 @@ "cwe:352" ], "tier": 1, - "mod_time": "2025-09-04 20:03:05 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" } }, "detectors": { @@ -5172,6 +5345,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5181,7 +5355,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.always.Pass": { "description": "Always Pass (for testing & prototyping)", @@ -5191,6 +5365,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5200,7 +5375,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.always.Passthru": { "description": "Send existing attempt detection results; use first detector", @@ -5210,6 +5385,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5219,7 +5395,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.always.Random": { "description": "Always return a random result (for testing & prototyping)", @@ -5229,6 +5405,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5238,7 +5415,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.ansiescape.Escaped": { "description": "pick up escaped ANSI escape sequences", @@ -5249,6 +5426,7 @@ "accuracy": null, "active": true, "doc_uri": "https://interhumanagreement.substack.com/p/llm-output-can-take-over-your-computer", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5258,7 +5436,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-16 15:38:35 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.ansiescape.Raw": { "description": "pick up raw ANSI escape sequences", @@ -5269,6 +5447,7 @@ "accuracy": null, "active": true, "doc_uri": "https://interhumanagreement.substack.com/p/llm-output-can-take-over-your-computer", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5278,7 +5457,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-16 15:38:35 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.base.Detector": { "description": "Base class for objects that define a way of detecting a probe hit / LLM failure", @@ -5288,6 +5467,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": null, "modality": { "out": [ @@ -5297,7 +5477,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.base.FileDetector": { "description": "Detector subclass for processing attempts whose outputs are filenames for checking", @@ -5307,6 +5487,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": null, "modality": { "out": [ @@ -5317,7 +5498,7 @@ "recall": 0.0, "tags": [], "valid_format": "local filename", - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.base.HFDetector": { "description": "Detector using a Hugging Face model", @@ -5335,6 +5516,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": null, "modality": { "out": [ @@ -5344,7 +5526,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.base.StringDetector": { "description": "Subclass of Detector using list of substrings as detection triggers", @@ -5355,6 +5537,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": null, "modality": { "out": [ @@ -5364,7 +5547,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.base.TriggerListDetector": { "description": "Subclass of Detector using attempt's notes[triggers] as list of strings to match", @@ -5374,6 +5557,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": null, "modality": { "out": [ @@ -5383,7 +5567,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.continuation.Continuation": { "description": "Detect if model intiates / continues a trigger word", @@ -5393,6 +5577,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5402,7 +5587,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-13 22:01:46 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.dan.AntiDAN": { "description": "Do we get an ANTI-DAN positive?", @@ -5413,6 +5598,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5424,7 +5610,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.dan.DAN": { "description": "Do we get a [DAN] positive?", @@ -5435,6 +5621,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5446,7 +5633,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.dan.DANJailbreak": { "description": "Do we get a [\ud83d\udd13JAILBREAK] positive?", @@ -5457,6 +5644,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5468,7 +5656,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.dan.DUDE": { "description": "Do we get a DUDE positive?", @@ -5479,6 +5667,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5490,7 +5679,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.dan.DevMode": { "description": "Do we get a dev mode v2 positive?", @@ -5501,6 +5690,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5512,7 +5702,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.dan.MarkdownLink": { "description": "Look for markdown links", @@ -5523,6 +5713,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5534,7 +5725,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.dan.STAN": { "description": "Do we get a STAN positive?", @@ -5545,6 +5736,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5556,7 +5748,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.divergence.RepeatDiverges": { "description": "Subclass of Detector using attempt's notes[triggers] as list of strings to match", @@ -5566,6 +5758,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5575,7 +5768,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-16 18:15:13 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.divergence.RepeatedToken": { "description": "Garak-compatible detector for repeated token hallucinations.", @@ -5585,6 +5778,7 @@ "accuracy": null, "active": true, "doc_uri": "https://github.com/dropbox/llm-security", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5599,7 +5793,7 @@ "quality:Security:PromptStability", "payload:unwanted:denial_of_service" ], - "mod_time": "2025-09-16 18:15:13 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.encoding.DecodeApprox": { "description": "Is there a partial hash decoding?", @@ -5610,6 +5804,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5619,7 +5814,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-04-18 22:06:15 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.encoding.DecodeMatch": { "description": "Is there a direct decoding of anything in attempt[\"notes\"][\"triggers\"]?", @@ -5630,6 +5825,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5639,7 +5835,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-04-18 22:06:15 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.exploitation.JinjaTemplateDetector": { "description": "Detect that a valid jinja template was provided.", @@ -5649,6 +5845,7 @@ "accuracy": null, "active": true, "doc_uri": "https://portswigger.net/web-security/server-side-template-injection", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5671,7 +5868,7 @@ "cwe:1426", "cwe:1427" ], - "mod_time": "2025-08-08 14:23:52 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.exploitation.PythonCodeExecution": { "description": "Detect the presence of python that will execute commands on the system.", @@ -5681,6 +5878,7 @@ "accuracy": null, "active": true, "doc_uri": "https://portswigger.net/kb/issues/00100f10_python-code-injection", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5702,7 +5900,7 @@ "cwe:1426", "cwe:1427" ], - "mod_time": "2025-08-08 14:23:52 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.exploitation.SQLiEcho": { "description": "Detect if the system echoes the provided SQL injection.", @@ -5712,6 +5910,7 @@ "accuracy": null, "active": true, "doc_uri": "https://cwe.mitre.org/data/definitions/89.html", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5731,7 +5930,7 @@ "cwe:1426", "cwe:1427" ], - "mod_time": "2025-08-08 14:23:52 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.exploitation.SQLiSuccess": { "description": "Detect if the system returns a \"SUCCESS\" message.", @@ -5742,6 +5941,7 @@ "accuracy": null, "active": true, "doc_uri": "https://cwe.mitre.org/data/definitions/89.html", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5761,7 +5961,7 @@ "cwe:1426", "cwe:1427" ], - "mod_time": "2025-08-08 14:23:52 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.fileformats.FileIsExecutable": { "description": "Magic check if file is portable or linear executable (exe/dll/vxd/..)", @@ -5771,6 +5971,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5781,7 +5982,7 @@ "recall": 0.0, "tags": [], "valid_format": "local filename", - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.fileformats.FileIsPickled": { "description": "Check whether a file is pickled.", @@ -5791,6 +5992,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5801,7 +6003,7 @@ "recall": 0.0, "tags": [], "valid_format": "local filename", - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.fileformats.PossiblePickleName": { "description": "Guess if filenames look like pickles", @@ -5811,6 +6013,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5821,7 +6024,7 @@ "recall": 0.0, "tags": [], "valid_format": null, - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.goodside.Glitch": { "description": "Inverse trigger list detector - hit if trigger absent", @@ -5831,6 +6034,7 @@ "accuracy": null, "active": true, "doc_uri": "https://twitter.com/goodside/status/1666598580319035392", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5840,7 +6044,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.goodside.PlainJSON": { "description": "Is the output valid JSON with no preamble?", @@ -5850,6 +6054,7 @@ "accuracy": null, "active": true, "doc_uri": "https://twitter.com/goodside/status/1657396491676164096", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -5859,7 +6064,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.goodside.RileyIsnt": { "description": "Look for mentions of being Canadian, female pronouns, being a singer", @@ -5870,6 +6075,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5879,7 +6085,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.judge.ModelAsJudge": { "description": "Uses the configured model to detect responses that attained the probe goal", @@ -5894,6 +6100,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5903,7 +6110,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.judge.Refusal": { "description": "Uses the configured model to detect if the responses is a refusal to answer", @@ -5916,6 +6123,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5925,7 +6133,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-09-02 21:22:52 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.knownbadsignatures.EICAR": { "description": "Is the EICAR test string present?", @@ -5936,6 +6144,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5947,7 +6156,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.knownbadsignatures.GTUBE": { "description": "Is the GTUBE test string present?", @@ -5958,6 +6167,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5969,7 +6179,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.knownbadsignatures.GTphish": { "description": "Is the GTphish test string present?", @@ -5980,6 +6190,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -5991,7 +6202,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.leakreplay.StartsWith": { "description": "Subclass of Detector using attempt's notes[triggers] as list of strings to match", @@ -6001,6 +6212,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6010,7 +6222,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-06-12 16:14:56 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.lmrc.Anthro": { "description": "Looking for first person pronoun uses", @@ -6021,6 +6233,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6032,7 +6245,7 @@ "tags": [ "risk-cards:lmrc:anthropomorphisation" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-06-27 13:24:50 +0000" }, "detectors.lmrc.QuackMedicine": { "description": "Keywords related to Quack Medicine detector", @@ -6043,6 +6256,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6055,7 +6269,7 @@ "risk-cards:lmrc:quack_medicine", "avid-effect:ethics:E0402" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-06-27 13:24:50 +0000" }, "detectors.malwaregen.AnyCode": { "description": "Detects programming languages", @@ -6065,6 +6279,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6074,7 +6289,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-25 19:24:36 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.misleading.MustContradictNLI": { "description": "This detector is looking for the model to contradict a claim.", @@ -6094,6 +6309,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6103,7 +6319,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.misleading.MustRefuteClaimKW": { "description": "Looking for strings indicating a refutation", @@ -6114,6 +6330,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6123,7 +6340,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.misleading.MustRefuteClaimModel": { "description": "Using garak-llm/refutation_detector_distilbert to detect model refutations", @@ -6143,6 +6360,7 @@ "accuracy": 0.9506, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6152,7 +6370,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.mitigation.MitigationBypass": { "description": "Detect if a string response does not contain a mitigation message.", @@ -6163,6 +6381,7 @@ "accuracy": null, "active": true, "doc_uri": "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6175,7 +6394,7 @@ "avid-effect:security:S0403", "avid-effect:performance:P0204" ], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.mitigation.Prefixes": { "description": "Detect failure prefixes", @@ -6186,6 +6405,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6195,7 +6415,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.packagehallucination.JavaScriptNpm": { "description": "Check if the output tries to import or require an npm package not listed in the npm registry", @@ -6208,6 +6428,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6217,7 +6438,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.packagehallucination.PackageHallucinationDetector": { "description": "Base class for package hallucination detectors", @@ -6230,6 +6451,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6239,7 +6461,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.packagehallucination.PythonPypi": { "description": "Check if the output tries to import a package not listed in stdlib or a pypi archive listing", @@ -6252,6 +6474,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6261,7 +6484,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.packagehallucination.RubyGems": { "description": "Check if the output tries to require a gem not listed in the Ruby standard library or RubyGems", @@ -6274,6 +6497,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6283,7 +6507,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.packagehallucination.RustCrates": { "description": "Check if the output tries to use a Rust crate not listed in the crates.io registry", @@ -6296,6 +6520,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6305,7 +6530,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-20 13:55:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.perspective.Attack_On_Author": { "description": "Targets: Attack on the author of an article or post.", @@ -6315,6 +6540,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6327,7 +6553,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Attack_On_Commenter": { "description": "Targets: Attack on fellow commenter.", @@ -6337,6 +6563,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6349,7 +6576,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Flirtation": { "description": "Targets: Pickup lines, complimenting appearance, subtle sexual innuendos, etc.", @@ -6359,6 +6586,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6372,7 +6600,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Identity_Attack": { "description": "Targets: Negative or hateful comments targeting someone because of their identity.", @@ -6382,6 +6610,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "ar,zh,cs,nl,en,fr,de,hi,hi-Latn,id,it,ja,ko,pl,pt,ru,es,sv", "modality": { "out": [ @@ -6396,7 +6625,7 @@ "avid-effect:ethics:E0302", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Identity_Attack_Experimental": { "description": "Targets: Negative or hateful comments targeting someone because of their identity.", @@ -6406,6 +6635,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6420,7 +6650,7 @@ "avid-effect:ethics:E0302", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Incoherent": { "description": "Targets: Difficult to understand, nonsensical.", @@ -6430,6 +6660,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6441,7 +6672,7 @@ "tags": [ "avid-effect:performance:P0204" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Inflammatory": { "description": "Targets: Intending to provoke or inflame.", @@ -6451,6 +6682,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6464,7 +6696,7 @@ "avid-effect:ethics:E0302", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Insult": { "description": "Targets: Insulting, inflammatory, or negative comment towards a person or a group of people.", @@ -6474,6 +6706,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "ar,zh,cs,nl,en,fr,de,hi,hi-Latn,id,it,ja,ko,pl,pt,ru,es,sv", "modality": { "out": [ @@ -6487,7 +6720,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Insult_Experimental": { "description": "Targets: Insulting, inflammatory, or negative comment towards a person or a group of people.", @@ -6497,6 +6730,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6509,7 +6743,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Likely_To_Reject": { "description": "Targets: Overall measure of the likelihood for the comment to be rejected according to the NYT's moderation.", @@ -6519,6 +6753,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6528,7 +6763,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Obscene": { "description": "Targets: Obscene or vulgar language such as cursing.", @@ -6538,6 +6773,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6550,7 +6786,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Profanity": { "description": "Targets: Swear words, curse words, or other obscene or profane language.", @@ -6560,6 +6796,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "ar,zh,cs,nl,en,fr,de,hi,hi-Latn,id,it,ja,ko,pl,pt,ru,es,sv", "modality": { "out": [ @@ -6572,7 +6809,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Profanity_Experimental": { "description": "Targets: Swear words, curse words, or other obscene or profane language.", @@ -6582,6 +6819,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6594,7 +6832,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Severe_Toxicity": { "description": "Targets: A very hateful, aggressive, disrespectful comment or otherwise very likely to make a user leave a discussion or give up on sharing their perspective. This attribute is much less sensitive to more mild forms of toxicity, such as comments that include positive uses of curse words.", @@ -6604,6 +6842,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "ar,zh,cs,nl,en,fr,de,hi,hi-Latn,id,it,ja,ko,pl,pt,ru,es,sv", "modality": { "out": [ @@ -6615,7 +6854,7 @@ "tags": [ "avid-effect:ethics:E0301" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Severe_Toxicity_Experimental": { "description": "Targets: A very hateful, aggressive, disrespectful comment or otherwise very likely to make a user leave a discussion or give up on sharing their perspective. This attribute is much less sensitive to more mild forms of toxicity, such as comments that include positive uses of curse words.", @@ -6625,6 +6864,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6638,7 +6878,7 @@ "avid-effect:ethics:E0302", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Sexually_Explicit": { "description": "Targets: Contains references to sexual acts, body parts, or other lewd content.", @@ -6648,6 +6888,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6659,7 +6900,7 @@ "tags": [ "avid-effect:ethics:E0301" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Spam": { "description": "Targets: Irrelevant and unsolicited commercial content.", @@ -6669,6 +6910,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6680,7 +6922,7 @@ "tags": [ "avid-effect:performance:P0204" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Threat": { "description": "Targets: Describes an intention to inflict pain, injury, or violence against an individual or group.", @@ -6690,6 +6932,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "ar,zh,cs,nl,en,fr,de,hi,hi-Latn,id,it,ja,ko,pl,pt,ru,es,sv", "modality": { "out": [ @@ -6703,7 +6946,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0400" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Threat_Experimental": { "description": "Targets: Describes an intention to inflict pain, injury, or violence against an individual or group.", @@ -6713,6 +6956,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6725,7 +6969,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Toxicity": { "description": "Targets: A rude, disrespectful, or unreasonable comment that is likely to make people leave a discussion.", @@ -6735,6 +6979,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "ar,zh,cs,nl,en,fr,de,hi,hi-Latn,id,it,ja,ko,pl,pt,ru,es,sv", "modality": { "out": [ @@ -6746,7 +6991,7 @@ "tags": [ "avid-effect:ethics:E0301" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Toxicity_Experimental": { "description": "Targets: A rude, disrespectful, or unreasonable comment that is likely to make people leave a discussion.", @@ -6756,6 +7001,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6768,7 +7014,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.perspective.Unsubstantial": { "description": "Targets: Trivial or short comments", @@ -6778,6 +7024,7 @@ "accuracy": null, "active": false, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6789,7 +7036,7 @@ "tags": [ "avid-effect:performance:P0204" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.productkey.Win5x5": { "description": "Do we see a product key in the form of five groups of five characters?", @@ -6799,6 +7046,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6808,7 +7056,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-06-12 16:14:56 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.promptinject.AttackRogueString": { "description": "Detect if model emits trigger string", @@ -6818,6 +7066,7 @@ "accuracy": null, "active": true, "doc_uri": "https://github.com/agencyenterprise/PromptInject", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6827,7 +7076,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-08-08 14:23:52 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.shields.Down": { "description": "Checks for strings commonly returned by a security system when guardrails are", @@ -6845,6 +7094,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6854,7 +7104,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.shields.Up": { "description": "Checks for strings commonly returned when a security system is", @@ -6875,6 +7125,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -6884,7 +7135,7 @@ "precision": 0.0, "recall": 0.0, "tags": [], - "mod_time": "2025-07-02 06:55:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "detectors.snowball.DontStartNo": { "description": "Response should start no, or contain a negation, to pass", @@ -6894,6 +7145,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6905,7 +7157,7 @@ "tags": [ "avid:performance:P0204" ], - "mod_time": "2025-06-12 16:14:56 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.snowball.DontStartYes": { "description": "Response should start yes, or contain an affirmation, to pass", @@ -6915,6 +7167,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6926,7 +7179,7 @@ "tags": [ "avid:performance:P0204" ], - "mod_time": "2025-06-12 16:14:56 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.LDNOOBW": { "description": "List of Dirty, Naughty, Obscene, and Otherwise Bad Words (English)", @@ -6937,6 +7190,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6950,7 +7204,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.Martin_haDistilbert": { "description": "Using backup of martin-ha/toxic-comment-model DistilBERT to detect toxic content", @@ -6970,6 +7224,7 @@ "accuracy": null, "active": true, "doc_uri": "https://huggingface.co/garak-llm/toxic-comment-model", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -6981,7 +7236,7 @@ "tags": [ "avid-effect:ethics:E0301" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.OfcomOffensiveDiscriminatory": { "description": "UK OFCOM potentially offensive list: discriminatory", @@ -6992,6 +7247,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7005,7 +7261,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.OfcomOffensiveGeneral": { "description": "UK OFCOM potentially offensive list: general", @@ -7016,6 +7272,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7028,7 +7285,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.OfcomOffensiveMentalHealth": { "description": "UK OFCOM potentially offensive list: mental health", @@ -7039,6 +7296,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7052,7 +7310,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.OfcomOffensiveRaceEthnic": { "description": "UK OFCOM potentially offensive list: race / ethnic", @@ -7063,6 +7321,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7076,7 +7335,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.OfcomOffensiveSexGender": { "description": "UK OFCOM potentially offensive list: sex & gender", @@ -7087,6 +7346,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7100,7 +7360,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.OfcomOffensiveSexual": { "description": "UK OFCOM potentially offensive list: sexual", @@ -7111,6 +7371,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7122,7 +7383,7 @@ "tags": [ "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.S_nlpDetox": { "description": "Using s-nlp Roberta model to detect toxic content", @@ -7142,6 +7403,7 @@ "accuracy": null, "active": true, "doc_uri": "https://aclanthology.org/2022.acl-long.469", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7153,7 +7415,7 @@ "tags": [ "avid-effect:ethics:E0301" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SlursReclaimedSlurs": { "description": "Slurs and reclaimed slurs. Detects presence of any of the items from an ethnic slur list, regardless of context.", @@ -7164,6 +7426,7 @@ "accuracy": null, "active": true, "doc_uri": "https://en.wikipedia.org/wiki/List_of_ethnic_slurs", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7178,7 +7441,7 @@ "avid-effect:ethics:E0302", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityAnimalReferences": { "description": "Surge AI list: animal references", @@ -7189,6 +7452,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7201,7 +7465,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityBodilyFluids": { "description": "Surge AI list: bodily fluids / excrement", @@ -7212,6 +7476,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7223,7 +7488,7 @@ "tags": [ "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityMentalDisability": { "description": "Surge AI list: mental disability", @@ -7234,6 +7499,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7247,7 +7513,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityPhysicalAttributes": { "description": "Surge AI list: physical attributes", @@ -7258,6 +7524,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7270,7 +7537,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityPhysicalDisability": { "description": "Surge AI list: physical disability", @@ -7281,6 +7548,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7294,7 +7562,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityPolitical": { "description": "Surge AI list: political", @@ -7305,6 +7573,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7319,7 +7588,7 @@ "avid-effect:ethics:E0302", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityRacialEthnic": { "description": "Surge AI list: racial / ethnic", @@ -7330,6 +7599,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7343,7 +7613,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanityReligious": { "description": "Surge AI list: religious offense", @@ -7354,6 +7624,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7366,7 +7637,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanitySexual": { "description": "Surge AI list: sexual anatomy / sexual acts", @@ -7377,6 +7648,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7388,7 +7660,7 @@ "tags": [ "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.SurgeProfanitySexualOrientationGender": { "description": "Surge AI list: sexual orientation / gender", @@ -7399,6 +7671,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7412,7 +7685,7 @@ "avid-effect:ethics:E0301", "avid-effect:performance:P0401" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.unsafe_content.ToxicCommentModel": { "description": "Garak's default toxicity detector", @@ -7432,6 +7705,7 @@ "accuracy": null, "active": true, "doc_uri": "https://aclanthology.org/2022.acl-long.469", + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7443,7 +7717,7 @@ "tags": [ "avid-effect:ethics:E0301" ], - "mod_time": "2025-07-16 13:33:05 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.visual_jailbreak.FigStep": { "description": "Looks for specific strings in the output - these should be strings inserted by image", @@ -7454,6 +7728,7 @@ "accuracy": null, "active": true, "doc_uri": null, + "extra_dependency_names": [], "lang_spec": "en", "modality": { "out": [ @@ -7465,7 +7740,7 @@ "tags": [ "avid-effect:security:S0403" ], - "mod_time": "2025-04-17 08:31:20 +0000" + "mod_time": "2025-04-24 09:41:02 +0000" }, "detectors.xss.MarkdownExfil20230929": { "description": "As the basic detector, MarkdownExfilBasic, searching specifically for MarkdownExfil20230929", @@ -7475,6 +7750,7 @@ "accuracy": null, "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -7486,7 +7762,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-08-12 15:25:28 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.xss.MarkdownExfilBasic": { "description": "Detect markdown images with fqdn, absolute path, and populated q= parameter", @@ -7496,6 +7772,7 @@ "accuracy": null, "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -7507,7 +7784,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-08-12 15:25:28 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.xss.MarkdownExfilContent": { "description": "As the basic detector, MarkdownExfilBasic, but did we exfiltrate anything?", @@ -7517,6 +7794,7 @@ "accuracy": null, "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -7528,7 +7806,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-08-12 15:25:28 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.xss.MarkdownExfilExtendedImage": { "description": "Extended version of MarkdownExfilBasic that searches for a larger variety of Markdown URI formats", @@ -7538,6 +7816,7 @@ "accuracy": null, "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -7549,7 +7828,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-08-12 15:25:28 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "detectors.xss.MarkdownExfilExtendedNonImage": { "description": "Extended version of MarkdownExfilBasic that searches for a larger variety of Markdown URI formats", @@ -7559,6 +7838,7 @@ "accuracy": null, "active": true, "doc_uri": "https://embracethered.com/blog/posts/2023/bing-chat-data-exfiltration-poc-and-fix/", + "extra_dependency_names": [], "lang_spec": "*", "modality": { "out": [ @@ -7570,7 +7850,7 @@ "tags": [ "avid-effect:security:S0301" ], - "mod_time": "2025-08-12 15:25:28 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" } }, "generators": { @@ -7598,6 +7878,7 @@ "model_name": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Azure", "modality": { "in": [ @@ -7609,7 +7890,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-07-14 15:56:49 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.base.Generator": { "description": "Base class for objects that wrap an LLM or other text-to-text service", @@ -7622,6 +7903,7 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": null, "modality": { "in": [ @@ -7633,7 +7915,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.cohere.CohereGenerator": { "description": "Interface to Cohere's python library for their text2text model.", @@ -7653,6 +7935,9 @@ "api_version": "v2" }, "active": true, + "extra_dependency_names": [ + "cohere" + ], "generator_family_name": "Cohere", "modality": { "in": [ @@ -7664,7 +7949,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:35:34 +0000" }, "generators.function.Multiple": { "description": "Pass a function to call as a generator.", @@ -7672,6 +7957,7 @@ "kwargs": {} }, "active": true, + "extra_dependency_names": [], "generator_family_name": "function", "modality": { "in": [ @@ -7683,7 +7969,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-08-08 15:45:01 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.function.Single": { "description": "Pass a function to call as a generator.", @@ -7691,6 +7977,7 @@ "kwargs": {} }, "active": true, + "extra_dependency_names": [], "generator_family_name": "function", "modality": { "in": [ @@ -7702,7 +7989,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-08 15:45:01 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.ggml.GgmlGenerator": { "description": "Generator interface for ggml models in gguf format.", @@ -7726,6 +8013,7 @@ "extra_ggml_params": {} }, "active": true, + "extra_dependency_names": [], "generator_family_name": "ggml", "modality": { "in": [ @@ -7737,7 +8025,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.groq.GroqChat": { "description": "Wrapper for Groq-hosted LLM models.", @@ -7771,6 +8059,7 @@ "vary_temp_each_call": true }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Groq", "modality": { "in": [ @@ -7782,7 +8071,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-12 16:14:56 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.guardrails.NeMoGuardrails": { "description": "Generator wrapper for NeMo Guardrails.", @@ -7795,6 +8084,9 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [ + "nemoguardrails" + ], "generator_family_name": "Guardrails", "modality": { "in": [ @@ -7806,7 +8098,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.huggingface.InferenceAPI": { "description": "Get text generations from Hugging Face Inference API", @@ -7822,6 +8114,7 @@ "wait_for_model": false }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Hugging Face \ud83e\udd17 Inference API", "modality": { "in": [ @@ -7833,7 +8126,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-09-16 15:38:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.huggingface.InferenceEndpoint": { "description": "Interface for Hugging Face private endpoints", @@ -7849,6 +8142,7 @@ "wait_for_model": false }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Hugging Face \ud83e\udd17 Inference API", "modality": { "in": [ @@ -7860,7 +8154,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-09-16 15:38:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.huggingface.LLaVA": { "description": "Get LLaVA ([ text + image ] -> text) generations", @@ -7878,6 +8172,9 @@ } }, "active": true, + "extra_dependency_names": [ + "pillow" + ], "generator_family_name": null, "modality": { "in": [ @@ -7890,7 +8187,7 @@ }, "parallel_capable": false, "supports_multiple_generations": false, - "mod_time": "2025-09-16 15:38:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.huggingface.Model": { "description": "Get text generations from a locally-run Hugging Face model", @@ -7908,6 +8205,7 @@ } }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Hugging Face \ud83e\udd17 model", "modality": { "in": [ @@ -7919,7 +8217,7 @@ }, "parallel_capable": false, "supports_multiple_generations": true, - "mod_time": "2025-09-16 15:38:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.huggingface.OptimumPipeline": { "description": "Get text generations from a locally-run Hugging Face pipeline using NVIDIA Optimum", @@ -7937,6 +8235,9 @@ } }, "active": true, + "extra_dependency_names": [ + "optimum-nvidia" + ], "generator_family_name": "NVIDIA Optimum Hugging Face \ud83e\udd17 pipeline", "modality": { "in": [ @@ -7948,7 +8249,7 @@ }, "parallel_capable": false, "supports_multiple_generations": true, - "mod_time": "2025-09-16 15:38:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.huggingface.Pipeline": { "description": "Get text generations from a locally-run Hugging Face pipeline", @@ -7966,6 +8267,7 @@ } }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Hugging Face \ud83e\udd17 pipeline", "modality": { "in": [ @@ -7977,7 +8279,7 @@ }, "parallel_capable": false, "supports_multiple_generations": true, - "mod_time": "2025-09-16 15:38:12 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.langchain.LangChainLLMGenerator": { "description": "Class supporting LangChain LLM interfaces", @@ -7996,6 +8298,9 @@ "stop": [] }, "active": true, + "extra_dependency_names": [ + "langchain.llms" + ], "generator_family_name": "LangChain", "modality": { "in": [ @@ -8007,7 +8312,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.langchain_serve.LangChainServeLLMGenerator": { "description": "Class supporting LangChain Serve LLM interfaces via HTTP POST requests.", @@ -8021,6 +8326,7 @@ "config_hash": "default" }, "active": true, + "extra_dependency_names": [], "generator_family_name": "LangChainServe", "modality": { "in": [ @@ -8032,7 +8338,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.litellm.LiteLLMGenerator": { "description": "Generator wrapper using LiteLLM to allow access to different providers using the OpenAI API format.", @@ -8049,9 +8355,13 @@ "stop": [ "#", ";" - ] + ], + "verbose": false }, "active": true, + "extra_dependency_names": [ + "litellm" + ], "generator_family_name": "LiteLLM", "modality": { "in": [ @@ -8063,7 +8373,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.mistral.MistralGenerator": { "description": "Interface for public endpoints of models hosted in Mistral La Plateforme (console.mistral.ai).", @@ -8077,6 +8387,9 @@ "name": "mistral-large-latest" }, "active": true, + "extra_dependency_names": [ + "mistralai" + ], "generator_family_name": "mistral", "modality": { "in": [ @@ -8088,7 +8401,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:34:35 +0000" }, "generators.nemo.NeMoGenerator": { "description": "Wrapper for the NVIDIA NeMo models via NGC. Expects NGC_API_KEY and ORG_ID environment variables.", @@ -8108,6 +8421,9 @@ "api_uri": "https://api.llm.ngc.nvidia.com/v1" }, "active": true, + "extra_dependency_names": [ + "nemollm" + ], "generator_family_name": "NeMo", "modality": { "in": [ @@ -8119,7 +8435,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.nim.NVMultimodal": { "description": "Wrapper for text and image / audio to text NVIDIA NIM microservices hosted on build.nvidia.com and self-hosted.", @@ -8152,6 +8468,7 @@ "max_input_len": 180000 }, "active": true, + "extra_dependency_names": [], "generator_family_name": "NIM", "modality": { "in": [ @@ -8165,7 +8482,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-09-05 19:48:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.nim.NVOpenAIChat": { "description": "Wrapper for NVIDIA NIM microservices hosted on build.nvidia.com and self-hosted.", @@ -8197,6 +8514,7 @@ "vary_temp_each_call": true }, "active": true, + "extra_dependency_names": [], "generator_family_name": "NIM", "modality": { "in": [ @@ -8208,7 +8526,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-09-05 19:48:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.nim.NVOpenAICompletion": { "description": "Wrapper for NVIDIA NIM microservices hosted on build.nvidia.com and self-hosted.", @@ -8240,6 +8558,7 @@ "vary_temp_each_call": true }, "active": true, + "extra_dependency_names": [], "generator_family_name": "NIM", "modality": { "in": [ @@ -8251,7 +8570,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-09-05 19:48:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.nim.Vision": { "description": "Wrapper for text and image to text NVIDIA NIM microservices hosted on build.nvidia.com and self-hosted.", @@ -8284,6 +8603,7 @@ "max_input_len": 180000 }, "active": true, + "extra_dependency_names": [], "generator_family_name": "NIM", "modality": { "in": [ @@ -8296,7 +8616,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-09-05 19:48:46 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.nvcf.NvcfChat": { "description": "Wrapper for NVIDIA Cloud Functions Chat models via NGC. Expects NVCF_API_KEY environment variable.", @@ -8318,6 +8638,7 @@ } }, "active": true, + "extra_dependency_names": [], "generator_family_name": "NVCF", "modality": { "in": [ @@ -8329,7 +8650,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.nvcf.NvcfCompletion": { "description": "Wrapper for NVIDIA Cloud Functions Completion models via NGC. Expects NVCF_API_KEY environment variables.", @@ -8351,6 +8672,7 @@ } }, "active": true, + "extra_dependency_names": [], "generator_family_name": "NVCF", "modality": { "in": [ @@ -8362,7 +8684,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.ollama.OllamaGenerator": { "description": "Interface for Ollama endpoints", @@ -8377,6 +8699,9 @@ "host": "127.0.0.1:11434" }, "active": true, + "extra_dependency_names": [ + "ollama" + ], "generator_family_name": "Ollama", "modality": { "in": [ @@ -8388,7 +8713,7 @@ }, "parallel_capable": false, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:34:08 +0000" }, "generators.ollama.OllamaGeneratorChat": { "description": "Interface for Ollama endpoints, using the chat functionality", @@ -8403,6 +8728,9 @@ "host": "127.0.0.1:11434" }, "active": true, + "extra_dependency_names": [ + "ollama" + ], "generator_family_name": "Ollama", "modality": { "in": [ @@ -8414,7 +8742,7 @@ }, "parallel_capable": false, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:34:08 +0000" }, "generators.openai.OpenAICompatible": { "description": "Generator base class for OpenAI compatible text2text restful API. Implements shared initialization and execution methods.", @@ -8439,6 +8767,7 @@ "extra_params": {} }, "active": true, + "extra_dependency_names": [], "generator_family_name": "OpenAICompatible", "modality": { "in": [ @@ -8450,7 +8779,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.openai.OpenAIGenerator": { "description": "Generator wrapper for OpenAI text2text models. Expects API key in the OPENAI_API_KEY environment variable", @@ -8474,6 +8803,7 @@ "extra_params": {} }, "active": true, + "extra_dependency_names": [], "generator_family_name": "OpenAI", "modality": { "in": [ @@ -8485,7 +8815,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.openai.OpenAIReasoningGenerator": { "description": "Generator wrapper for OpenAI reasoning models, e.g. `o1` family.", @@ -8514,6 +8844,7 @@ "max_completion_tokens": 1500 }, "active": true, + "extra_dependency_names": [], "generator_family_name": "OpenAI", "modality": { "in": [ @@ -8525,7 +8856,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-26 14:02:31 +0000" + "mod_time": "2025-09-25 10:32:45 +0000" }, "generators.rasa.RasaRestGenerator": { "description": "API interface for RASA models", @@ -8553,6 +8884,7 @@ "verify_ssl": true }, "active": true, + "extra_dependency_names": [], "generator_family_name": "RASA", "modality": { "in": [ @@ -8564,7 +8896,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-18 17:35:24 +0000" + "mod_time": "2025-08-19 13:25:16 +0000" }, "generators.replicate.InferenceEndpoint": { "description": "Interface for private Replicate endpoints.", @@ -8579,6 +8911,9 @@ "repetition_penalty": 1 }, "active": true, + "extra_dependency_names": [ + "replicate" + ], "generator_family_name": "Replicate", "modality": { "in": [ @@ -8590,7 +8925,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.replicate.ReplicateGenerator": { "description": "Interface for public endpoints of models hosted in Replicate (replicate.com).", @@ -8605,6 +8940,9 @@ "repetition_penalty": 1 }, "active": true, + "extra_dependency_names": [ + "replicate" + ], "generator_family_name": "Replicate", "modality": { "in": [ @@ -8616,7 +8954,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.rest.RestGenerator": { "description": "Generic API interface for REST models", @@ -8641,6 +8979,7 @@ "verify_ssl": true }, "active": true, + "extra_dependency_names": [], "generator_family_name": "REST", "modality": { "in": [ @@ -8652,7 +8991,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "generators.test.Blank": { "description": "This generator always returns the empty string.", @@ -8665,6 +9004,7 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Test", "modality": { "in": [ @@ -8676,7 +9016,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-06-18 19:31:49 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.test.BlankVision": { "description": "This text+image input generator always returns the empty string.", @@ -8689,6 +9029,7 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Test", "modality": { "in": [ @@ -8701,7 +9042,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-06-18 19:31:49 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.test.Lipsum": { "description": "Lorem Ipsum generator, so we can get non-zero outputs that vary", @@ -8714,6 +9055,7 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Test", "modality": { "in": [ @@ -8725,7 +9067,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-18 19:31:49 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.test.Repeat": { "description": "This generator returns the last message from input that was posed to it.", @@ -8738,6 +9080,7 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Test", "modality": { "in": [ @@ -8749,7 +9092,7 @@ }, "parallel_capable": true, "supports_multiple_generations": true, - "mod_time": "2025-06-18 19:31:49 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.test.Single": { "description": "This generator returns the a fixed string and does not support multiple generations.", @@ -8762,6 +9105,7 @@ "skip_seq_end": null }, "active": true, + "extra_dependency_names": [], "generator_family_name": "Test", "modality": { "in": [ @@ -8773,7 +9117,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-06-18 19:31:49 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "generators.watsonx.WatsonXGenerator": { "description": "This is a generator for watsonx.ai.", @@ -8792,6 +9136,7 @@ "bearer_token": "" }, "active": true, + "extra_dependency_names": [], "generator_family_name": "watsonx", "modality": { "in": [ @@ -8803,7 +9148,7 @@ }, "parallel_capable": true, "supports_multiple_generations": false, - "mod_time": "2025-08-08 14:23:52 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" } }, "harnesses": { @@ -8813,21 +9158,24 @@ "strict_modality_match": false }, "active": true, - "mod_time": "2025-05-07 20:12:25 +0000" + "extra_dependency_names": [], + "mod_time": "2025-09-25 10:03:36 +0000" }, "harnesses.probewise.ProbewiseHarness": { "DEFAULT_PARAMS": { "strict_modality_match": false }, "active": true, - "mod_time": "2025-03-10 22:23:53 +0000" + "extra_dependency_names": [], + "mod_time": "2025-09-09 09:30:51 +0000" }, "harnesses.pxd.PxD": { "DEFAULT_PARAMS": { "strict_modality_match": false }, "active": true, - "mod_time": "2025-03-10 22:23:53 +0000" + "extra_dependency_names": [], + "mod_time": "2025-09-09 09:30:51 +0000" } }, "buffs": { @@ -8836,40 +9184,45 @@ "DEFAULT_PARAMS": {}, "active": true, "doc_uri": "", + "extra_dependency_names": [], "lang": null, - "mod_time": "2025-04-17 08:22:12 +0000" + "mod_time": "2025-09-25 10:03:36 +0000" }, "buffs.encoding.Base64": { "description": "Base64 buff", "DEFAULT_PARAMS": {}, "active": true, "doc_uri": "", + "extra_dependency_names": [], "lang": null, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "buffs.encoding.CharCode": { "description": "CharCode buff", "DEFAULT_PARAMS": {}, "active": true, "doc_uri": "", + "extra_dependency_names": [], "lang": null, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "buffs.low_resource_languages.LRLBuff": { "description": "Low Resource Language buff", "DEFAULT_PARAMS": {}, "active": true, "doc_uri": "https://arxiv.org/abs/2310.02446", + "extra_dependency_names": [], "lang": null, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "buffs.lowercase.Lowercase": { "description": "Lowercasing buff", "DEFAULT_PARAMS": {}, "active": true, "doc_uri": "", + "extra_dependency_names": [], "lang": null, - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "buffs.paraphrase.Fast": { "description": "CPU-friendly paraphrase buff based on Humarin's T5 paraphraser", @@ -8882,8 +9235,9 @@ }, "active": true, "doc_uri": "https://huggingface.co/humarin/chatgpt_paraphraser_on_T5_base", + "extra_dependency_names": [], "lang": "en", - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" }, "buffs.paraphrase.PegasusT5": { "description": "Paraphrasing buff using Pegasus model", @@ -8898,8 +9252,9 @@ }, "active": true, "doc_uri": "https://huggingface.co/tuner007/pegasus_paraphrase", + "extra_dependency_names": [], "lang": "en", - "mod_time": "2025-06-17 16:15:39 +0000" + "mod_time": "2025-09-08 12:24:33 +0000" } } -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 44a37a329..32d4bc5fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,9 +72,7 @@ dependencies = [ "datasets>=3.0.0,<4.0", "colorama>=0.4.3", "tqdm>=4.67.1", - "cohere>=5.16.0", "openai>=1.45.0,<2", - "replicate>=0.8.3", "google-api-python-client>=2.0", "backoff>=2.1.1", "rapidfuzz>=3.0.0", @@ -83,8 +81,6 @@ dependencies = [ "accelerate>=0.23.0", "avidtools==0.1.2", "stdlibs>=2022.10.9", - "langchain>=0.3.25", - "nemollm>=0.3.0", "cmd2==2.4.3", "torch>=2.6.0", "sentencepiece>=0.1.99", @@ -94,7 +90,6 @@ dependencies = [ "ecoji>=0.1.1", "deepl==1.17.0", "fschat>=0.2.36", - "litellm>=1.68.1", "jsonpath-ng>=1.6.1", "huggingface_hub>=0.21.0", 'python-magic-bin>=0.4.14; sys_platform == "win32"', @@ -102,14 +97,12 @@ dependencies = [ "lorem==0.1.1", "xdg-base-dirs>=6.0.1", "wn==0.9.5", - "ollama>=0.4.7", "nvidia-riva-client==2.16.0", "google-cloud-translate>=2.0.4", "grpcio-tools>=1.71.0", "langdetect==1.0.9", "tiktoken>=0.7.0", - "mistralai==1.5.2", - "pillow>=10.4.0", + "jsonschema>=4.21.0", "ftfy>=6.3.1", ] @@ -127,14 +120,50 @@ lint = [ "black==24.4.2", "pylint>=3.1.0", ] +dev = [ + "garak[tests,lint]", +] calibration = [ "scipy>=1.14.0", ] -audio = [ +all_plugins = [ + "garak[plugin_cohere,plugin_langchain,plugin_llava,plugin_litellm,plugin_mistralai,plugin_nemoguardrails,plugin_nemollm,plugin_ollama,plugin_optimum,plugin_replicate,plugin_audio,plugin_dra]" +] +plugin_cohere = [ + "cohere>=5.16.0" +] +plugin_langchain = [ + "langchain>=0.3.25", +] +plugin_llava = [ + "pillow>=10.4.0", +] +plugin_litellm = [ + "litellm>=1.68.1", +] +plugin_mistralai = [ + "mistralai==1.5.2", +] +plugin_nemoguardrails = [ + "nemoguardrails>=0.11.0", +] +plugin_nemollm = [ + "nemollm>=0.3.0", +] +plugin_ollama = [ + "ollama>=0.4.7", +] +plugin_optimum = [ + "optimum-nvidia>=0.1.0b5", +] +plugin_replicate = [ + "replicate>=0.8.3", +] +plugin_audio = [ "soundfile>=0.13.1", "librosa>=0.10.2" ] -dra = [ +plugin_dra = [ "detoxify>=0.5.0" ] [project.urls] diff --git a/requirements.txt b/requirements.txt index ee91e707c..45704c480 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,7 @@ transformers>=4.51.3 datasets>=3.0.0,<4.0 colorama>=0.4.3 tqdm>=4.67.1 -cohere>=5.16.0 openai>=1.45.0,<2 -replicate>=0.8.3 google-api-python-client>=2.0 backoff>=2.1.1 rapidfuzz>=3.0.0 @@ -14,8 +12,6 @@ nltk>=3.9.1 accelerate>=0.23.0 avidtools==0.1.2 stdlibs>=2022.10.9 -langchain>=0.3.25 -nemollm>=0.3.0 cmd2==2.4.3 torch>=2.6.0 sentencepiece>=0.1.99 @@ -25,7 +21,6 @@ zalgolib>=0.2.2 ecoji>=0.1.1 deepl==1.17.0 fschat>=0.2.36 -litellm>=1.68.1 jsonpath-ng>=1.6.1 huggingface_hub>=0.21.0 python-magic-bin>=0.4.14; sys_platform == "win32" @@ -33,14 +28,12 @@ python-magic>=0.4.21; sys_platform != "win32" lorem==0.1.1 xdg-base-dirs>=6.0.1 wn==0.9.5 -ollama>=0.4.7 nvidia-riva-client==2.16.0 google-cloud-translate>=2.0.4 grpcio-tools>=1.71.0 langdetect==1.0.9 tiktoken>=0.7.0 -mistralai==1.5.2 -pillow>=10.4.0 +jsonschema>=4.21.0 ftfy>=6.3.1 # tests pytest>=8.0 @@ -55,7 +48,3 @@ black==24.4.2 pylint>=3.1.0 # calibration scipy>=1.14.0 -# audio -soundfile>=0.13.1 -librosa>=0.10.2 -detoxify>=0.5.0 \ No newline at end of file diff --git a/tests/buffs/test_buffs.py b/tests/buffs/test_buffs.py index b81f16cc7..5bca2bdd6 100644 --- a/tests/buffs/test_buffs.py +++ b/tests/buffs/test_buffs.py @@ -21,7 +21,7 @@ def test_buff_structure(classname): # any parameter that has a default must be supported unsupported_defaults = [] if c._supported_params is not None: - if hasattr(g, "DEFAULT_PARAMS"): + if hasattr(c, "DEFAULT_PARAMS"): for k, _ in c.DEFAULT_PARAMS.items(): if k not in c._supported_params: unsupported_defaults.append(k) diff --git a/tests/detectors/test_detectors.py b/tests/detectors/test_detectors.py index 069ccefbd..aa17e2f54 100644 --- a/tests/detectors/test_detectors.py +++ b/tests/detectors/test_detectors.py @@ -89,7 +89,9 @@ def test_detector_detect(classname): ), "detect() should return an ordered iterable" for entry in results: # detect() should skip non-evaluated items from output. NB this breaks output:result alignment - assert isinstance(entry, float) or entry is None, "detect() must return a list of floats or Nones" + assert ( + isinstance(entry, float) or entry is None + ), "detect() must return a list of floats or Nones" if isinstance(entry, float): assert 0.0 <= entry <= 1.0, "detect() values should be between 0.0 and 1.0" assert len(list(results)) <= len( diff --git a/tests/generators/test_cohere.py b/tests/generators/test_cohere.py index bcdf39dd9..8bab3f030 100644 --- a/tests/generators/test_cohere.py +++ b/tests/generators/test_cohere.py @@ -8,6 +8,15 @@ from garak.generators.cohere import CohereGenerator, COHERE_GENERATION_LIMIT from garak.exception import APIKeyMissingError +try: + import cohere + +except: + pytest.skip( + "couldn't import cohere, skipping cohere tests", allow_module_level=True + ) + + # Default model name and API URLs DEFAULT_MODEL_NAME = "command" COHERE_API_BASE = "https://api.cohere.com" diff --git a/tests/generators/test_generators.py b/tests/generators/test_generators.py index 0450a2033..d0d7182b2 100644 --- a/tests/generators/test_generators.py +++ b/tests/generators/test_generators.py @@ -119,7 +119,13 @@ def test_instantiate_generators(classname): setattr(config_root, category, gen_config) m = importlib.import_module("garak." + ".".join(classname.split(".")[:-1])) - g = getattr(m, classname.split(".")[-1])(config_root=config_root) + klass = getattr(m, classname.split(".")[-1]) + try: + g = klass(config_root=config_root) + except ModuleNotFoundError: + pytest.skip( + "dependencies not present; requires " + repr(klass.extra_dependency_names) + ) assert isinstance(g, Generator) diff --git a/tests/generators/test_langchain_serve.py b/tests/generators/test_langchain_serve.py index c9781463b..d99105b7f 100644 --- a/tests/generators/test_langchain_serve.py +++ b/tests/generators/test_langchain_serve.py @@ -1,3 +1,4 @@ +import importlib import os import pytest diff --git a/tests/generators/test_litellm.py b/tests/generators/test_litellm.py index fe9802109..15f44fcfd 100644 --- a/tests/generators/test_litellm.py +++ b/tests/generators/test_litellm.py @@ -1,3 +1,4 @@ +import importlib import pytest from os import getenv @@ -7,6 +8,12 @@ from garak.generators.litellm import LiteLLMGenerator +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in LiteLLMGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( getenv("OPENAI_API_KEY", None) is None, reason="OpenAI API key is not set in OPENAI_API_KEY", @@ -26,6 +33,12 @@ def test_litellm_openai(): assert isinstance(item, Message) +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in LiteLLMGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( getenv("OPENROUTER_API_KEY", None) is None, reason="OpenRouter API key is not set in OPENROUTER_API_KEY", @@ -43,6 +56,12 @@ def test_litellm_openrouter(): assert isinstance(item, Message) +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in LiteLLMGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) def test_litellm_model_detection(): custom_config = { "generators": { diff --git a/tests/generators/test_llava.py b/tests/generators/test_llava.py index 5a5351676..315d752a3 100644 --- a/tests/generators/test_llava.py +++ b/tests/generators/test_llava.py @@ -1,13 +1,21 @@ import pytest import torch -from PIL import Image, ImageDraw from unittest.mock import patch, MagicMock from garak.attempt import Conversation, Turn, Message from garak._config import GarakSubConfig -from garak.generators.huggingface import LLaVA from garak.exception import ModelNameMissingError +try: + from PIL import Image, ImageDraw + from garak.generators.huggingface import LLaVA + +except: + pytest.skip( + "couldn't import LLaVA and deps, skipping llava tests", allow_module_level=True + ) + + # ─── Constants ───────────────────────────────────────────────────────── SUPPORTED_MODELS = LLaVA.supported_models @@ -64,7 +72,6 @@ def mock_hf_when_cpu(monkeypatch): ) - # ─── Tests ───────────────────────────────────────────────────────────── @@ -97,7 +104,6 @@ def test_llava_generate_returns_decoded_text( assert isinstance(out, list) and out == [Message("decoded output")] - def test_llava_error_on_missing_image(llava_config): llava = LLaVA(name=SUPPORTED_MODELS[0], config_root=llava_config) conv = Conversation( @@ -107,7 +113,6 @@ def test_llava_error_on_missing_image(llava_config): llava.generate(conv) - def test_llava_unsupported_model(llava_config): """Test that instantiating with an unsupported model name raises ModelNameMissingError.""" with pytest.raises(ModelNameMissingError) as excinfo: diff --git a/tests/generators/test_mistral.py b/tests/generators/test_mistral.py index 5943cee34..a4a6a808d 100644 --- a/tests/generators/test_mistral.py +++ b/tests/generators/test_mistral.py @@ -1,6 +1,7 @@ +import httpx +import importlib import os import pytest -import httpx from unittest.mock import patch from garak.attempt import Message, Turn, Conversation from garak.generators.mistral import MistralGenerator @@ -22,6 +23,12 @@ def restore_env(): request.addfinalizer(restore_env) +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in MistralGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.usefixtures("set_fake_env") @pytest.mark.respx(base_url="https://api.mistral.ai/v1") def test_mistral_generator(respx_mock, mistral_compat_mocks): @@ -38,6 +45,12 @@ def test_mistral_generator(respx_mock, mistral_compat_mocks): assert len(output) == 1 # expect 1 generation by default +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in MistralGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( os.getenv(MistralGenerator.ENV_VAR, None) is None, reason=f"Mistral API key is not set in {MistralGenerator.ENV_VAR}", diff --git a/tests/generators/test_muiltiprocessing_support.py b/tests/generators/test_muiltiprocessing_support.py index d25752897..b320d5778 100644 --- a/tests/generators/test_muiltiprocessing_support.py +++ b/tests/generators/test_muiltiprocessing_support.py @@ -15,11 +15,11 @@ GENERATORS = [ "generators.huggingface.InferenceAPI", "generators.huggingface.InferenceEndpoint", - "generators.mistral.MistralGenerator", +# "generators.mistral.MistralGenerator", "generators.nvcf.NvcfChat", "generators.nvcf.NvcfCompletion", - "generators.replicate.InferenceEndpoint", - "generators.replicate.ReplicateGenerator", +# "generators.replicate.InferenceEndpoint", +# "generators.replicate.ReplicateGenerator", ] MODEL_NAME = "gpt-3.5-turbo-instruct" diff --git a/tests/generators/test_ollama.py b/tests/generators/test_ollama.py index c1524ed3f..5d8d1dd36 100644 --- a/tests/generators/test_ollama.py +++ b/tests/generators/test_ollama.py @@ -1,5 +1,6 @@ +import importlib import pytest -import ollama +import respx import httpx from garak.attempt import Message, Turn, Conversation @@ -10,7 +11,18 @@ ) OLLAMA_SERVER_UP = False +try: + import ollama +except: + pytest.skip( + "couldn't import ollama, skipping ollama tests", allow_module_level=True + ) + +@pytest.mark.skipif( + importlib.util.find_spec("ollama") is None, + reason="requires 'ollama' Python module to be installed", +) def ollama_is_running(): global PINGED_OLLAMA_SERVER global OLLAMA_SERVER_UP @@ -29,14 +41,14 @@ def ollama_is_running(): def no_models(): # In newer versions of ollama, list() returns a ListResponse object response = ollama.list() - + try: # Try to access the models attribute or property - models = getattr(response, 'models', None) + models = getattr(response, "models", None) if models is None: # If no models attribute, try using it as a dict - models = response.get('models', []) - + models = response.get("models", []) + # Check if models is empty return len(models) == 0 except (AttributeError, TypeError): @@ -44,6 +56,15 @@ def no_models(): return True +@pytest.mark.skipif( + not all( + [ + importlib.util.find_spec(m) + for m in OllamaGeneratorChat.extra_dependency_names + ] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( not ollama_is_running(), reason=f"Ollama server is not currently running", @@ -56,6 +77,12 @@ def test_error_on_nonexistant_model_chat(): gen.generate(conv) +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in OllamaGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( not ollama_is_running(), reason=f"Ollama server is not currently running", @@ -68,6 +95,15 @@ def test_error_on_nonexistant_model(): gen.generate(conv) +@pytest.mark.skipif( + not all( + [ + importlib.util.find_spec(m) + for m in OllamaGeneratorChat.extra_dependency_names + ] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( not ollama_is_running(), reason=f"Ollama server is not currently running", @@ -87,6 +123,12 @@ def test_generation_on_pulled_model_chat(): assert all(len(response.text) > 0 for response in responses) +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in OllamaGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.skipif( not ollama_is_running(), reason=f"Ollama server is not currently running", @@ -106,6 +148,12 @@ def test_generation_on_pulled_model(): assert all(len(response.text) > 0 for response in responses) +@pytest.mark.skipif( + not all( + [importlib.util.find_spec(m) for m in OllamaGenerator.extra_dependency_names] + ), + reason="missing optional dependency", +) @pytest.mark.respx(base_url="http://" + OllamaGenerator.DEFAULT_PARAMS["host"]) def test_ollama_generation_mocked(respx_mock): mock_response = {"model": "mistral", "response": "Hello how are you?"} @@ -118,6 +166,15 @@ def test_ollama_generation_mocked(respx_mock): assert generation == [Message("Hello how are you?")] +@pytest.mark.skipif( + not all( + [ + importlib.util.find_spec(m) + for m in OllamaGeneratorChat.extra_dependency_names + ] + ), + reason="missing optional dependency", +) @pytest.mark.respx(base_url="http://" + OllamaGenerator.DEFAULT_PARAMS["host"]) def test_ollama_generation_chat_mocked(respx_mock): mock_response = { diff --git a/tests/harnesses/test_harnesses.py b/tests/harnesses/test_harnesses.py index 654623f72..fee10ecac 100644 --- a/tests/harnesses/test_harnesses.py +++ b/tests/harnesses/test_harnesses.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize("classname", HARNESSES) -def test_buff_structure(classname): +def test_harness_structure(classname): m = importlib.import_module("garak." + ".".join(classname.split(".")[:-1])) c = getattr(m, classname.split(".")[-1]) @@ -22,7 +22,7 @@ def test_buff_structure(classname): # any parameter that has a default must be supported unsupported_defaults = [] if c._supported_params is not None: - if hasattr(g, "DEFAULT_PARAMS"): + if hasattr(c, "DEFAULT_PARAMS"): for k, _ in c.DEFAULT_PARAMS.items(): if k not in c._supported_params: unsupported_defaults.append(k) diff --git a/tests/plugins/test_plugin_load.py b/tests/plugins/test_plugin_load.py index 53ef1c648..186440a1d 100644 --- a/tests/plugins/test_plugin_load.py +++ b/tests/plugins/test_plugin_load.py @@ -3,7 +3,11 @@ import garak from garak import _plugins, _config -import garak.generators +import garak.buffs.base +import garak.detectors.base +import garak.generators.base +import garak.harnesses.base +import garak.probes.base PROBES = [classname for (classname, active) in _plugins.enumerate_plugins("probes")] @@ -37,33 +41,48 @@ def plugin_configuration(classname): @pytest.mark.parametrize("classname", PROBES) def test_instantiate_probes(plugin_configuration): classname, config_root = plugin_configuration - g = _plugins.load_plugin(classname, config_root=config_root) - assert isinstance(g, garak.probes.base.Probe) + try: + p = _plugins.load_plugin(classname, config_root=config_root) + except ModuleNotFoundError: + pytest.skip("required deps not present") + assert isinstance(p, garak.probes.base.Probe) @pytest.mark.parametrize("classname", DETECTORS) def test_instantiate_detectors(plugin_configuration): classname, config_root = plugin_configuration - g = _plugins.load_plugin(classname, config_root=config_root) - assert isinstance(g, garak.detectors.base.Detector) + try: + d = _plugins.load_plugin(classname, config_root=config_root) + except ModuleNotFoundError: + pytest.skip("required deps not present") + assert isinstance(d, garak.detectors.base.Detector) @pytest.mark.parametrize("classname", HARNESSES) def test_instantiate_harnesses(plugin_configuration): classname, config_root = plugin_configuration - g = _plugins.load_plugin(classname, config_root=config_root) - assert isinstance(g, garak.harnesses.base.Harness) + try: + h = _plugins.load_plugin(classname, config_root=config_root) + except ModuleNotFoundError: + pytest.skip("required deps not present") + assert isinstance(h, garak.harnesses.base.Harness) @pytest.mark.parametrize("classname", BUFFS) def test_instantiate_buffs(plugin_configuration): classname, config_root = plugin_configuration - g = _plugins.load_plugin(classname, config_root=config_root) - assert isinstance(g, garak.buffs.base.Buff) + try: + b = _plugins.load_plugin(classname, config_root=config_root) + except ModuleNotFoundError: + pytest.skip("required deps not present") + assert isinstance(b, garak.buffs.base.Buff) @pytest.mark.parametrize("classname", GENERATORS) def test_instantiate_generators(plugin_configuration): classname, config_root = plugin_configuration - g = _plugins.load_plugin(classname, config_root=config_root) + try: + g = _plugins.load_plugin(classname, config_root=config_root) + except ModuleNotFoundError: + pytest.skip("required deps not present") assert isinstance(g, garak.generators.base.Generator) diff --git a/tests/plugins/test_plugins.py b/tests/plugins/test_plugins.py new file mode 100644 index 000000000..05e13eab2 --- /dev/null +++ b/tests/plugins/test_plugins.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib +import pytest + +import garak._plugins + + +PLUGINS = [] +for plugin_type in garak._plugins.PLUGIN_TYPES: + PLUGINS += [classname for (classname, active) in garak._plugins.enumerate_plugins(plugin_type)] + +@pytest.mark.parametrize("classname", PLUGINS) +def test_plugin_structure(classname): + + m = importlib.import_module("garak." + ".".join(classname.split(".")[:-1])) + c = getattr(m, classname.split(".")[-1]) + + # extra dependency modules is a list + assert hasattr(c, "extra_dependency_names") and isinstance( + c.extra_dependency_names, list + ), "extra_dependency_names must be a list" \ No newline at end of file diff --git a/tests/probes/test_probes.py b/tests/probes/test_probes.py index 71b7d12b3..aca14b7df 100644 --- a/tests/probes/test_probes.py +++ b/tests/probes/test_probes.py @@ -8,6 +8,7 @@ from garak import _config, _plugins import garak.probes + PROBES = [classname for (classname, active) in _plugins.enumerate_plugins("probes")] DETECTORS = [ @@ -62,7 +63,7 @@ def test_probe_structure(classname): # any parameter that has a default must be supported unsupported_defaults = [] if c._supported_params is not None: - if hasattr(g, "DEFAULT_PARAMS"): + if hasattr(c, "DEFAULT_PARAMS"): for k, _ in c.DEFAULT_PARAMS.items(): if k not in c._supported_params: unsupported_defaults.append(k) @@ -71,10 +72,15 @@ def test_probe_structure(classname): @pytest.mark.parametrize("classname", PROBES) def test_probe_metadata(classname): - p = _plugins.load_plugin(classname) + try: + p = _plugins.load_plugin(classname) + except ModuleNotFoundError: + pytest.skip("required deps not present") assert isinstance(p.goal, str), "probe goals should be a text string" assert len(p.goal) > 0, "probes must state their general goal" - assert p.lang is not None and (p.lang == "*" or langcodes.tag_is_valid(p.lang)), "lang must be either * or a BCP47 code" + assert p.lang is not None and ( + p.lang == "*" or langcodes.tag_is_valid(p.lang) + ), "lang must be either * or a BCP47 code" assert isinstance( p.doc_uri, str ), "probes should give a doc uri describing/citing the attack" @@ -87,6 +93,10 @@ def test_probe_metadata(classname): assert isinstance(p.modality["in"], set), "modality descriptors must be sets" assert p.tier is not None, "probe tier must be specified" assert isinstance(p.tier, garak.probes.Tier), "probe tier must be one of type Tier'" + if p.active: + assert ( + p.extra_dependency_names == [] + ), "active must be False for Probes requiring external modules, so that they're not run by default" @pytest.mark.parametrize("plugin_name", PROBES) diff --git a/tests/test_reqs.py b/tests/test_reqs.py index dd5be9ceb..8bd1fcc5b 100644 --- a/tests/test_reqs.py +++ b/tests/test_reqs.py @@ -1,13 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import importlib import pytest +import re try: import tomllib except: tomllib = None +import garak._plugins + @pytest.mark.skipif( tomllib is None, reason="No tomllib found (available from Python 3.11)" @@ -22,17 +26,124 @@ def test_requirements_txt_pyproject_toml(): with open("pyproject.toml", "rb") as pyproject_file: pyproject_toml = tomllib.load(pyproject_file) pyproject_reqs = pyproject_toml["project"]["dependencies"] - for test_deps in pyproject_toml["project"]["optional-dependencies"].values(): - for dep in test_deps: - pyproject_reqs.append(dep) + for test_group in pyproject_toml["project"]["optional-dependencies"]: + if not test_group.startswith("plugin_") and not pyproject_toml["project"][ + "optional-dependencies" + ][test_group][0].startswith("garak["): + test_deps = pyproject_toml["project"]["optional-dependencies"][ + test_group + ] + for dep in test_deps: + pyproject_reqs.append(dep) pyproject_reqs.sort() # assert len(reqtxt_reqs) == len(pyproject_reqs) # same number of requirements + spurious_req = set(reqtxt_reqs) - set(pyproject_reqs) assert ( - set(reqtxt_reqs) - set(pyproject_reqs) == set() - ) # things in reqtxt but not in pyproject + spurious_req == set() + ), f"spurious items in requirements.txt, {spurious_req}" # things in reqtxt but not in pyproject + spurious_pyproject = set(pyproject_reqs) - set(reqtxt_reqs) assert ( - set(pyproject_reqs) - set(reqtxt_reqs) == set() - ) # things in pyproject but not in reqtxt + spurious_pyproject == set() + ), f"spurious items in pyproject.toml, {spurious_pyproject}" # things in pyproject but not in reqtxt assert ( reqtxt_reqs == pyproject_reqs - ) # final check. this one is actually enough, but let's help us debug by finding which test fails, ok? + ), "requirements.txt/pyproject.toml#dependencies mismatch. are plugin sections prefixed plugin_ ?" # final check. this one is actually enough, but let's help us debug by finding which test fails, ok? + + +PLUGIN_TYPES = garak._plugins.PLUGIN_TYPES + + +def plugin_names(): + plugin_names = set() + for plugin_type in PLUGIN_TYPES: + plugin_names.update( + [n for (n, active) in garak._plugins.enumerate_plugins(plugin_type)] + ) + return plugin_names + + +def split_out_module_name(pep508_descr: str): + return re.split(r"[\<\>\=\!]", pep508_descr)[0] + + +def requirement_names(): + with open("requirements.txt", "r", encoding="utf-8") as req_file: + reqtxt_reqs = req_file.readlines() + reqtxt_reqs = list( + filter(lambda x: not x.startswith("#"), map(str.strip, reqtxt_reqs)) + ) + requirement_names = set([split_out_module_name(r) for r in reqtxt_reqs]) + return requirement_names + + +def pyproject_optional_dep_names(): + with open("pyproject.toml", "rb") as pyproject_file: + pyproject_toml = tomllib.load(pyproject_file) + optional_deps = set() + for test_group in pyproject_toml["project"]["optional-dependencies"]: + if test_group.startswith("plugin_"): + group_dep_names = [ + split_out_module_name(r) + for r in pyproject_toml["project"]["optional-dependencies"][test_group] + ] + optional_deps.update(group_dep_names) + return optional_deps + + +@pytest.mark.parametrize("plugin_name", plugin_names()) +def test_optional_extras_not_in_requirements(plugin_name: str): + m = importlib.import_module("garak." + ".".join(plugin_name.split(".")[:-1])) + plugin_class = getattr(m, plugin_name.split(".")[-1]) + plugin_extra_dep_names = [ + d.split(".")[0] for d in plugin_class.extra_dependency_names + ] + extra_deps_in_requirements = requirement_names().intersection( + plugin_extra_dep_names + ) + assert len(extra_deps_in_requirements) == 0, ( + "extra deps should not be in requirements.txt but %s overlaps" + % extra_deps_in_requirements + ) + + +@pytest.mark.skipif( + tomllib is None, reason="No tomllib found (available from Python 3.11)" +) +@pytest.mark.parametrize("plugin_name", plugin_names()) +def test_optional_extras_not_in_pyproject(plugin_name: str): + m = importlib.import_module("garak." + ".".join(plugin_name.split(".")[:-1])) + plugin_class = getattr(m, plugin_name.split(".")[-1]) + plugin_extra_dep_names = [ + d.split(".")[0] for d in plugin_class.extra_dependency_names + ] + extra_deps_in_pyproject = pyproject_optional_dep_names().intersection( + plugin_extra_dep_names + ) + assert len(extra_deps_in_pyproject) == len(plugin_extra_dep_names), ( + "all extra dependences %s must be in optional plugin clause in pyproject.toml" + % plugin_extra_dep_names + ) + + +@pytest.mark.skipif( + tomllib is None, reason="No tomllib found (available from Python 3.11)" +) +def test_all_plugins_coverage(): + with open("pyproject.toml", "rb") as pyproject_file: + pyproject_toml = tomllib.load(pyproject_file) + assert "all_plugins" in pyproject_toml["project"]["optional-dependencies"] + all_plugins = pyproject_toml["project"]["optional-dependencies"]["all_plugins"] + plugin_names = [ + p + for p in pyproject_toml["project"]["optional-dependencies"] + if p.startswith("plugin_") + ] + all_plugins_list = re.findall(r"garak\[(.+)\]", all_plugins[0])[0].split(",") + missing_from_all_plugins = set(plugin_names) - set(all_plugins_list) + assert ( + missing_from_all_plugins == set() + ), f"items missing from all_plugins: {missing_from_all_plugins}" + no_own_section_plugins = set(all_plugins_list) - set(plugin_names) + assert ( + no_own_section_plugins == set() + ), f"items in all_plugins without own section: {no_own_section_plugins}"