feature: deferred loading and requirement pruning

.github/workflows/lint.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .[tests]
       - name: Pylint
         run: |
           pylint -v garak

.github/workflows/maintain_cache.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .[all_plugins]
       - name: Build a local cache
         run: |
           export TZ=UTC

.github/workflows/remote_package_install.yml

@@ -37,7 +37,7 @@ jobs:
       - name: pip install from repo
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -U git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}
+          python -m pip install -U "git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}"
       - name: Sanity Test
         run: |
           python -m garak --model_type test.Blank --probes test.Test

.github/workflows/test_linux.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

.github/workflows/test_macos.yml

@@ -53,7 +53,7 @@ jobs:
           brew install libmagic
           cd garak
           python -m pip install --upgrade pip
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

.github/workflows/test_windows.yml

@@ -46,7 +46,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           cd garak
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

garak/_plugins.py

@@ -402,6 +402,28 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             ) from ve
         else:
             return False
+
+    full_plugin_name = ".".join((category, module_name, plugin_class_name))
+
+    # check cache for optional imports
+    if category in PLUGIN_TYPES:
+        extra_dependency_names = PluginCache.instance()[category][full_plugin_name][
+            "extra_dependency_names"
+        ]
+        if len(extra_dependency_names) > 0:
+            absent_modules = []
+            for dependency_module_name in extra_dependency_names:
+                for (
+                    dependency_path
+                ) in [  # support both plain names and also multi-point names e.g. langchain.llms
+                    ".".join(dependency_module_name.split(".")[: n + 1])
+                    for n in range(dependency_module_name.count(".") + 1)
+                ]:
+                    if importlib.util.find_spec(dependency_path) is None:
+                        absent_modules.append(dependency_module_name)
+            if len(absent_modules):
+                _import_failed(absent_modules, full_plugin_name)
+
     module_path = f"garak.{category}.{module_name}"
     try:
         mod = importlib.import_module(module_path)
@@ -428,6 +450,7 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
         if plugin_instance is None:
             plugin_instance = klass(config_root=config_root)
             PluginProvider.storeInstance(plugin_instance, config_root)
+
     except Exception as e:
         logging.warning(
             "Exception instantiating %s.%s: %s",
@@ -442,3 +465,44 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             return False
 
     return plugin_instance
+
+
+def load_optional_module(module_name: str):
+    try:
+        m = importlib.import_module(module_name)
+    except ModuleNotFoundError:
+        requesting_module = Path(inspect.stack()[1].filename).name.replace(".py", "")
+        _import_failed([module_name], requesting_module)
+    return m
+
+
+def _import_failed(absent_modules: [str], calling_module: str):
+    quoted_module_list = "'" + "', '".join(absent_modules) + "'"
+    module_list = " ".join(absent_modules)
+    msg = f"⛔ Plugin '{calling_module}' requires Python modules which aren't installed/available: {quoted_module_list}"
+    hint = f"💡 Try 'pip install {module_list}' to get missing module."
+    logging.critical(msg)
+    print(msg + "\n" + hint)
+    raise ModuleNotFoundError(msg)
+
+
+def _load_deps(self, deps_override=list()):
+    # load external dependencies. should be invoked at construction and
+    # in _client_load (if used)
+    dep_names = deps_override if deps_override else self.extra_dependency_names
+    for extra_dependency in dep_names:
+        extra_dep_name = extra_dependency.replace(".", "_").replace("-", "_")
+        if not hasattr(self, extra_dep_name) or getattr(self, extra_dep_name) is None:
+            setattr(
+                self,
+                extra_dep_name,
+                load_optional_module(extra_dependency),
+            )
+
+
+def _clear_deps(self):
+    # unload external dependencies from class. should be invoked before
+    # serialisation, esp. in _clear_client (if used)
+    for extra_dependency in self.extra_dependency_names:
+        extra_dep_name = extra_dependency.replace(".", "_")
+        setattr(self, extra_dep_name, None)

garak/buffs/base.py

@@ -27,6 +27,8 @@ class Buff(Configurable):
     doc_uri = ""
     lang = None  # set of languages this buff should be constrained to
     active = True
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     DEFAULT_PARAMS = {}
 

garak/detectors/base.py

@@ -27,6 +27,8 @@ class Detector(Configurable):
     accuracy = None
     active = True
     tags = []  # list of taxonomy categories per the MISP format
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     # support mainstream any-to-any large models
     # legal element for str list `modality['in']`: 'text', 'image', 'audio', 'video', '3d'

garak/exception.py

@@ -14,7 +14,7 @@ class ModelNameMissingError(GarakException):
     """A generator requires model_name to be set, but it wasn't"""
 
 
-class GarakBackoffTrigger(GarakException):
+class GeneratorBackoffTrigger(GarakException):
     """Thrown when backoff should be triggered"""
 
 

garak/generators/azure.py

@@ -83,6 +83,7 @@ def _validate_env_var(self):
         return super()._validate_env_var()
 
     def _load_client(self):
+        self._load_deps()
         if self.model_name in openai_model_mapping:
             self.model_name = openai_model_mapping[self.model_name]
 

garak/generators/base.py

@@ -10,7 +10,7 @@
 from colorama import Fore, Style
 import tqdm
 
-from garak import _config
+from garak import _config, _plugins
 from garak.attempt import Message, Conversation
 from garak.configurable import Configurable
 from garak.exception import GarakException
@@ -45,6 +45,8 @@ class Generator(Configurable):
     supports_multiple_generations = (
         False  # can more than one generation be extracted per request?
     )
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     def __init__(self, name="", config_root=_config):
         self._load_config(config_root)
@@ -64,6 +66,10 @@ def __init__(self, name="", config_root=_config):
             f"🦜 loading {Style.BRIGHT}{Fore.LIGHTMAGENTA_EX}generator{Style.RESET_ALL}: {self.generator_family_name}: {self.name}"
         )
         logging.info("generator init: %s", self)
+        self._load_deps()
+
+    _load_deps = _plugins._load_deps
+    _clear_deps = _plugins._clear_deps
 
     def _call_model(
         self, prompt: Conversation, generations_this_call: int = 1

garak/generators/cohere.py

@@ -15,12 +15,11 @@
 from typing import List, Union
 
 import backoff
-from cohere.core.api_error import ApiError
-import cohere
 import tqdm
 
 from garak import _config
 from garak.attempt import Message, Conversation
+from garak.exception import GeneratorBackoffTrigger
 from garak.generators.base import Generator
 
 
@@ -51,6 +50,8 @@ class CohereGenerator(Generator):
         "api_version": "v2",  # "v1" for legacy generate API, "v2" for chat API (recommended)
     }
 
+    extra_dependency_names = ["cohere"]
+
     generator_family_name = "Cohere"
 
     def __init__(self, name="command", config_root=_config):
@@ -73,11 +74,11 @@ def __init__(self, name="command", config_root=_config):
         # Initialize appropriate client based on API version
         # Following Cohere's guidance to use Client() for v1 and ClientV2() for v2
         if self.api_version == "v1":
-            self.generator = cohere.Client(api_key=self.api_key)
+            self.generator = self.cohere.Client(api_key=self.api_key)
         else:  # api_version == "v2"
-            self.generator = cohere.ClientV2(api_key=self.api_key)
+            self.generator = self.cohere.ClientV2(api_key=self.api_key)
 
-    @backoff.on_exception(backoff.fibo, ApiError, max_value=70)
+    @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70)
     def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT):
         """Empty prompts raise API errors (e.g. invalid request: prompt must be at least 1 token long).
         We catch these using the ApiError base class in Cohere v5+.
@@ -126,9 +127,17 @@ def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT):
                                 "Chat response structure doesn't match expected format"
                             )
                             responses.append(str(response))
-                    except ApiError as e:
-                        raise e  # bubble up ApiError for backoff handling
                     except Exception as e:
+
+                        backoff_exception_types = (
+                            self.cohere.errors.GatewayTimeoutError,
+                            self.cohere.errors.TooManyRequestsError,
+                            self.cohere.errors.ServiceUnavailableError,
+                            self.cohere.errors.InternalServerError,
+                        )
+                        for backoff_exception in backoff_exception_types:
+                            if isinstance(e, backoff_exception):
+                                raise GeneratorBackoffTrigger from e  # bubble up ApiError for backoff handling
                         logging.error(f"Chat API error: {e}")
                         responses.append(None)
 

garak/generators/groq.py

@@ -40,6 +40,7 @@ class GroqChat(OpenAICompatible):
     generator_family_name = "Groq"
 
     def _load_client(self):
+        self._load_deps()
         self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key)
         if self.name in ("", None):
             raise ValueError(

garak/generators/guardrails.py

@@ -17,26 +17,21 @@ class NeMoGuardrails(Generator):
 
     supports_multiple_generations = False
     generator_family_name = "Guardrails"
+    extra_dependency_names = ["nemoguardrails"]
 
     def __init__(self, name="", config_root=_config):
-        # another class that may need to skip testing due to non required dependency
-        try:
-            from nemoguardrails import RailsConfig, LLMRails
-        except ImportError as e:
-            raise NameError(
-                "You must first install NeMo Guardrails using `pip install nemoguardrails`."
-            ) from e
 
         self.name = name
         self._load_config(config_root)
         self.fullname = f"Guardrails {self.name}"
 
+        super().__init__(self.name, config_root=config_root)
+
+        set_verbose = self.nemoguardrails.logging.verbose.set_verbose
         # Currently, we use the model_name as the path to the config
         with redirect_stderr(io.StringIO()) as f:  # quieten the tqdm
-            config = RailsConfig.from_path(self.name)
-            self.rails = LLMRails(config=config)
-
-        super().__init__(self.name, config_root=config_root)
+            config = self.nemoguardrails.RailsConfig.from_path(self.name)
+            self.rails = self.nemoguardrails.LLMRails(config=config)
 
     def _call_model(
         self, prompt: Conversation, generations_this_call: int = 1

garak/generators/huggingface.py

@@ -21,7 +21,6 @@
 
 import backoff
 import torch
-from PIL import Image
 
 from garak import _config
 from garak.attempt import Message, Conversation
@@ -71,6 +70,7 @@ def __init__(self, name="", config_root=_config):
         self._load_client()
 
     def _load_client(self):
+        self._load_deps()
         if hasattr(self, "generator") and self.generator is not None:
             return
 
@@ -106,6 +106,7 @@ def _load_client(self):
         self._set_hf_context_len(self.generator.model.config)
 
     def _clear_client(self):
+        self._clear_deps()
         self.generator = None
         self.tokenizer = None
 
@@ -168,19 +169,15 @@ class OptimumPipeline(Pipeline, HFCompatible):
     generator_family_name = "NVIDIA Optimum Hugging Face 🤗 pipeline"
     supports_multiple_generations = True
     doc_uri = "https://huggingface.co/blog/optimum-nvidia"
+    extra_dependency_names = ["optimum-nvidia"]
 
     def _load_client(self):
+        self._load_deps()
         if hasattr(self, "generator") and self.generator is not None:
             return
 
-        try:
-            from optimum.nvidia.pipelines import pipeline
-            from transformers import set_seed
-        except Exception as e:
-            logging.exception(e)
-            raise GarakException(
-                f"Missing required dependencies for {self.__class__.__name__}"
-            )
+        pipeline = self.optimum.nvidia.pipelines.pipeline
+        from transformers import set_seed
 
         if self.seed is not None:
             set_seed(self.seed)
@@ -399,6 +396,7 @@ class Model(Pipeline, HFCompatible):
     supports_multiple_generations = True
 
     def _load_client(self):
+        self._load_deps()
         if hasattr(self, "model") and self.model is not None:
             return
 
@@ -446,6 +444,7 @@ def _load_client(self):
         self.generation_config.pad_token_id = self.model.config.eos_token_id
 
     def _clear_client(self):
+        self._clear_deps()
         self.model = None
         self.config = None
         self.tokenizer = None
@@ -522,6 +521,11 @@ class LLaVA(Generator, HFCompatible):
     NB. This should be use with strict modality matching - generate() doesn't
     support text-only prompts."""
 
+    extra_dependency_names = ["pillow"]
+
+    def _load_deps(self):
+        return super()._load_deps(["PIL"])
+
     DEFAULT_PARAMS = Generator.DEFAULT_PARAMS | {
         "max_tokens": 4000,
         # "exist_tokens + max_new_tokens < 4K is the golden rule."
@@ -577,7 +581,7 @@ def generate(
 
         text_prompt = prompt.last_message().text
         try:
-            image_prompt = Image.open(prompt.last_message().data_path)
+            image_prompt = self.PIL.Image.open(prompt.last_message().data_path)
         except FileNotFoundError:
             file_path = prompt.last_message().data_path
             raise FileNotFoundError(f"Cannot open image {file_path}.")