feature: deferred loading and requirement pruning

.github/workflows/lint.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .[tests]
       - name: Pylint
         run: |
           pylint -v garak

.github/workflows/maintain_cache.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .
       - name: Build a local cache
         run: |
           export TZ=UTC

.github/workflows/remote_package_install.yml

@@ -37,7 +37,7 @@ jobs:
       - name: pip install from repo
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -U git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}
+          python -m pip install -U "git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}"
       - name: Sanity Test
         run: |
           python -m garak --model_type test.Blank --probes test.Test

.github/workflows/test_linux.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

.github/workflows/test_macos.yml

@@ -53,7 +53,7 @@ jobs:
           brew install libmagic
           cd garak
           python -m pip install --upgrade pip
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

.github/workflows/test_windows.yml

@@ -46,7 +46,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           cd garak
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

garak/_plugins.py

@@ -402,6 +402,23 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             ) from ve
         else:
             return False
+
+    full_plugin_name = ".".join((category, module_name, plugin_class_name))
+
+    # check cache for optional imports
+    if category in PLUGIN_TYPES:
+        extra_dependency_names = PluginCache.instance()[category][full_plugin_name][
+            "extra_dependency_names"
+        ]
+        if len(extra_dependency_names) > 0:
+            for dependency_module_name in extra_dependency_names:
+                for dependency_path in [ # support both plain names and also multi-point names e.g. langchain.llms
+                    ".".join(dependency_module_name.split(".")[: n + 1])
+                    for n in range(dependency_module_name.count(".") + 1)
+                ]:
+                    if importlib.util.find_spec(dependency_path) is None:
+                        _import_failed(dependency_path, full_plugin_name)
+
     module_path = f"garak.{category}.{module_name}"
     try:
         mod = importlib.import_module(module_path)
@@ -428,6 +445,7 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
         if plugin_instance is None:
             plugin_instance = klass(config_root=config_root)
             PluginProvider.storeInstance(plugin_instance, config_root)
+
     except Exception as e:
         logging.warning(
             "Exception instantiating %s.%s: %s",
@@ -442,3 +460,20 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             return False
 
     return plugin_instance
+
+
+def load_optional_module(module_name: str):
+    try:
+        m = importlib.import_module(module_name)
+    except ModuleNotFoundError:
+        requesting_module = Path(inspect.stack()[1].filename).name.replace(".py", "")
+        _import_failed(module_name, requesting_module)
+    return m
+
+
+def _import_failed(import_module: str, calling_module: str):
+    msg = f"⛔ Plugin '{calling_module}' requires Python module '{import_module}' but this isn't installed/available."
+    hint = f"💡 Try 'pip install {import_module}' to get it."
+    logging.critical(msg)
+    print(msg + "\n" + hint)
+    raise ModuleNotFoundError(msg)

garak/buffs/base.py

@@ -27,6 +27,8 @@ class Buff(Configurable):
     doc_uri = ""
     lang = None  # set of languages this buff should be constrained to
     active = True
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     DEFAULT_PARAMS = {}
 

garak/detectors/base.py

@@ -27,6 +27,8 @@ class Detector(Configurable):
     accuracy = None
     active = True
     tags = []  # list of taxonomy categories per the MISP format
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     # support mainstream any-to-any large models
     # legal element for str list `modality['in']`: 'text', 'image', 'audio', 'video', '3d'

garak/exception.py

@@ -14,7 +14,7 @@ class ModelNameMissingError(GarakException):
     """A generator requires model_name to be set, but it wasn't"""
 
 
-class GarakBackoffTrigger(GarakException):
+class GeneratorBackoffTrigger(GarakException):
     """Thrown when backoff should be triggered"""
 
 
@@ -36,3 +36,4 @@ class ConfigFailure(GarakException):
 
 class PayloadFailure(GarakException):
     """Problem instantiating/using payloads"""
+

garak/generators/azure.py

@@ -82,6 +82,7 @@ def _validate_env_var(self):
         return super()._validate_env_var()
 
     def _load_client(self):
+        self._load_deps()
         if self.model_name in openai_model_mapping:
             self.model_name = openai_model_mapping[self.model_name]
 

garak/generators/base.py

@@ -44,6 +44,8 @@ class Generator(Configurable):
     supports_multiple_generations = (
         False  # can more than one generation be extracted per request?
     )
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     def __init__(self, name="", config_root=_config):
         self._load_config(config_root)
@@ -63,6 +65,29 @@ def __init__(self, name="", config_root=_config):
             f"🦜 loading {Style.BRIGHT}{Fore.LIGHTMAGENTA_EX}generator{Style.RESET_ALL}: {self.generator_family_name}: {self.name}"
         )
         logging.info("generator init: %s", self)
+        self._load_deps()
+
+    def _load_deps(self):
+        # load external dependencies. should be invoked at construction and
+        # in _client_load (if used)
+        for extra_dependency in self.extra_dependency_names:
+            extra_dep_name = extra_dependency.replace(".", "_").replace("-", "_")
+            if (
+                not hasattr(self, extra_dep_name)
+                or getattr(self, extra_dep_name) is None
+            ):
+                setattr(
+                    self,
+                    extra_dep_name,
+                    garak._plugins.load_optional_module(extra_dependency),
+                )
+
+    def _clear_deps(self):
+        # unload external dependencies from class. should be invoked before
+        # serialisation, esp. in _clear_client (if used)
+        for extra_dependency in self.extra_dependency_names:
+            extra_dep_name = extra_dependency.replace(".", "_")
+            setattr(self, extra_dep_name, None)
 
     def _call_model(
         self, prompt: str, generations_this_call: int = 1
@@ -101,7 +126,7 @@ def _prune_skip_sequences(self, outputs: List[str | None]) -> List[str | None]:
         )
         rx_missing_final = re.escape(self.skip_seq_start) + ".*?$"
         rx_missing_start = ".*?" + re.escape(self.skip_seq_end)
-        
+
         if self.skip_seq_start == "":
             complete_seqs_removed = [
                 (

garak/generators/cohere.py

@@ -10,10 +10,10 @@
 from typing import List, Union
 
 import backoff
-import cohere
 import tqdm
 
 from garak import _config
+from garak.exception import GeneratorBackoffTrigger
 from garak.generators.base import Generator
 
 
@@ -38,11 +38,13 @@ class CohereGenerator(Generator):
         "presence_penalty": 0.0,
         "stop": [],
     }
+    extra_dependency_names = ["cohere"]
 
     supports_multiple_generations = True
     generator_family_name = "Cohere"
 
     def __init__(self, name="command", config_root=_config):
+
         self.name = name
         self.fullname = f"Cohere {self.name}"
 
@@ -51,9 +53,9 @@ def __init__(self, name="command", config_root=_config):
         logging.debug(
             "Cohere generation request limit capped at %s", COHERE_GENERATION_LIMIT
         )
-        self.generator = cohere.Client(self.api_key)
+        self.generator = self.cohere.Client(self.api_key)
 
-    @backoff.on_exception(backoff.fibo, cohere.error.CohereAPIError, max_value=70)
+    @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70)
     def _call_cohere_api(self, prompt, request_size=COHERE_GENERATION_LIMIT):
         """as of jun 2 2023, empty prompts raise:
         cohere.error.CohereAPIError: invalid request: prompt must be at least 1 token long
@@ -63,19 +65,26 @@ def _call_cohere_api(self, prompt, request_size=COHERE_GENERATION_LIMIT):
         if prompt == "":
             return [""] * request_size
         else:
-            response = self.generator.generate(
-                model=self.name,
-                prompt=prompt,
-                temperature=self.temperature,
-                num_generations=request_size,
-                max_tokens=self.max_tokens,
-                preset=self.preset,
-                k=self.k,
-                p=self.p,
-                frequency_penalty=self.frequency_penalty,
-                presence_penalty=self.presence_penalty,
-                end_sequences=self.stop,
-            )
+            try:
+                response = self.generator.generate(
+                    model=self.name,
+                    prompt=prompt,
+                    temperature=self.temperature,
+                    num_generations=request_size,
+                    max_tokens=self.max_tokens,
+                    preset=self.preset,
+                    k=self.k,
+                    p=self.p,
+                    frequency_penalty=self.frequency_penalty,
+                    presence_penalty=self.presence_penalty,
+                    end_sequences=self.stop,
+                )
+            except Exception as e:
+                backoff_exception_types = [self.cohere.error.CohereAPIError]
+                for backoff_exception in backoff_exception_types:
+                    if isinstance(e, backoff_exception):
+                        raise GeneratorBackoffTrigger from e
+                raise e
             return [g.text for g in response]
 
     def _call_model(

garak/generators/groq.py

@@ -39,6 +39,7 @@ class GroqChat(OpenAICompatible):
     generator_family_name = "Groq"
 
     def _load_client(self):
+        self._load_deps()
         self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key)
         if self.name in ("", None):
             raise ValueError(

garak/generators/guardrails.py

@@ -16,27 +16,21 @@ class NeMoGuardrails(Generator):
 
     supports_multiple_generations = False
     generator_family_name = "Guardrails"
+    extra_dependency_names = ["nemoguardrails"]
 
     def __init__(self, name="", config_root=_config):
-        # another class that may need to skip testing due to non required dependency
-        try:
-            from nemoguardrails import RailsConfig, LLMRails
-            from nemoguardrails.logging.verbose import set_verbose
-        except ImportError as e:
-            raise NameError(
-                "You must first install NeMo Guardrails using `pip install nemoguardrails`."
-            ) from e
 
         self.name = name
         self._load_config(config_root)
         self.fullname = f"Guardrails {self.name}"
 
+        super().__init__(self.name, config_root=config_root)
+
+        set_verbose = self.nemoguardrails.logging.verbose.set_verbose
         # Currently, we use the model_name as the path to the config
         with redirect_stderr(io.StringIO()) as f:  # quieten the tqdm
-            config = RailsConfig.from_path(self.name)
-            self.rails = LLMRails(config=config)
-
-        super().__init__(self.name, config_root=config_root)
+            config = self.nemoguardrails.RailsConfig.from_path(self.name)
+            self.rails = self.nemoguardrails.LLMRails(config=config)
 
     def _call_model(
         self, prompt: str, generations_this_call: int = 1