Release v2.6.1 (#550)

Author: gabegma

.github/workflows/pythonci.yml

@@ -1,10 +1,7 @@
 name: Python CI
 
 on:
-  push:
-    branches: [ main, dev ]
-  pull_request:
-    branches: [ main, dev ]
+  push
 
 jobs:
   backend:
@@ -14,13 +11,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8" ]
+        python-version: [ "3.9" ]
 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.9
       - uses: Gr1N/setup-poetry@v7
       - uses: actions/cache@v2
         with:

.github/workflows/webapp_ci.yml

@@ -1,10 +1,7 @@
 name: Webapp CI
 
 on:
-  push:
-    branches: [ main, dev ]
-  pull_request:
-    branches: [ main, dev ]
+  push
 
 jobs:
   frontend:

Dockerfile

@@ -4,7 +4,7 @@ FROM pytorch/pytorch:1.8.1-cuda11.1-cudnn8-runtime as build_gpu
 # Copy binaries from other images here
 RUN pip install --upgrade pip
 
-FROM python:3.8 as build_cpu
+FROM python:3.9 as build_cpu
 # NOOP step on CPU
 FROM build_${DEVICE}
 

azimuth/app.py

@@ -25,15 +25,13 @@
 
 from azimuth.config import AzimuthConfig, load_azimuth_config
 from azimuth.dataset_split_manager import DatasetSplitManager
-from azimuth.modules.base_classes import DaskModule
-from azimuth.modules.utilities.validation import ValidationModule
+from azimuth.modules.base_classes import ArtifactManager, DaskModule
 from azimuth.startup import startup_tasks
 from azimuth.task_manager import TaskManager
-from azimuth.types import DatasetSplitName, ModuleOptions
+from azimuth.types import DatasetSplitName, ModuleOptions, SupportedModule
 from azimuth.utils.cluster import default_cluster
 from azimuth.utils.conversion import JSONResponseIgnoreNan
 from azimuth.utils.logs import set_logger_config
-from azimuth.utils.project import load_dataset_split_managers_from_config
 from azimuth.utils.validation import assert_not_none
 
 _dataset_split_managers: Dict[DatasetSplitName, Optional[DatasetSplitManager]] = {}
@@ -297,6 +295,32 @@ def create_app() -> FastAPI:
     return app
 
 
+def load_dataset_split_managers_from_config(
+    azimuth_config: AzimuthConfig,
+) -> Dict[DatasetSplitName, Optional[DatasetSplitManager]]:
+    """
+    Load all dataset splits for the application.
+
+    Args:
+        azimuth_config: Azimuth Configuration.
+
+    Returns:
+        For all DatasetSplitName, None or a dataset_split manager.
+
+    """
+    artifact_manager = ArtifactManager.instance()
+    dataset = artifact_manager.get_dataset_dict(azimuth_config)
+
+    return {
+        dataset_split_name: None
+        if dataset_split_name not in dataset
+        else artifact_manager.get_dataset_split_manager(
+            azimuth_config, DatasetSplitName[dataset_split_name]
+        )
+        for dataset_split_name in [DatasetSplitName.eval, DatasetSplitName.train]
+    }
+
+
 def initialize_managers(azimuth_config: AzimuthConfig, cluster: SpecCluster):
     """Initialize DatasetSplitManagers and TaskManagers.
 
@@ -334,22 +358,19 @@ def run_validation(
     """
 
     def run_validation_module(pipeline_index=None):
-        validation_module = ValidationModule(
-            config=config,
+        _, task = task_manager.get_task(
+            task_name=SupportedModule.Validation,
             dataset_split_name=dataset_split,
             mod_options=ModuleOptions(pipeline_index=pipeline_index),
         )
-        validation_module.start_task_on_dataset_split(task_manager.client)
         # Will raise exceptions as needed.
-        validation_module.result()
+        task.result()
 
     if config.pipelines is None:
         run_validation_module()
     else:
         for pipeline_index in range(len(config.pipelines)):
             run_validation_module(pipeline_index)
-    task_manager.clear_worker_cache()
-    task_manager.restart()
 
 
 def run_startup_tasks(azimuth_config: AzimuthConfig, cluster: SpecCluster):

azimuth/config.py

@@ -200,24 +200,27 @@ class SyntaxOptions(AzimuthBaseSettings):
 
 
 class NeutralTokenOptions(AzimuthBaseSettings):
-    threshold: float = 1
+    threshold: float = Field(1, ge=0, le=1)
     suffix_list: List[str] = []  # Language-based default value
     prefix_list: List[str] = []  # Language-based default value
 
 
 class PunctuationTestOptions(AzimuthBaseSettings):
-    threshold: float = 1
+    threshold: float = Field(1, ge=0, le=1)
 
 
 class FuzzyMatchingTestOptions(AzimuthBaseSettings):
-    threshold: float = 1
+    threshold: float = Field(1, ge=0, le=1)
 
 
 class TypoTestOptions(AzimuthBaseSettings):
-    threshold: float = 1
-    # Ex: if nb_typos_per_utterance = 2, this will create both tests with 1 typo and 2 typos per
-    # utterance.
-    nb_typos_per_utterance: int = 1
+    threshold: float = Field(1, ge=0, le=1)
+    nb_typos_per_utterance: int = Field(
+        1,
+        ge=1,
+        description="For example, the value 2 would create both tests with 1 typo and with 2 typos "
+        "per utterance.",
+    )
 
 
 class BehavioralTestingOptions(AzimuthBaseSettings):
@@ -355,6 +358,16 @@ def check_pipeline_names(cls, pipeline_definitions):
             raise ValueError(f"Duplicated pipeline names {pipeline_names}.")
         return pipeline_definitions
 
+    def get_model_contract_hash(self):
+        """Hash for fields related to model contract only (excluding fields from the parents)."""
+        return md5_hash(
+            self.dict(
+                include=ModelContractConfig.__fields__.keys()
+                - CommonFieldsConfig.__fields__.keys(),
+                by_alias=True,
+            )
+        )
+
 
 class MetricsConfig(ModelContractConfig):
     # Custom HuggingFace metrics
@@ -445,10 +458,9 @@ def dynamic_language_config_values(cls, values):
 
     @classmethod
     def load(cls, config_path: Optional[str], load_config_history: bool) -> "AzimuthConfig":
-        # Loading config from config_path if specified, or else from environment variables only.
-        cfg = ArtifactsConfig.parse_file(config_path) if config_path else ArtifactsConfig()
-
         if load_config_history:
+            # Load artifact_path from config_path if specified, or else from env var ARTIFACT_PATH.
+            cfg = ArtifactsConfig.parse_file(config_path) if config_path else ArtifactsConfig()
             config_history_path = cfg.get_config_history_path()
             last_config = cls.load_last_from_config_history(config_history_path)
             if last_config:
@@ -457,6 +469,7 @@ def load(cls, config_path: Optional[str], load_config_history: bool) -> "Azimuth
 
             log.info("Empty or invalid config history.")
 
+        # Load config from config_path if specified, or else from env vars only.
         return cls.parse_file(config_path) if config_path else cls()
 
     @classmethod