Add --include_path option to include external folders in Docker build context

.github/workflows/test_cli.yml

@@ -33,26 +33,29 @@ jobs:
         fetch-depth: 0
         ref: ${{ github.event.pull_request.head.ref }}
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
     - name: Clear runner space
       run: |
         rm -rf /opt/hostedtoolcache
 
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
     - name: Install csc-mlops
       run: |
         python -m pip install --upgrade pip build
         python -m build 
-        pip install dist/*.whl
+        python -m pip install dist/*.whl
 
     - name: start MLOps server
+      # --wait blocks until all services are healthy or completed before returning.
+      # createbuckets depends on minio being healthy, so the mlflow bucket is guaranteed
+      # to exist before the test runs.
       run: |
         mkdir -p ~/MLOps_data/mlflow_minio
         mkdir -p ~/MLOps_data/mlflow_db
-        docker compose -f mlflow_server/docker-compose.yml up -d --build
+        docker compose -f mlflow_server/docker-compose.yml up -d --build --wait
 
     - name: Test run CLI
       run: |

mlflow_server/docker-compose.yml

@@ -43,9 +43,10 @@ services:
               target: /data
 
     createbuckets:
-        image: minio/mc
+        image: minio/mc:RELEASE.2022-11-07T23-47-39Z
         depends_on:
-            - minio
+            minio:
+                condition: service_healthy
         environment:
             - NO_PROXY=minio,db,localhost
         networks:
@@ -54,7 +55,7 @@ services:
             /bin/sh -c "
             /usr/bin/mc alias set myminio http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY};
             /usr/bin/mc mb myminio/mlflow;
-            /usr/bin/mc policy set public myminio/mlflow;
+            /usr/bin/mc anonymous set public myminio/mlflow;
             exit 0;"
 
     db:
@@ -81,8 +82,12 @@ services:
         image: mlflow_server
         container_name: mlflow_server
         depends_on:
-            - "minio"
-            - "db"
+            minio:
+                condition: service_healthy
+            db:
+                condition: service_started
+            createbuckets:
+                condition: service_completed_successfully
         networks:
             - frontend
             - backend

mlops/Experiment.py

@@ -1,6 +1,8 @@
+import atexit
 import configparser
 import logging
 import os
+import shutil
 import subprocess
 import sys
 
@@ -76,7 +78,7 @@
         if self.auth is None:
             logger.debug(f'Found minio credentials in {self.auth.method}')
             raise Exception(
                 f'minio credentials not found - either specify in ~/.aws/credentials or using environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)')
 
     def check_dirty(self) -> bool:
         """
@@ -247,6 +249,15 @@
         """
         rebuild_docker = kwargs.get('rebuild_docker', False)
         shared_memory = kwargs.get('shared_memory', '8gb')
+        include_path = kwargs.get('include_path', None)
+
+        if include_path:
+            folder_name = os.path.basename(os.path.abspath(include_path))
+            included_folder_dest = os.path.join(self.project_path, folder_name)
+            shutil.rmtree(included_folder_dest, ignore_errors=True)
+            shutil.copytree(include_path, included_folder_dest)
+            logger.info(f'Copied {include_path} to {included_folder_dest}')
+            atexit.register(shutil.rmtree, included_folder_dest, ignore_errors=True) # remove folder at exit
 
         logger.info(f'Starting experiment: {self.experiment_name}')
 

mlops/cli.py

@@ -47,7 +47,8 @@ def cli(ctx):
               show_default=True, default=False)
 @click.option('--ignore_git_check', is_flag=True, show_default=True, default=False,
               help='TESTING ONLY - ignore git checks, occasionally it might be necessary to ignore the git checks for example, offline testing, do not use this feature if working on tracked models')
-def run(script, config_path, run_name, ignore_git_check, shared_memory, logging_level, rebuild_docker):
+@click.option('-i', '--include_path', 'include_path', help='Path to a folder to copy into the project directory before building the Docker image', default=None, type=click.Path(exists=True, file_okay=False))
+def run(script, config_path, run_name, ignore_git_check, shared_memory, logging_level, rebuild_docker, include_path):
     """
     Runs python project using csc-mlops framework.
 
@@ -65,6 +66,7 @@ def run(script, config_path, run_name, ignore_git_check, shared_memory, logging_
             run_name=run_name,
             rebuild_docker=rebuild_docker,
             shared_memory=shared_memory,
+            include_path=include_path,
             )
 
 

tests/data/requirements.txt

@@ -1,6 +1,6 @@
 numpy
 -f https://download.pytorch.org/whl/torch_stable.html
 torch==2.0.1+cpu
-mlflow
+mlflow==2.10.0
 boto3
 requests==2.31.0