use minio as storage for persistent packages

Author: mlabeeb03

changelog.d/20250806_160115_muhammad.labeeb_install_xblocks_at_runtime.md

@@ -1 +1 @@
-- [Feature] Add Xblocks at runtime without rebuilding image. (by @mlabeeb03)
+- [Feature] Install python packages at runtime without rebuilding image. (by @mlabeeb03)

tutor/commands/cli.py

@@ -16,7 +16,6 @@
 from tutor.commands.local import local
 from tutor.commands.mounts import mounts_command
 from tutor.commands.plugins import plugins_command
-from tutor.commands.packages import packages_command
 
 
 def main() -> None:
@@ -66,7 +65,7 @@ def ensure_plugins_enabled(self, ctx: click.Context) -> None:
         """
         We enable plugins as soon as possible to have access to commands.
         """
-        if not "root" in ctx.params:
+        if "root" not in ctx.params:
             # When generating docs, this function is called with empty args.
             # That's ok, we just ignore it.
             return
@@ -130,7 +129,6 @@ def help_command(context: click.Context) -> None:
         local,
         mounts_command,
         plugins_command,
-        packages_command,
     ]
 )
 

tutor/commands/jobs.py

@@ -13,7 +13,6 @@
 
 from tutor import config as tutor_config
 from tutor import env, fmt, hooks
-from tutor.commands.config import save as config_save_command
 from tutor.commands.context import Context
 from tutor.commands.jobs_utils import (
     create_user_template,
@@ -541,68 +540,57 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         runner.run_task_from_str(service, command)
 
 
-@click.command(help="Install a pip package at runtime")
-@click.pass_context
-@click.argument("package")
-def pip_install(context: click.Context, package: str) -> t.Iterable[tuple[str, str]]:
+@click.command(help="Build all persistent pip packages and upload to MinIO")
+@click.pass_obj
+def build_packages(context: Context) -> t.Iterable[tuple[str, str]]:
     """
-    Installs a pip package persistently in the lms and cms container and
-    restarts with uwsgi server in both containers.
+    Build the persistent pip packages and upload to MinIO.
+    You need to update the `PERSISTENT_PIP_PACKAGES` variable
+    in the config file to add/remove packages.
     """
-
-    # TODO Only add package to config if pip install is successful
-    fmt.echo_info(f"Adding {package} to config...")
-    context.invoke(
-        config_save_command,
-        interactive=False,
-        set_vars=[],
-        append_vars=[("PERSISTENT_PIP_PACKAGES", package)],
-        remove_vars=[],
-        unset_vars=[],
-        env_only=False,
-        clean_env=False,
+    config = tutor_config.load(context.root)
+    all_packages = " ".join(
+        package for package in t.cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
     )
 
     script = f"""
     pip install \
-    --prefix=/mnt/persistent-python-packages \
-    {package} \
-    && echo \"$(date)\" > /mnt/persistent-python-packages/.uwsgi_trigger
-    """
-
-    yield ("lms", script)
-
+    --prefix=/openedx/persistent-python-packages/deps \
+    {all_packages} \
+    && python3 -c '
+import os, shutil, tempfile, boto3, botocore, datetime, zipfile
+
+DEPS_DIR = "/openedx/persistent-python-packages/deps"
+MINIO_KEY = "deps.zip"
+DEPS_ZIP_PATH = DEPS_DIR[:-4] + MINIO_KEY
+MINIO_BUCKET = "tutor-deps"
+
+s3 = boto3.client(
+    "s3",
+    endpoint_url="http://" + os.environ.get("MINIO_HOST"),
+    aws_access_key_id=os.environ.get("OPENEDX_AWS_ACCESS_KEY"),
+    aws_secret_access_key=os.environ.get("OPENEDX_AWS_SECRET_ACCESS_KEY"),
+)
 
-@click.command(help="Remove a pip package at runtime")
-@click.pass_obj
-@click.pass_context
-@click.argument("package")
-def pip_uninstall(
-    click_context: click.Context, context: Context, package: str
-) -> t.Iterable[tuple[str, str]]:
+def _upload_to_minio(local_path):
+    try:
+        s3.head_bucket(Bucket=MINIO_BUCKET)
+    except botocore.exceptions.ClientError:
+        s3.create_bucket(Bucket=MINIO_BUCKET)
+    s3.upload_file(local_path, MINIO_BUCKET, MINIO_KEY)
+    print(f"Uploaded {{local_path}} → MinIO:{{MINIO_BUCKET}}/{{MINIO_KEY}}")
+    os.remove(local_path)
+
+def _make_zip_archive(src_dir):
+    with tempfile.TemporaryDirectory(prefix="tutor-depszip-") as zip_dir:
+        path = os.path.join(zip_dir, "deps.zip")
+        shutil.make_archive(path[:-4], format="zip", root_dir=src_dir)
+        shutil.move(path, DEPS_ZIP_PATH)
+
+_make_zip_archive(DEPS_DIR)
+_upload_to_minio(DEPS_ZIP_PATH)
+'
     """
-    Deletes the persistently installed pip package along with its dependencies
-    """
-
-    fmt.echo_info(f"Removing {package} from config...")
-    click_context.invoke(
-        config_save_command,
-        interactive=False,
-        set_vars=[],
-        append_vars=[],
-        remove_vars=[("PERSISTENT_PIP_PACKAGES", package)],
-        unset_vars=[],
-        env_only=False,
-        clean_env=False,
-    )
-
-    script = "rm -rf /mnt/persistent-python-packages/lib/"
-    config = tutor_config.load(context.root)
-    values = t.cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
-    remaining_packages = " ".join(values)
-    if len(values) > 0:
-        script += f" && pip install --prefix=/mnt/persistent-python-packages {remaining_packages}"
-    script += ' && echo "$(date)" > /mnt/persistent-python-packages/.uwsgi_trigger'
 
     yield ("lms", script)
 
@@ -631,8 +619,7 @@ def run_migrations(package: str) -> t.Iterable[tuple[str, str]]:
         settheme,
         sqlshell,
         update_mysql_authentication_plugin,
-        pip_install,
-        pip_uninstall,
+        build_packages,
         run_migrations,
     ]
 )

tutor/commands/packages.py

@@ -179,7 +179,7 @@ ENV VIRTUAL_ENV=/openedx/venv/
 ENV COMPREHENSIVE_THEME_DIRS=/openedx/themes
 ENV STATIC_ROOT_LMS=/openedx/staticfiles
 ENV STATIC_ROOT_CMS=/openedx/staticfiles/studio
-ENV PYTHONPATH=/mnt/persistent-python-packages/lib/python3.11/site-packages
+ENV PYTHONPATH=/openedx/persistent-python-packages/deps/lib/python3.11/site-packages
 
 WORKDIR /openedx/edx-platform
 
@@ -314,8 +314,14 @@ ENV UWSGI_WORKERS=2
 # Copy the default uWSGI configuration
 COPY --chown=app:app settings/uwsgi.ini /openedx
 
+# Copy the download script that fetches dependencies from MinIO
+COPY --chown=app:app settings/download_packages_from_minio.py /openedx
+
 # Run server
-CMD ["uwsgi", "/openedx/uwsgi.ini"]
+CMD sh -c "python /openedx/download_packages_from_minio.py & \
+           while true; do sleep 30; python /openedx/download_packages_from_minio.py; done & \
+           uwsgi /openedx/uwsgi.ini"
+
 
 {{ patch("openedx-dockerfile-final") }}
 

tutor/templates/build/openedx/Dockerfile

@@ -0,0 +1,46 @@
+import datetime
+import os
+import shutil
+import zipfile
+
+import boto3
+
+DEPS_DIR = "/openedx/persistent-python-packages/deps"
+MINIO_KEY = "deps.zip"
+DEPS_ZIP_PATH = DEPS_DIR[:-4] + MINIO_KEY
+MINIO_BUCKET = "tutor-deps"
+TRIGGER_FILE = "/openedx/persistent-python-packages/.uwsgi_trigger"
+
+s3 = boto3.client(
+    "s3",
+    endpoint_url="http://" + os.environ.get("MINIO_HOST"),
+    aws_access_key_id=os.environ.get("OPENEDX_AWS_ACCESS_KEY"),
+    aws_secret_access_key=os.environ.get("OPENEDX_AWS_SECRET_ACCESS_KEY"),
+)
+
+
+def _download_from_minio():
+    head = s3.head_object(Bucket=MINIO_BUCKET, Key=MINIO_KEY)
+    remote_ts = head["LastModified"].astimezone(datetime.timezone.utc)
+
+    if os.path.exists(DEPS_ZIP_PATH):
+        local_ts = os.path.getmtime(DEPS_ZIP_PATH)
+        local_ts = datetime.datetime.fromtimestamp(local_ts, tz=datetime.timezone.utc)
+
+        if local_ts >= remote_ts:
+            return
+
+    if os.path.exists(DEPS_DIR):
+        shutil.rmtree(DEPS_DIR)
+    os.makedirs(DEPS_DIR, exist_ok=True)
+
+    s3.download_file(MINIO_BUCKET, MINIO_KEY, DEPS_ZIP_PATH)
+
+    with zipfile.ZipFile(DEPS_ZIP_PATH, "r") as zip_ref:
+        zip_ref.extractall(DEPS_DIR)
+
+    with open(TRIGGER_FILE, "a"):
+        os.utime(TRIGGER_FILE, None)
+
+
+_download_from_minio()

tutor/templates/build/openedx/settings/download_packages_from_minio.py

@@ -22,4 +22,4 @@ master = true
 py-call-osafterfork = true
 vacuum = true
 # Touch this file to initiate a reload
-touch-reload = /mnt/persistent-python-packages/.uwsgi_trigger
+touch-reload = /openedx/persistent-python-packages/.uwsgi_trigger

tutor/templates/build/openedx/settings/uwsgi.ini

@@ -14,20 +14,24 @@ x-openedx-job-service:
     - ../apps/openedx/config:/openedx/config:ro
     # theme files
     - ../build/openedx/themes:/openedx/themes
-    # third party xblocks and their dependencies
-    - ../../data/persistent-python-packages:/mnt/persistent-python-packages
 
 services:
 
   lms-job:
     <<: *openedx-job-service
     environment:
       DJANGO_SETTINGS_MODULE: lms.envs.tutor.development
+      MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
+      OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
+      OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
 
 
   cms-job:
     <<: *openedx-job-service
     environment:
       DJANGO_SETTINGS_MODULE: cms.envs.tutor.development
+      MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
+      OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
+      OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
 
   {{ patch("dev-docker-compose-jobs-services")|indent(2) }}