Skip to content

Commit 36345c2

Browse files
mlabeeb03mlabeeb03
committed
use django storages api
1 parent 44f50d4 commit 36345c2

File tree

11 files changed

+82
-122
lines changed

11 files changed

+82
-122
lines changed

‎tutor/commands/jobs.py‎

Lines changed: 20 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -540,55 +540,37 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
540540
runner.run_task_from_str(service, command)
541541

542542

543-
@click.command(help="Build all persistent pip packages and upload to MinIO")
543+
@click.command(help="Build all live dependencies, zip them and upload to storage backend")
544544
@click.pass_obj
545-
def build_packages(context: Context) -> t.Iterable[tuple[str, str]]:
545+
def build_live_dependencies(context: Context) -> t.Iterable[tuple[str, str]]:
546546
"""
547-
Build the persistent pip packages and upload to MinIO.
548-
You need to update the `PERSISTENT_PIP_PACKAGES` variable
549-
in the config file to add/remove packages.
547+
Build the live dependencies and upload using Django's storage API.
548+
You need to update the `LIVE_DEPENDENCIES` variable in the config file to add/remove packages.
550549
"""
551550
config = tutor_config.load(context.root)
552551
all_packages = " ".join(
553-
package for package in t.cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
552+
package for package in t.cast(list[str], config["LIVE_DEPENDENCIES"])
554553
)
555554

556555
script = f"""
557556
pip install \
558-
--prefix=/openedx/persistent-python-packages/deps \
557+
--prefix=/openedx/live-dependencies/deps \
559558
{all_packages} \
560559
&& python3 -c '
561-
import os, shutil, tempfile, boto3, botocore, datetime, zipfile
562-
563-
DEPS_DIR = "/openedx/persistent-python-packages/deps"
564-
MINIO_KEY = "deps.zip"
565-
DEPS_ZIP_PATH = DEPS_DIR[:-4] + MINIO_KEY
566-
MINIO_BUCKET = "tutor-deps"
567-
568-
s3 = boto3.client(
569-
"s3",
570-
endpoint_url="http://" + os.environ.get("MINIO_HOST"),
571-
aws_access_key_id=os.environ.get("OPENEDX_AWS_ACCESS_KEY"),
572-
aws_secret_access_key=os.environ.get("OPENEDX_AWS_SECRET_ACCESS_KEY"),
573-
)
560+
import os, shutil, tempfile
561+
from django.core.files.storage import storages
562+
from django.core.files.base import File
563+
564+
DEPS_DIR = "/openedx/live-dependencies/deps"
565+
DEPS_KEY = "deps.zip"
566+
567+
with tempfile.TemporaryDirectory(prefix="tutor-livedeps-") as zip_dir:
568+
base = os.path.join(zip_dir, DEPS_KEY)
569+
archive_path = shutil.make_archive(base[:-4], format="zip", root_dir=DEPS_DIR)
574570
575-
def _upload_to_minio(local_path):
576-
try:
577-
s3.head_bucket(Bucket=MINIO_BUCKET)
578-
except botocore.exceptions.ClientError:
579-
s3.create_bucket(Bucket=MINIO_BUCKET)
580-
s3.upload_file(local_path, MINIO_BUCKET, MINIO_KEY)
581-
print(f"Uploaded {{local_path}} → MinIO:{{MINIO_BUCKET}}/{{MINIO_KEY}}")
582-
os.remove(local_path)
583-
584-
def _make_zip_archive(src_dir):
585-
with tempfile.TemporaryDirectory(prefix="tutor-depszip-") as zip_dir:
586-
path = os.path.join(zip_dir, "deps.zip")
587-
shutil.make_archive(path[:-4], format="zip", root_dir=src_dir)
588-
shutil.move(path, DEPS_ZIP_PATH)
589-
590-
_make_zip_archive(DEPS_DIR)
591-
_upload_to_minio(DEPS_ZIP_PATH)
571+
with open(archive_path, "rb") as f:
572+
# TODO User a separate storage for live dependencies
573+
storages["default"].save(DEPS_KEY, File(f))
592574
'
593575
"""
594576

@@ -619,7 +601,7 @@ def run_migrations(package: str) -> t.Iterable[tuple[str, str]]:
619601
settheme,
620602
sqlshell,
621603
update_mysql_authentication_plugin,
622-
build_packages,
604+
build_live_dependencies,
623605
run_migrations,
624606
]
625607
)

‎tutor/templates/build/openedx/Dockerfile‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ ENV VIRTUAL_ENV=/openedx/venv/
179179
ENV COMPREHENSIVE_THEME_DIRS=/openedx/themes
180180
ENV STATIC_ROOT_LMS=/openedx/staticfiles
181181
ENV STATIC_ROOT_CMS=/openedx/staticfiles/studio
182-
ENV PYTHONPATH=/openedx/persistent-python-packages/deps/lib/python3.11/site-packages
182+
ENV PYTHONPATH=/openedx/live-dependencies/deps/lib/python3.11/site-packages
183183

184184
WORKDIR /openedx/edx-platform
185185

@@ -314,14 +314,14 @@ ENV UWSGI_WORKERS=2
314314
# Copy the default uWSGI configuration
315315
COPY --chown=app:app settings/uwsgi.ini /openedx
316316

317-
# Copy the download script that fetches dependencies from MinIO
318-
COPY --chown=app:app settings/download_packages_from_minio.py /openedx
317+
# Copy live dependencies scripts and trigger file
318+
RUN mkdir -p /openedx/live-dependencies
319+
RUN touch /openedx/live-dependencies/uwsgi_trigger
320+
COPY --chown=app:app settings/monitor_livedeps.py /openedx/live-dependencies/
321+
COPY --chown=app:app settings/update_livedeps.py /openedx/live-dependencies/
319322

320323
# Run server
321-
CMD sh -c "python /openedx/download_packages_from_minio.py & \
322-
while true; do sleep 30; python /openedx/download_packages_from_minio.py; done & \
323-
uwsgi /openedx/uwsgi.ini"
324+
CMD ["uwsgi", "/openedx/uwsgi.ini"]
324325

325326

326327
{{ patch("openedx-dockerfile-final") }}
327-

‎tutor/templates/build/openedx/settings/download_packages_from_minio.py‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎tutor/templates/build/openedx/settings/monitor_livedeps.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import datetime
2+
import os
3+
import time
4+
from django.core.files.storage import storages
5+
6+
7+
DEPS_DIR = "/openedx/live-dependencies/deps"
8+
DEPS_KEY = "deps.zip"
9+
DEPS_ZIP_PATH = DEPS_DIR[:-4] + DEPS_KEY
10+
TRIGGER_FILE = "/openedx/live-dependencies/uwsgi_trigger"
11+
12+
# TODO User a separate storage for live dependencies
13+
storage = storages["default"]
14+
15+
while True:
16+
if storage.exists(DEPS_KEY):
17+
remote_ts = storage.get_modified_time(DEPS_KEY)
18+
19+
if os.path.exists(DEPS_ZIP_PATH):
20+
local_ts = os.path.getmtime(DEPS_ZIP_PATH)
21+
local_ts = datetime.datetime.fromtimestamp(local_ts, tz=datetime.timezone.utc)
22+
23+
if local_ts < remote_ts:
24+
with open(TRIGGER_FILE, "a"):
25+
os.utime(TRIGGER_FILE, None)
26+
# TODO What happens if download takes more than 10 seconds? This could loop forever.
27+
time.sleep(10)

‎tutor/templates/build/openedx/settings/update_livedeps.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import os
2+
import shutil
3+
import zipfile
4+
5+
from django.core.files.storage import storages
6+
7+
DEPS_DIR = "/openedx/live-dependencies/deps"
8+
DEPS_KEY = "deps.zip"
9+
DEPS_ZIP_PATH = DEPS_DIR[:-4] + DEPS_KEY
10+
11+
# TODO User a separate storage for live dependencies
12+
storage = storages["default"]
13+
14+
if storage.exists(DEPS_KEY):
15+
if os.path.exists(DEPS_DIR):
16+
shutil.rmtree(DEPS_DIR)
17+
os.makedirs(DEPS_DIR, exist_ok=True)
18+
19+
with storage.open(DEPS_KEY, "rb") as remote_f, open(DEPS_ZIP_PATH, "wb") as local_f:
20+
shutil.copyfileobj(remote_f, local_f)
21+
22+
with zipfile.ZipFile(DEPS_ZIP_PATH, "r") as zip_ref:
23+
zip_ref.extractall(DEPS_DIR)

‎tutor/templates/build/openedx/settings/uwsgi.ini‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,8 @@ master = true
2222
py-call-osafterfork = true
2323
vacuum = true
2424
# Touch this file to initiate a reload
25-
touch-reload = /openedx/persistent-python-packages/.uwsgi_trigger
25+
touch-reload = /openedx/live-dependencies/uwsgi_trigger
26+
# Run the update checker as a daemon
27+
attach-daemon = python /openedx/live-dependencies/monitor_livedeps.py
28+
# Download persistent packages on server starts/reload
29+
exec-asap = python /openedx/live-dependencies/update_livedeps.py

‎tutor/templates/dev/docker-compose.jobs.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,11 @@ services:
2121
<<: *openedx-job-service
2222
environment:
2323
DJANGO_SETTINGS_MODULE: lms.envs.tutor.development
24-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
25-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
26-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
2724

2825

2926
cms-job:
3027
<<: *openedx-job-service
3128
environment:
3229
DJANGO_SETTINGS_MODULE: cms.envs.tutor.development
33-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
34-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
35-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
3630

3731
{{ patch("dev-docker-compose-jobs-services")|indent(2) }}

‎tutor/templates/dev/docker-compose.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,6 @@ services:
1717
command: ./manage.py lms runserver 0.0.0.0:8000
1818
environment:
1919
DJANGO_SETTINGS_MODULE: lms.envs.tutor.development
20-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
21-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
22-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
2320
ports:
2421
- "8000:8000"
2522
networks:
@@ -32,9 +29,6 @@ services:
3229
command: ./manage.py cms runserver 0.0.0.0:8000
3330
environment:
3431
DJANGO_SETTINGS_MODULE: cms.envs.tutor.development
35-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
36-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
37-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
3832
ports:
3933
- "8001:8000"
4034

‎tutor/templates/local/docker-compose.jobs.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,6 @@ services:
2222
environment:
2323
SERVICE_VARIANT: lms
2424
DJANGO_SETTINGS_MODULE: lms.envs.tutor.production
25-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
26-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
27-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
2825
volumes:
2926
- ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
3027
- ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
@@ -39,9 +36,6 @@ services:
3936
environment:
4037
SERVICE_VARIANT: cms
4138
DJANGO_SETTINGS_MODULE: cms.envs.tutor.production
42-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
43-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
44-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
4539
volumes:
4640
- ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
4741
- ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro

‎tutor/templates/local/docker-compose.prod.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,6 @@ services:
3131
environment:
3232
SERVICE_VARIANT: lms
3333
DJANGO_SETTINGS_MODULE: lms.envs.tutor.production
34-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
35-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
36-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
3734
command: {% for value in iter_lms_celery_parameters() %}
3835
- "{{value}}"{% endfor %}
3936
restart: unless-stopped
@@ -55,9 +52,6 @@ services:
5552
environment:
5653
SERVICE_VARIANT: cms
5754
DJANGO_SETTINGS_MODULE: cms.envs.tutor.production
58-
MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
59-
OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
60-
OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
6155
command: {% for value in iter_cms_celery_parameters() %}
6256
- "{{value}}"{% endfor %}
6357
restart: unless-stopped

0 commit comments

Comments
 (0)