From 5bec7d092b7d1135d6790068948e849d79ff8584 Mon Sep 17 00:00:00 2001 From: andrii-hantkovskyi <131773947+andrii-hantkovskyi@users.noreply.github.com> Date: Fri, 7 Mar 2025 16:12:28 +0200 Subject: [PATCH 01/23] feat: [AXM-1607] create initial DB layer (#2601) * feat: [AXM-1607] create initial DB layer * refactor: [AXM-1607] resolve lint errors, clean up code & add migrations * refactor: [AXM-1607] simplify raw user_id in admin & improve validation by checking for duplicate keys * refactor: [AXM-1607] lint fixes #2 --------- Co-authored-by: Andrii --- .github/workflows/unit-test-shards.json | 1 + .../course_to_library_import/__init__.py | 0 .../course_to_library_import/admin.py | 37 ++++++++++ .../course_to_library_import/apps.py | 13 ++++ .../course_to_library_import/data.py | 22 ++++++ .../migrations/0001_initial.py | 55 +++++++++++++++ .../migrations/__init__.py | 0 .../course_to_library_import/models.py | 67 +++++++++++++++++++ .../tests/__init__.py | 0 .../tests/test_validators.py | 28 ++++++++ .../course_to_library_import/validators.py | 29 ++++++++ cms/envs/common.py | 1 + 12 files changed, 253 insertions(+) create mode 100644 cms/djangoapps/course_to_library_import/__init__.py create mode 100644 cms/djangoapps/course_to_library_import/admin.py create mode 100644 cms/djangoapps/course_to_library_import/apps.py create mode 100644 cms/djangoapps/course_to_library_import/data.py create mode 100644 cms/djangoapps/course_to_library_import/migrations/0001_initial.py create mode 100644 cms/djangoapps/course_to_library_import/migrations/__init__.py create mode 100644 cms/djangoapps/course_to_library_import/models.py create mode 100644 cms/djangoapps/course_to_library_import/tests/__init__.py create mode 100644 cms/djangoapps/course_to_library_import/tests/test_validators.py create mode 100644 cms/djangoapps/course_to_library_import/validators.py diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 784f607f06ba..dfd1e41de1e5 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -237,6 +237,7 @@ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", "cms/djangoapps/course_creators/", + "cms/djangoapps/course_to_library_import/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/maintenance/", "cms/djangoapps/models/", diff --git a/cms/djangoapps/course_to_library_import/__init__.py b/cms/djangoapps/course_to_library_import/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/course_to_library_import/admin.py b/cms/djangoapps/course_to_library_import/admin.py new file mode 100644 index 000000000000..67c3a7ea2488 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/admin.py @@ -0,0 +1,37 @@ +""" +This module contains the admin configuration for the CourseToLibraryImport model. +""" + +from django.contrib import admin + +from .models import CourseToLibraryImport + +# Run a task to import courses to the library +# This task is run by the celery worker to import courses to the library. + + +class CourseToLibraryImportAdmin(admin.ModelAdmin): + """ + Admin configuration for the CourseToLibraryImport model. + """ + + list_display = ( + 'id', + 'status', + 'course_ids', + 'library_key', + 'source_type', + ) + list_filter = ( + 'status', + 'source_type', + ) + search_fields = ( + 'course_ids', + 'library_key', + ) + + raw_id_fields = ('user',) + + +admin.site.register(CourseToLibraryImport, CourseToLibraryImportAdmin) diff --git a/cms/djangoapps/course_to_library_import/apps.py b/cms/djangoapps/course_to_library_import/apps.py new file mode 100644 index 000000000000..b82c7fdf90c8 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/apps.py @@ -0,0 +1,13 @@ +""" +App for importing Courses to the Content Library +""" + +from django.apps import AppConfig + + +class CourseToLibraryImportConfig(AppConfig): + """ + App for importing Courses to the Content Library + """ + + name = 'cms.djangoapps.course_to_library_import' diff --git a/cms/djangoapps/course_to_library_import/data.py b/cms/djangoapps/course_to_library_import/data.py new file mode 100644 index 000000000000..af14fb77de65 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/data.py @@ -0,0 +1,22 @@ +""" +This module contains the data models for the course_to_library_import app. +""" + +from django.db.models import TextChoices +from django.utils.translation import gettext_lazy as _ + + +class CourseToLibraryImportStatus(TextChoices): + """ + The status of this course import. + """ + + # PENDING: The import has been created, but the OLX and related data are not yet in the library. + # It is not ready to be read. + PENDING = 'pending', _('Pending') + # READY: The content is staged and ready to be read. + READY = 'ready', _('Ready') + # IMPORTED: The content has been imported into the library. + IMPORTED = 'imported', _('Imported') + # ERROR: The content could not be imported. + ERROR = 'error', _('Error') diff --git a/cms/djangoapps/course_to_library_import/migrations/0001_initial.py b/cms/djangoapps/course_to_library_import/migrations/0001_initial.py new file mode 100644 index 000000000000..1c9fd2d95aae --- /dev/null +++ b/cms/djangoapps/course_to_library_import/migrations/0001_initial.py @@ -0,0 +1,55 @@ +# Generated by Django 4.2.18 on 2025-03-07 09:20 + +import cms.djangoapps.course_to_library_import.validators +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import opaque_keys.edx.django.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'), + ] + + operations = [ + migrations.CreateModel( + name='CourseToLibraryImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('imported', 'Imported'), ('error', 'Error')], default='pending', max_length=100)), + ('course_ids', models.TextField(help_text='Whitespace-separated list of course keys for which to compute grades.', validators=[cms.djangoapps.course_to_library_import.validators.validate_course_ids])), + ('library_key', models.CharField(max_length=100)), + ('source_type', models.CharField(max_length=30)), + ('metadata', models.JSONField(blank=True, default=dict, null=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Course to Library Import', + 'verbose_name_plural': 'Course to Library Imports', + }, + ), + migrations.CreateModel( + name='ComponentVersionImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), + ('component_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='oel_components.componentversion')), + ('library_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='course_to_library_import.coursetolibraryimport')), + ], + options={ + 'verbose_name': 'Component Version Import', + 'verbose_name_plural': 'Component Version Imports', + }, + ), + ] diff --git a/cms/djangoapps/course_to_library_import/migrations/__init__.py b/cms/djangoapps/course_to_library_import/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py new file mode 100644 index 000000000000..f722b80be412 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/models.py @@ -0,0 +1,67 @@ +""" +Models for the course to library import app. +""" + +import logging +from django.contrib.auth import get_user_model +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from opaque_keys.edx.django.models import UsageKeyField + +from model_utils.models import TimeStampedModel + +from .data import CourseToLibraryImportStatus +from .validators import validate_course_ids + +logger = logging.getLogger(__name__) +User = get_user_model() + + +class CourseToLibraryImport(TimeStampedModel): + """ + Represents a course import into a content library. + """ + + status = models.CharField( + max_length=100, + choices=CourseToLibraryImportStatus.choices, + default=CourseToLibraryImportStatus.PENDING + ) + course_ids = models.TextField( + blank=False, + help_text=_('Whitespace-separated list of course keys for which to compute grades.'), + validators=[validate_course_ids] + ) + library_key = models.CharField(max_length=100) + source_type = models.CharField(max_length=30) + metadata = models.JSONField(default=dict, blank=True, null=True) + user = models.ForeignKey(User, on_delete=models.CASCADE) + + def __str__(self): + return f'{self.course_ids} - {self.library_key}' + + class Meta: + verbose_name = _('Course to Library Import') + verbose_name_plural = _('Course to Library Imports') + + +class ComponentVersionImport(TimeStampedModel): + """ + Represents a component version that has been imported into a content library. + This is a many-to-many relationship between a component version and a course to library import. + """ + + component_version = models.OneToOneField( + to='oel_components.ComponentVersion', + on_delete=models.CASCADE + ) + source_usage_key = UsageKeyField(max_length=255) + library_import = models.ForeignKey(CourseToLibraryImport, on_delete=models.CASCADE) + + def __str__(self): + return f'{self.component_version} - {self.source_usage_key}' + + class Meta: + verbose_name = _('Component Version Import') + verbose_name_plural = _('Component Version Imports') diff --git a/cms/djangoapps/course_to_library_import/tests/__init__.py b/cms/djangoapps/course_to_library_import/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/course_to_library_import/tests/test_validators.py b/cms/djangoapps/course_to_library_import/tests/test_validators.py new file mode 100644 index 000000000000..c2d415e77d1d --- /dev/null +++ b/cms/djangoapps/course_to_library_import/tests/test_validators.py @@ -0,0 +1,28 @@ +""" +Tests for course_to_library_import validators +""" + +from unittest import TestCase + +import pytest + +from cms.djangoapps.course_to_library_import.validators import validate_course_ids + + +class TestValidateCourseIds(TestCase): + """ + Tests for the validate_course_ids function. + """ + + def test_valid_course_ids(self): + validate_course_ids('course-v1:edX+DemoX+Demo_Course course-v1:edX+DemoX+Demo_Course2') + + def test_invalid_course_ids(self): + with pytest.raises(ValueError) as exc: + validate_course_ids('course-v1:edX+DemoX+Demo_Course invalid_course_id') + assert str(exc.value) == 'Invalid course key: invalid_course_id' + + def test_duplicate_course_ids(self): + with pytest.raises(ValueError) as exc: + validate_course_ids('course-v1:edX+DemoX+Demo_Course course-v1:edX+DemoX+Demo_Course') + assert str(exc.value) == 'Duplicate course keys are not allowed' diff --git a/cms/djangoapps/course_to_library_import/validators.py b/cms/djangoapps/course_to_library_import/validators.py new file mode 100644 index 000000000000..775a5e382e34 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/validators.py @@ -0,0 +1,29 @@ +""" +Validators for the course_to_library_import app. +""" + +from django.utils.translation import gettext_lazy as _ +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey + + +def validate_course_ids(value: str): + """ + Validate that the course_ids are valid course keys. + + Args: + value (str): A string containing course IDs separated by spaces. + + Raises: + ValueError: If the course IDs are not valid course keys or if there are duplicate course keys. + """ + + course_ids = value.split() + if len(course_ids) != len(set(course_ids)): + raise ValueError(_('Duplicate course keys are not allowed')) + + for course_id in course_ids: + try: + CourseKey.from_string(course_id) + except InvalidKeyError as exc: + raise ValueError(_('Invalid course key: {course_id}').format(course_id=course_id)) from exc diff --git a/cms/envs/common.py b/cms/envs/common.py index 9be3eb9e0956..c60bf264d99b 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1667,6 +1667,7 @@ 'openedx.core.djangoapps.course_groups', # not used in cms (yet), but tests run 'cms.djangoapps.xblock_config.apps.XBlockConfig', 'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig', + 'cms.djangoapps.course_to_library_import.apps.CourseToLibraryImportConfig', # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig', From d76554321e801f39b19ddd7897aecd6acf5f2a41 Mon Sep 17 00:00:00 2001 From: andrii-hantkovskyi <131773947+andrii-hantkovskyi@users.noreply.github.com> Date: Wed, 12 Mar 2025 09:37:55 +0200 Subject: [PATCH 02/23] feat: [AXM-1621] add staged content creation (#2602) * feat: [AXM-1621] add staged content creation * refactor: [AXM-1621] revert unnecessary changes & functions refactor * feat: [AXM-1621] add CourseToLibraryImport creation & update + refactor to process multiple course ids --------- Co-authored-by: Andrii --- .../course_to_library_import/api.py | 19 ++++++++ .../course_to_library_import/models.py | 2 - .../course_to_library_import/tasks.py | 44 +++++++++++++++++++ .../tests/test_api.py | 37 ++++++++++++++++ .../tests/test_tasks.py | 38 ++++++++++++++++ .../tests/test_validators.py | 9 ++-- .../core/djangoapps/content_staging/api.py | 23 +++++----- 7 files changed, 156 insertions(+), 16 deletions(-) create mode 100644 cms/djangoapps/course_to_library_import/api.py create mode 100644 cms/djangoapps/course_to_library_import/tasks.py create mode 100644 cms/djangoapps/course_to_library_import/tests/test_api.py create mode 100644 cms/djangoapps/course_to_library_import/tests/test_tasks.py diff --git a/cms/djangoapps/course_to_library_import/api.py b/cms/djangoapps/course_to_library_import/api.py new file mode 100644 index 000000000000..12f6833fbec6 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/api.py @@ -0,0 +1,19 @@ +"""" +API for course to library import. +""" + +from .tasks import save_courses_to_staged_content_task + +COURSE_TO_LIBRARY_IMPORT_PURPOSE = "course_to_library_import" + + +def save_courses_to_staged_content( + course_ids: list[str], + user_id: int, + purpose: str = COURSE_TO_LIBRARY_IMPORT_PURPOSE, + version_num: int | None = None, +) -> None: + """ + Save courses to staged content. + """ + save_courses_to_staged_content_task.delay(course_ids, user_id, purpose, version_num) diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py index f722b80be412..757a86dec44a 100644 --- a/cms/djangoapps/course_to_library_import/models.py +++ b/cms/djangoapps/course_to_library_import/models.py @@ -2,7 +2,6 @@ Models for the course to library import app. """ -import logging from django.contrib.auth import get_user_model from django.db import models from django.utils.translation import gettext_lazy as _ @@ -14,7 +13,6 @@ from .data import CourseToLibraryImportStatus from .validators import validate_course_ids -logger = logging.getLogger(__name__) User = get_user_model() diff --git a/cms/djangoapps/course_to_library_import/tasks.py b/cms/djangoapps/course_to_library_import/tasks.py new file mode 100644 index 000000000000..dfe347899a3a --- /dev/null +++ b/cms/djangoapps/course_to_library_import/tasks.py @@ -0,0 +1,44 @@ +""" +Tasks for course to library import. +""" + +from celery import shared_task +from django.db import transaction +from edx_django_utils.monitoring import set_code_owner_attribute +from opaque_keys.edx.keys import CourseKey + +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.django import modulestore + +from .data import CourseToLibraryImportStatus +from .models import CourseToLibraryImport + + +@shared_task +@set_code_owner_attribute +def save_courses_to_staged_content_task( + course_ids: list[str], user_id: int, purpose: str, version_num: int | None +) -> None: + """ + Save courses to staged content task. + """ + course_to_library_import = CourseToLibraryImport.objects.create( + course_ids=' '.join(course_ids), + user_id=user_id, + ) + + with transaction.atomic(): + for course_id in course_ids: + course_key = CourseKey.from_string(course_id) + sections = modulestore().get_items(course_key, qualifiers={"category": "chapter"}) + + for section in sections: + content_staging_api.stage_xblock_temporarily( + section, + user_id, + purpose=purpose, + version_num=version_num, + ) + + course_to_library_import.status = CourseToLibraryImportStatus.READY + course_to_library_import.save() diff --git a/cms/djangoapps/course_to_library_import/tests/test_api.py b/cms/djangoapps/course_to_library_import/tests/test_api.py new file mode 100644 index 000000000000..f18fb7dba161 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/tests/test_api.py @@ -0,0 +1,37 @@ +""" +Test cases for course_to_library_import.api module. +""" + +from unittest.mock import patch + +import pytest + +from cms.djangoapps.course_to_library_import.api import ( + save_courses_to_staged_content, + COURSE_TO_LIBRARY_IMPORT_PURPOSE +) + + +@pytest.mark.parametrize("purpose, expected_purpose", [ + ('custom_purpose', 'custom_purpose'), + (None, COURSE_TO_LIBRARY_IMPORT_PURPOSE), +]) +@patch('cms.djangoapps.course_to_library_import.api.save_courses_to_staged_content_task') +def test_save_courses_to_staged_content(mock_task, purpose, expected_purpose): + """ + Test save_course_to_staged_content function. + + Case 1: Purpose is provided. + Case 2: Purpose is not provided + """ + + course_ids = ('course-v1:edX+DemoX+Demo_Course', 'course-v1:edX+DemoX+Demo_Course2') + user_id = 1 + version_num = 1 + + if purpose: + save_courses_to_staged_content(course_ids, user_id, purpose, version_num) + else: + save_courses_to_staged_content(course_ids, user_id, version_num=version_num) + + mock_task.delay.assert_called_once_with(course_ids, user_id, expected_purpose, version_num) diff --git a/cms/djangoapps/course_to_library_import/tests/test_tasks.py b/cms/djangoapps/course_to_library_import/tests/test_tasks.py new file mode 100644 index 000000000000..6db5ca3b64d6 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/tests/test_tasks.py @@ -0,0 +1,38 @@ +""" +Tests for tasks in course_to_library_import app. +""" + +from unittest.mock import patch + +from django.test import TestCase +from opaque_keys.edx.keys import CourseKey + +from cms.djangoapps.course_to_library_import.tasks import save_courses_to_staged_content_task +from common.djangoapps.student.tests.factories import UserFactory + + +class TestSaveCourseSectionsToStagedContentTask(TestCase): + """ + Test cases for save_course_sections_to_staged_content_task. + """ + + @patch('cms.djangoapps.course_to_library_import.tasks.modulestore') + @patch('openedx.core.djangoapps.content_staging.api.stage_xblock_temporarily') + def test_save_courses_to_staged_content_task(self, mock_stage_xblock_temporarily, mock_modulestore): + + course_ids = ('course-v1:edX+DemoX+Demo_Course', 'course-v1:edX+DemoX+Demo_Course2') + user_id = UserFactory().id + purpose = 'test_purpose' + version_num = 1 + + mock_course_keys = [CourseKey.from_string(course_id) for course_id in course_ids] + mock_modulestore().get_items.return_value = sections = ['section1', 'section2'] + + save_courses_to_staged_content_task(course_ids, user_id, purpose, version_num) + + for mock_course_key in mock_course_keys: + mock_modulestore().get_items.assert_any_call(mock_course_key, qualifiers={"category": "chapter"}) + + self.assertEqual(mock_stage_xblock_temporarily.call_count, len(sections) * len(course_ids)) + for section in sections: + mock_stage_xblock_temporarily.assert_any_call(section, user_id, purpose=purpose, version_num=version_num) diff --git a/cms/djangoapps/course_to_library_import/tests/test_validators.py b/cms/djangoapps/course_to_library_import/tests/test_validators.py index c2d415e77d1d..e843cb324e79 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_validators.py +++ b/cms/djangoapps/course_to_library_import/tests/test_validators.py @@ -2,8 +2,7 @@ Tests for course_to_library_import validators """ -from unittest import TestCase - +from django.test import TestCase import pytest from cms.djangoapps.course_to_library_import.validators import validate_course_ids @@ -11,7 +10,11 @@ class TestValidateCourseIds(TestCase): """ - Tests for the validate_course_ids function. + Test cases for validate_course_ids function. + + Case 1: Valid course ids + Case 2: Invalid course ids + Case 3: Duplicate course ids """ def test_valid_course_ids(self): diff --git a/openedx/core/djangoapps/content_staging/api.py b/openedx/core/djangoapps/content_staging/api.py index acc920fd6cb4..271b89871eaa 100644 --- a/openedx/core/djangoapps/content_staging/api.py +++ b/openedx/core/djangoapps/content_staging/api.py @@ -53,17 +53,18 @@ def _save_xblock_to_staged_content( expired_ids = [] with transaction.atomic(): - # Mark all of the user's existing StagedContent rows as EXPIRED - to_expire = _StagedContent.objects.filter( - user_id=user_id, - purpose=purpose, - ).exclude( - status=StagedContentStatus.EXPIRED, - ) - for sc in to_expire: - expired_ids.append(sc.id) - sc.status = StagedContentStatus.EXPIRED - sc.save() + if purpose == CLIPBOARD_PURPOSE: + # Mark all of the user's existing StagedContent rows as EXPIRED + to_expire = _StagedContent.objects.filter( + user_id=user_id, + purpose=purpose, + ).exclude( + status=StagedContentStatus.EXPIRED, + ) + for sc in to_expire: + expired_ids.append(sc.id) + sc.status = StagedContentStatus.EXPIRED + sc.save() # Insert a new StagedContent row for this staged_content = _StagedContent.objects.create( user_id=user_id, From 2f13155399a76c878c70db4903084fe76cf5c01b Mon Sep 17 00:00:00 2001 From: andrii-hantkovskyi <131773947+andrii-hantkovskyi@users.noreply.github.com> Date: Mon, 17 Mar 2025 09:41:40 +0200 Subject: [PATCH 03/23] feat: [AXM-1614] add course to library import feature (#2603) * feat: [AXM-1614] add course to library import feature --- .../course_to_library_import/api.py | 39 +- .../course_to_library_import/constants.py | 5 + .../course_to_library_import/helpers.py | 224 +++++++ .../course_to_library_import/models.py | 9 + .../course_to_library_import/tasks.py | 61 +- .../tests/factories.py | 24 + .../tests/test_api.py | 69 ++- .../tests/test_helpers.py | 557 ++++++++++++++++++ .../tests/test_tasks.py | 125 +++- .../tests/test_validators.py | 33 +- .../course_to_library_import/validators.py | 9 + .../core/djangoapps/content_libraries/api.py | 0 .../core/djangoapps/content_staging/api.py | 8 + 13 files changed, 1117 insertions(+), 46 deletions(-) create mode 100644 cms/djangoapps/course_to_library_import/constants.py create mode 100644 cms/djangoapps/course_to_library_import/helpers.py create mode 100644 cms/djangoapps/course_to_library_import/tests/factories.py create mode 100644 cms/djangoapps/course_to_library_import/tests/test_helpers.py create mode 100644 openedx/core/djangoapps/content_libraries/api.py diff --git a/cms/djangoapps/course_to_library_import/api.py b/cms/djangoapps/course_to_library_import/api.py index 12f6833fbec6..c1f3f756bd12 100644 --- a/cms/djangoapps/course_to_library_import/api.py +++ b/cms/djangoapps/course_to_library_import/api.py @@ -2,18 +2,39 @@ API for course to library import. """ -from .tasks import save_courses_to_staged_content_task +from .constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE +from .models import CourseToLibraryImport +from .tasks import ( + import_library_from_staged_content_task, + save_courses_to_staged_content_task, +) -COURSE_TO_LIBRARY_IMPORT_PURPOSE = "course_to_library_import" +def import_library_from_staged_content( + library_key: str, user_id: int, usage_ids: list[str], course_id: str, override: bool +) -> None: + """ + Import staged content to a library. + """ + import_library_from_staged_content_task.delay( + user_id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, override + ) -def save_courses_to_staged_content( - course_ids: list[str], - user_id: int, - purpose: str = COURSE_TO_LIBRARY_IMPORT_PURPOSE, - version_num: int | None = None, + +def create_import( + course_ids: list[str], user_id: int, library_key: str, source_type: str ) -> None: """ - Save courses to staged content. + Create a new import task to import a course to a library. """ - save_courses_to_staged_content_task.delay(course_ids, user_id, purpose, version_num) + import_task = CourseToLibraryImport( + course_ids=" ".join(course_ids), + library_key=library_key, + source_type=source_type, + user_id=user_id, + ) + import_task.save() + + save_courses_to_staged_content_task.delay( + course_ids, user_id, import_task.id, COURSE_TO_LIBRARY_IMPORT_PURPOSE + ) diff --git a/cms/djangoapps/course_to_library_import/constants.py b/cms/djangoapps/course_to_library_import/constants.py new file mode 100644 index 000000000000..f619bf9e2697 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/constants.py @@ -0,0 +1,5 @@ +""" +Constants for course_to_library_import app +""" + +COURSE_TO_LIBRARY_IMPORT_PURPOSE = "course_{course_id}_to_library_import" diff --git a/cms/djangoapps/course_to_library_import/helpers.py b/cms/djangoapps/course_to_library_import/helpers.py new file mode 100644 index 000000000000..317cf435b206 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/helpers.py @@ -0,0 +1,224 @@ +""" +Helper functions for importing course content into a library. +""" + +import logging +import mimetypes +from datetime import datetime, timezone + +from django.db import transaction +from django.db.utils import IntegrityError +from lxml import etree + +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 +from openedx_learning.api import authoring as authoring_api + +from openedx.core.djangoapps.content_libraries import api +from openedx.core.djangoapps.content_libraries.api import ContentLibrary +from openedx.core.djangoapps.content_staging import api as content_staging_api + +from .data import CourseToLibraryImportStatus +from .models import ComponentVersionImport, CourseToLibraryImport +import os + + +log = logging.getLogger(__name__) + + +def create_block_in_library(block_to_import, usage_key, library_key, user_id, staged_content_id, override): + """ + Create a block in a library from a staged content block. + """ + now = datetime.now(tz=timezone.utc) + staged_content_files = content_staging_api.get_staged_content_static_files(staged_content_id) + + assert isinstance(library_key, LibraryLocatorV2) + content_library = ContentLibrary.objects.get_by_key(library_key) + + with transaction.atomic(): + component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type) + component_version = None + does_component_exist = authoring_api.get_components( + content_library.learning_package.id + ).filter(local_key=usage_key.block_id).exists() + + if does_component_exist: + if not override: + log.info(f"Component {usage_key.block_id} already exists in library {library_key}, skipping.") + return + else: + component_version = _handle_component_override( + content_library, usage_key, etree.tostring(block_to_import) + ) + + if not override: + # Create component (regardless of override path) + _, library_usage_key = api.validate_can_add_block_to_library( + library_key, + block_to_import.tag, + usage_key.block_id, + ) + authoring_api.create_component( + content_library.learning_package.id, + component_type=component_type, + local_key=usage_key.block_id, + created=now, + created_by=user_id, + ) + + component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import)) + + # Handle component version import records for overrides + overrided_component_version_import = False + if override: + _update_component_version_import( + component_version, usage_key, library_key, user_id + ) + overrided_component_version_import = True + + _process_staged_content_files( + component_version, staged_content_files, staged_content_id, usage_key, + content_library, now, block_to_import, overrided_component_version_import, library_key, user_id + ) + + +def _handle_component_override(content_library, usage_key, new_content): + """ + Create new ComponentVersion for overridden component. + """ + component_version = None + component = content_library.learning_package.component_set.filter(local_key=usage_key.block_id).first() + + if component: + lib_usage_key = LibraryUsageLocatorV2( # type: ignore[abstract] + lib_key=content_library.library_key, + block_type=component.component_type.name, + usage_id=component.local_key, + ) + component_version = api.set_library_block_olx(lib_usage_key, new_content) + + return component_version + + +def _update_component_version_import(component_version, usage_key, library_key, user_id): + """ + Update component version import records for overridden components. + """ + return ComponentVersionImport.objects.create( + component_version=component_version, + source_usage_key=usage_key, + library_import=CourseToLibraryImport.objects.get( + library_key=library_key, + user_id=user_id, + status=CourseToLibraryImportStatus.READY + ), + ) + + +def _process_staged_content_files( + component_version, + staged_content_files, + staged_content_id, + usage_key, + content_library, + now, + block_to_import, + overrided_component_version_import, + library_key, + user_id, +): + """ + Process staged content files for a component. + """ + block_olx = etree.tostring(block_to_import).decode('utf-8') + + for staged_content_file_data in staged_content_files: + original_filename = staged_content_file_data.filename + file_basename = os.path.basename(original_filename) + + # Skip files not referenced in the block + if file_basename not in block_olx: + log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") + continue + + file_data = content_staging_api.get_staged_content_static_file_data( + staged_content_id, + original_filename, + ) + if not file_data: + log.error( + f"Staged content {staged_content_id} included referenced " + f"file {original_filename}, but no file data was found." + ) + continue + + filename = f"static/{str(usage_key)}" + media_type_str, _ = mimetypes.guess_type(filename) + if not media_type_str: + media_type_str = "application/octet-stream" + + media_type = authoring_api.get_or_create_media_type(media_type_str) + content = authoring_api.get_or_create_file_content( + content_library.learning_package.id, + media_type.id, + data=file_data, + created=now, + ) + + try: + authoring_api.create_component_version_content( + component_version.pk, + content.id, + key=filename, + ) + except IntegrityError: + pass # Content already exists + + if not overrided_component_version_import: + ComponentVersionImport.objects.get_or_create( + component_version=component_version, + source_usage_key=usage_key, + library_import=CourseToLibraryImport.objects.get( + library_key=library_key, + user_id=user_id, + status=CourseToLibraryImportStatus.READY + ), + ) + + +def flat_import_children(block_to_import, library_key, user_id, staged_content, override): + """ + Import children of a block from staged content into a library. + """ + staged_keys = [UsageKey.from_string(key) for key in staged_content.tags.keys()] + block_id_to_usage_key = {key.block_id: key for key in staged_keys} + + for child in block_to_import.getchildren(): + if child.tag in ('chapter', 'sequential', 'vertical'): + flat_import_children(child, library_key, user_id, staged_content, override) + else: + usage_key_str = child.get('url_name') + + if usage_key_str: + usage_key = block_id_to_usage_key.get(usage_key_str) + + if usage_key and usage_key in staged_keys: + library = ContentLibrary.objects.filter( + org__short_name=library_key.org, slug=library_key.slug + ).first() + if not library: + raise ValueError(f"Library {library_key} does not exist.") + create_block_in_library(child, usage_key, library_key, user_id, staged_content.id, override) + + +def get_block_to_import(node, usage_key): + """ + Get the block to import from a node. + """ + if node.get('url_name') == usage_key.block_id: + return node + + for child in node.getchildren(): + if found := get_block_to_import(child, usage_key): + return found diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py index 757a86dec44a..8abd7508461c 100644 --- a/cms/djangoapps/course_to_library_import/models.py +++ b/cms/djangoapps/course_to_library_import/models.py @@ -2,6 +2,8 @@ Models for the course to library import app. """ +from typing import Self + from django.contrib.auth import get_user_model from django.db import models from django.utils.translation import gettext_lazy as _ @@ -43,6 +45,13 @@ class Meta: verbose_name = _('Course to Library Import') verbose_name_plural = _('Course to Library Imports') + @classmethod + def get_by_id(cls, import_id: int) -> Self: + """ + Get an import task by its ID. + """ + return cls.objects.filter(id=import_id).first() + class ComponentVersionImport(TimeStampedModel): """ diff --git a/cms/djangoapps/course_to_library_import/tasks.py b/cms/djangoapps/course_to_library_import/tasks.py index dfe347899a3a..378078d6bcc5 100644 --- a/cms/djangoapps/course_to_library_import/tasks.py +++ b/cms/djangoapps/course_to_library_import/tasks.py @@ -5,40 +5,85 @@ from celery import shared_task from django.db import transaction from edx_django_utils.monitoring import set_code_owner_attribute -from opaque_keys.edx.keys import CourseKey +from lxml import etree +from opaque_keys.edx.keys import CourseKey, UsageKey +from opaque_keys.edx.locator import LibraryLocatorV2 from openedx.core.djangoapps.content_staging import api as content_staging_api from xmodule.modulestore.django import modulestore from .data import CourseToLibraryImportStatus +from .helpers import flat_import_children, get_block_to_import from .models import CourseToLibraryImport +from .validators import validate_usage_ids @shared_task @set_code_owner_attribute def save_courses_to_staged_content_task( - course_ids: list[str], user_id: int, purpose: str, version_num: int | None + course_ids: list[str], + user_id: int, + import_task_id, + purpose: str, + version_num: int | None = None, ) -> None: """ Save courses to staged content task. """ - course_to_library_import = CourseToLibraryImport.objects.create( - course_ids=' '.join(course_ids), - user_id=user_id, - ) + course_to_library_import = CourseToLibraryImport.get_by_id(import_task_id) + if not course_to_library_import: + return with transaction.atomic(): for course_id in course_ids: course_key = CourseKey.from_string(course_id) - sections = modulestore().get_items(course_key, qualifiers={"category": "chapter"}) + sections = modulestore().get_items( + course_key, qualifiers={"category": "chapter"} + ) for section in sections: content_staging_api.stage_xblock_temporarily( section, user_id, - purpose=purpose, + purpose=purpose.format(course_id=course_id), version_num=version_num, ) course_to_library_import.status = CourseToLibraryImportStatus.READY course_to_library_import.save() + + +@shared_task +@set_code_owner_attribute +def import_library_from_staged_content_task( + user_id: int, usage_ids: list[str], library_key: str, purpose: str, course_id: str, override: bool +) -> None: + """ + Import staged content to a library task. + """ + staged_content = content_staging_api.get_ready_staged_content_by_user_and_purpose( + user_id, purpose.format(course_id=course_id) + ) + validate_usage_ids(usage_ids, staged_content) + parser = etree.XMLParser(strip_cdata=False) + library_key = LibraryLocatorV2.from_string(library_key) + + with transaction.atomic(): + for usage_key in usage_ids: + if staged_content_item := staged_content.filter( + tags__icontains=usage_key, + ).first(): + node = etree.fromstring(staged_content_item.olx, parser=parser) + usage_key = UsageKey.from_string(usage_key) + block_to_import = get_block_to_import(node, usage_key) + if not block_to_import: + continue + flat_import_children( + block_to_import, library_key, user_id, staged_content_item, override + ) + + CourseToLibraryImport.objects.filter( + user_id=user_id, library_key=library_key + ).update(status=CourseToLibraryImportStatus.IMPORTED) + + staged_content.delete() diff --git a/cms/djangoapps/course_to_library_import/tests/factories.py b/cms/djangoapps/course_to_library_import/tests/factories.py new file mode 100644 index 000000000000..c4dd4e22d5a6 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/tests/factories.py @@ -0,0 +1,24 @@ +""" +Factories for CourseToLibraryImport model. +""" + +import factory +from factory.django import DjangoModelFactory + +from common.djangoapps.student.tests.factories import UserFactory + +from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport + + +class CourseToLibraryImportFactory(DjangoModelFactory): + """ + Factory for CourseToLibraryImport model. + """ + class Meta: + model = CourseToLibraryImport + + course_ids = ' '.join([f'course-v1:edX+DemoX+Demo_Course{i}' for i in range(1, 3)]) + library_key = 'library-key' + source_type = 'source-type' + metadata = {} + user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/course_to_library_import/tests/test_api.py b/cms/djangoapps/course_to_library_import/tests/test_api.py index f18fb7dba161..7cd4014bf5e4 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_api.py +++ b/cms/djangoapps/course_to_library_import/tests/test_api.py @@ -6,32 +6,61 @@ import pytest +from common.djangoapps.student.tests.factories import UserFactory from cms.djangoapps.course_to_library_import.api import ( - save_courses_to_staged_content, - COURSE_TO_LIBRARY_IMPORT_PURPOSE + create_import, + import_library_from_staged_content, ) +from cms.djangoapps.course_to_library_import.constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE +from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport -@pytest.mark.parametrize("purpose, expected_purpose", [ - ('custom_purpose', 'custom_purpose'), - (None, COURSE_TO_LIBRARY_IMPORT_PURPOSE), -]) -@patch('cms.djangoapps.course_to_library_import.api.save_courses_to_staged_content_task') -def test_save_courses_to_staged_content(mock_task, purpose, expected_purpose): +@pytest.mark.django_db +def test_create_import(): """ - Test save_course_to_staged_content function. - - Case 1: Purpose is provided. - Case 2: Purpose is not provided + Test create_import function. """ + course_ids = [ + "course-v1:edX+DemoX+Demo_Course", + "course-v1:edX+DemoX+Demo_Course_2", + ] + user = UserFactory() + library_key = "lib:edX:DemoLib" + source_type = "test_source_type" + with patch( + "cms.djangoapps.course_to_library_import.api.save_courses_to_staged_content_task" + ) as save_courses_to_staged_content_task_mock: + create_import(course_ids, user.id, library_key, source_type) + + import_task = CourseToLibraryImport.objects.get() + assert import_task.course_ids == " ".join(course_ids) + assert import_task.library_key == library_key + assert import_task.source_type == source_type + assert import_task.user_id == user.id + save_courses_to_staged_content_task_mock.delay.assert_called_once_with( + course_ids, user.id, import_task.id, COURSE_TO_LIBRARY_IMPORT_PURPOSE + ) + - course_ids = ('course-v1:edX+DemoX+Demo_Course', 'course-v1:edX+DemoX+Demo_Course2') - user_id = 1 - version_num = 1 +@pytest.mark.django_db +@pytest.mark.parametrize("override", [True, False]) +def test_import_library_from_staged_content(override): + """ + Test import_library_from_staged_content function with different override values. + """ + library_key = "lib:edX:DemoLib" + user = UserFactory() + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@html+block@123", + "block-v1:edX+DemoX+Demo_Course+type@html+block@456", + ] + course_id = "course-v1:edX+DemoX+Demo_Course" - if purpose: - save_courses_to_staged_content(course_ids, user_id, purpose, version_num) - else: - save_courses_to_staged_content(course_ids, user_id, version_num=version_num) + with patch( + "cms.djangoapps.course_to_library_import.api.import_library_from_staged_content_task" + ) as import_library_from_staged_content_task_mock: + import_library_from_staged_content(library_key, user.id, usage_ids, course_id, override) - mock_task.delay.assert_called_once_with(course_ids, user_id, expected_purpose, version_num) + import_library_from_staged_content_task_mock.delay.assert_called_once_with( + user.id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, override + ) diff --git a/cms/djangoapps/course_to_library_import/tests/test_helpers.py b/cms/djangoapps/course_to_library_import/tests/test_helpers.py new file mode 100644 index 000000000000..55469deb3e66 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/tests/test_helpers.py @@ -0,0 +1,557 @@ +""" +Tests for the course_to_library_import helper functions. +""" + +from datetime import datetime, timezone +from unittest import mock + + +from lxml import etree +from django.db.utils import IntegrityError +from django.test import TestCase +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 + +from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus +from cms.djangoapps.course_to_library_import.helpers import ( + _handle_component_override, + _process_staged_content_files, + create_block_in_library, + flat_import_children, +) +from common.djangoapps.student.tests.factories import UserFactory + +from .factories import CourseToLibraryImportFactory + + +class TestFlatImportChildren(TestCase): + """ + Tests for the flat_import_children helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = "test_user" + + self.staged_content = mock.MagicMock() + self.staged_content.id = "staged-content-id" + self.staged_content.tags = { + "block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1": {}, + "block-v1:TestOrg+TestCourse+Run1+type@html+block@html1": {}, + "block-v1:TestOrg+TestCourse+Run1+type@video+block@video1": {}, + } + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_flat_import_children_basic(self, mock_content_library, mock_create_block): + xml = """ + + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + + self.assertEqual(mock_create_block.call_count, 2) + + usage_key_problem = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1") + usage_key_html = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@html+block@html1") + + mock_create_block.assert_any_call( + mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, False + ) + mock_create_block.assert_any_call( + mock.ANY, usage_key_html, self.library_key, self.user_id, self.staged_content.id, False + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_flat_import_children_nested_structure(self, mock_content_library, mock_create_block): + xml = """ + + + + + + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + + self.assertEqual(mock_create_block.call_count, 2) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_flat_import_children_with_override(self, mock_content_library, mock_create_block): + xml = """ + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, True) + + usage_key_problem = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1") + mock_create_block.assert_called_with( + mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, True + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_flat_import_children_library_not_found(self, mock_content_library): + xml = """ + + + + """ + block_to_import = etree.fromstring(xml) + + mock_content_library.objects.filter.return_value.first.return_value = None + + with self.assertRaises(ValueError): + flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_flat_import_children_ignores_unmatched_url_names(self, mock_content_library, mock_create_block): + xml = """ + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + + mock_create_block.assert_not_called() + + +class TestCreateBlockInLibrary(TestCase): + """ + Tests for the create_block_in_library helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = UserFactory().id + self.block_id = "problem1" + self.block_type = "problem" + self.staged_content_id = "staged-content-id" + self.usage_key = UsageKey.from_string( + f"block-v1:TestOrg+TestCourse+Run1+type@{self.block_type}+block@{self.block_id}" + ) + + self.xml_content = "Test problem content" + self.block_to_import = etree.fromstring(self.xml_content) + self.mock_library = mock.MagicMock() + self.mock_library.library_key = self.library_key + self.mock_learning_package = mock.MagicMock() + self.mock_library.learning_package = self.mock_learning_package + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_create_block_in_library_new_component( + self, mock_content_library, mock_api, mock_authoring_api, mock_content_staging_api + ): + mock_content_library.objects.get_by_key.return_value = self.mock_library + mock_component_type = mock.MagicMock() + mock_authoring_api.get_or_create_component_type.return_value = mock_component_type + mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = False + mock_library_usage_key = mock.MagicMock() + mock_api.validate_can_add_block_to_library.return_value = (None, mock_library_usage_key) + mock_component_version = mock.MagicMock() + mock_api.set_library_block_olx.return_value = mock_component_version + mock_content_staging_api.get_staged_content_static_files.return_value = [] + + create_block_in_library( + self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, False + ) + + mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) + mock_authoring_api.get_or_create_component_type.assert_called_once_with("xblock.v1", self.block_type) + mock_authoring_api.get_components.assert_called_once_with(self.mock_learning_package.id) + mock_api.validate_can_add_block_to_library.assert_called_once_with( + self.library_key, self.block_to_import.tag, self.block_id + ) + mock_authoring_api.create_component.assert_called_once() + mock_api.set_library_block_olx.assert_called_once_with( + mock_library_usage_key, etree.tostring(self.block_to_import) + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + @mock.patch('cms.djangoapps.course_to_library_import.helpers._handle_component_override') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport') + def test_create_block_in_library_existing_component_with_override( + self, + mock_component_version_import, + mock_handle_override, + mock_content_library, + mock_api, + mock_authoring_api, + mock_content_staging_api + ): + mock_content_library.objects.get_by_key.return_value = self.mock_library + mock_component_type = mock.MagicMock() + mock_authoring_api.get_or_create_component_type.return_value = mock_component_type + mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = True + + mock_component_version = mock.MagicMock(spec=['id', 'component_id']) + mock_handle_override.return_value = mock_component_version + + mock_component_version_import.return_value = mock.MagicMock() + + mock_content_staging_api.get_staged_content_static_files.return_value = [] + + CourseToLibraryImportFactory( + status=CourseToLibraryImportStatus.READY, library_key=self.library_key, user_id=self.user_id + ) + create_block_in_library( + self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, True + ) + + mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) + mock_authoring_api.get_or_create_component_type.assert_called_once_with("xblock.v1", self.block_type) + mock_authoring_api.get_components.assert_called_once_with(self.mock_learning_package.id) + mock_handle_override.assert_called_once_with( + self.mock_library, self.usage_key, etree.tostring(self.block_to_import) + ) + mock_api.validate_can_add_block_to_library.assert_not_called() + mock_authoring_api.create_component.assert_not_called() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_create_block_in_library_existing_component_without_override( + self, + mock_content_library, + mock_api, + mock_authoring_api, + mock_content_staging_api + ): + mock_content_library.objects.get_by_key.return_value = self.mock_library + mock_component_type = mock.MagicMock() + mock_authoring_api.get_or_create_component_type.return_value = mock_component_type + mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = True + mock_content_staging_api.get_staged_content_static_files.return_value = [] + + create_block_in_library( + self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, False + ) + + mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) + mock_authoring_api.get_or_create_component_type.assert_called_once_with("xblock.v1", self.block_type) + mock_authoring_api.get_components.assert_called_once_with(self.mock_learning_package.id) + mock_api.validate_can_add_block_to_library.assert_not_called() + mock_authoring_api.create_component.assert_not_called() + mock_api.set_library_block_olx.assert_not_called() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_component_version_import') + @mock.patch('cms.djangoapps.course_to_library_import.helpers._process_staged_content_files') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + @mock.patch('cms.djangoapps.course_to_library_import.helpers._handle_component_override') + def test_create_block_in_library_with_files_and_override( + self, mock_handle_override, mock_content_library, + mock_authoring_api, mock_process_files, + mock_update_component, mock_content_staging_api + ): + mock_content_library.objects.get_by_key.return_value = self.mock_library + mock_component_type = mock.MagicMock() + mock_authoring_api.get_or_create_component_type.return_value = mock_component_type + mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = True + mock_component_version = mock.MagicMock() + mock_handle_override.return_value = mock_component_version + mock_file_data = [mock.MagicMock()] + mock_content_staging_api.get_staged_content_static_files.return_value = mock_file_data + + create_block_in_library( + self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, True + ) + + mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) + mock_update_component.assert_called_once_with( + mock_component_version, self.usage_key, self.library_key, self.user_id + ) + mock_process_files.assert_called_once() + + +class TestProcessStagedContentFiles(TestCase): + """ + Tests for the _process_staged_content_files helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = UserFactory().id + self.block_id = "problem1" + self.block_type = "problem" + self.staged_content_id = "staged-content-id" + self.usage_key = UsageKey.from_string( + f"block-v1:TestOrg+TestCourse+Run1+type@{self.block_type}+block@{self.block_id}" + ) + + self.xml_content = "Test problem content" + self.block_to_import = etree.fromstring(self.xml_content) + self.mock_library = mock.MagicMock() + self.mock_library.library_key = self.library_key + self.mock_learning_package = mock.MagicMock() + self.mock_library.learning_package = self.mock_learning_package + self.now = datetime.now(tz=timezone.utc) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport.objects.get') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport.objects.get_or_create') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_process_staged_content_files_with_reference_in_block( + self, mock_authoring_api, mock_content_staging_api, mock_get_or_create, mock_get_import + ): + mock_component_version = mock.MagicMock() + mock_file_data = mock.MagicMock() + mock_file_data.filename = "test_file.txt" + + xml_content = 'Test problem with ' + block_to_import = etree.fromstring(xml_content) + + mock_content_staging_api.get_staged_content_static_file_data.return_value = b"file data" + mock_media_type = mock.MagicMock(id=1) + mock_authoring_api.get_or_create_media_type.return_value = mock_media_type + mock_content = mock.MagicMock(id=1) + mock_authoring_api.get_or_create_file_content.return_value = mock_content + mock_import = mock.MagicMock() + mock_get_import.return_value = mock_import + + _process_staged_content_files( + mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, + self.mock_library, self.now, block_to_import, False, self.library_key, self.user_id + ) + + mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with( + self.staged_content_id, mock_file_data.filename + ) + mock_authoring_api.get_or_create_media_type.assert_called_once() + mock_authoring_api.get_or_create_file_content.assert_called_once_with( + self.mock_library.learning_package.id, + mock_media_type.id, + data=b"file data", + created=self.now, + ) + mock_authoring_api.create_component_version_content.assert_called_once_with( + mock_component_version.pk, + mock_content.id, + key=f"static/{str(self.usage_key)}" + ) + mock_get_or_create.assert_called_once() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport.objects.get') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_process_staged_content_files_missing_file_data( + self, mock_authoring_api, mock_content_staging_api, mock_get_import + ): + mock_component_version = mock.MagicMock() + mock_file_data = mock.MagicMock() + mock_file_data.filename = "test_file.txt" + + xml_content = 'Test problem with ' + block_to_import = etree.fromstring(xml_content) + + mock_content_staging_api.get_staged_content_static_file_data.return_value = None + mock_import = mock.MagicMock() + mock_get_import.return_value = mock_import + + _process_staged_content_files( + mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, + self.mock_library, self.now, block_to_import, False, self.library_key, self.user_id + ) + + mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with( + self.staged_content_id, mock_file_data.filename + ) + mock_authoring_api.get_or_create_file_content.assert_not_called() + mock_authoring_api.create_component_version_content.assert_not_called() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport.objects.get') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport.objects.get_or_create') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_process_staged_content_files_integrity_error( + self, mock_authoring_api, mock_content_staging_api, mock_get_or_create, mock_get_import + ): + mock_component_version = mock.MagicMock() + mock_file_data = mock.MagicMock() + mock_file_data.filename = "test_file.txt" + + xml_content = 'Test problem with ' + block_to_import = etree.fromstring(xml_content) + + mock_content_staging_api.get_staged_content_static_file_data.return_value = b"file data" + mock_media_type = mock.MagicMock(id=1) + mock_authoring_api.get_or_create_media_type.return_value = mock_media_type + mock_content = mock.MagicMock(id=1) + mock_authoring_api.get_or_create_file_content.return_value = mock_content + + mock_authoring_api.create_component_version_content.side_effect = IntegrityError("Duplicate content") + + mock_import = mock.MagicMock() + mock_get_import.return_value = mock_import + mock_get_or_create.return_value = (mock.MagicMock(), True) + + _process_staged_content_files( + mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, + self.mock_library, self.now, block_to_import, False, self.library_key, self.user_id + ) + + mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with( + self.staged_content_id, mock_file_data.filename + ) + mock_authoring_api.create_component_version_content.assert_called_once() + mock_get_or_create.assert_called_once() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_process_staged_content_files_no_files(self, mock_authoring_api): + mock_component_version = mock.MagicMock() + + _process_staged_content_files( + mock_component_version, [], self.staged_content_id, self.usage_key, + self.mock_library, self.now, self.block_to_import, False, self.library_key, self.user_id + ) + + mock_authoring_api.get_or_create_media_type.assert_not_called() + mock_authoring_api.get_or_create_file_content.assert_not_called() + mock_authoring_api.create_component_version_content.assert_not_called() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_process_staged_content_files_file_not_referenced(self, mock_authoring_api): + mock_component_version = mock.MagicMock() + mock_file_data = mock.MagicMock() + mock_file_data.filename = "unreferenced_file.txt" + + _process_staged_content_files( + mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, + self.mock_library, self.now, self.block_to_import, False, self.library_key, self.user_id + ) + + mock_authoring_api.get_or_create_file_content.assert_not_called() + mock_authoring_api.create_component_version_content.assert_not_called() + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport.objects.get_or_create') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') + def test_process_staged_content_files_with_override( + self, mock_content_staging_api, mock_authoring_api, mock_get_or_create + ): + mock_component_version = mock.MagicMock() + mock_file_data = mock.MagicMock() + mock_file_data.filename = "test_file.txt" + mock_content_staging_api.get_staged_content_static_file_data.return_value = b"file data" + mock_authoring_api.get_or_create_media_type.return_value = mock.MagicMock(id=1) + mock_authoring_api.get_or_create_file_content.return_value = mock.MagicMock(id=1) + mock_get_or_create.return_value = (mock.MagicMock(), True) + + self.xml_content = '' + self.block_to_import = etree.fromstring(self.xml_content) + _process_staged_content_files( + mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, + self.mock_library, self.now, self.block_to_import, True, self.library_key, self.user_id + ) + + mock_authoring_api.get_or_create_media_type.assert_called_once() + mock_authoring_api.get_or_create_file_content.assert_called_once() + mock_authoring_api.create_component_version_content.assert_called_once() + + +class TestHandleComponentOverride(TestCase): + """ + Tests for the _handle_component_override helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = UserFactory().id + self.block_id = "problem1" + self.block_type = "problem" + self.usage_key = UsageKey.from_string( + f"block-v1:TestOrg+TestCourse+Run1+type@{self.block_type}+block@{self.block_id}" + ) + self.xml_content = b"Test problem content" + + self.mock_library = mock.MagicMock() + self.mock_library.library_key = self.library_key + self.mock_learning_package = mock.MagicMock() + self.mock_library.learning_package = self.mock_learning_package + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') + def test_handle_component_override_existing_component(self, mock_api): + mock_component = mock.MagicMock() + mock_component.component_type.name = self.block_type + mock_component.local_key = self.block_id + + self.mock_learning_package.component_set.filter.return_value.first.return_value = mock_component + + expected_lib_usage_key = LibraryUsageLocatorV2( + lib_key=self.library_key, + block_type=self.block_type, + usage_id=self.block_id, + ) + + mock_component_version = mock.MagicMock() + mock_api.set_library_block_olx.return_value = mock_component_version + + result = _handle_component_override(self.mock_library, self.usage_key, self.xml_content) + + self.mock_learning_package.component_set.filter.assert_called_once_with(local_key=self.block_id) + mock_api.set_library_block_olx.assert_called_once_with(expected_lib_usage_key, self.xml_content) + self.assertEqual(result, mock_component_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') + def test_handle_component_override_nonexistent_component(self, mock_api): + self.mock_learning_package.component_set.filter.return_value.first.return_value = None + + result = _handle_component_override(self.mock_library, self.usage_key, self.xml_content) + + self.mock_learning_package.component_set.filter.assert_called_once_with(local_key=self.block_id) + mock_api.set_library_block_olx.assert_not_called() + self.assertIsNone(result) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') + def test_handle_component_override_api_error(self, mock_api): + mock_component = mock.MagicMock() + mock_component.component_type.name = self.block_type + mock_component.local_key = self.block_id + + self.mock_learning_package.component_set.filter.return_value.first.return_value = mock_component + + mock_api.set_library_block_olx.side_effect = Exception("API error") + + with self.assertRaises(Exception): + _handle_component_override(self.mock_library, self.usage_key, self.xml_content) + + mock_api.set_library_block_olx.assert_called_once() diff --git a/cms/djangoapps/course_to_library_import/tests/test_tasks.py b/cms/djangoapps/course_to_library_import/tests/test_tasks.py index 6db5ca3b64d6..71bdf12f3700 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_tasks.py +++ b/cms/djangoapps/course_to_library_import/tests/test_tasks.py @@ -2,14 +2,21 @@ Tests for tasks in course_to_library_import app. """ -from unittest.mock import patch +from unittest.mock import Mock, patch from django.test import TestCase -from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.keys import CourseKey, UsageKey +from opaque_keys.edx.locator import LibraryLocatorV2 -from cms.djangoapps.course_to_library_import.tasks import save_courses_to_staged_content_task +from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus +from cms.djangoapps.course_to_library_import.tasks import ( + import_library_from_staged_content_task, + save_courses_to_staged_content_task, +) from common.djangoapps.student.tests.factories import UserFactory +from .factories import CourseToLibraryImportFactory + class TestSaveCourseSectionsToStagedContentTask(TestCase): """ @@ -19,16 +26,18 @@ class TestSaveCourseSectionsToStagedContentTask(TestCase): @patch('cms.djangoapps.course_to_library_import.tasks.modulestore') @patch('openedx.core.djangoapps.content_staging.api.stage_xblock_temporarily') def test_save_courses_to_staged_content_task(self, mock_stage_xblock_temporarily, mock_modulestore): - - course_ids = ('course-v1:edX+DemoX+Demo_Course', 'course-v1:edX+DemoX+Demo_Course2') - user_id = UserFactory().id + course_to_library_import = CourseToLibraryImportFactory() + course_ids = course_to_library_import.course_ids.split(' ') + user_id = course_to_library_import.user.id purpose = 'test_purpose' version_num = 1 mock_course_keys = [CourseKey.from_string(course_id) for course_id in course_ids] mock_modulestore().get_items.return_value = sections = ['section1', 'section2'] - save_courses_to_staged_content_task(course_ids, user_id, purpose, version_num) + self.assertEqual(course_to_library_import.status, CourseToLibraryImportStatus.PENDING) + + save_courses_to_staged_content_task(course_ids, user_id, course_to_library_import.id, purpose, version_num) for mock_course_key in mock_course_keys: mock_modulestore().get_items.assert_any_call(mock_course_key, qualifiers={"category": "chapter"}) @@ -36,3 +45,105 @@ def test_save_courses_to_staged_content_task(self, mock_stage_xblock_temporarily self.assertEqual(mock_stage_xblock_temporarily.call_count, len(sections) * len(course_ids)) for section in sections: mock_stage_xblock_temporarily.assert_any_call(section, user_id, purpose=purpose, version_num=version_num) + + +class TestImportLibraryFromStagedContentTask(TestCase): + """ + Test cases for import_library_from_staged_content_task. + """ + + @patch('cms.djangoapps.course_to_library_import.tasks.validate_usage_ids') + @patch( + 'cms.djangoapps.course_to_library_import.tasks.content_staging_api.get_ready_staged_content_by_user_and_purpose' + ) + @patch('cms.djangoapps.course_to_library_import.tasks.get_block_to_import') + @patch('cms.djangoapps.course_to_library_import.tasks.flat_import_children') + @patch('cms.djangoapps.course_to_library_import.tasks.etree') + def test_import_library_from_staged_content_task( + self, mock_etree, mock_flat_import_children, mock_get_block_to_import, + mock_get_ready_staged_content, mock_validate_usage_ids + ): + user = UserFactory() + usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] + library_key = 'lib:TestOrg:TestLib' + purpose = 'import_from_{course_id}' + course_id = 'course-v1:edX+Demo+2023' + override = True + + mock_staged_content = mock_get_ready_staged_content.return_value + mock_content_item = Mock() + mock_content_item.olx = 'content' + mock_staged_content.filter.return_value.first.return_value = mock_content_item + + mock_node = Mock() + mock_etree.fromstring.return_value = mock_node + mock_etree.XMLParser.return_value = Mock() + + mock_block = Mock() + mock_get_block_to_import.return_value = mock_block + + library_locator = LibraryLocatorV2.from_string(library_key) + course_to_library_import = CourseToLibraryImportFactory( + user=user, + library_key=library_locator, + status=CourseToLibraryImportStatus.READY + ) + + import_library_from_staged_content_task( + user.id, usage_ids, library_key, purpose, course_id, override + ) + + mock_get_ready_staged_content.assert_called_once_with( + user.id, purpose.format(course_id=course_id) + ) + mock_validate_usage_ids.assert_called_once_with(usage_ids, mock_staged_content) + mock_etree.XMLParser.assert_called_once_with(strip_cdata=False) + mock_etree.fromstring.assert_called_once_with(mock_content_item.olx, parser=mock_etree.XMLParser()) + + mock_staged_content.filter.assert_called_once_with(tags__icontains=usage_ids[0]) + mock_get_block_to_import.assert_called_once_with(mock_node, UsageKey.from_string(usage_ids[0])) + mock_flat_import_children.assert_called_once_with( + mock_block, library_locator, user.id, mock_content_item, override + ) + + course_to_library_import.refresh_from_db() + self.assertEqual(course_to_library_import.status, CourseToLibraryImportStatus.IMPORTED) + mock_staged_content.delete.assert_called_once() + + @patch( + 'cms.djangoapps.course_to_library_import.tasks.content_staging_api.get_ready_staged_content_by_user_and_purpose' + ) + @patch('cms.djangoapps.course_to_library_import.tasks.get_block_to_import') + @patch('cms.djangoapps.course_to_library_import.tasks.etree') + def test_import_library_block_not_found( + self, mock_etree, mock_get_block_to_import, mock_get_ready_staged_content + ): + user = UserFactory() + usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] + library_key = 'lib:TestOrg:TestLib' + purpose = 'import_from_{course_id}' + course_id = 'course-v1:edX+Demo+2023' + override = True + + mock_staged_content = mock_get_ready_staged_content.return_value + mock_content_item = Mock() + mock_content_item.olx = 'content' + mock_staged_content.filter.return_value.first.return_value = mock_content_item + + mock_node = Mock() + mock_etree.fromstring.return_value = mock_node + mock_etree.XMLParser.return_value = Mock() + + mock_get_block_to_import.return_value = None + + library_locator = LibraryLocatorV2.from_string(library_key) + CourseToLibraryImportFactory( + user=user, + library_key=library_locator, + status=CourseToLibraryImportStatus.READY + ) + + with self.assertRaises(ValueError): + import_library_from_staged_content_task( + user.id, usage_ids, library_key, purpose, course_id, override + ) diff --git a/cms/djangoapps/course_to_library_import/tests/test_validators.py b/cms/djangoapps/course_to_library_import/tests/test_validators.py index e843cb324e79..d8cf9f8cd335 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_validators.py +++ b/cms/djangoapps/course_to_library_import/tests/test_validators.py @@ -2,10 +2,12 @@ Tests for course_to_library_import validators """ +from unittest.mock import MagicMock + from django.test import TestCase import pytest -from cms.djangoapps.course_to_library_import.validators import validate_course_ids +from cms.djangoapps.course_to_library_import.validators import validate_course_ids, validate_usage_ids class TestValidateCourseIds(TestCase): @@ -28,4 +30,31 @@ def test_invalid_course_ids(self): def test_duplicate_course_ids(self): with pytest.raises(ValueError) as exc: validate_course_ids('course-v1:edX+DemoX+Demo_Course course-v1:edX+DemoX+Demo_Course') - assert str(exc.value) == 'Duplicate course keys are not allowed' + assert str(exc.value) == 'Duplicate course keys are not allowed' + + +class TestValidateUsageIds(TestCase): + """ + Test cases for validate_usage_ids function. + + Case 1: Valid usage ids + Case 2: Invalid usage ids + """ + + def test_valid_usage_ids(self): + staged_content = MagicMock() + staged_content.values_list.return_value = [ + ['block-v1:edX+DemoX+type@problem+block@12345'], + ['block-v1:edX+DemoX+type@video+block@67890'], + ] + validate_usage_ids(['block-v1:edX+DemoX+type@problem+block@12345'], staged_content) + + def test_invalid_usage_ids(self): + staged_content = MagicMock() + staged_content.values_list.return_value = [ + ['block-v1:edX+DemoX+type@problem+block@12345'], + ['block-v1:edX+DemoX+type@video+block@67890'], + ] + with pytest.raises(ValueError) as exc: + validate_usage_ids(['block-v1:edX+DemoX+type@discussion+block@54321'], staged_content) + assert str(exc.value) == 'Block block-v1:edX+DemoX+type@discussion+block@54321 is not available for import' diff --git a/cms/djangoapps/course_to_library_import/validators.py b/cms/djangoapps/course_to_library_import/validators.py index 775a5e382e34..cf65f3055501 100644 --- a/cms/djangoapps/course_to_library_import/validators.py +++ b/cms/djangoapps/course_to_library_import/validators.py @@ -2,6 +2,8 @@ Validators for the course_to_library_import app. """ +from collections import ChainMap + from django.utils.translation import gettext_lazy as _ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey @@ -27,3 +29,10 @@ def validate_course_ids(value: str): CourseKey.from_string(course_id) except InvalidKeyError as exc: raise ValueError(_('Invalid course key: {course_id}').format(course_id=course_id)) from exc + + +def validate_usage_ids(usage_ids, staged_content): + available_block_keys = ChainMap(*staged_content.values_list('tags', flat=True)) + for usage_key in usage_ids: + if usage_key not in available_block_keys: + raise ValueError(f'Block {usage_key} is not available for import') diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/core/djangoapps/content_staging/api.py b/openedx/core/djangoapps/content_staging/api.py index 271b89871eaa..83a2e7f386ac 100644 --- a/openedx/core/djangoapps/content_staging/api.py +++ b/openedx/core/djangoapps/content_staging/api.py @@ -8,6 +8,7 @@ from django.core.files.base import ContentFile from django.db import transaction +from django.db.models import QuerySet from django.http import HttpRequest from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import AssetKey, UsageKey @@ -251,6 +252,13 @@ def _user_clipboard_model_to_data(clipboard: _UserClipboard) -> UserClipboardDat ) +def get_ready_staged_content_by_user_and_purpose(user_id: int, purpose: str) -> QuerySet[_StagedContent]: + """ + Get all staged content for the given user and purpose that are READY to use. + """ + return _StagedContent.objects.filter(user_id=user_id, purpose=purpose, status=StagedContentStatus.READY) + + def get_staged_content_olx(staged_content_id: int) -> str | None: """ Get the OLX (as a string) for the given StagedContent. From c75c0a4e7cd73cd910a4ab1c33824bb7b111c238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=86=D0=B2=D0=B0=D0=BD=20=D0=9D=D1=94=D0=B4=D1=94=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=96=D1=86=D0=B5=D0=B2?= Date: Mon, 17 Mar 2025 14:55:09 +0200 Subject: [PATCH 04/23] docs: add readme --- .../course_to_library_import/README.rst | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 cms/djangoapps/course_to_library_import/README.rst diff --git a/cms/djangoapps/course_to_library_import/README.rst b/cms/djangoapps/course_to_library_import/README.rst new file mode 100644 index 000000000000..b2be9e67af7e --- /dev/null +++ b/cms/djangoapps/course_to_library_import/README.rst @@ -0,0 +1,32 @@ +======================== +Course to Library Import +======================== + +The new Django application `course_to_library_import` is designed to +automate the process of importing course OLX content from Modulestore to +Content Libraries. The application allows users to easily and quickly +migrate existing course content, minimizing the manual work and potential +errors associated with manual migration. +The new app makes the import process automated and easy to manage. + +The main problems solved by the application: + +* Reducing the time to import course content. +* Ensuring data integrity during the transfer. +* Ability to choose which content to import before the final import. + +------------------------------ +Course to Library Import Usage +------------------------------ + +* Import course elements at the level of sections, subsections, units, + and xblocks into the Content Libraries. +* Choose the structure of this import, whether it will be only xblocks + from a particular course or full sections/subsections/units. +* Store the history of imports. +* Synchronize the course content with the library content (when re-importing, + the blocks can be updated according to changes in the original course). +* The new import mechanism ensures data integrity at the time of importing + by saving the course in StagedContent. + + From 05c41958af3bce0ddb05a6d5e7c4556f0c6ef95f Mon Sep 17 00:00:00 2001 From: andrii-hantkovskyi <131773947+andrii-hantkovskyi@users.noreply.github.com> Date: Fri, 21 Mar 2025 12:41:07 +0200 Subject: [PATCH 05/23] feat: [AXM-1726] add container feature for import (#2607) * feat: [AXM-1726] add container feature for import * refactor: [AXM-1726] post-review changes --------- Co-authored-by: Andrii --- .../course_to_library_import/api.py | 10 +- .../course_to_library_import/helpers.py | 180 ++++++- .../migrations/0002_containerversionimport.py | 33 ++ .../course_to_library_import/models.py | 21 + .../course_to_library_import/tasks.py | 21 +- .../tests/test_api.py | 4 +- .../tests/test_helpers.py | 498 +++++++++++++++++- .../tests/test_tasks.py | 15 +- .../tests/test_validators.py | 27 +- .../course_to_library_import/types.py | 6 + .../course_to_library_import/validators.py | 10 + 11 files changed, 763 insertions(+), 62 deletions(-) create mode 100644 cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py create mode 100644 cms/djangoapps/course_to_library_import/types.py diff --git a/cms/djangoapps/course_to_library_import/api.py b/cms/djangoapps/course_to_library_import/api.py index c1f3f756bd12..4f00ee5a331c 100644 --- a/cms/djangoapps/course_to_library_import/api.py +++ b/cms/djangoapps/course_to_library_import/api.py @@ -8,16 +8,22 @@ import_library_from_staged_content_task, save_courses_to_staged_content_task, ) +from .types import CompositionLevel def import_library_from_staged_content( - library_key: str, user_id: int, usage_ids: list[str], course_id: str, override: bool + library_key: str, + user_id: int, + usage_ids: list[str], + course_id: str, + composition_level: CompositionLevel, + override: bool ) -> None: """ Import staged content to a library. """ import_library_from_staged_content_task.delay( - user_id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, override + user_id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, composition_level, override ) diff --git a/cms/djangoapps/course_to_library_import/helpers.py b/cms/djangoapps/course_to_library_import/helpers.py index 317cf435b206..f612e2c17664 100644 --- a/cms/djangoapps/course_to_library_import/helpers.py +++ b/cms/djangoapps/course_to_library_import/helpers.py @@ -2,9 +2,10 @@ Helper functions for importing course content into a library. """ +from datetime import datetime, timezone import logging import mimetypes -from datetime import datetime, timezone +import secrets from django.db import transaction from django.db.utils import IntegrityError @@ -13,13 +14,14 @@ from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import ContainerVersion from openedx.core.djangoapps.content_libraries import api from openedx.core.djangoapps.content_libraries.api import ContentLibrary from openedx.core.djangoapps.content_staging import api as content_staging_api from .data import CourseToLibraryImportStatus -from .models import ComponentVersionImport, CourseToLibraryImport +from .models import ComponentVersionImport, ContainerVersionImport, CourseToLibraryImport import os @@ -33,7 +35,6 @@ def create_block_in_library(block_to_import, usage_key, library_key, user_id, st now = datetime.now(tz=timezone.utc) staged_content_files = content_staging_api.get_staged_content_static_files(staged_content_id) - assert isinstance(library_key, LibraryLocatorV2) content_library = ContentLibrary.objects.get_by_key(library_key) with transaction.atomic(): @@ -82,6 +83,8 @@ def create_block_in_library(block_to_import, usage_key, library_key, user_id, st content_library, now, block_to_import, overrided_component_version_import, library_key, user_id ) + return component_version + def _handle_component_override(content_library, usage_key, new_content): """ @@ -187,38 +190,181 @@ def _process_staged_content_files( ) -def flat_import_children(block_to_import, library_key, user_id, staged_content, override): +def _update_container_components(container_version, component_versions, user_id): + return authoring_api.create_next_container_version( + container_pk=container_version.container.pk, + title=container_version.title, + publishable_entities_pks=[ + cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk for cv in component_versions + ], + entity_version_pks=[cv.pk for cv in component_versions], + created=datetime.now(tz=timezone.utc), + created_by=user_id, + container_version_cls=container_version.__class__, + ) + + +def _process_xblock(child, library_key, user_id, staged_content, override): """ - Import children of a block from staged content into a library. + Process an xblock and create a block in the library. """ staged_keys = [UsageKey.from_string(key) for key in staged_content.tags.keys()] block_id_to_usage_key = {key.block_id: key for key in staged_keys} + usage_key = block_id_to_usage_key.get(child.get('url_name')) + + if usage_key in staged_keys: + return create_block_in_library( + child, + usage_key, + library_key, + user_id, + staged_content.id, + override, + ) + + +def import_children(block_to_import, library_key, user_id, staged_content, composition_level, override): + """ + Import children of a block from staged content into a library. + Creates appropriate container hierarchy based on composition_level. + """ + result = [] + + if block_to_import.tag not in ('chapter', 'sequential', 'vertical'): + component_version = _process_xblock(block_to_import, library_key, user_id, staged_content, override) + if component_version: + return [component_version] + for child in block_to_import.getchildren(): if child.tag in ('chapter', 'sequential', 'vertical'): - flat_import_children(child, library_key, user_id, staged_content, override) + container_version = create_container( + child.tag, + child.get('url_name'), + child.get('display_name', ''), + library_key, + user_id, + ) + + child_component_versions = import_children( + child, + library_key, + user_id, + staged_content, + composition_level, + override, + ) + + if child_component_versions: + _update_container_components(container_version, child_component_versions, user_id) + + result.append(container_version) else: - usage_key_str = child.get('url_name') + component_version = _process_xblock(child, library_key, user_id, staged_content, override) + if component_version is not None: + result.append(component_version) - if usage_key_str: - usage_key = block_id_to_usage_key.get(usage_key_str) + if composition_level == 'xblock': + return [component for component in result if not isinstance(component, ContainerVersion)] + else: + return result - if usage_key and usage_key in staged_keys: - library = ContentLibrary.objects.filter( - org__short_name=library_key.org, slug=library_key.slug - ).first() - if not library: - raise ValueError(f"Library {library_key} does not exist.") - create_block_in_library(child, usage_key, library_key, user_id, staged_content.id, override) + +def create_container(container_type, key, display_name, library_key, user_id): + """ + Create a container of the specified type. + """ + assert isinstance(library_key, LibraryLocatorV2) + content_library = ContentLibrary.objects.get_by_key(library_key) + + container_creators_map = { + 'chapter': authoring_api.create_unit_and_version, # TODO: replace with create_module_and_version + 'sequential': authoring_api.create_unit_and_version, # TODO: replace with create_section_and_version + 'vertical': authoring_api.create_unit_and_version, + } + + if container_type not in container_creators_map: + raise ValueError(f"Unknown container type: {container_type}") + + if not key: + key = secrets.token_hex(16) + + if not display_name: + display_name = f"New {container_type}" + + if container_creator_func := container_creators_map.get(container_type): + _, container_version = container_creator_func( + content_library.learning_package.id, + key=key, + title=display_name, + components=[], + created=datetime.now(tz=timezone.utc), + created_by=user_id, + ) + + return container_version + + +def import_container(usage_key, block_to_import, library_key, user_id, staged_content, composition_level, override): + """ + Import a blocks hierarchy into a library, creating proper container structure. + """ + container_type = block_to_import.tag + + if composition_level in ('chapter', 'sequential', 'vertical'): + key = block_to_import.get('url_name') + display_name = block_to_import.get('display_name', '') + + top_container_version = create_container( + container_type, + key, + display_name, + library_key, + user_id, + ) + + component_versions = import_children( + block_to_import, + library_key, + user_id, + staged_content, + composition_level, + override, + ) + + if component_versions: + _update_container_components(top_container_version, component_versions, user_id) + + with transaction.atomic(): + ContainerVersionImport.objects.create( + section_version=top_container_version, + source_usage_key=usage_key, + library_import=CourseToLibraryImport.objects.get( + library_key=library_key, + user_id=user_id, + status=CourseToLibraryImportStatus.READY + ), + ) + else: # xblock level + import_children( + block_to_import, + library_key, + user_id, + staged_content, + composition_level, + override, + ) def get_block_to_import(node, usage_key): """ Get the block to import from a node. """ + if node.get('url_name') == usage_key.block_id: return node for child in node.getchildren(): - if found := get_block_to_import(child, usage_key): + found = get_block_to_import(child, usage_key) + if found is not None: return found diff --git a/cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py b/cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py new file mode 100644 index 000000000000..67880b183df3 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.19 on 2025-03-19 17:47 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import opaque_keys.edx.django.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_publishing', '0003_containers'), + ('course_to_library_import', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ContainerVersionImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), + ('container_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.containerversion')), + ('library_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='course_to_library_import.coursetolibraryimport')), + ], + options={ + 'verbose_name': 'Container Version Import', + 'verbose_name_plural': 'Container Version Imports', + }, + ), + ] diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py index 8abd7508461c..a30cfaa6fa45 100644 --- a/cms/djangoapps/course_to_library_import/models.py +++ b/cms/djangoapps/course_to_library_import/models.py @@ -72,3 +72,24 @@ def __str__(self): class Meta: verbose_name = _('Component Version Import') verbose_name_plural = _('Component Version Imports') + + +class ContainerVersionImport(TimeStampedModel): + """ + Represents a container version that has been imported into a content library. + This is a many-to-many relationship between a container version and a course to library import. + """ + + container_version = models.OneToOneField( + to='oel_publishing.ContainerVersion', + on_delete=models.CASCADE + ) + source_usage_key = UsageKeyField(max_length=255) + library_import = models.ForeignKey(CourseToLibraryImport, on_delete=models.CASCADE) + + def __str__(self): + return f'{self.container_version} - {self.source_usage_key}' + + class Meta: + verbose_name = _('Container Version Import') + verbose_name_plural = _('Container Version Imports') diff --git a/cms/djangoapps/course_to_library_import/tasks.py b/cms/djangoapps/course_to_library_import/tasks.py index 378078d6bcc5..6ddf23730ca5 100644 --- a/cms/djangoapps/course_to_library_import/tasks.py +++ b/cms/djangoapps/course_to_library_import/tasks.py @@ -13,9 +13,10 @@ from xmodule.modulestore.django import modulestore from .data import CourseToLibraryImportStatus -from .helpers import flat_import_children, get_block_to_import +from .helpers import get_block_to_import, import_container from .models import CourseToLibraryImport -from .validators import validate_usage_ids +from .types import CompositionLevel +from .validators import validate_composition_level, validate_usage_ids @shared_task @@ -56,11 +57,18 @@ def save_courses_to_staged_content_task( @shared_task @set_code_owner_attribute def import_library_from_staged_content_task( - user_id: int, usage_ids: list[str], library_key: str, purpose: str, course_id: str, override: bool + user_id: int, + usage_ids: list[str], + library_key: str, + purpose: str, + course_id: str, + composition_level: CompositionLevel, + override: bool ) -> None: """ Import staged content to a library task. """ + validate_composition_level(composition_level) staged_content = content_staging_api.get_ready_staged_content_by_user_and_purpose( user_id, purpose.format(course_id=course_id) ) @@ -76,10 +84,11 @@ def import_library_from_staged_content_task( node = etree.fromstring(staged_content_item.olx, parser=parser) usage_key = UsageKey.from_string(usage_key) block_to_import = get_block_to_import(node, usage_key) - if not block_to_import: + + if block_to_import is None: continue - flat_import_children( - block_to_import, library_key, user_id, staged_content_item, override + import_container( + usage_key, block_to_import, library_key, user_id, staged_content_item, composition_level, override ) CourseToLibraryImport.objects.filter( diff --git a/cms/djangoapps/course_to_library_import/tests/test_api.py b/cms/djangoapps/course_to_library_import/tests/test_api.py index 7cd4014bf5e4..1111120feba6 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_api.py +++ b/cms/djangoapps/course_to_library_import/tests/test_api.py @@ -59,8 +59,8 @@ def test_import_library_from_staged_content(override): with patch( "cms.djangoapps.course_to_library_import.api.import_library_from_staged_content_task" ) as import_library_from_staged_content_task_mock: - import_library_from_staged_content(library_key, user.id, usage_ids, course_id, override) + import_library_from_staged_content(library_key, user.id, usage_ids, course_id, 'xblock', override) import_library_from_staged_content_task_mock.delay.assert_called_once_with( - user.id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, override + user.id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, 'xblock', override ) diff --git a/cms/djangoapps/course_to_library_import/tests/test_helpers.py b/cms/djangoapps/course_to_library_import/tests/test_helpers.py index 55469deb3e66..4c0f460758f0 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_helpers.py +++ b/cms/djangoapps/course_to_library_import/tests/test_helpers.py @@ -11,13 +11,17 @@ from django.test import TestCase from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 +from openedx_learning.api.authoring_models import ContainerVersion from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus from cms.djangoapps.course_to_library_import.helpers import ( _handle_component_override, _process_staged_content_files, + _update_container_components, create_block_in_library, - flat_import_children, + create_container, + import_children, + import_container, ) from common.djangoapps.student.tests.factories import UserFactory @@ -56,7 +60,7 @@ def test_flat_import_children_basic(self, mock_content_library, mock_create_bloc mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library - flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False) self.assertEqual(mock_create_block.call_count, 2) @@ -70,28 +74,6 @@ def test_flat_import_children_basic(self, mock_content_library, mock_create_bloc mock.ANY, usage_key_html, self.library_key, self.user_id, self.staged_content.id, False ) - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_flat_import_children_nested_structure(self, mock_content_library, mock_create_block): - xml = """ - - - - - - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) - - self.assertEqual(mock_create_block.call_count, 2) - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') def test_flat_import_children_with_override(self, mock_content_library, mock_create_block): @@ -105,7 +87,7 @@ def test_flat_import_children_with_override(self, mock_content_library, mock_cre mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library - flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, True) + import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', True) usage_key_problem = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1") mock_create_block.assert_called_with( @@ -124,7 +106,7 @@ def test_flat_import_children_library_not_found(self, mock_content_library): mock_content_library.objects.filter.return_value.first.return_value = None with self.assertRaises(ValueError): - flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False) @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') @@ -139,7 +121,7 @@ def test_flat_import_children_ignores_unmatched_url_names(self, mock_content_lib mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library - flat_import_children(block_to_import, self.library_key, self.user_id, self.staged_content, False) + import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False) mock_create_block.assert_not_called() @@ -555,3 +537,465 @@ def test_handle_component_override_api_error(self, mock_api): _handle_component_override(self.mock_library, self.usage_key, self.xml_content) mock_api.set_library_block_olx.assert_called_once() + + +class TestUpdateContainerComponents(TestCase): + """ + Tests for the _update_container_components helper function. + """ + + def setUp(self): + super().setUp() + self.user_id = UserFactory().id + self.mock_container_version = mock.MagicMock() + self.mock_container_version.container.pk = "container_pk" + self.mock_container_version.title = "Container Title" + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_update_container_components_with_mixed_components(self, mock_authoring_api): + mock_component_version = mock.MagicMock() + mock_component_version.component.pk = "component_pk" + mock_container_child_version = mock.MagicMock(spec=ContainerVersion) + mock_container_child_version.container.pk = "container_child_pk" + + component_versions = [mock_component_version, mock_container_child_version] + + _update_container_components(self.mock_container_version, component_versions, self.user_id) + + mock_authoring_api.create_next_container_version.assert_called_once_with( + container_pk=self.mock_container_version.container.pk, + title=self.mock_container_version.title, + publishable_entities_pks=["component_pk", "container_child_pk"], + entity_version_pks=[mock_component_version.pk, mock_container_child_version.pk], + created=mock.ANY, + created_by=self.user_id, + container_version_cls=self.mock_container_version.__class__, + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + def test_update_container_components_empty_list(self, mock_authoring_api): + _update_container_components(self.mock_container_version, [], self.user_id) + + mock_authoring_api.create_next_container_version.assert_called_once_with( + container_pk=self.mock_container_version.container.pk, + title=self.mock_container_version.title, + publishable_entities_pks=[], + entity_version_pks=[], + created=mock.ANY, + created_by=self.user_id, + container_version_cls=self.mock_container_version.__class__, + ) + + +class TestCreateContainer(TestCase): + """ + Tests for the create_container helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = UserFactory().id + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_create_container_chapter(self, mock_content_library, mock_authoring_api): + mock_library = mock.MagicMock() + mock_content_library.objects.get_by_key.return_value = mock_library + + mock_container = mock.MagicMock() + mock_container_version = mock.MagicMock() + mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) + + result = create_container('chapter', 'test_key', 'Test Chapter', self.library_key, self.user_id) + + mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) + mock_authoring_api.create_unit_and_version.assert_called_once_with( + mock_library.learning_package.id, + key='test_key', + title='Test Chapter', + components=[], + created=mock.ANY, + created_by=self.user_id, + ) + self.assertEqual(result, mock_container_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_create_container_sequential(self, mock_content_library, mock_authoring_api): + mock_library = mock.MagicMock() + mock_content_library.objects.get_by_key.return_value = mock_library + + mock_container = mock.MagicMock() + mock_container_version = mock.MagicMock() + mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) + + result = create_container('sequential', 'test_key', 'Test Sequential', self.library_key, self.user_id) + + mock_authoring_api.create_unit_and_version.assert_called_once_with( + mock_library.learning_package.id, + key='test_key', + title='Test Sequential', + components=[], + created=mock.ANY, + created_by=self.user_id, + ) + self.assertEqual(result, mock_container_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.secrets') + def test_create_container_no_key(self, mock_secrets, mock_content_library, mock_authoring_api): + mock_library = mock.MagicMock() + mock_content_library.objects.get_by_key.return_value = mock_library + + mock_container = mock.MagicMock() + mock_container_version = mock.MagicMock() + mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) + + mock_secrets.token_hex.return_value = "generated_key" + + result = create_container('vertical', None, 'Test Vertical', self.library_key, self.user_id) + + mock_secrets.token_hex.assert_called_once_with(16) + mock_authoring_api.create_unit_and_version.assert_called_once_with( + mock_library.learning_package.id, + key='generated_key', + title='Test Vertical', + components=[], + created=mock.ANY, + created_by=self.user_id, + ) + self.assertEqual(result, mock_container_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_create_container_no_display_name(self, mock_content_library, mock_authoring_api): + mock_library = mock.MagicMock() + mock_content_library.objects.get_by_key.return_value = mock_library + + mock_container = mock.MagicMock() + mock_container_version = mock.MagicMock() + mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) + + result = create_container('vertical', 'test_key', '', self.library_key, self.user_id) + + mock_authoring_api.create_unit_and_version.assert_called_once_with( + mock_library.learning_package.id, + key='test_key', + title='New vertical', + components=[], + created=mock.ANY, + created_by=self.user_id, + ) + self.assertEqual(result, mock_container_version) + + +class TestImportContainer(TestCase): + """ + Tests for the import_container helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = UserFactory().id + self.usage_key = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@chapter+block@chapter1") + self.staged_content = mock.MagicMock() + self.staged_content.id = "staged-content-id" + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContainerVersionImport') + @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.import_children') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport') + def test_import_container_with_container_composition( + self, mock_course_import, mock_import_children, mock_create_container, + mock_update_container, mock_section_version_import + ): + xml = """ + + + + + + + + """ + block_to_import = etree.fromstring(xml) + + mock_container_version = mock.MagicMock() + mock_create_container.return_value = mock_container_version + + mock_component_versions = [mock.MagicMock()] + mock_import_children.return_value = mock_component_versions + + mock_section_version_import.objects.create = mock.MagicMock() + mock_get_import = mock.MagicMock() + mock_course_import.objects.get.return_value = mock_get_import + + import_container( + self.usage_key, block_to_import, self.library_key, self.user_id, + self.staged_content, 'chapter', False + ) + + mock_create_container.assert_called_once_with( + 'chapter', 'chapter1', 'Test Chapter', self.library_key, self.user_id + ) + mock_import_children.assert_called_once_with( + block_to_import, self.library_key, self.user_id, self.staged_content, + 'chapter', False + ) + mock_update_container.assert_called_once_with( + mock_container_version, mock_component_versions, self.user_id + ) + mock_section_version_import.objects.create.assert_called_once_with( + section_version=mock_container_version, + source_usage_key=self.usage_key, + library_import=mock_get_import + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.import_children') + def test_import_container_xblock_level(self, mock_import_children): + xml = """ + + + + + + + + """ + block_to_import = etree.fromstring(xml) + + import_container( + self.usage_key, block_to_import, self.library_key, self.user_id, + self.staged_content, 'xblock', False + ) + + mock_import_children.assert_called_once_with( + block_to_import, self.library_key, self.user_id, self.staged_content, + 'xblock', False + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContainerVersionImport') + @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.import_children') + def test_import_container_no_children( + self, mock_import_children, mock_create_container, mock_update_container, mock_section_version_import + ): + xml = """""" + block_to_import = etree.fromstring(xml) + + CourseToLibraryImportFactory( + library_key=self.library_key, + user_id=self.user_id, + status=CourseToLibraryImportStatus.READY, + ) + + mock_container_version = mock.MagicMock() + mock_create_container.return_value = mock_container_version + + mock_import_children.return_value = [] + + import_container( + self.usage_key, block_to_import, self.library_key, self.user_id, + self.staged_content, 'chapter', False + ) + + mock_create_container.assert_called_once() + mock_import_children.assert_called_once() + mock_update_container.assert_not_called() + + +class TestImportChildren(TestCase): + """ + Tests for the import_children helper function. + """ + + def setUp(self): + super().setUp() + self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") + self.user_id = UserFactory().id + + self.staged_content = mock.MagicMock() + self.staged_content.id = "staged-content-id" + self.staged_content.tags = { + "block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1": {}, + "block-v1:TestOrg+TestCourse+Run1+type@html+block@html1": {}, + "block-v1:TestOrg+TestCourse+Run1+type@video+block@video1": {}, + } + + self.problem_usage_key = UsageKey.from_string( + "block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1" + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_import_children_xblock_level(self, mock_content_library, mock_create_block): + xml = """ + + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + mock_component_version = mock.MagicMock(spec=['id', 'component_id']) + mock_create_block.return_value = mock_component_version + + result = import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + ) + + self.assertEqual(mock_create_block.call_count, 2) + self.assertEqual(len(result), 2) + self.assertEqual(result[0], mock_component_version) + self.assertEqual(result[1], mock_component_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_import_children_with_containers( + self, mock_content_library, mock_create_block, mock_create_container, mock_update_container + ): + xml = """ + + + + + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + mock_component_version = mock.MagicMock(spec=['id', 'component_id']) + mock_create_block.return_value = mock_component_version + + mock_container_version = mock.MagicMock(spec=ContainerVersion) + mock_create_container.return_value = mock_container_version + + result = import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', False + ) + + self.assertEqual(mock_create_container.call_count, 2) + self.assertEqual(mock_create_block.call_count, 1) + self.assertEqual(len(result), 1) + self.assertEqual(result[0], mock_container_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_import_children_empty_block_xblock_level(self, mock_content_library, mock_create_block): + xml = '' + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + mock_component_version = mock.MagicMock() + mock_create_block.return_value = mock_component_version + + result = import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + ) + + mock_create_block.assert_called_once() + self.assertEqual(len(result), 1) + self.assertEqual(result[0], mock_component_version) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_import_children_with_override(self, mock_content_library, mock_create_block): + xml = '' + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', True + ) + + mock_create_block.assert_called_once_with( + block_to_import, + self.problem_usage_key, + self.library_key, + self.user_id, + self.staged_content.id, + True, + ) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_import_children_library_not_found(self, mock_content_library): + xml = '' + block_to_import = etree.fromstring(xml) + + mock_content_library.objects.filter.return_value.first.return_value = None + + with self.assertRaises(ValueError): + import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + ) + + def test_import_children_no_matching_children(self): + xml = """ + + + + + """ + block_to_import = etree.fromstring(xml) + + result = import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + ) + + self.assertEqual(result, []) + + @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') + @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') + def test_import_children_filter_by_composition_level( + self, mock_content_library, mock_create_block, mock_create_container, mock_update_container + ): + xml = """ + + + + + + """ + block_to_import = etree.fromstring(xml) + + mock_library = mock.MagicMock() + mock_content_library.objects.filter.return_value.first.return_value = mock_library + + mock_component_version = mock.MagicMock(spec=['id', 'component_id']) + mock_create_block.return_value = mock_component_version + + mock_container_version = mock.MagicMock(spec=ContainerVersion) + mock_create_container.return_value = mock_container_version + + result_xblock = import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + ) + + result_chapter = import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', False + ) + + self.assertTrue(all(not isinstance(item, ContainerVersion) for item in result_xblock)) + + self.assertEqual(len(result_chapter), 1) + self.assertEqual(result_chapter[0], mock_container_version) diff --git a/cms/djangoapps/course_to_library_import/tests/test_tasks.py b/cms/djangoapps/course_to_library_import/tests/test_tasks.py index 71bdf12f3700..8785fe06fa15 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_tasks.py +++ b/cms/djangoapps/course_to_library_import/tests/test_tasks.py @@ -57,14 +57,15 @@ class TestImportLibraryFromStagedContentTask(TestCase): 'cms.djangoapps.course_to_library_import.tasks.content_staging_api.get_ready_staged_content_by_user_and_purpose' ) @patch('cms.djangoapps.course_to_library_import.tasks.get_block_to_import') - @patch('cms.djangoapps.course_to_library_import.tasks.flat_import_children') + @patch('cms.djangoapps.course_to_library_import.tasks.import_container') @patch('cms.djangoapps.course_to_library_import.tasks.etree') def test_import_library_from_staged_content_task( - self, mock_etree, mock_flat_import_children, mock_get_block_to_import, + self, mock_etree, mock_import_container, mock_get_block_to_import, mock_get_ready_staged_content, mock_validate_usage_ids ): user = UserFactory() usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] + usage_key = UsageKey.from_string(usage_ids[0]) library_key = 'lib:TestOrg:TestLib' purpose = 'import_from_{course_id}' course_id = 'course-v1:edX+Demo+2023' @@ -90,7 +91,7 @@ def test_import_library_from_staged_content_task( ) import_library_from_staged_content_task( - user.id, usage_ids, library_key, purpose, course_id, override + user.id, usage_ids, library_key, purpose, course_id, 'xblock', override ) mock_get_ready_staged_content.assert_called_once_with( @@ -101,9 +102,9 @@ def test_import_library_from_staged_content_task( mock_etree.fromstring.assert_called_once_with(mock_content_item.olx, parser=mock_etree.XMLParser()) mock_staged_content.filter.assert_called_once_with(tags__icontains=usage_ids[0]) - mock_get_block_to_import.assert_called_once_with(mock_node, UsageKey.from_string(usage_ids[0])) - mock_flat_import_children.assert_called_once_with( - mock_block, library_locator, user.id, mock_content_item, override + mock_get_block_to_import.assert_called_once_with(mock_node, usage_key) + mock_import_container.assert_called_once_with( + usage_key, mock_block, library_locator, user.id, mock_content_item, 'xblock', override ) course_to_library_import.refresh_from_db() @@ -145,5 +146,5 @@ def test_import_library_block_not_found( with self.assertRaises(ValueError): import_library_from_staged_content_task( - user.id, usage_ids, library_key, purpose, course_id, override + user.id, usage_ids, library_key, purpose, course_id, 'xblock', override ) diff --git a/cms/djangoapps/course_to_library_import/tests/test_validators.py b/cms/djangoapps/course_to_library_import/tests/test_validators.py index d8cf9f8cd335..823222fba23f 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_validators.py +++ b/cms/djangoapps/course_to_library_import/tests/test_validators.py @@ -2,12 +2,18 @@ Tests for course_to_library_import validators """ +from typing import get_args from unittest.mock import MagicMock from django.test import TestCase import pytest -from cms.djangoapps.course_to_library_import.validators import validate_course_ids, validate_usage_ids +from cms.djangoapps.course_to_library_import.validators import ( + validate_course_ids, + validate_usage_ids, + validate_composition_level +) +from cms.djangoapps.course_to_library_import.types import CompositionLevel class TestValidateCourseIds(TestCase): @@ -58,3 +64,22 @@ def test_invalid_usage_ids(self): with pytest.raises(ValueError) as exc: validate_usage_ids(['block-v1:edX+DemoX+type@discussion+block@54321'], staged_content) assert str(exc.value) == 'Block block-v1:edX+DemoX+type@discussion+block@54321 is not available for import' + + +class TestValidateCompositionLevel(TestCase): + """ + Test cases for validate_composition_level function. + + Case 1: Valid composition level + Case 2: Invalid composition level + """ + + def test_valid_composition_level(self): + for level in get_args(CompositionLevel): + # Should not raise an exception for valid levels + validate_composition_level(level) + + def test_invalid_composition_level(self): + with pytest.raises(ValueError) as exc: + validate_composition_level('invalid_composition_level') + assert 'Invalid composition level: invalid_composition_level' in str(exc.value) diff --git a/cms/djangoapps/course_to_library_import/types.py b/cms/djangoapps/course_to_library_import/types.py new file mode 100644 index 000000000000..6a76d9ee1f5f --- /dev/null +++ b/cms/djangoapps/course_to_library_import/types.py @@ -0,0 +1,6 @@ +""" +This module contains type definitions for the course to library import process. +""" +from typing import Literal + +CompositionLevel = Literal["chapter", "sequential", "vertical", "xblock"] diff --git a/cms/djangoapps/course_to_library_import/validators.py b/cms/djangoapps/course_to_library_import/validators.py index cf65f3055501..20fc9ed58b75 100644 --- a/cms/djangoapps/course_to_library_import/validators.py +++ b/cms/djangoapps/course_to_library_import/validators.py @@ -3,11 +3,14 @@ """ from collections import ChainMap +from typing import get_args from django.utils.translation import gettext_lazy as _ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey +from .types import CompositionLevel + def validate_course_ids(value: str): """ @@ -36,3 +39,10 @@ def validate_usage_ids(usage_ids, staged_content): for usage_key in usage_ids: if usage_key not in available_block_keys: raise ValueError(f'Block {usage_key} is not available for import') + + +def validate_composition_level(composition_level): + if composition_level not in get_args(CompositionLevel): + raise ValueError( + _('Invalid composition level: {composition_level}').format(composition_level=composition_level) + ) From 530157ddec04ebaf10202ad071c229e0491f8005 Mon Sep 17 00:00:00 2001 From: Ivan Niedielnitsev <81557788+NiedielnitsevIvan@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:57:13 +0200 Subject: [PATCH 06/23] refactor: add uuid and remove source_type field to CourseToLibraryImport model (#2610) --- .../course_to_library_import/admin.py | 2 -- .../course_to_library_import/api.py | 6 ++--- ...rsetolibraryimport_source_type_and_more.py | 23 +++++++++++++++++++ .../course_to_library_import/models.py | 3 ++- .../tests/factories.py | 1 - .../tests/test_api.py | 4 +--- 6 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py diff --git a/cms/djangoapps/course_to_library_import/admin.py b/cms/djangoapps/course_to_library_import/admin.py index 67c3a7ea2488..a12a14bc2131 100644 --- a/cms/djangoapps/course_to_library_import/admin.py +++ b/cms/djangoapps/course_to_library_import/admin.py @@ -20,11 +20,9 @@ class CourseToLibraryImportAdmin(admin.ModelAdmin): 'status', 'course_ids', 'library_key', - 'source_type', ) list_filter = ( 'status', - 'source_type', ) search_fields = ( 'course_ids', diff --git a/cms/djangoapps/course_to_library_import/api.py b/cms/djangoapps/course_to_library_import/api.py index 4f00ee5a331c..5ba6c39f89c8 100644 --- a/cms/djangoapps/course_to_library_import/api.py +++ b/cms/djangoapps/course_to_library_import/api.py @@ -27,16 +27,13 @@ def import_library_from_staged_content( ) -def create_import( - course_ids: list[str], user_id: int, library_key: str, source_type: str -) -> None: +def create_import(course_ids: list[str], user_id: int, library_key: str) -> None: """ Create a new import task to import a course to a library. """ import_task = CourseToLibraryImport( course_ids=" ".join(course_ids), library_key=library_key, - source_type=source_type, user_id=user_id, ) import_task.save() @@ -44,3 +41,4 @@ def create_import( save_courses_to_staged_content_task.delay( course_ids, user_id, import_task.id, COURSE_TO_LIBRARY_IMPORT_PURPOSE ) + return import_task diff --git a/cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py b/cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py new file mode 100644 index 000000000000..9122325ef942 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.18 on 2025-03-21 11:01 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('course_to_library_import', '0002_containerversionimport'), + ] + + operations = [ + migrations.RemoveField( + model_name='coursetolibraryimport', + name='source_type', + ), + migrations.AddField( + model_name='coursetolibraryimport', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True), + ), + ] diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py index a30cfaa6fa45..fbcf9b160cdf 100644 --- a/cms/djangoapps/course_to_library_import/models.py +++ b/cms/djangoapps/course_to_library_import/models.py @@ -2,6 +2,7 @@ Models for the course to library import app. """ +import uuid as uuid_tools from typing import Self from django.contrib.auth import get_user_model @@ -23,6 +24,7 @@ class CourseToLibraryImport(TimeStampedModel): Represents a course import into a content library. """ + uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True, db_index=True) status = models.CharField( max_length=100, choices=CourseToLibraryImportStatus.choices, @@ -34,7 +36,6 @@ class CourseToLibraryImport(TimeStampedModel): validators=[validate_course_ids] ) library_key = models.CharField(max_length=100) - source_type = models.CharField(max_length=30) metadata = models.JSONField(default=dict, blank=True, null=True) user = models.ForeignKey(User, on_delete=models.CASCADE) diff --git a/cms/djangoapps/course_to_library_import/tests/factories.py b/cms/djangoapps/course_to_library_import/tests/factories.py index c4dd4e22d5a6..d93e9941c1da 100644 --- a/cms/djangoapps/course_to_library_import/tests/factories.py +++ b/cms/djangoapps/course_to_library_import/tests/factories.py @@ -19,6 +19,5 @@ class Meta: course_ids = ' '.join([f'course-v1:edX+DemoX+Demo_Course{i}' for i in range(1, 3)]) library_key = 'library-key' - source_type = 'source-type' metadata = {} user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/course_to_library_import/tests/test_api.py b/cms/djangoapps/course_to_library_import/tests/test_api.py index 1111120feba6..4b4d0183e7b7 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_api.py +++ b/cms/djangoapps/course_to_library_import/tests/test_api.py @@ -26,16 +26,14 @@ def test_create_import(): ] user = UserFactory() library_key = "lib:edX:DemoLib" - source_type = "test_source_type" with patch( "cms.djangoapps.course_to_library_import.api.save_courses_to_staged_content_task" ) as save_courses_to_staged_content_task_mock: - create_import(course_ids, user.id, library_key, source_type) + create_import(course_ids, user.id, library_key) import_task = CourseToLibraryImport.objects.get() assert import_task.course_ids == " ".join(course_ids) assert import_task.library_key == library_key - assert import_task.source_type == source_type assert import_task.user_id == user.id save_courses_to_staged_content_task_mock.delay.assert_called_once_with( course_ids, user.id, import_task.id, COURSE_TO_LIBRARY_IMPORT_PURPOSE From 71579817bd95b13e7c0d240eb45f4f9b67da971e Mon Sep 17 00:00:00 2001 From: andrii-hantkovskyi <131773947+andrii-hantkovskyi@users.noreply.github.com> Date: Mon, 24 Mar 2025 14:39:04 +0200 Subject: [PATCH 07/23] feat: [AXM-1635] create block importing route (#2611) * feat: [AXM-1635] create block importing route --- .../course_to_library_import/api.py | 16 +++- .../course_to_library_import/helpers.py | 3 +- .../course_to_library_import/models.py | 9 +- .../course_to_library_import/tasks.py | 7 +- .../tests/factories.py | 5 +- .../tests/test_api.py | 10 ++- .../tests/test_tasks.py | 25 +++--- .../views/__init__.py | 0 .../course_to_library_import/views/urls.py | 10 +++ .../views/v0/__init__.py | 0 .../views/v0/serializers.py | 32 +++++++ .../views/v0/tests/__init__.py | 0 .../views/v0/tests/test_serializers.py | 60 +++++++++++++ .../views/v0/tests/test_views.py | 88 +++++++++++++++++++ .../course_to_library_import/views/v0/urls.py | 12 +++ .../views/v0/views.py | 54 ++++++++++++ cms/urls.py | 3 + 17 files changed, 305 insertions(+), 29 deletions(-) create mode 100644 cms/djangoapps/course_to_library_import/views/__init__.py create mode 100644 cms/djangoapps/course_to_library_import/views/urls.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/__init__.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/serializers.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/tests/__init__.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/urls.py create mode 100644 cms/djangoapps/course_to_library_import/views/v0/views.py diff --git a/cms/djangoapps/course_to_library_import/api.py b/cms/djangoapps/course_to_library_import/api.py index 5ba6c39f89c8..22c045ba057a 100644 --- a/cms/djangoapps/course_to_library_import/api.py +++ b/cms/djangoapps/course_to_library_import/api.py @@ -1,4 +1,4 @@ -"""" +""" API for course to library import. """ @@ -16,6 +16,7 @@ def import_library_from_staged_content( user_id: int, usage_ids: list[str], course_id: str, + import_id: str, composition_level: CompositionLevel, override: bool ) -> None: @@ -23,11 +24,20 @@ def import_library_from_staged_content( Import staged content to a library. """ import_library_from_staged_content_task.delay( - user_id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, composition_level, override + user_id, + usage_ids, + library_key, + COURSE_TO_LIBRARY_IMPORT_PURPOSE, + course_id, + import_id, + composition_level, + override, ) -def create_import(course_ids: list[str], user_id: int, library_key: str) -> None: +def create_import( + course_ids: list[str], user_id: int, library_key: str +) -> None: """ Create a new import task to import a course to a library. """ diff --git a/cms/djangoapps/course_to_library_import/helpers.py b/cms/djangoapps/course_to_library_import/helpers.py index f612e2c17664..aa00d04c0ca3 100644 --- a/cms/djangoapps/course_to_library_import/helpers.py +++ b/cms/djangoapps/course_to_library_import/helpers.py @@ -52,8 +52,7 @@ def create_block_in_library(block_to_import, usage_key, library_key, user_id, st component_version = _handle_component_override( content_library, usage_key, etree.tostring(block_to_import) ) - - if not override: + else: # Create component (regardless of override path) _, library_usage_key = api.validate_can_add_block_to_library( library_key, diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py index fbcf9b160cdf..8c815e347871 100644 --- a/cms/djangoapps/course_to_library_import/models.py +++ b/cms/djangoapps/course_to_library_import/models.py @@ -47,12 +47,19 @@ class Meta: verbose_name_plural = _('Course to Library Imports') @classmethod - def get_by_id(cls, import_id: int) -> Self: + def get_by_id(cls, import_id: int) -> Self | None: """ Get an import task by its ID. """ return cls.objects.filter(id=import_id).first() + @classmethod + def get_ready_by_uuid(cls, import_uuid: str) -> Self | None: + """ + Get an import task by its UUID. + """ + return cls.objects.filter(uuid=import_uuid, status=CourseToLibraryImportStatus.READY).first() + class ComponentVersionImport(TimeStampedModel): """ diff --git a/cms/djangoapps/course_to_library_import/tasks.py b/cms/djangoapps/course_to_library_import/tasks.py index 6ddf23730ca5..eddc69dc24f6 100644 --- a/cms/djangoapps/course_to_library_import/tasks.py +++ b/cms/djangoapps/course_to_library_import/tasks.py @@ -62,6 +62,7 @@ def import_library_from_staged_content_task( library_key: str, purpose: str, course_id: str, + import_id: str, composition_level: CompositionLevel, override: bool ) -> None: @@ -91,8 +92,8 @@ def import_library_from_staged_content_task( usage_key, block_to_import, library_key, user_id, staged_content_item, composition_level, override ) - CourseToLibraryImport.objects.filter( - user_id=user_id, library_key=library_key - ).update(status=CourseToLibraryImportStatus.IMPORTED) + ctli = CourseToLibraryImport.get_ready_by_uuid(import_id) + ctli.status = CourseToLibraryImportStatus.IMPORTED + ctli.save() staged_content.delete() diff --git a/cms/djangoapps/course_to_library_import/tests/factories.py b/cms/djangoapps/course_to_library_import/tests/factories.py index d93e9941c1da..4caa9a9bb1ed 100644 --- a/cms/djangoapps/course_to_library_import/tests/factories.py +++ b/cms/djangoapps/course_to_library_import/tests/factories.py @@ -2,6 +2,8 @@ Factories for CourseToLibraryImport model. """ +import uuid + import factory from factory.django import DjangoModelFactory @@ -18,6 +20,7 @@ class Meta: model = CourseToLibraryImport course_ids = ' '.join([f'course-v1:edX+DemoX+Demo_Course{i}' for i in range(1, 3)]) - library_key = 'library-key' + uuid = factory.LazyFunction(lambda: str(uuid.uuid4())) + library_key = 'lib:TestOrg:TestLib' metadata = {} user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/course_to_library_import/tests/test_api.py b/cms/djangoapps/course_to_library_import/tests/test_api.py index 4b4d0183e7b7..fa864bc97ab0 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_api.py +++ b/cms/djangoapps/course_to_library_import/tests/test_api.py @@ -13,6 +13,7 @@ ) from cms.djangoapps.course_to_library_import.constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport +from .factories import CourseToLibraryImportFactory @pytest.mark.django_db @@ -46,8 +47,9 @@ def test_import_library_from_staged_content(override): """ Test import_library_from_staged_content function with different override values. """ - library_key = "lib:edX:DemoLib" - user = UserFactory() + ctli = CourseToLibraryImportFactory() + library_key = ctli.library_key + user = ctli.user usage_ids = [ "block-v1:edX+DemoX+Demo_Course+type@html+block@123", "block-v1:edX+DemoX+Demo_Course+type@html+block@456", @@ -57,8 +59,8 @@ def test_import_library_from_staged_content(override): with patch( "cms.djangoapps.course_to_library_import.api.import_library_from_staged_content_task" ) as import_library_from_staged_content_task_mock: - import_library_from_staged_content(library_key, user.id, usage_ids, course_id, 'xblock', override) + import_library_from_staged_content(library_key, user.id, usage_ids, course_id, ctli.uuid, 'xblock', override) import_library_from_staged_content_task_mock.delay.assert_called_once_with( - user.id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, 'xblock', override + user.id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, ctli.uuid, 'xblock', override ) diff --git a/cms/djangoapps/course_to_library_import/tests/test_tasks.py b/cms/djangoapps/course_to_library_import/tests/test_tasks.py index 8785fe06fa15..ee21e8e462a9 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_tasks.py +++ b/cms/djangoapps/course_to_library_import/tests/test_tasks.py @@ -13,8 +13,6 @@ import_library_from_staged_content_task, save_courses_to_staged_content_task, ) -from common.djangoapps.student.tests.factories import UserFactory - from .factories import CourseToLibraryImportFactory @@ -63,10 +61,11 @@ def test_import_library_from_staged_content_task( self, mock_etree, mock_import_container, mock_get_block_to_import, mock_get_ready_staged_content, mock_validate_usage_ids ): - user = UserFactory() + ctli = CourseToLibraryImportFactory(status=CourseToLibraryImportStatus.READY) + library_key = ctli.library_key + user = ctli.user usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] usage_key = UsageKey.from_string(usage_ids[0]) - library_key = 'lib:TestOrg:TestLib' purpose = 'import_from_{course_id}' course_id = 'course-v1:edX+Demo+2023' override = True @@ -84,14 +83,9 @@ def test_import_library_from_staged_content_task( mock_get_block_to_import.return_value = mock_block library_locator = LibraryLocatorV2.from_string(library_key) - course_to_library_import = CourseToLibraryImportFactory( - user=user, - library_key=library_locator, - status=CourseToLibraryImportStatus.READY - ) import_library_from_staged_content_task( - user.id, usage_ids, library_key, purpose, course_id, 'xblock', override + user.id, usage_ids, library_key, purpose, course_id, ctli.uuid, 'xblock', override ) mock_get_ready_staged_content.assert_called_once_with( @@ -107,8 +101,8 @@ def test_import_library_from_staged_content_task( usage_key, mock_block, library_locator, user.id, mock_content_item, 'xblock', override ) - course_to_library_import.refresh_from_db() - self.assertEqual(course_to_library_import.status, CourseToLibraryImportStatus.IMPORTED) + ctli.refresh_from_db() + self.assertEqual(ctli.status, CourseToLibraryImportStatus.IMPORTED) mock_staged_content.delete.assert_called_once() @patch( @@ -119,9 +113,10 @@ def test_import_library_from_staged_content_task( def test_import_library_block_not_found( self, mock_etree, mock_get_block_to_import, mock_get_ready_staged_content ): - user = UserFactory() + ctli = CourseToLibraryImportFactory() + library_key = ctli.library_key + user = ctli.user usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] - library_key = 'lib:TestOrg:TestLib' purpose = 'import_from_{course_id}' course_id = 'course-v1:edX+Demo+2023' override = True @@ -146,5 +141,5 @@ def test_import_library_block_not_found( with self.assertRaises(ValueError): import_library_from_staged_content_task( - user.id, usage_ids, library_key, purpose, course_id, 'xblock', override + user.id, usage_ids, library_key, purpose, course_id, ctli.uuid, 'xblock', override ) diff --git a/cms/djangoapps/course_to_library_import/views/__init__.py b/cms/djangoapps/course_to_library_import/views/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/course_to_library_import/views/urls.py b/cms/djangoapps/course_to_library_import/views/urls.py new file mode 100644 index 000000000000..ef82a37edd2f --- /dev/null +++ b/cms/djangoapps/course_to_library_import/views/urls.py @@ -0,0 +1,10 @@ +""" +Course to Library Import API URLs. +""" + +from django.urls import include, path + +app_name = 'course_to_library_import' +urlpatterns = [ + path('v0/', include('cms.djangoapps.course_to_library_import.views.v0.urls', namespace='v0')), +] diff --git a/cms/djangoapps/course_to_library_import/views/v0/__init__.py b/cms/djangoapps/course_to_library_import/views/v0/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/course_to_library_import/views/v0/serializers.py b/cms/djangoapps/course_to_library_import/views/v0/serializers.py new file mode 100644 index 000000000000..110df58420f7 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/views/v0/serializers.py @@ -0,0 +1,32 @@ +""" +Serializers for the Course to Library Import API. +""" + +from rest_framework import serializers + +from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport +from cms.djangoapps.course_to_library_import.validators import validate_composition_level + + +class ImportBlocksSerializer(serializers.Serializer): + """ + Serializer for the import blocks API. + """ + + library_key = serializers.CharField(required=True) + usage_ids = serializers.ListField( + child=serializers.CharField(), + required=True, + ) + course_id = serializers.CharField(required=True) + import_id = serializers.CharField(required=True) + composition_level = serializers.CharField( + required=True, + validators=[validate_composition_level], + ) + override = serializers.BooleanField(default=False, required=False) + + def validate(self, data): + if not CourseToLibraryImport.get_ready_by_uuid(data['import_id']): + raise serializers.ValidationError({'import_id': 'Invalid import ID.'}) + return data diff --git a/cms/djangoapps/course_to_library_import/views/v0/tests/__init__.py b/cms/djangoapps/course_to_library_import/views/v0/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py b/cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py new file mode 100644 index 000000000000..924fdc246422 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py @@ -0,0 +1,60 @@ +""" +Unit tests for the ImportBlocksSerializer. +""" + +from unittest.mock import patch + +from django.test import TestCase +from rest_framework.exceptions import ValidationError + +from cms.djangoapps.course_to_library_import.views.v0.serializers import ImportBlocksSerializer + + +class TestImportBlocksSerializer(TestCase): + """ + Tests for the ImportBlocksSerializer. + """ + + def setUp(self): + """ + Set up common test data. + """ + self.valid_data = { + 'library_key': 'lib:v1:org+lib+2023', + 'usage_ids': ['block-v1:org+course+2023+type@html+block@123'], + 'course_id': 'course-v1:org+course+2023', + 'import_id': 'valid-import-id', + 'composition_level': 'vertical', + 'override': False, + } + + @patch('cms.djangoapps.course_to_library_import.models.CourseToLibraryImport.get_ready_by_uuid') + def test_validate_with_valid_import_id(self, mock_get_ready_by_uuid): + """ + Test that validation passes when a valid import_id is provided. + """ + mock_get_ready_by_uuid.return_value = {'some': 'object'} + + serializer = ImportBlocksSerializer(data=self.valid_data) + self.assertTrue(serializer.is_valid()) + + mock_get_ready_by_uuid.assert_called_once_with(self.valid_data['import_id']) + + @patch('cms.djangoapps.course_to_library_import.models.CourseToLibraryImport.get_ready_by_uuid') + def test_validate_with_invalid_import_id(self, mock_get_ready_by_uuid): + """ + Test that validation fails when an invalid import_id is provided. + """ + mock_get_ready_by_uuid.return_value = None + + serializer = ImportBlocksSerializer(data=self.valid_data) + + with self.assertRaises(ValidationError) as context: + serializer.is_valid(raise_exception=True) + + self.assertEqual( + context.exception.detail, + {'import_id': ['Invalid import ID.']} + ) + + mock_get_ready_by_uuid.assert_called_once_with(self.valid_data['import_id']) diff --git a/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py b/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py new file mode 100644 index 000000000000..fc088d9f79b9 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py @@ -0,0 +1,88 @@ +""" +Unit tests for the ImportBlocksView API endpoint. +""" + +from unittest import mock + +from django.test import TestCase +from django.urls import reverse +from rest_framework.test import APIClient + +from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus +from cms.djangoapps.course_to_library_import.tests.factories import CourseToLibraryImportFactory +from common.djangoapps.student.tests.factories import UserFactory + + +class ImportBlocksViewTest(TestCase): + """ + Tests for ImportBlocksView. + """ + + def setUp(self): + super().setUp() + self.client = APIClient() + self.url = reverse('course_to_library_import:v0:import_blocks') + + self.admin_user = UserFactory(is_staff=True) + self.non_admin_user = UserFactory() + + self.ctli = CourseToLibraryImportFactory(user_id=self.admin_user.pk, status=CourseToLibraryImportStatus.READY) + + self.valid_data = { + 'library_key': 'lib:org:lib1', + 'usage_ids': ['block-v1:org+course+run+type@problem+block@123'], + 'course_id': 'course-v1:org+course+run', + 'import_id': self.ctli.uuid, + 'composition_level': 'xblock', + 'override': False, + } + + def test_permissions(self): + """ + Test that only admin users can access the endpoint. + """ + self.client.force_authenticate(user=self.non_admin_user) + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, 403) + + self.client.force_authenticate(user=self.admin_user) + with mock.patch('cms.djangoapps.course_to_library_import.views.v0.views.import_library_from_staged_content'): + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, 200) + + def test_invalid_data(self): + """ + Test that invalid data returns appropriate errors. + """ + self.client.force_authenticate(user=self.admin_user) + + response = self.client.post(self.url, {}, format='json') + self.assertEqual(response.status_code, 400) + + invalid_data = self.valid_data.copy() + invalid_data.pop('library_key') + response = self.client.post(self.url, invalid_data, format='json') + self.assertEqual(response.status_code, 400) + + @mock.patch('cms.djangoapps.course_to_library_import.views.v0.views.import_library_from_staged_content') + def test_successful_import(self, mock_import): + """ + Test successful import returns a success response. + """ + self.client.force_authenticate(user=self.admin_user) + + mock_import.return_value = None + response = self.client.post(self.url, self.valid_data, format='json') + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, {'status': 'success'}) + + mock_import.assert_called_once_with( + library_key=self.valid_data['library_key'], + user_id=self.admin_user.pk, + usage_ids=self.valid_data['usage_ids'], + course_id=self.valid_data['course_id'], + import_id=self.valid_data['import_id'], + composition_level=self.valid_data['composition_level'], + override=self.valid_data['override'], + ) diff --git a/cms/djangoapps/course_to_library_import/views/v0/urls.py b/cms/djangoapps/course_to_library_import/views/v0/urls.py new file mode 100644 index 000000000000..9810969e1382 --- /dev/null +++ b/cms/djangoapps/course_to_library_import/views/v0/urls.py @@ -0,0 +1,12 @@ +""" +Course to Library Import API v0 URLs. +""" + +from django.urls import path + +from .views import ImportBlocksView + +app_name = 'v0' +urlpatterns = [ + path('import_blocks/', ImportBlocksView.as_view(), name='import_blocks'), +] diff --git a/cms/djangoapps/course_to_library_import/views/v0/views.py b/cms/djangoapps/course_to_library_import/views/v0/views.py new file mode 100644 index 000000000000..242c2690664f --- /dev/null +++ b/cms/djangoapps/course_to_library_import/views/v0/views.py @@ -0,0 +1,54 @@ +""" +API v0 views. +""" + +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response +from rest_framework.views import APIView + +from cms.djangoapps.course_to_library_import.api import import_library_from_staged_content +from .serializers import ImportBlocksSerializer + + +class ImportBlocksView(APIView): + """ + Import blocks from a course to a library. + """ + + permission_classes = (IsAdminUser,) + serializer_class = ImportBlocksSerializer + + def post(self, request, *args, **kwargs): + """ + Import blocks from a course to a library. + + API endpoint: POST /api/v0/course-to-library-import/ + + Request: + { + "library_key": "lib:org:code:run", + "course_id": "course-v1:org+course+run", + "usage_ids": ["block-v1:org+course+run+type@problem+block@12345"], + "import_id": "78df3b2c-4e5a-4d6b-8c7e-1f2a3b4c5d6e", + "composition_level": "xblock", + "override": false + } + + Response: + { + "status": "success" + } + """ + data = self.serializer_class(data=request.data) + data.is_valid(raise_exception=True) + + import_library_from_staged_content( + library_key=data.validated_data['library_key'], + user_id=request.user.pk, + usage_ids=data.validated_data['usage_ids'], + course_id=data.validated_data['course_id'], + import_id=data.validated_data['import_id'], + composition_level=data.validated_data['composition_level'], + override=data.validated_data['override'], + ) + return Response({'status': 'success'}) diff --git a/cms/urls.py b/cms/urls.py index d01e89d9d276..fe7dc965fcc8 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -141,6 +141,9 @@ # rest api for course import/export path('api/courses/', include('cms.djangoapps.contentstore.api.urls', namespace='courses_api') ), + path('api/course_to_library_import/', + include('cms.djangoapps.course_to_library_import.views.urls', namespace='course_to_library_import_api'), + ), re_path(fr'^export/{COURSELIKE_KEY_PATTERN}$', contentstore_views.export_handler, name='export_handler'), re_path(fr'^export_output/{COURSELIKE_KEY_PATTERN}$', contentstore_views.export_output_handler, From 8405b21757c2bf8e2eb0bfde5765360c0ad116a7 Mon Sep 17 00:00:00 2001 From: andrii-hantkovskyi <131773947+andrii-hantkovskyi@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:52:49 +0200 Subject: [PATCH 08/23] feat: [AXM-1780] add static tabs importing (#2614) * feat: [AXM-1780] add static tabs importing * refactor: [AXM-1780] rename to --------- Co-authored-by: Andrii --- .../course_to_library_import/helpers.py | 33 +++--- .../course_to_library_import/tasks.py | 18 +++- .../tests/test_helpers.py | 101 +++++++++++++----- .../tests/test_tasks.py | 12 ++- 4 files changed, 113 insertions(+), 51 deletions(-) diff --git a/cms/djangoapps/course_to_library_import/helpers.py b/cms/djangoapps/course_to_library_import/helpers.py index aa00d04c0ca3..bbe71019046b 100644 --- a/cms/djangoapps/course_to_library_import/helpers.py +++ b/cms/djangoapps/course_to_library_import/helpers.py @@ -28,7 +28,7 @@ log = logging.getLogger(__name__) -def create_block_in_library(block_to_import, usage_key, library_key, user_id, staged_content_id, override): +def create_block_in_library(block_to_import, usage_key, library_key, user_id, staged_content_id, import_id, override): """ Create a block in a library from a staged content block. """ @@ -73,7 +73,7 @@ def create_block_in_library(block_to_import, usage_key, library_key, user_id, st overrided_component_version_import = False if override: _update_component_version_import( - component_version, usage_key, library_key, user_id + component_version, usage_key, import_id ) overrided_component_version_import = True @@ -103,18 +103,14 @@ def _handle_component_override(content_library, usage_key, new_content): return component_version -def _update_component_version_import(component_version, usage_key, library_key, user_id): +def _update_component_version_import(component_version, usage_key, import_id): """ Update component version import records for overridden components. """ return ComponentVersionImport.objects.create( component_version=component_version, source_usage_key=usage_key, - library_import=CourseToLibraryImport.objects.get( - library_key=library_key, - user_id=user_id, - status=CourseToLibraryImportStatus.READY - ), + library_import=CourseToLibraryImport.get_ready_by_uuid(import_id), ) @@ -138,9 +134,10 @@ def _process_staged_content_files( for staged_content_file_data in staged_content_files: original_filename = staged_content_file_data.filename file_basename = os.path.basename(original_filename) + file_basename_no_ext, _ = os.path.splitext(file_basename) # Skip files not referenced in the block - if file_basename not in block_olx: + if file_basename not in block_olx and file_basename_no_ext not in block_olx: log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") continue @@ -155,7 +152,7 @@ def _process_staged_content_files( ) continue - filename = f"static/{str(usage_key)}" + filename = f"static/{file_basename}" media_type_str, _ = mimetypes.guess_type(filename) if not media_type_str: media_type_str = "application/octet-stream" @@ -203,7 +200,7 @@ def _update_container_components(container_version, component_versions, user_id) ) -def _process_xblock(child, library_key, user_id, staged_content, override): +def _process_xblock(child, library_key, user_id, staged_content, import_id, override): """ Process an xblock and create a block in the library. """ @@ -219,11 +216,12 @@ def _process_xblock(child, library_key, user_id, staged_content, override): library_key, user_id, staged_content.id, + import_id, override, ) -def import_children(block_to_import, library_key, user_id, staged_content, composition_level, override): +def import_children(block_to_import, library_key, user_id, staged_content, composition_level, import_id, override): """ Import children of a block from staged content into a library. Creates appropriate container hierarchy based on composition_level. @@ -231,7 +229,7 @@ def import_children(block_to_import, library_key, user_id, staged_content, compo result = [] if block_to_import.tag not in ('chapter', 'sequential', 'vertical'): - component_version = _process_xblock(block_to_import, library_key, user_id, staged_content, override) + component_version = _process_xblock(block_to_import, library_key, user_id, staged_content, import_id, override) if component_version: return [component_version] @@ -251,6 +249,7 @@ def import_children(block_to_import, library_key, user_id, staged_content, compo user_id, staged_content, composition_level, + import_id, override, ) @@ -259,7 +258,7 @@ def import_children(block_to_import, library_key, user_id, staged_content, compo result.append(container_version) else: - component_version = _process_xblock(child, library_key, user_id, staged_content, override) + component_version = _process_xblock(child, library_key, user_id, staged_content, import_id, override) if component_version is not None: result.append(component_version) @@ -304,7 +303,9 @@ def create_container(container_type, key, display_name, library_key, user_id): return container_version -def import_container(usage_key, block_to_import, library_key, user_id, staged_content, composition_level, override): +def import_container( + usage_key, block_to_import, library_key, user_id, staged_content, composition_level, import_id, override +): """ Import a blocks hierarchy into a library, creating proper container structure. """ @@ -328,6 +329,7 @@ def import_container(usage_key, block_to_import, library_key, user_id, staged_co user_id, staged_content, composition_level, + import_id, override, ) @@ -351,6 +353,7 @@ def import_container(usage_key, block_to_import, library_key, user_id, staged_co user_id, staged_content, composition_level, + import_id, override, ) diff --git a/cms/djangoapps/course_to_library_import/tasks.py b/cms/djangoapps/course_to_library_import/tasks.py index eddc69dc24f6..749199460079 100644 --- a/cms/djangoapps/course_to_library_import/tasks.py +++ b/cms/djangoapps/course_to_library_import/tasks.py @@ -40,11 +40,14 @@ def save_courses_to_staged_content_task( course_key = CourseKey.from_string(course_id) sections = modulestore().get_items( course_key, qualifiers={"category": "chapter"} - ) + ) or [] + static_tabs = modulestore().get_items( + course_key, qualifiers={"category": "static_tab"} + ) or [] - for section in sections: + for item in sections + static_tabs: content_staging_api.stage_xblock_temporarily( - section, + item, user_id, purpose=purpose.format(course_id=course_id), version_num=version_num, @@ -89,7 +92,14 @@ def import_library_from_staged_content_task( if block_to_import is None: continue import_container( - usage_key, block_to_import, library_key, user_id, staged_content_item, composition_level, override + usage_key, + block_to_import, + library_key, + user_id, + staged_content_item, + composition_level, + import_id, + override, ) ctli = CourseToLibraryImport.get_ready_by_uuid(import_id) diff --git a/cms/djangoapps/course_to_library_import/tests/test_helpers.py b/cms/djangoapps/course_to_library_import/tests/test_helpers.py index 4c0f460758f0..61b1b087fee8 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_helpers.py +++ b/cms/djangoapps/course_to_library_import/tests/test_helpers.py @@ -4,7 +4,7 @@ from datetime import datetime, timezone from unittest import mock - +from uuid import uuid4 from lxml import etree from django.db.utils import IntegrityError @@ -46,6 +46,8 @@ def setUp(self): "block-v1:TestOrg+TestCourse+Run1+type@video+block@video1": {}, } + self.import_id = str(uuid4()) + @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') def test_flat_import_children_basic(self, mock_content_library, mock_create_block): @@ -60,7 +62,9 @@ def test_flat_import_children_basic(self, mock_content_library, mock_create_bloc mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library - import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False) + import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, False + ) self.assertEqual(mock_create_block.call_count, 2) @@ -68,10 +72,10 @@ def test_flat_import_children_basic(self, mock_content_library, mock_create_bloc usage_key_html = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@html+block@html1") mock_create_block.assert_any_call( - mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, False + mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, self.import_id, False ) mock_create_block.assert_any_call( - mock.ANY, usage_key_html, self.library_key, self.user_id, self.staged_content.id, False + mock.ANY, usage_key_html, self.library_key, self.user_id, self.staged_content.id, self.import_id, False ) @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') @@ -87,11 +91,13 @@ def test_flat_import_children_with_override(self, mock_content_library, mock_cre mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library - import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', True) + import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, True + ) usage_key_problem = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1") mock_create_block.assert_called_with( - mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, True + mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, self.import_id, True ) @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') @@ -106,7 +112,9 @@ def test_flat_import_children_library_not_found(self, mock_content_library): mock_content_library.objects.filter.return_value.first.return_value = None with self.assertRaises(ValueError): - import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False) + import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, False + ) @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') @@ -121,7 +129,9 @@ def test_flat_import_children_ignores_unmatched_url_names(self, mock_content_lib mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library - import_children(block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False) + import_children( + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, False + ) mock_create_block.assert_not_called() @@ -167,7 +177,13 @@ def test_create_block_in_library_new_component( mock_content_staging_api.get_staged_content_static_files.return_value = [] create_block_in_library( - self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, False + self.block_to_import, + self.usage_key, + self.library_key, + self.user_id, + self.staged_content_id, + str(uuid4()), + False, ) mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) @@ -208,11 +224,17 @@ def test_create_block_in_library_existing_component_with_override( mock_content_staging_api.get_staged_content_static_files.return_value = [] - CourseToLibraryImportFactory( + ctli = CourseToLibraryImportFactory( status=CourseToLibraryImportStatus.READY, library_key=self.library_key, user_id=self.user_id ) create_block_in_library( - self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, True + self.block_to_import, + self.usage_key, + self.library_key, + self.user_id, + self.staged_content_id, + str(ctli.uuid), + True, ) mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) @@ -242,7 +264,13 @@ def test_create_block_in_library_existing_component_without_override( mock_content_staging_api.get_staged_content_static_files.return_value = [] create_block_in_library( - self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, False + self.block_to_import, + self.usage_key, + self.library_key, + self.user_id, + self.staged_content_id, + str(uuid4()), + False, ) mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) @@ -272,13 +300,21 @@ def test_create_block_in_library_with_files_and_override( mock_file_data = [mock.MagicMock()] mock_content_staging_api.get_staged_content_static_files.return_value = mock_file_data + import_id = str(uuid4()) + create_block_in_library( - self.block_to_import, self.usage_key, self.library_key, self.user_id, self.staged_content_id, True + self.block_to_import, + self.usage_key, + self.library_key, + self.user_id, + self.staged_content_id, + import_id, + True, ) mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) mock_update_component.assert_called_once_with( - mock_component_version, self.usage_key, self.library_key, self.user_id + mock_component_version, self.usage_key, import_id, ) mock_process_files.assert_called_once() @@ -347,7 +383,7 @@ def test_process_staged_content_files_with_reference_in_block( mock_authoring_api.create_component_version_content.assert_called_once_with( mock_component_version.pk, mock_content.id, - key=f"static/{str(self.usage_key)}" + key=f"static/{mock_file_data.filename}", ) mock_get_or_create.assert_called_once() @@ -734,9 +770,11 @@ def test_import_container_with_container_composition( mock_get_import = mock.MagicMock() mock_course_import.objects.get.return_value = mock_get_import + import_id = str(uuid4()) + import_container( self.usage_key, block_to_import, self.library_key, self.user_id, - self.staged_content, 'chapter', False + self.staged_content, 'chapter', import_id, False ) mock_create_container.assert_called_once_with( @@ -744,7 +782,7 @@ def test_import_container_with_container_composition( ) mock_import_children.assert_called_once_with( block_to_import, self.library_key, self.user_id, self.staged_content, - 'chapter', False + 'chapter', import_id, False ) mock_update_container.assert_called_once_with( mock_container_version, mock_component_versions, self.user_id @@ -768,14 +806,16 @@ def test_import_container_xblock_level(self, mock_import_children): """ block_to_import = etree.fromstring(xml) + import_id = str(uuid4()) + import_container( self.usage_key, block_to_import, self.library_key, self.user_id, - self.staged_content, 'xblock', False + self.staged_content, 'xblock', import_id, False ) mock_import_children.assert_called_once_with( block_to_import, self.library_key, self.user_id, self.staged_content, - 'xblock', False + 'xblock', import_id, False ) @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContainerVersionImport') @@ -788,7 +828,7 @@ def test_import_container_no_children( xml = """""" block_to_import = etree.fromstring(xml) - CourseToLibraryImportFactory( + ctli = CourseToLibraryImportFactory( library_key=self.library_key, user_id=self.user_id, status=CourseToLibraryImportStatus.READY, @@ -801,7 +841,7 @@ def test_import_container_no_children( import_container( self.usage_key, block_to_import, self.library_key, self.user_id, - self.staged_content, 'chapter', False + self.staged_content, 'chapter', str(ctli.uuid), False ) mock_create_container.assert_called_once() @@ -849,7 +889,7 @@ def test_import_children_xblock_level(self, mock_content_library, mock_create_bl mock_create_block.return_value = mock_component_version result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False ) self.assertEqual(mock_create_block.call_count, 2) @@ -885,7 +925,7 @@ def test_import_children_with_containers( mock_create_container.return_value = mock_container_version result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', str(uuid4()), False ) self.assertEqual(mock_create_container.call_count, 2) @@ -906,7 +946,7 @@ def test_import_children_empty_block_xblock_level(self, mock_content_library, mo mock_create_block.return_value = mock_component_version result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False ) mock_create_block.assert_called_once() @@ -922,8 +962,10 @@ def test_import_children_with_override(self, mock_content_library, mock_create_b mock_library = mock.MagicMock() mock_content_library.objects.filter.return_value.first.return_value = mock_library + import_id = str(uuid4()) + import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', True + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', import_id, True ) mock_create_block.assert_called_once_with( @@ -932,6 +974,7 @@ def test_import_children_with_override(self, mock_content_library, mock_create_b self.library_key, self.user_id, self.staged_content.id, + import_id, True, ) @@ -944,7 +987,7 @@ def test_import_children_library_not_found(self, mock_content_library): with self.assertRaises(ValueError): import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False ) def test_import_children_no_matching_children(self): @@ -957,7 +1000,7 @@ def test_import_children_no_matching_children(self): block_to_import = etree.fromstring(xml) result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False ) self.assertEqual(result, []) @@ -988,11 +1031,11 @@ def test_import_children_filter_by_composition_level( mock_create_container.return_value = mock_container_version result_xblock = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False ) result_chapter = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', False + block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', str(uuid4()), False ) self.assertTrue(all(not isinstance(item, ContainerVersion) for item in result_xblock)) diff --git a/cms/djangoapps/course_to_library_import/tests/test_tasks.py b/cms/djangoapps/course_to_library_import/tests/test_tasks.py index ee21e8e462a9..5c9e2f63b510 100644 --- a/cms/djangoapps/course_to_library_import/tests/test_tasks.py +++ b/cms/djangoapps/course_to_library_import/tests/test_tasks.py @@ -24,14 +24,20 @@ class TestSaveCourseSectionsToStagedContentTask(TestCase): @patch('cms.djangoapps.course_to_library_import.tasks.modulestore') @patch('openedx.core.djangoapps.content_staging.api.stage_xblock_temporarily') def test_save_courses_to_staged_content_task(self, mock_stage_xblock_temporarily, mock_modulestore): - course_to_library_import = CourseToLibraryImportFactory() + course_to_library_import = CourseToLibraryImportFactory(status=CourseToLibraryImportStatus.PENDING) course_ids = course_to_library_import.course_ids.split(' ') user_id = course_to_library_import.user.id purpose = 'test_purpose' version_num = 1 mock_course_keys = [CourseKey.from_string(course_id) for course_id in course_ids] - mock_modulestore().get_items.return_value = sections = ['section1', 'section2'] + + def mock_get_items_side_effect(*args, **kwargs): + if kwargs.get('qualifiers', {}).get('category') == 'chapter': + return ['section1', 'section2'] + + mock_modulestore().get_items.side_effect = mock_get_items_side_effect + sections = ['section1', 'section2'] self.assertEqual(course_to_library_import.status, CourseToLibraryImportStatus.PENDING) @@ -98,7 +104,7 @@ def test_import_library_from_staged_content_task( mock_staged_content.filter.assert_called_once_with(tags__icontains=usage_ids[0]) mock_get_block_to_import.assert_called_once_with(mock_node, usage_key) mock_import_container.assert_called_once_with( - usage_key, mock_block, library_locator, user.id, mock_content_item, 'xblock', override + usage_key, mock_block, library_locator, user.id, mock_content_item, 'xblock', ctli.uuid, override ) ctli.refresh_from_db() From a0c88f5d112e9c907734535b3ba4a61ac759d51f Mon Sep 17 00:00:00 2001 From: Ivan Niedielnitsev <81557788+NiedielnitsevIvan@users.noreply.github.com> Date: Fri, 28 Mar 2025 16:51:21 +0200 Subject: [PATCH 09/23] feat: REST API to get and create imports (#2612) * feat: REST API to get and create imports * test: add tests for new APIs --- .../views/v0/serializers.py | 37 +++ .../views/v0/tests/test_views.py | 208 +++++++++++++++-- .../course_to_library_import/views/v0/urls.py | 8 +- .../views/v0/views.py | 220 +++++++++++++++++- 4 files changed, 453 insertions(+), 20 deletions(-) diff --git a/cms/djangoapps/course_to_library_import/views/v0/serializers.py b/cms/djangoapps/course_to_library_import/views/v0/serializers.py index 110df58420f7..4c107e0be808 100644 --- a/cms/djangoapps/course_to_library_import/views/v0/serializers.py +++ b/cms/djangoapps/course_to_library_import/views/v0/serializers.py @@ -4,6 +4,7 @@ from rest_framework import serializers +from cms.djangoapps.course_to_library_import import api from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport from cms.djangoapps.course_to_library_import.validators import validate_composition_level @@ -30,3 +31,39 @@ def validate(self, data): if not CourseToLibraryImport.get_ready_by_uuid(data['import_id']): raise serializers.ValidationError({'import_id': 'Invalid import ID.'}) return data + + +class CourseToLibraryImportSerializer(serializers.ModelSerializer): + """ + Serializer for CourseToLibraryImport model. + """ + + course_ids = serializers.ListField() + status = serializers.CharField(allow_blank=True, required=False) + library_key = serializers.CharField(allow_blank=True, required=False) + uuid = serializers.CharField(allow_blank=True, required=False) + + class Meta: + model = CourseToLibraryImport + fields = ('course_ids', 'status', 'library_key', 'uuid') + + def create(self, validated_data): + """ + Run the import creation logic. + Creates a new CourseToLibraryImport instance and related data such as StagedContent. + """ + user = getattr(self.context.get('request'), 'user', None) + course_to_library_import = api.create_import( + validated_data['course_ids'], + getattr(user, 'pk', None), + self.context.get('content_library_id'), + ) + return course_to_library_import + + def to_representation(self, instance): + """ + Converts a string with course IDs into a list of strings with course IDs. + """ + representation = super().to_representation(instance) + representation['course_ids'] = ''.join(representation['course_ids']).split() + return representation diff --git a/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py b/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py index fc088d9f79b9..af250dba037e 100644 --- a/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py +++ b/cms/djangoapps/course_to_library_import/views/v0/tests/test_views.py @@ -4,29 +4,63 @@ from unittest import mock -from django.test import TestCase from django.urls import reverse +from organizations.models import Organization from rest_framework.test import APIClient +from rest_framework import status -from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus -from cms.djangoapps.course_to_library_import.tests.factories import CourseToLibraryImportFactory from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.course_to_library_import import api +from cms.djangoapps.course_to_library_import.constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE +from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus +from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport +from openedx.core.djangoapps.content_libraries import api as content_libraries_api +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory -class ImportBlocksViewTest(TestCase): +class TestCourseToLibraryImportViewsMixin(SharedModuleStoreTestCase): """ - Tests for ImportBlocksView. + Mixin for tests that require a CourseToLibraryImport instance. """ def setUp(self): super().setUp() self.client = APIClient() - self.url = reverse('course_to_library_import:v0:import_blocks') + + self.library = content_libraries_api.create_library( + org=Organization.objects.create(name='Organization 1', short_name='org1'), + slug='lib_1', + title='Library Org 1', + description='This is a library from Org 1', + ) + self.library_id = str(self.library.key) self.admin_user = UserFactory(is_staff=True) self.non_admin_user = UserFactory() - self.ctli = CourseToLibraryImportFactory(user_id=self.admin_user.pk, status=CourseToLibraryImportStatus.READY) + self.course = CourseFactory.create() + self.chapter = BlockFactory.create(category='chapter', parent=self.course) + self.sequential = BlockFactory.create(category='sequential', parent=self.chapter) + self.vertical = BlockFactory.create(category='vertical', parent=self.sequential) + self.problem = BlockFactory.create(category='problem', parent=self.vertical) + + self.ctli = api.create_import( + user_id=self.admin_user.pk, + library_key=str(self.library.key), + course_ids=[str(self.course.id)], + ) + + +class ImportBlocksViewTest(TestCourseToLibraryImportViewsMixin): + """ + Tests for ImportBlocksView. + """ + + def setUp(self): + super().setUp() + self.url = reverse('course_to_library_import:v0:import_blocks') self.valid_data = { 'library_key': 'lib:org:lib1', @@ -43,12 +77,11 @@ def test_permissions(self): """ self.client.force_authenticate(user=self.non_admin_user) response = self.client.post(self.url, self.valid_data, format='json') - self.assertEqual(response.status_code, 403) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.client.force_authenticate(user=self.admin_user) - with mock.patch('cms.djangoapps.course_to_library_import.views.v0.views.import_library_from_staged_content'): - response = self.client.post(self.url, self.valid_data, format='json') - self.assertEqual(response.status_code, 200) + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_200_OK) def test_invalid_data(self): """ @@ -57,14 +90,14 @@ def test_invalid_data(self): self.client.force_authenticate(user=self.admin_user) response = self.client.post(self.url, {}, format='json') - self.assertEqual(response.status_code, 400) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) invalid_data = self.valid_data.copy() invalid_data.pop('library_key') response = self.client.post(self.url, invalid_data, format='json') - self.assertEqual(response.status_code, 400) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - @mock.patch('cms.djangoapps.course_to_library_import.views.v0.views.import_library_from_staged_content') + @mock.patch('cms.djangoapps.course_to_library_import.views.v0.views.api.import_library_from_staged_content') def test_successful_import(self, mock_import): """ Test successful import returns a success response. @@ -74,7 +107,7 @@ def test_successful_import(self, mock_import): mock_import.return_value = None response = self.client.post(self.url, self.valid_data, format='json') - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, {'status': 'success'}) mock_import.assert_called_once_with( @@ -82,7 +115,150 @@ def test_successful_import(self, mock_import): user_id=self.admin_user.pk, usage_ids=self.valid_data['usage_ids'], course_id=self.valid_data['course_id'], - import_id=self.valid_data['import_id'], + import_id=str(self.valid_data['import_id']), composition_level=self.valid_data['composition_level'], override=self.valid_data['override'], ) + + +class TestCreateCourseToLibraryImportView(TestCourseToLibraryImportViewsMixin): + """ + Tests for the CreateCourseToLibraryImportView API endpoint. + """ + + def setUp(self): + super().setUp() + + self.url = reverse('course_to_library_import:v0:create_import', args=[self.library_id]) + self.valid_data = { + 'course_ids': ['course-v1:org+course+run', 'course-v1:org2+course2+run2'], + } + + def test_permissions(self): + """ + Test that only admin users can access the endpoint. + """ + self.client.force_authenticate(user=self.non_admin_user) + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + self.client.force_authenticate(user=self.admin_user) + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + def test_invalid_data(self): + """ + Test that invalid data returns appropriate errors. + """ + self.client.force_authenticate(user=self.admin_user) + + response = self.client.post( + self.url, + {'course_ids': 'course-v1:org+course+run course-v1:org2+course2+run2'}, + format='json' + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_successful_import(self): + """ + Test successful import returns a success response. + """ + self.client.force_authenticate(user=self.admin_user) + + response = self.client.post(self.url, self.valid_data, format='json') + expected_response = { + 'course_ids': self.valid_data['course_ids'], + 'status': 'pending', + 'library_key': self.library_id, + 'uuid': str(CourseToLibraryImport.objects.last().uuid), + } + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.data, expected_response) + + def test_non_existent_library(self): + """ + Test that a non-existent library returns a 404 response. + """ + self.client.force_authenticate(user=self.admin_user) + + response = self.client.post( + reverse('course_to_library_import:v0:create_import', args=['lib:org:lib2']), + self.valid_data, + format='json' + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + +class GetCourseStructureToLibraryImportView(TestCourseToLibraryImportViewsMixin): + """ + Tests for the GetCourseStructureToLibraryImportView API endpoint. + """ + + def setUp(self): + super().setUp() + self.url = reverse('course_to_library_import:v0:get_import', args=[str(self.ctli.uuid)]) + + def test_get_course_structure(self): + """ + Test that the endpoint returns the correct course structure. + """ + expected_course_structure = [{ + str(self.chapter.location): self.chapter.display_name, + 'children': [{ + str(self.sequential.location): self.sequential.display_name, + 'children': [{ + str(self.vertical.location): self.vertical.display_name, + 'children': [{ + str(self.problem.location): self.problem.display_name, + }] + }] + }] + }] + + self.client.force_authenticate(user=self.admin_user) + + response = self.client.get(self.url) + course_structure = response.data[str(self.course.id)] + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(list(response.data.keys()), self.ctli.course_ids.split()) + self.assertTrue(course_structure, expected_course_structure) + + def test_get_course_structure_not_found(self): + """ + Test that the endpoint returns a 404 response when the import is not found. + """ + self.client.force_authenticate(user=self.admin_user) + + response = self.client.get(reverse( + 'course_to_library_import:v0:get_import', + kwargs={'course_to_lib_uuid': '593e93d7-ed64-4147-bb5c-4cfcb1cf80b1'}) + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_get_course_structure_no_permissions(self): + """ + Test that the endpoint returns a 403 response when the user does not have permissions. + """ + self.client.force_authenticate(user=self.non_admin_user) + + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_get_course_structure_for_imported_course(self): + """ + Test that the endpoint returns an empty course structure for an imported course. + """ + self.client.force_authenticate(user=self.admin_user) + + self.ctli.status = CourseToLibraryImportStatus.IMPORTED + self.ctli.save() + + content_staging_api.get_ready_staged_content_by_user_and_purpose( + self.admin_user.pk, + COURSE_TO_LIBRARY_IMPORT_PURPOSE.format(course_id=str(self.course.id)) + ).delete() + + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, {str(self.course.id): []}) diff --git a/cms/djangoapps/course_to_library_import/views/v0/urls.py b/cms/djangoapps/course_to_library_import/views/v0/urls.py index 9810969e1382..e23fb8300726 100644 --- a/cms/djangoapps/course_to_library_import/views/v0/urls.py +++ b/cms/djangoapps/course_to_library_import/views/v0/urls.py @@ -4,9 +4,15 @@ from django.urls import path -from .views import ImportBlocksView +from .views import ( + CreateCourseToLibraryImportView, + ImportBlocksView, + GetCourseStructureToLibraryImportView, +) app_name = 'v0' urlpatterns = [ path('import_blocks/', ImportBlocksView.as_view(), name='import_blocks'), + path('create_import//', CreateCourseToLibraryImportView.as_view(), name='create_import'), + path('get_import//', GetCourseStructureToLibraryImportView.as_view(), name='get_import'), ] diff --git a/cms/djangoapps/course_to_library_import/views/v0/views.py b/cms/djangoapps/course_to_library_import/views/v0/views.py index 242c2690664f..27960234b499 100644 --- a/cms/djangoapps/course_to_library_import/views/v0/views.py +++ b/cms/djangoapps/course_to_library_import/views/v0/views.py @@ -1,12 +1,27 @@ """ API v0 views. """ +from lxml import etree + +from django.shortcuts import get_object_or_404 + +from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication +from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import LibraryLocatorV2 from rest_framework.permissions import IsAdminUser +from rest_framework.generics import CreateAPIView, RetrieveAPIView from rest_framework.response import Response from rest_framework.views import APIView -from cms.djangoapps.course_to_library_import.api import import_library_from_staged_content +from cms.djangoapps.course_to_library_import import api +from cms.djangoapps.course_to_library_import.constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE +from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport +from cms.djangoapps.course_to_library_import.views.v0.serializers import CourseToLibraryImportSerializer +from openedx.core.djangoapps.content_libraries.api import ContentLibrary +from openedx.core.djangoapps.content_staging import api as content_staging_api +from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser from .serializers import ImportBlocksSerializer @@ -22,7 +37,7 @@ def post(self, request, *args, **kwargs): """ Import blocks from a course to a library. - API endpoint: POST /api/v0/course-to-library-import/ + API endpoint: POST /api/course_to_library_import/v0/import_blocks/ Request: { @@ -42,7 +57,7 @@ def post(self, request, *args, **kwargs): data = self.serializer_class(data=request.data) data.is_valid(raise_exception=True) - import_library_from_staged_content( + api.import_library_from_staged_content( library_key=data.validated_data['library_key'], user_id=request.user.pk, usage_ids=data.validated_data['usage_ids'], @@ -52,3 +67,202 @@ def post(self, request, *args, **kwargs): override=data.validated_data['override'], ) return Response({'status': 'success'}) + + +class CreateCourseToLibraryImportView(CreateAPIView): + """ + **Use Case** + Allows to create course to library import. + **Example Request** + POST /api/course_to_library_import/v0/create_import// + **POST Parameters** + * course_ids (list) - A list of course IDs whose content will be saved + in Staged Content for further import. + **POST Response Values** + If the request is successful, an HTTP 201 "Created" response + is returned with the newly created CourseToLibraryImport details. + The HTTP 201 response has the following values. + { + "course_ids": ["course-v1:edX+DemoX+Demo_Course", "course-v1:edX+DemoX+Demo_Course2"], + "status": "pending", + "library_key": "lib:edX:1", + "uuid": "89b71d29-2135-4cf2-991d-e4e13b5a959a" + } + """ + + serializer_class = CourseToLibraryImportSerializer + + permission_classes = (IsAdminUser,) + authentication_classes = ( + JwtAuthentication, + BearerAuthenticationAllowInactiveUser, + SessionAuthenticationAllowInactiveUser, + ) + + def get_serializer_context(self) -> dict: + """ + Add library_id to the serializer context. + """ + context = super().get_serializer_context() + context['content_library_id'] = self.kwargs['content_library_id'] + return context + + def post(self, request, *args, **kwargs): + """ + Create course to library import. + """ + library_key = LibraryLocatorV2.from_string(self.kwargs['content_library_id']) + try: + ContentLibrary.objects.get_by_key(library_key) + except ContentLibrary.DoesNotExist: + return Response(status=404) + else: + return super().post(request, *args, **kwargs) + + +class GetCourseStructureToLibraryImportView(RetrieveAPIView): + """ + **Use Case** + Get the course structure saved when creating the import. + **Example Request** + GET /api/course_to_library_import/v0/get_import/{course-to-library-uuid}/ + **GET Response Values** + The query returns a list of hierarchical structures of + courses that are related to the import in the format: + [ + course_id1: [ + { + chapter_id1: chapter_display_name, + children: [ + { + sequential_id1: chapter_display_name + children: [...] + } + ] + }, + { + chapter_id2: chapter_display_name, + children: [ + { + sequential_id2: chapter_display_name + children: [...] + } + ] + }, + ], + ... + ] + **Example GET Response** + { + "course-v1:edX+DemoX+Demo_Course": [ + { + "block-v1:edX+DemoX+Demo_Course+type@chapter+block@3f8c073c6bf74096b9a4033227de01d3": "Section 1", + "children": [ + { + "block-v1:edX+DemoX+Demo_Course+type@sequential+block@194836ad915645d684828d4e48dbc09e": "Subsection", + "children": [ + { + "block-v1:edX+DemoX+Demo_Course+type@vertical+block@07a5b2fb186f4a47ac2d1afe3ef91850": "Unit 1", + "children": [ + { + "block-v1:edX+DemoX+Demo_Course+type@problem+block@a9c78c9ad3a148c2939091f5fbdd0eeb": "Block" + }, + { + "block-v1:edX+DemoX+Demo_Course+type@video+block@195f37e99f1b4fedb607c621f239debb": "Video" + }, + { + "block-v1:edX+DemoX+Demo_Course+type@lti+block@1700d68eae7d438aacf66fc8203efcda": "lti" + } + ] + }, + { + "block-v1:edX+DemoX+Demo_Course+type@vertical+block@c6b19a1c7136483f9dd037a14641c289": "Unit 2", + "children": [ + { + "block-v1:edX+DemoX+Demo_Course+type@html+block@330fcd9b9fa6476b8d39629dbc5cf20b": "HTML" + } + ] + } + ] + } + ] + } + ] + } + """ + + queryset = CourseToLibraryImport.objects.all() + lookup_field = 'uuid' + lookup_url_kwarg = 'course_to_lib_uuid' + + permission_classes = (IsAdminUser,) + authentication_classes = ( + JwtAuthentication, + BearerAuthenticationAllowInactiveUser, + SessionAuthenticationAllowInactiveUser, + ) + + def get(self, request, *args, **kwargs) -> Response: + """ + Get the course structure saved when creating the import. + """ + ctl = get_object_or_404(CourseToLibraryImport, uuid=self.kwargs['course_to_lib_uuid']) + courses = {} + for course_id in ctl.course_ids.split(): + courses[course_id] = self.get_structure_for_course_from_stage_content(course_id) + + return Response(courses) + + def get_structure_for_course_from_stage_content(self, course_id: str) -> list[dict]: + """ + Build course structure of the course from staged content. + + This method retrieves the staged content for the given course ID and constructs + a hierarchical structure representing the course's content. The structure is built + by parsing the OLX fragments and mapping them to their respective usage keys. + """ + parser = etree.XMLParser(strip_cdata=False) + staged_content = content_staging_api.get_ready_staged_content_by_user_and_purpose( + self.request.user.pk, COURSE_TO_LIBRARY_IMPORT_PURPOSE.format(course_id=course_id) + ) + + courses_structure = [] + for staged_content_item in staged_content: + staged_keys = [UsageKey.from_string(key) for key in staged_content_item.tags.keys()] + block_id_usage_key_map = {key.block_id: key for key in staged_keys} + olx_fragment = etree.fromstring(staged_content_item.olx, parser=parser) + courses_structure.append( + self.build_hierarchical_course_fragment_structure(olx_fragment, block_id_usage_key_map) + ) + + return courses_structure + + def build_hierarchical_course_fragment_structure( + self, + olx_fragment: 'etree._Element', + block_id_usage_key_map: dict[str, UsageKey] + ) -> dict[str, list[dict[str, dict[str, str]]]] | None: + """ + Creates a hierarchical structure of course parts recursively. + + This method takes an OLX fragment and a mapping of block IDs to usage keys, + and constructs a nested dictionary representing the hierarchical structure + of the course. It processes each OLX element, mapping it to its usage key, + and recursively processes its children if they exist. + """ + usage_key = block_id_usage_key_map.get(olx_fragment.get('url_name')) + if usage_key: + node_dict = { + str(usage_key): olx_fragment.get('display_name') or olx_fragment.tag, + } + + children = olx_fragment.getchildren() + if children and olx_fragment.tag in ('chapter', 'sequential', 'vertical'): + node_dict.update({ + 'children': [ + self.build_hierarchical_course_fragment_structure(child, block_id_usage_key_map) + for child in children + ] + }) + + return node_dict From 39160be4ca699a10d622316fc123ed10a692335d Mon Sep 17 00:00:00 2001 From: Ivan Niedielnitsev <81557788+NiedielnitsevIvan@users.noreply.github.com> Date: Thu, 3 Apr 2025 13:44:05 +0300 Subject: [PATCH 10/23] refactor: refactor importing models and related functionality (#2618) * refactor: refactor import creation api and task * test: refactor and improve tests * style: fix code style * refactor: refactor relation between CourseToLibraryImport and StagedContent * refactor: fix tests after refactoring * refactor: add CourseToLibraryImport relation to ContentLibrary * feat: add ContentLibraryFactory * test: refactor tests after change relations * feat: add CANCELED status * refactor: rename app, refactor models and Python APIs * fix: fix REST APIs after refactoring * fix: fix tests after refactoring * test: add LearningPackageFactory * style: remove extra import * chore: update migration --- .github/workflows/unit-test-shards.json | 2 +- .../course_to_library_import/admin.py | 35 - .../course_to_library_import/api.py | 54 - .../course_to_library_import/apps.py | 13 - .../course_to_library_import/constants.py | 5 - .../course_to_library_import/data.py | 22 - .../course_to_library_import/helpers.py | 372 ------ .../migrations/0001_initial.py | 55 - .../migrations/0002_containerversionimport.py | 33 - ...rsetolibraryimport_source_type_and_more.py | 23 - .../course_to_library_import/models.py | 103 -- .../course_to_library_import/tasks.py | 109 -- .../tests/factories.py | 26 - .../tests/test_api.py | 66 -- .../tests/test_helpers.py | 1044 ----------------- .../tests/test_tasks.py | 151 --- .../course_to_library_import/views/urls.py | 10 - .../views/v0/serializers.py | 69 -- .../views/v0/tests/test_serializers.py | 60 - .../README.rst | 2 +- .../__init__.py | 0 .../import_from_modulestore/admin.py | 33 + cms/djangoapps/import_from_modulestore/api.py | 38 + .../import_from_modulestore/apps.py | 19 + .../import_from_modulestore/constants.py | 5 + .../import_from_modulestore/data.py | 50 + .../import_from_modulestore/helpers.py | 370 ++++++ .../migrations/0001_initial.py | 81 ++ .../migrations/__init__.py | 0 .../import_from_modulestore/models.py | 175 +++ .../import_from_modulestore/permissions.py | 17 + .../import_from_modulestore/signals.py | 25 + .../import_from_modulestore/tasks.py | 84 ++ .../tests/__init__.py | 0 .../tests/factories.py | 30 + .../import_from_modulestore/tests/test_api.py | 77 ++ .../tests/test_helpers.py | 168 +++ .../tests/test_tasks.py | 158 +++ .../tests/test_validators.py | 37 +- .../types.py | 0 .../validators.py | 17 +- .../views/__init__.py | 0 .../import_from_modulestore/views/urls.py | 10 + .../views/v0/__init__.py | 0 .../views/v0/serializers.py | 35 + .../views/v0/tests/__init__.py | 0 .../views/v0/tests/test_views.py | 77 +- .../views/v0/urls.py | 0 .../views/v0/views.py | 182 +-- cms/envs/common.py | 2 +- cms/urls.py | 5 +- .../content_libraries/tests/factories.py | 39 + .../core/djangoapps/content_staging/api.py | 8 - setup.cfg | 1 + 54 files changed, 1558 insertions(+), 2439 deletions(-) delete mode 100644 cms/djangoapps/course_to_library_import/admin.py delete mode 100644 cms/djangoapps/course_to_library_import/api.py delete mode 100644 cms/djangoapps/course_to_library_import/apps.py delete mode 100644 cms/djangoapps/course_to_library_import/constants.py delete mode 100644 cms/djangoapps/course_to_library_import/data.py delete mode 100644 cms/djangoapps/course_to_library_import/helpers.py delete mode 100644 cms/djangoapps/course_to_library_import/migrations/0001_initial.py delete mode 100644 cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py delete mode 100644 cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py delete mode 100644 cms/djangoapps/course_to_library_import/models.py delete mode 100644 cms/djangoapps/course_to_library_import/tasks.py delete mode 100644 cms/djangoapps/course_to_library_import/tests/factories.py delete mode 100644 cms/djangoapps/course_to_library_import/tests/test_api.py delete mode 100644 cms/djangoapps/course_to_library_import/tests/test_helpers.py delete mode 100644 cms/djangoapps/course_to_library_import/tests/test_tasks.py delete mode 100644 cms/djangoapps/course_to_library_import/views/urls.py delete mode 100644 cms/djangoapps/course_to_library_import/views/v0/serializers.py delete mode 100644 cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/README.rst (94%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/__init__.py (100%) create mode 100644 cms/djangoapps/import_from_modulestore/admin.py create mode 100644 cms/djangoapps/import_from_modulestore/api.py create mode 100644 cms/djangoapps/import_from_modulestore/apps.py create mode 100644 cms/djangoapps/import_from_modulestore/constants.py create mode 100644 cms/djangoapps/import_from_modulestore/data.py create mode 100644 cms/djangoapps/import_from_modulestore/helpers.py create mode 100644 cms/djangoapps/import_from_modulestore/migrations/0001_initial.py rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/migrations/__init__.py (100%) create mode 100644 cms/djangoapps/import_from_modulestore/models.py create mode 100644 cms/djangoapps/import_from_modulestore/permissions.py create mode 100644 cms/djangoapps/import_from_modulestore/signals.py create mode 100644 cms/djangoapps/import_from_modulestore/tasks.py rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/tests/__init__.py (100%) create mode 100644 cms/djangoapps/import_from_modulestore/tests/factories.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_api.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_helpers.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_tasks.py rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/tests/test_validators.py (54%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/types.py (100%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/validators.py (67%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/views/__init__.py (100%) create mode 100644 cms/djangoapps/import_from_modulestore/views/urls.py rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/views/v0/__init__.py (100%) create mode 100644 cms/djangoapps/import_from_modulestore/views/v0/serializers.py rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/views/v0/tests/__init__.py (100%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/views/v0/tests/test_views.py (76%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/views/v0/urls.py (100%) rename cms/djangoapps/{course_to_library_import => import_from_modulestore}/views/v0/views.py (60%) create mode 100644 openedx/core/djangoapps/content_libraries/tests/factories.py diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index dfd1e41de1e5..00215fde949c 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -237,7 +237,7 @@ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", "cms/djangoapps/course_creators/", - "cms/djangoapps/course_to_library_import/", + "cms/djangoapps/import_from_modulestore/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/maintenance/", "cms/djangoapps/models/", diff --git a/cms/djangoapps/course_to_library_import/admin.py b/cms/djangoapps/course_to_library_import/admin.py deleted file mode 100644 index a12a14bc2131..000000000000 --- a/cms/djangoapps/course_to_library_import/admin.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -This module contains the admin configuration for the CourseToLibraryImport model. -""" - -from django.contrib import admin - -from .models import CourseToLibraryImport - -# Run a task to import courses to the library -# This task is run by the celery worker to import courses to the library. - - -class CourseToLibraryImportAdmin(admin.ModelAdmin): - """ - Admin configuration for the CourseToLibraryImport model. - """ - - list_display = ( - 'id', - 'status', - 'course_ids', - 'library_key', - ) - list_filter = ( - 'status', - ) - search_fields = ( - 'course_ids', - 'library_key', - ) - - raw_id_fields = ('user',) - - -admin.site.register(CourseToLibraryImport, CourseToLibraryImportAdmin) diff --git a/cms/djangoapps/course_to_library_import/api.py b/cms/djangoapps/course_to_library_import/api.py deleted file mode 100644 index 22c045ba057a..000000000000 --- a/cms/djangoapps/course_to_library_import/api.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -API for course to library import. -""" - -from .constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE -from .models import CourseToLibraryImport -from .tasks import ( - import_library_from_staged_content_task, - save_courses_to_staged_content_task, -) -from .types import CompositionLevel - - -def import_library_from_staged_content( - library_key: str, - user_id: int, - usage_ids: list[str], - course_id: str, - import_id: str, - composition_level: CompositionLevel, - override: bool -) -> None: - """ - Import staged content to a library. - """ - import_library_from_staged_content_task.delay( - user_id, - usage_ids, - library_key, - COURSE_TO_LIBRARY_IMPORT_PURPOSE, - course_id, - import_id, - composition_level, - override, - ) - - -def create_import( - course_ids: list[str], user_id: int, library_key: str -) -> None: - """ - Create a new import task to import a course to a library. - """ - import_task = CourseToLibraryImport( - course_ids=" ".join(course_ids), - library_key=library_key, - user_id=user_id, - ) - import_task.save() - - save_courses_to_staged_content_task.delay( - course_ids, user_id, import_task.id, COURSE_TO_LIBRARY_IMPORT_PURPOSE - ) - return import_task diff --git a/cms/djangoapps/course_to_library_import/apps.py b/cms/djangoapps/course_to_library_import/apps.py deleted file mode 100644 index b82c7fdf90c8..000000000000 --- a/cms/djangoapps/course_to_library_import/apps.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -App for importing Courses to the Content Library -""" - -from django.apps import AppConfig - - -class CourseToLibraryImportConfig(AppConfig): - """ - App for importing Courses to the Content Library - """ - - name = 'cms.djangoapps.course_to_library_import' diff --git a/cms/djangoapps/course_to_library_import/constants.py b/cms/djangoapps/course_to_library_import/constants.py deleted file mode 100644 index f619bf9e2697..000000000000 --- a/cms/djangoapps/course_to_library_import/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Constants for course_to_library_import app -""" - -COURSE_TO_LIBRARY_IMPORT_PURPOSE = "course_{course_id}_to_library_import" diff --git a/cms/djangoapps/course_to_library_import/data.py b/cms/djangoapps/course_to_library_import/data.py deleted file mode 100644 index af14fb77de65..000000000000 --- a/cms/djangoapps/course_to_library_import/data.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This module contains the data models for the course_to_library_import app. -""" - -from django.db.models import TextChoices -from django.utils.translation import gettext_lazy as _ - - -class CourseToLibraryImportStatus(TextChoices): - """ - The status of this course import. - """ - - # PENDING: The import has been created, but the OLX and related data are not yet in the library. - # It is not ready to be read. - PENDING = 'pending', _('Pending') - # READY: The content is staged and ready to be read. - READY = 'ready', _('Ready') - # IMPORTED: The content has been imported into the library. - IMPORTED = 'imported', _('Imported') - # ERROR: The content could not be imported. - ERROR = 'error', _('Error') diff --git a/cms/djangoapps/course_to_library_import/helpers.py b/cms/djangoapps/course_to_library_import/helpers.py deleted file mode 100644 index bbe71019046b..000000000000 --- a/cms/djangoapps/course_to_library_import/helpers.py +++ /dev/null @@ -1,372 +0,0 @@ -""" -Helper functions for importing course content into a library. -""" - -from datetime import datetime, timezone -import logging -import mimetypes -import secrets - -from django.db import transaction -from django.db.utils import IntegrityError -from lxml import etree - -from opaque_keys.edx.keys import UsageKey -from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 -from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import ContainerVersion - -from openedx.core.djangoapps.content_libraries import api -from openedx.core.djangoapps.content_libraries.api import ContentLibrary -from openedx.core.djangoapps.content_staging import api as content_staging_api - -from .data import CourseToLibraryImportStatus -from .models import ComponentVersionImport, ContainerVersionImport, CourseToLibraryImport -import os - - -log = logging.getLogger(__name__) - - -def create_block_in_library(block_to_import, usage_key, library_key, user_id, staged_content_id, import_id, override): - """ - Create a block in a library from a staged content block. - """ - now = datetime.now(tz=timezone.utc) - staged_content_files = content_staging_api.get_staged_content_static_files(staged_content_id) - - content_library = ContentLibrary.objects.get_by_key(library_key) - - with transaction.atomic(): - component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type) - component_version = None - does_component_exist = authoring_api.get_components( - content_library.learning_package.id - ).filter(local_key=usage_key.block_id).exists() - - if does_component_exist: - if not override: - log.info(f"Component {usage_key.block_id} already exists in library {library_key}, skipping.") - return - else: - component_version = _handle_component_override( - content_library, usage_key, etree.tostring(block_to_import) - ) - else: - # Create component (regardless of override path) - _, library_usage_key = api.validate_can_add_block_to_library( - library_key, - block_to_import.tag, - usage_key.block_id, - ) - authoring_api.create_component( - content_library.learning_package.id, - component_type=component_type, - local_key=usage_key.block_id, - created=now, - created_by=user_id, - ) - - component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import)) - - # Handle component version import records for overrides - overrided_component_version_import = False - if override: - _update_component_version_import( - component_version, usage_key, import_id - ) - overrided_component_version_import = True - - _process_staged_content_files( - component_version, staged_content_files, staged_content_id, usage_key, - content_library, now, block_to_import, overrided_component_version_import, library_key, user_id - ) - - return component_version - - -def _handle_component_override(content_library, usage_key, new_content): - """ - Create new ComponentVersion for overridden component. - """ - component_version = None - component = content_library.learning_package.component_set.filter(local_key=usage_key.block_id).first() - - if component: - lib_usage_key = LibraryUsageLocatorV2( # type: ignore[abstract] - lib_key=content_library.library_key, - block_type=component.component_type.name, - usage_id=component.local_key, - ) - component_version = api.set_library_block_olx(lib_usage_key, new_content) - - return component_version - - -def _update_component_version_import(component_version, usage_key, import_id): - """ - Update component version import records for overridden components. - """ - return ComponentVersionImport.objects.create( - component_version=component_version, - source_usage_key=usage_key, - library_import=CourseToLibraryImport.get_ready_by_uuid(import_id), - ) - - -def _process_staged_content_files( - component_version, - staged_content_files, - staged_content_id, - usage_key, - content_library, - now, - block_to_import, - overrided_component_version_import, - library_key, - user_id, -): - """ - Process staged content files for a component. - """ - block_olx = etree.tostring(block_to_import).decode('utf-8') - - for staged_content_file_data in staged_content_files: - original_filename = staged_content_file_data.filename - file_basename = os.path.basename(original_filename) - file_basename_no_ext, _ = os.path.splitext(file_basename) - - # Skip files not referenced in the block - if file_basename not in block_olx and file_basename_no_ext not in block_olx: - log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") - continue - - file_data = content_staging_api.get_staged_content_static_file_data( - staged_content_id, - original_filename, - ) - if not file_data: - log.error( - f"Staged content {staged_content_id} included referenced " - f"file {original_filename}, but no file data was found." - ) - continue - - filename = f"static/{file_basename}" - media_type_str, _ = mimetypes.guess_type(filename) - if not media_type_str: - media_type_str = "application/octet-stream" - - media_type = authoring_api.get_or_create_media_type(media_type_str) - content = authoring_api.get_or_create_file_content( - content_library.learning_package.id, - media_type.id, - data=file_data, - created=now, - ) - - try: - authoring_api.create_component_version_content( - component_version.pk, - content.id, - key=filename, - ) - except IntegrityError: - pass # Content already exists - - if not overrided_component_version_import: - ComponentVersionImport.objects.get_or_create( - component_version=component_version, - source_usage_key=usage_key, - library_import=CourseToLibraryImport.objects.get( - library_key=library_key, - user_id=user_id, - status=CourseToLibraryImportStatus.READY - ), - ) - - -def _update_container_components(container_version, component_versions, user_id): - return authoring_api.create_next_container_version( - container_pk=container_version.container.pk, - title=container_version.title, - publishable_entities_pks=[ - cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk for cv in component_versions - ], - entity_version_pks=[cv.pk for cv in component_versions], - created=datetime.now(tz=timezone.utc), - created_by=user_id, - container_version_cls=container_version.__class__, - ) - - -def _process_xblock(child, library_key, user_id, staged_content, import_id, override): - """ - Process an xblock and create a block in the library. - """ - staged_keys = [UsageKey.from_string(key) for key in staged_content.tags.keys()] - block_id_to_usage_key = {key.block_id: key for key in staged_keys} - - usage_key = block_id_to_usage_key.get(child.get('url_name')) - - if usage_key in staged_keys: - return create_block_in_library( - child, - usage_key, - library_key, - user_id, - staged_content.id, - import_id, - override, - ) - - -def import_children(block_to_import, library_key, user_id, staged_content, composition_level, import_id, override): - """ - Import children of a block from staged content into a library. - Creates appropriate container hierarchy based on composition_level. - """ - result = [] - - if block_to_import.tag not in ('chapter', 'sequential', 'vertical'): - component_version = _process_xblock(block_to_import, library_key, user_id, staged_content, import_id, override) - if component_version: - return [component_version] - - for child in block_to_import.getchildren(): - if child.tag in ('chapter', 'sequential', 'vertical'): - container_version = create_container( - child.tag, - child.get('url_name'), - child.get('display_name', ''), - library_key, - user_id, - ) - - child_component_versions = import_children( - child, - library_key, - user_id, - staged_content, - composition_level, - import_id, - override, - ) - - if child_component_versions: - _update_container_components(container_version, child_component_versions, user_id) - - result.append(container_version) - else: - component_version = _process_xblock(child, library_key, user_id, staged_content, import_id, override) - if component_version is not None: - result.append(component_version) - - if composition_level == 'xblock': - return [component for component in result if not isinstance(component, ContainerVersion)] - else: - return result - - -def create_container(container_type, key, display_name, library_key, user_id): - """ - Create a container of the specified type. - """ - assert isinstance(library_key, LibraryLocatorV2) - content_library = ContentLibrary.objects.get_by_key(library_key) - - container_creators_map = { - 'chapter': authoring_api.create_unit_and_version, # TODO: replace with create_module_and_version - 'sequential': authoring_api.create_unit_and_version, # TODO: replace with create_section_and_version - 'vertical': authoring_api.create_unit_and_version, - } - - if container_type not in container_creators_map: - raise ValueError(f"Unknown container type: {container_type}") - - if not key: - key = secrets.token_hex(16) - - if not display_name: - display_name = f"New {container_type}" - - if container_creator_func := container_creators_map.get(container_type): - _, container_version = container_creator_func( - content_library.learning_package.id, - key=key, - title=display_name, - components=[], - created=datetime.now(tz=timezone.utc), - created_by=user_id, - ) - - return container_version - - -def import_container( - usage_key, block_to_import, library_key, user_id, staged_content, composition_level, import_id, override -): - """ - Import a blocks hierarchy into a library, creating proper container structure. - """ - container_type = block_to_import.tag - - if composition_level in ('chapter', 'sequential', 'vertical'): - key = block_to_import.get('url_name') - display_name = block_to_import.get('display_name', '') - - top_container_version = create_container( - container_type, - key, - display_name, - library_key, - user_id, - ) - - component_versions = import_children( - block_to_import, - library_key, - user_id, - staged_content, - composition_level, - import_id, - override, - ) - - if component_versions: - _update_container_components(top_container_version, component_versions, user_id) - - with transaction.atomic(): - ContainerVersionImport.objects.create( - section_version=top_container_version, - source_usage_key=usage_key, - library_import=CourseToLibraryImport.objects.get( - library_key=library_key, - user_id=user_id, - status=CourseToLibraryImportStatus.READY - ), - ) - else: # xblock level - import_children( - block_to_import, - library_key, - user_id, - staged_content, - composition_level, - import_id, - override, - ) - - -def get_block_to_import(node, usage_key): - """ - Get the block to import from a node. - """ - - if node.get('url_name') == usage_key.block_id: - return node - - for child in node.getchildren(): - found = get_block_to_import(child, usage_key) - if found is not None: - return found diff --git a/cms/djangoapps/course_to_library_import/migrations/0001_initial.py b/cms/djangoapps/course_to_library_import/migrations/0001_initial.py deleted file mode 100644 index 1c9fd2d95aae..000000000000 --- a/cms/djangoapps/course_to_library_import/migrations/0001_initial.py +++ /dev/null @@ -1,55 +0,0 @@ -# Generated by Django 4.2.18 on 2025-03-07 09:20 - -import cms.djangoapps.course_to_library_import.validators -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone -import model_utils.fields -import opaque_keys.edx.django.models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'), - ] - - operations = [ - migrations.CreateModel( - name='CourseToLibraryImport', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ('status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('imported', 'Imported'), ('error', 'Error')], default='pending', max_length=100)), - ('course_ids', models.TextField(help_text='Whitespace-separated list of course keys for which to compute grades.', validators=[cms.djangoapps.course_to_library_import.validators.validate_course_ids])), - ('library_key', models.CharField(max_length=100)), - ('source_type', models.CharField(max_length=30)), - ('metadata', models.JSONField(blank=True, default=dict, null=True)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'verbose_name': 'Course to Library Import', - 'verbose_name_plural': 'Course to Library Imports', - }, - ), - migrations.CreateModel( - name='ComponentVersionImport', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), - ('component_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='oel_components.componentversion')), - ('library_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='course_to_library_import.coursetolibraryimport')), - ], - options={ - 'verbose_name': 'Component Version Import', - 'verbose_name_plural': 'Component Version Imports', - }, - ), - ] diff --git a/cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py b/cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py deleted file mode 100644 index 67880b183df3..000000000000 --- a/cms/djangoapps/course_to_library_import/migrations/0002_containerversionimport.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.2.19 on 2025-03-19 17:47 - -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone -import model_utils.fields -import opaque_keys.edx.django.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('oel_publishing', '0003_containers'), - ('course_to_library_import', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='ContainerVersionImport', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), - ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), - ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255)), - ('container_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.containerversion')), - ('library_import', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='course_to_library_import.coursetolibraryimport')), - ], - options={ - 'verbose_name': 'Container Version Import', - 'verbose_name_plural': 'Container Version Imports', - }, - ), - ] diff --git a/cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py b/cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py deleted file mode 100644 index 9122325ef942..000000000000 --- a/cms/djangoapps/course_to_library_import/migrations/0003_remove_coursetolibraryimport_source_type_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.18 on 2025-03-21 11:01 - -from django.db import migrations, models -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - ('course_to_library_import', '0002_containerversionimport'), - ] - - operations = [ - migrations.RemoveField( - model_name='coursetolibraryimport', - name='source_type', - ), - migrations.AddField( - model_name='coursetolibraryimport', - name='uuid', - field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True), - ), - ] diff --git a/cms/djangoapps/course_to_library_import/models.py b/cms/djangoapps/course_to_library_import/models.py deleted file mode 100644 index 8c815e347871..000000000000 --- a/cms/djangoapps/course_to_library_import/models.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Models for the course to library import app. -""" - -import uuid as uuid_tools -from typing import Self - -from django.contrib.auth import get_user_model -from django.db import models -from django.utils.translation import gettext_lazy as _ - -from opaque_keys.edx.django.models import UsageKeyField - -from model_utils.models import TimeStampedModel - -from .data import CourseToLibraryImportStatus -from .validators import validate_course_ids - -User = get_user_model() - - -class CourseToLibraryImport(TimeStampedModel): - """ - Represents a course import into a content library. - """ - - uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True, db_index=True) - status = models.CharField( - max_length=100, - choices=CourseToLibraryImportStatus.choices, - default=CourseToLibraryImportStatus.PENDING - ) - course_ids = models.TextField( - blank=False, - help_text=_('Whitespace-separated list of course keys for which to compute grades.'), - validators=[validate_course_ids] - ) - library_key = models.CharField(max_length=100) - metadata = models.JSONField(default=dict, blank=True, null=True) - user = models.ForeignKey(User, on_delete=models.CASCADE) - - def __str__(self): - return f'{self.course_ids} - {self.library_key}' - - class Meta: - verbose_name = _('Course to Library Import') - verbose_name_plural = _('Course to Library Imports') - - @classmethod - def get_by_id(cls, import_id: int) -> Self | None: - """ - Get an import task by its ID. - """ - return cls.objects.filter(id=import_id).first() - - @classmethod - def get_ready_by_uuid(cls, import_uuid: str) -> Self | None: - """ - Get an import task by its UUID. - """ - return cls.objects.filter(uuid=import_uuid, status=CourseToLibraryImportStatus.READY).first() - - -class ComponentVersionImport(TimeStampedModel): - """ - Represents a component version that has been imported into a content library. - This is a many-to-many relationship between a component version and a course to library import. - """ - - component_version = models.OneToOneField( - to='oel_components.ComponentVersion', - on_delete=models.CASCADE - ) - source_usage_key = UsageKeyField(max_length=255) - library_import = models.ForeignKey(CourseToLibraryImport, on_delete=models.CASCADE) - - def __str__(self): - return f'{self.component_version} - {self.source_usage_key}' - - class Meta: - verbose_name = _('Component Version Import') - verbose_name_plural = _('Component Version Imports') - - -class ContainerVersionImport(TimeStampedModel): - """ - Represents a container version that has been imported into a content library. - This is a many-to-many relationship between a container version and a course to library import. - """ - - container_version = models.OneToOneField( - to='oel_publishing.ContainerVersion', - on_delete=models.CASCADE - ) - source_usage_key = UsageKeyField(max_length=255) - library_import = models.ForeignKey(CourseToLibraryImport, on_delete=models.CASCADE) - - def __str__(self): - return f'{self.container_version} - {self.source_usage_key}' - - class Meta: - verbose_name = _('Container Version Import') - verbose_name_plural = _('Container Version Imports') diff --git a/cms/djangoapps/course_to_library_import/tasks.py b/cms/djangoapps/course_to_library_import/tasks.py deleted file mode 100644 index 749199460079..000000000000 --- a/cms/djangoapps/course_to_library_import/tasks.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Tasks for course to library import. -""" - -from celery import shared_task -from django.db import transaction -from edx_django_utils.monitoring import set_code_owner_attribute -from lxml import etree -from opaque_keys.edx.keys import CourseKey, UsageKey -from opaque_keys.edx.locator import LibraryLocatorV2 - -from openedx.core.djangoapps.content_staging import api as content_staging_api -from xmodule.modulestore.django import modulestore - -from .data import CourseToLibraryImportStatus -from .helpers import get_block_to_import, import_container -from .models import CourseToLibraryImport -from .types import CompositionLevel -from .validators import validate_composition_level, validate_usage_ids - - -@shared_task -@set_code_owner_attribute -def save_courses_to_staged_content_task( - course_ids: list[str], - user_id: int, - import_task_id, - purpose: str, - version_num: int | None = None, -) -> None: - """ - Save courses to staged content task. - """ - course_to_library_import = CourseToLibraryImport.get_by_id(import_task_id) - if not course_to_library_import: - return - - with transaction.atomic(): - for course_id in course_ids: - course_key = CourseKey.from_string(course_id) - sections = modulestore().get_items( - course_key, qualifiers={"category": "chapter"} - ) or [] - static_tabs = modulestore().get_items( - course_key, qualifiers={"category": "static_tab"} - ) or [] - - for item in sections + static_tabs: - content_staging_api.stage_xblock_temporarily( - item, - user_id, - purpose=purpose.format(course_id=course_id), - version_num=version_num, - ) - - course_to_library_import.status = CourseToLibraryImportStatus.READY - course_to_library_import.save() - - -@shared_task -@set_code_owner_attribute -def import_library_from_staged_content_task( - user_id: int, - usage_ids: list[str], - library_key: str, - purpose: str, - course_id: str, - import_id: str, - composition_level: CompositionLevel, - override: bool -) -> None: - """ - Import staged content to a library task. - """ - validate_composition_level(composition_level) - staged_content = content_staging_api.get_ready_staged_content_by_user_and_purpose( - user_id, purpose.format(course_id=course_id) - ) - validate_usage_ids(usage_ids, staged_content) - parser = etree.XMLParser(strip_cdata=False) - library_key = LibraryLocatorV2.from_string(library_key) - - with transaction.atomic(): - for usage_key in usage_ids: - if staged_content_item := staged_content.filter( - tags__icontains=usage_key, - ).first(): - node = etree.fromstring(staged_content_item.olx, parser=parser) - usage_key = UsageKey.from_string(usage_key) - block_to_import = get_block_to_import(node, usage_key) - - if block_to_import is None: - continue - import_container( - usage_key, - block_to_import, - library_key, - user_id, - staged_content_item, - composition_level, - import_id, - override, - ) - - ctli = CourseToLibraryImport.get_ready_by_uuid(import_id) - ctli.status = CourseToLibraryImportStatus.IMPORTED - ctli.save() - - staged_content.delete() diff --git a/cms/djangoapps/course_to_library_import/tests/factories.py b/cms/djangoapps/course_to_library_import/tests/factories.py deleted file mode 100644 index 4caa9a9bb1ed..000000000000 --- a/cms/djangoapps/course_to_library_import/tests/factories.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Factories for CourseToLibraryImport model. -""" - -import uuid - -import factory -from factory.django import DjangoModelFactory - -from common.djangoapps.student.tests.factories import UserFactory - -from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport - - -class CourseToLibraryImportFactory(DjangoModelFactory): - """ - Factory for CourseToLibraryImport model. - """ - class Meta: - model = CourseToLibraryImport - - course_ids = ' '.join([f'course-v1:edX+DemoX+Demo_Course{i}' for i in range(1, 3)]) - uuid = factory.LazyFunction(lambda: str(uuid.uuid4())) - library_key = 'lib:TestOrg:TestLib' - metadata = {} - user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/course_to_library_import/tests/test_api.py b/cms/djangoapps/course_to_library_import/tests/test_api.py deleted file mode 100644 index fa864bc97ab0..000000000000 --- a/cms/djangoapps/course_to_library_import/tests/test_api.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Test cases for course_to_library_import.api module. -""" - -from unittest.mock import patch - -import pytest - -from common.djangoapps.student.tests.factories import UserFactory -from cms.djangoapps.course_to_library_import.api import ( - create_import, - import_library_from_staged_content, -) -from cms.djangoapps.course_to_library_import.constants import COURSE_TO_LIBRARY_IMPORT_PURPOSE -from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport -from .factories import CourseToLibraryImportFactory - - -@pytest.mark.django_db -def test_create_import(): - """ - Test create_import function. - """ - course_ids = [ - "course-v1:edX+DemoX+Demo_Course", - "course-v1:edX+DemoX+Demo_Course_2", - ] - user = UserFactory() - library_key = "lib:edX:DemoLib" - with patch( - "cms.djangoapps.course_to_library_import.api.save_courses_to_staged_content_task" - ) as save_courses_to_staged_content_task_mock: - create_import(course_ids, user.id, library_key) - - import_task = CourseToLibraryImport.objects.get() - assert import_task.course_ids == " ".join(course_ids) - assert import_task.library_key == library_key - assert import_task.user_id == user.id - save_courses_to_staged_content_task_mock.delay.assert_called_once_with( - course_ids, user.id, import_task.id, COURSE_TO_LIBRARY_IMPORT_PURPOSE - ) - - -@pytest.mark.django_db -@pytest.mark.parametrize("override", [True, False]) -def test_import_library_from_staged_content(override): - """ - Test import_library_from_staged_content function with different override values. - """ - ctli = CourseToLibraryImportFactory() - library_key = ctli.library_key - user = ctli.user - usage_ids = [ - "block-v1:edX+DemoX+Demo_Course+type@html+block@123", - "block-v1:edX+DemoX+Demo_Course+type@html+block@456", - ] - course_id = "course-v1:edX+DemoX+Demo_Course" - - with patch( - "cms.djangoapps.course_to_library_import.api.import_library_from_staged_content_task" - ) as import_library_from_staged_content_task_mock: - import_library_from_staged_content(library_key, user.id, usage_ids, course_id, ctli.uuid, 'xblock', override) - - import_library_from_staged_content_task_mock.delay.assert_called_once_with( - user.id, usage_ids, library_key, COURSE_TO_LIBRARY_IMPORT_PURPOSE, course_id, ctli.uuid, 'xblock', override - ) diff --git a/cms/djangoapps/course_to_library_import/tests/test_helpers.py b/cms/djangoapps/course_to_library_import/tests/test_helpers.py deleted file mode 100644 index 61b1b087fee8..000000000000 --- a/cms/djangoapps/course_to_library_import/tests/test_helpers.py +++ /dev/null @@ -1,1044 +0,0 @@ -""" -Tests for the course_to_library_import helper functions. -""" - -from datetime import datetime, timezone -from unittest import mock -from uuid import uuid4 - -from lxml import etree -from django.db.utils import IntegrityError -from django.test import TestCase -from opaque_keys.edx.keys import UsageKey -from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 -from openedx_learning.api.authoring_models import ContainerVersion - -from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus -from cms.djangoapps.course_to_library_import.helpers import ( - _handle_component_override, - _process_staged_content_files, - _update_container_components, - create_block_in_library, - create_container, - import_children, - import_container, -) -from common.djangoapps.student.tests.factories import UserFactory - -from .factories import CourseToLibraryImportFactory - - -class TestFlatImportChildren(TestCase): - """ - Tests for the flat_import_children helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = "test_user" - - self.staged_content = mock.MagicMock() - self.staged_content.id = "staged-content-id" - self.staged_content.tags = { - "block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1": {}, - "block-v1:TestOrg+TestCourse+Run1+type@html+block@html1": {}, - "block-v1:TestOrg+TestCourse+Run1+type@video+block@video1": {}, - } - - self.import_id = str(uuid4()) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_flat_import_children_basic(self, mock_content_library, mock_create_block): - xml = """ - - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, False - ) - - self.assertEqual(mock_create_block.call_count, 2) - - usage_key_problem = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1") - usage_key_html = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@html+block@html1") - - mock_create_block.assert_any_call( - mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, self.import_id, False - ) - mock_create_block.assert_any_call( - mock.ANY, usage_key_html, self.library_key, self.user_id, self.staged_content.id, self.import_id, False - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_flat_import_children_with_override(self, mock_content_library, mock_create_block): - xml = """ - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, True - ) - - usage_key_problem = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1") - mock_create_block.assert_called_with( - mock.ANY, usage_key_problem, self.library_key, self.user_id, self.staged_content.id, self.import_id, True - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_flat_import_children_library_not_found(self, mock_content_library): - xml = """ - - - - """ - block_to_import = etree.fromstring(xml) - - mock_content_library.objects.filter.return_value.first.return_value = None - - with self.assertRaises(ValueError): - import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, False - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_flat_import_children_ignores_unmatched_url_names(self, mock_content_library, mock_create_block): - xml = """ - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', self.import_id, False - ) - - mock_create_block.assert_not_called() - - -class TestCreateBlockInLibrary(TestCase): - """ - Tests for the create_block_in_library helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = UserFactory().id - self.block_id = "problem1" - self.block_type = "problem" - self.staged_content_id = "staged-content-id" - self.usage_key = UsageKey.from_string( - f"block-v1:TestOrg+TestCourse+Run1+type@{self.block_type}+block@{self.block_id}" - ) - - self.xml_content = "Test problem content" - self.block_to_import = etree.fromstring(self.xml_content) - self.mock_library = mock.MagicMock() - self.mock_library.library_key = self.library_key - self.mock_learning_package = mock.MagicMock() - self.mock_library.learning_package = self.mock_learning_package - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_create_block_in_library_new_component( - self, mock_content_library, mock_api, mock_authoring_api, mock_content_staging_api - ): - mock_content_library.objects.get_by_key.return_value = self.mock_library - mock_component_type = mock.MagicMock() - mock_authoring_api.get_or_create_component_type.return_value = mock_component_type - mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = False - mock_library_usage_key = mock.MagicMock() - mock_api.validate_can_add_block_to_library.return_value = (None, mock_library_usage_key) - mock_component_version = mock.MagicMock() - mock_api.set_library_block_olx.return_value = mock_component_version - mock_content_staging_api.get_staged_content_static_files.return_value = [] - - create_block_in_library( - self.block_to_import, - self.usage_key, - self.library_key, - self.user_id, - self.staged_content_id, - str(uuid4()), - False, - ) - - mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) - mock_authoring_api.get_or_create_component_type.assert_called_once_with("xblock.v1", self.block_type) - mock_authoring_api.get_components.assert_called_once_with(self.mock_learning_package.id) - mock_api.validate_can_add_block_to_library.assert_called_once_with( - self.library_key, self.block_to_import.tag, self.block_id - ) - mock_authoring_api.create_component.assert_called_once() - mock_api.set_library_block_olx.assert_called_once_with( - mock_library_usage_key, etree.tostring(self.block_to_import) - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - @mock.patch('cms.djangoapps.course_to_library_import.helpers._handle_component_override') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport') - def test_create_block_in_library_existing_component_with_override( - self, - mock_component_version_import, - mock_handle_override, - mock_content_library, - mock_api, - mock_authoring_api, - mock_content_staging_api - ): - mock_content_library.objects.get_by_key.return_value = self.mock_library - mock_component_type = mock.MagicMock() - mock_authoring_api.get_or_create_component_type.return_value = mock_component_type - mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = True - - mock_component_version = mock.MagicMock(spec=['id', 'component_id']) - mock_handle_override.return_value = mock_component_version - - mock_component_version_import.return_value = mock.MagicMock() - - mock_content_staging_api.get_staged_content_static_files.return_value = [] - - ctli = CourseToLibraryImportFactory( - status=CourseToLibraryImportStatus.READY, library_key=self.library_key, user_id=self.user_id - ) - create_block_in_library( - self.block_to_import, - self.usage_key, - self.library_key, - self.user_id, - self.staged_content_id, - str(ctli.uuid), - True, - ) - - mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) - mock_authoring_api.get_or_create_component_type.assert_called_once_with("xblock.v1", self.block_type) - mock_authoring_api.get_components.assert_called_once_with(self.mock_learning_package.id) - mock_handle_override.assert_called_once_with( - self.mock_library, self.usage_key, etree.tostring(self.block_to_import) - ) - mock_api.validate_can_add_block_to_library.assert_not_called() - mock_authoring_api.create_component.assert_not_called() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_create_block_in_library_existing_component_without_override( - self, - mock_content_library, - mock_api, - mock_authoring_api, - mock_content_staging_api - ): - mock_content_library.objects.get_by_key.return_value = self.mock_library - mock_component_type = mock.MagicMock() - mock_authoring_api.get_or_create_component_type.return_value = mock_component_type - mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = True - mock_content_staging_api.get_staged_content_static_files.return_value = [] - - create_block_in_library( - self.block_to_import, - self.usage_key, - self.library_key, - self.user_id, - self.staged_content_id, - str(uuid4()), - False, - ) - - mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) - mock_authoring_api.get_or_create_component_type.assert_called_once_with("xblock.v1", self.block_type) - mock_authoring_api.get_components.assert_called_once_with(self.mock_learning_package.id) - mock_api.validate_can_add_block_to_library.assert_not_called() - mock_authoring_api.create_component.assert_not_called() - mock_api.set_library_block_olx.assert_not_called() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_component_version_import') - @mock.patch('cms.djangoapps.course_to_library_import.helpers._process_staged_content_files') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - @mock.patch('cms.djangoapps.course_to_library_import.helpers._handle_component_override') - def test_create_block_in_library_with_files_and_override( - self, mock_handle_override, mock_content_library, - mock_authoring_api, mock_process_files, - mock_update_component, mock_content_staging_api - ): - mock_content_library.objects.get_by_key.return_value = self.mock_library - mock_component_type = mock.MagicMock() - mock_authoring_api.get_or_create_component_type.return_value = mock_component_type - mock_authoring_api.get_components.return_value.filter.return_value.exists.return_value = True - mock_component_version = mock.MagicMock() - mock_handle_override.return_value = mock_component_version - mock_file_data = [mock.MagicMock()] - mock_content_staging_api.get_staged_content_static_files.return_value = mock_file_data - - import_id = str(uuid4()) - - create_block_in_library( - self.block_to_import, - self.usage_key, - self.library_key, - self.user_id, - self.staged_content_id, - import_id, - True, - ) - - mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) - mock_update_component.assert_called_once_with( - mock_component_version, self.usage_key, import_id, - ) - mock_process_files.assert_called_once() - - -class TestProcessStagedContentFiles(TestCase): - """ - Tests for the _process_staged_content_files helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = UserFactory().id - self.block_id = "problem1" - self.block_type = "problem" - self.staged_content_id = "staged-content-id" - self.usage_key = UsageKey.from_string( - f"block-v1:TestOrg+TestCourse+Run1+type@{self.block_type}+block@{self.block_id}" - ) - - self.xml_content = "Test problem content" - self.block_to_import = etree.fromstring(self.xml_content) - self.mock_library = mock.MagicMock() - self.mock_library.library_key = self.library_key - self.mock_learning_package = mock.MagicMock() - self.mock_library.learning_package = self.mock_learning_package - self.now = datetime.now(tz=timezone.utc) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport.objects.get') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport.objects.get_or_create') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_process_staged_content_files_with_reference_in_block( - self, mock_authoring_api, mock_content_staging_api, mock_get_or_create, mock_get_import - ): - mock_component_version = mock.MagicMock() - mock_file_data = mock.MagicMock() - mock_file_data.filename = "test_file.txt" - - xml_content = 'Test problem with ' - block_to_import = etree.fromstring(xml_content) - - mock_content_staging_api.get_staged_content_static_file_data.return_value = b"file data" - mock_media_type = mock.MagicMock(id=1) - mock_authoring_api.get_or_create_media_type.return_value = mock_media_type - mock_content = mock.MagicMock(id=1) - mock_authoring_api.get_or_create_file_content.return_value = mock_content - mock_import = mock.MagicMock() - mock_get_import.return_value = mock_import - - _process_staged_content_files( - mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, - self.mock_library, self.now, block_to_import, False, self.library_key, self.user_id - ) - - mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with( - self.staged_content_id, mock_file_data.filename - ) - mock_authoring_api.get_or_create_media_type.assert_called_once() - mock_authoring_api.get_or_create_file_content.assert_called_once_with( - self.mock_library.learning_package.id, - mock_media_type.id, - data=b"file data", - created=self.now, - ) - mock_authoring_api.create_component_version_content.assert_called_once_with( - mock_component_version.pk, - mock_content.id, - key=f"static/{mock_file_data.filename}", - ) - mock_get_or_create.assert_called_once() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport.objects.get') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_process_staged_content_files_missing_file_data( - self, mock_authoring_api, mock_content_staging_api, mock_get_import - ): - mock_component_version = mock.MagicMock() - mock_file_data = mock.MagicMock() - mock_file_data.filename = "test_file.txt" - - xml_content = 'Test problem with ' - block_to_import = etree.fromstring(xml_content) - - mock_content_staging_api.get_staged_content_static_file_data.return_value = None - mock_import = mock.MagicMock() - mock_get_import.return_value = mock_import - - _process_staged_content_files( - mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, - self.mock_library, self.now, block_to_import, False, self.library_key, self.user_id - ) - - mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with( - self.staged_content_id, mock_file_data.filename - ) - mock_authoring_api.get_or_create_file_content.assert_not_called() - mock_authoring_api.create_component_version_content.assert_not_called() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport.objects.get') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport.objects.get_or_create') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_process_staged_content_files_integrity_error( - self, mock_authoring_api, mock_content_staging_api, mock_get_or_create, mock_get_import - ): - mock_component_version = mock.MagicMock() - mock_file_data = mock.MagicMock() - mock_file_data.filename = "test_file.txt" - - xml_content = 'Test problem with ' - block_to_import = etree.fromstring(xml_content) - - mock_content_staging_api.get_staged_content_static_file_data.return_value = b"file data" - mock_media_type = mock.MagicMock(id=1) - mock_authoring_api.get_or_create_media_type.return_value = mock_media_type - mock_content = mock.MagicMock(id=1) - mock_authoring_api.get_or_create_file_content.return_value = mock_content - - mock_authoring_api.create_component_version_content.side_effect = IntegrityError("Duplicate content") - - mock_import = mock.MagicMock() - mock_get_import.return_value = mock_import - mock_get_or_create.return_value = (mock.MagicMock(), True) - - _process_staged_content_files( - mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, - self.mock_library, self.now, block_to_import, False, self.library_key, self.user_id - ) - - mock_content_staging_api.get_staged_content_static_file_data.assert_called_once_with( - self.staged_content_id, mock_file_data.filename - ) - mock_authoring_api.create_component_version_content.assert_called_once() - mock_get_or_create.assert_called_once() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_process_staged_content_files_no_files(self, mock_authoring_api): - mock_component_version = mock.MagicMock() - - _process_staged_content_files( - mock_component_version, [], self.staged_content_id, self.usage_key, - self.mock_library, self.now, self.block_to_import, False, self.library_key, self.user_id - ) - - mock_authoring_api.get_or_create_media_type.assert_not_called() - mock_authoring_api.get_or_create_file_content.assert_not_called() - mock_authoring_api.create_component_version_content.assert_not_called() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_process_staged_content_files_file_not_referenced(self, mock_authoring_api): - mock_component_version = mock.MagicMock() - mock_file_data = mock.MagicMock() - mock_file_data.filename = "unreferenced_file.txt" - - _process_staged_content_files( - mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, - self.mock_library, self.now, self.block_to_import, False, self.library_key, self.user_id - ) - - mock_authoring_api.get_or_create_file_content.assert_not_called() - mock_authoring_api.create_component_version_content.assert_not_called() - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ComponentVersionImport.objects.get_or_create') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.content_staging_api') - def test_process_staged_content_files_with_override( - self, mock_content_staging_api, mock_authoring_api, mock_get_or_create - ): - mock_component_version = mock.MagicMock() - mock_file_data = mock.MagicMock() - mock_file_data.filename = "test_file.txt" - mock_content_staging_api.get_staged_content_static_file_data.return_value = b"file data" - mock_authoring_api.get_or_create_media_type.return_value = mock.MagicMock(id=1) - mock_authoring_api.get_or_create_file_content.return_value = mock.MagicMock(id=1) - mock_get_or_create.return_value = (mock.MagicMock(), True) - - self.xml_content = '' - self.block_to_import = etree.fromstring(self.xml_content) - _process_staged_content_files( - mock_component_version, [mock_file_data], self.staged_content_id, self.usage_key, - self.mock_library, self.now, self.block_to_import, True, self.library_key, self.user_id - ) - - mock_authoring_api.get_or_create_media_type.assert_called_once() - mock_authoring_api.get_or_create_file_content.assert_called_once() - mock_authoring_api.create_component_version_content.assert_called_once() - - -class TestHandleComponentOverride(TestCase): - """ - Tests for the _handle_component_override helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = UserFactory().id - self.block_id = "problem1" - self.block_type = "problem" - self.usage_key = UsageKey.from_string( - f"block-v1:TestOrg+TestCourse+Run1+type@{self.block_type}+block@{self.block_id}" - ) - self.xml_content = b"Test problem content" - - self.mock_library = mock.MagicMock() - self.mock_library.library_key = self.library_key - self.mock_learning_package = mock.MagicMock() - self.mock_library.learning_package = self.mock_learning_package - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') - def test_handle_component_override_existing_component(self, mock_api): - mock_component = mock.MagicMock() - mock_component.component_type.name = self.block_type - mock_component.local_key = self.block_id - - self.mock_learning_package.component_set.filter.return_value.first.return_value = mock_component - - expected_lib_usage_key = LibraryUsageLocatorV2( - lib_key=self.library_key, - block_type=self.block_type, - usage_id=self.block_id, - ) - - mock_component_version = mock.MagicMock() - mock_api.set_library_block_olx.return_value = mock_component_version - - result = _handle_component_override(self.mock_library, self.usage_key, self.xml_content) - - self.mock_learning_package.component_set.filter.assert_called_once_with(local_key=self.block_id) - mock_api.set_library_block_olx.assert_called_once_with(expected_lib_usage_key, self.xml_content) - self.assertEqual(result, mock_component_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') - def test_handle_component_override_nonexistent_component(self, mock_api): - self.mock_learning_package.component_set.filter.return_value.first.return_value = None - - result = _handle_component_override(self.mock_library, self.usage_key, self.xml_content) - - self.mock_learning_package.component_set.filter.assert_called_once_with(local_key=self.block_id) - mock_api.set_library_block_olx.assert_not_called() - self.assertIsNone(result) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.api') - def test_handle_component_override_api_error(self, mock_api): - mock_component = mock.MagicMock() - mock_component.component_type.name = self.block_type - mock_component.local_key = self.block_id - - self.mock_learning_package.component_set.filter.return_value.first.return_value = mock_component - - mock_api.set_library_block_olx.side_effect = Exception("API error") - - with self.assertRaises(Exception): - _handle_component_override(self.mock_library, self.usage_key, self.xml_content) - - mock_api.set_library_block_olx.assert_called_once() - - -class TestUpdateContainerComponents(TestCase): - """ - Tests for the _update_container_components helper function. - """ - - def setUp(self): - super().setUp() - self.user_id = UserFactory().id - self.mock_container_version = mock.MagicMock() - self.mock_container_version.container.pk = "container_pk" - self.mock_container_version.title = "Container Title" - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_update_container_components_with_mixed_components(self, mock_authoring_api): - mock_component_version = mock.MagicMock() - mock_component_version.component.pk = "component_pk" - mock_container_child_version = mock.MagicMock(spec=ContainerVersion) - mock_container_child_version.container.pk = "container_child_pk" - - component_versions = [mock_component_version, mock_container_child_version] - - _update_container_components(self.mock_container_version, component_versions, self.user_id) - - mock_authoring_api.create_next_container_version.assert_called_once_with( - container_pk=self.mock_container_version.container.pk, - title=self.mock_container_version.title, - publishable_entities_pks=["component_pk", "container_child_pk"], - entity_version_pks=[mock_component_version.pk, mock_container_child_version.pk], - created=mock.ANY, - created_by=self.user_id, - container_version_cls=self.mock_container_version.__class__, - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - def test_update_container_components_empty_list(self, mock_authoring_api): - _update_container_components(self.mock_container_version, [], self.user_id) - - mock_authoring_api.create_next_container_version.assert_called_once_with( - container_pk=self.mock_container_version.container.pk, - title=self.mock_container_version.title, - publishable_entities_pks=[], - entity_version_pks=[], - created=mock.ANY, - created_by=self.user_id, - container_version_cls=self.mock_container_version.__class__, - ) - - -class TestCreateContainer(TestCase): - """ - Tests for the create_container helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = UserFactory().id - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_create_container_chapter(self, mock_content_library, mock_authoring_api): - mock_library = mock.MagicMock() - mock_content_library.objects.get_by_key.return_value = mock_library - - mock_container = mock.MagicMock() - mock_container_version = mock.MagicMock() - mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) - - result = create_container('chapter', 'test_key', 'Test Chapter', self.library_key, self.user_id) - - mock_content_library.objects.get_by_key.assert_called_once_with(self.library_key) - mock_authoring_api.create_unit_and_version.assert_called_once_with( - mock_library.learning_package.id, - key='test_key', - title='Test Chapter', - components=[], - created=mock.ANY, - created_by=self.user_id, - ) - self.assertEqual(result, mock_container_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_create_container_sequential(self, mock_content_library, mock_authoring_api): - mock_library = mock.MagicMock() - mock_content_library.objects.get_by_key.return_value = mock_library - - mock_container = mock.MagicMock() - mock_container_version = mock.MagicMock() - mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) - - result = create_container('sequential', 'test_key', 'Test Sequential', self.library_key, self.user_id) - - mock_authoring_api.create_unit_and_version.assert_called_once_with( - mock_library.learning_package.id, - key='test_key', - title='Test Sequential', - components=[], - created=mock.ANY, - created_by=self.user_id, - ) - self.assertEqual(result, mock_container_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.secrets') - def test_create_container_no_key(self, mock_secrets, mock_content_library, mock_authoring_api): - mock_library = mock.MagicMock() - mock_content_library.objects.get_by_key.return_value = mock_library - - mock_container = mock.MagicMock() - mock_container_version = mock.MagicMock() - mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) - - mock_secrets.token_hex.return_value = "generated_key" - - result = create_container('vertical', None, 'Test Vertical', self.library_key, self.user_id) - - mock_secrets.token_hex.assert_called_once_with(16) - mock_authoring_api.create_unit_and_version.assert_called_once_with( - mock_library.learning_package.id, - key='generated_key', - title='Test Vertical', - components=[], - created=mock.ANY, - created_by=self.user_id, - ) - self.assertEqual(result, mock_container_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.authoring_api') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_create_container_no_display_name(self, mock_content_library, mock_authoring_api): - mock_library = mock.MagicMock() - mock_content_library.objects.get_by_key.return_value = mock_library - - mock_container = mock.MagicMock() - mock_container_version = mock.MagicMock() - mock_authoring_api.create_unit_and_version.return_value = (mock_container, mock_container_version) - - result = create_container('vertical', 'test_key', '', self.library_key, self.user_id) - - mock_authoring_api.create_unit_and_version.assert_called_once_with( - mock_library.learning_package.id, - key='test_key', - title='New vertical', - components=[], - created=mock.ANY, - created_by=self.user_id, - ) - self.assertEqual(result, mock_container_version) - - -class TestImportContainer(TestCase): - """ - Tests for the import_container helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = UserFactory().id - self.usage_key = UsageKey.from_string("block-v1:TestOrg+TestCourse+Run1+type@chapter+block@chapter1") - self.staged_content = mock.MagicMock() - self.staged_content.id = "staged-content-id" - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContainerVersionImport') - @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.import_children') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.CourseToLibraryImport') - def test_import_container_with_container_composition( - self, mock_course_import, mock_import_children, mock_create_container, - mock_update_container, mock_section_version_import - ): - xml = """ - - - - - - - - """ - block_to_import = etree.fromstring(xml) - - mock_container_version = mock.MagicMock() - mock_create_container.return_value = mock_container_version - - mock_component_versions = [mock.MagicMock()] - mock_import_children.return_value = mock_component_versions - - mock_section_version_import.objects.create = mock.MagicMock() - mock_get_import = mock.MagicMock() - mock_course_import.objects.get.return_value = mock_get_import - - import_id = str(uuid4()) - - import_container( - self.usage_key, block_to_import, self.library_key, self.user_id, - self.staged_content, 'chapter', import_id, False - ) - - mock_create_container.assert_called_once_with( - 'chapter', 'chapter1', 'Test Chapter', self.library_key, self.user_id - ) - mock_import_children.assert_called_once_with( - block_to_import, self.library_key, self.user_id, self.staged_content, - 'chapter', import_id, False - ) - mock_update_container.assert_called_once_with( - mock_container_version, mock_component_versions, self.user_id - ) - mock_section_version_import.objects.create.assert_called_once_with( - section_version=mock_container_version, - source_usage_key=self.usage_key, - library_import=mock_get_import - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.import_children') - def test_import_container_xblock_level(self, mock_import_children): - xml = """ - - - - - - - - """ - block_to_import = etree.fromstring(xml) - - import_id = str(uuid4()) - - import_container( - self.usage_key, block_to_import, self.library_key, self.user_id, - self.staged_content, 'xblock', import_id, False - ) - - mock_import_children.assert_called_once_with( - block_to_import, self.library_key, self.user_id, self.staged_content, - 'xblock', import_id, False - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContainerVersionImport') - @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.import_children') - def test_import_container_no_children( - self, mock_import_children, mock_create_container, mock_update_container, mock_section_version_import - ): - xml = """""" - block_to_import = etree.fromstring(xml) - - ctli = CourseToLibraryImportFactory( - library_key=self.library_key, - user_id=self.user_id, - status=CourseToLibraryImportStatus.READY, - ) - - mock_container_version = mock.MagicMock() - mock_create_container.return_value = mock_container_version - - mock_import_children.return_value = [] - - import_container( - self.usage_key, block_to_import, self.library_key, self.user_id, - self.staged_content, 'chapter', str(ctli.uuid), False - ) - - mock_create_container.assert_called_once() - mock_import_children.assert_called_once() - mock_update_container.assert_not_called() - - -class TestImportChildren(TestCase): - """ - Tests for the import_children helper function. - """ - - def setUp(self): - super().setUp() - self.library_key = LibraryLocatorV2(org="TestOrg", slug="test-lib") - self.user_id = UserFactory().id - - self.staged_content = mock.MagicMock() - self.staged_content.id = "staged-content-id" - self.staged_content.tags = { - "block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1": {}, - "block-v1:TestOrg+TestCourse+Run1+type@html+block@html1": {}, - "block-v1:TestOrg+TestCourse+Run1+type@video+block@video1": {}, - } - - self.problem_usage_key = UsageKey.from_string( - "block-v1:TestOrg+TestCourse+Run1+type@problem+block@problem1" - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_import_children_xblock_level(self, mock_content_library, mock_create_block): - xml = """ - - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - mock_component_version = mock.MagicMock(spec=['id', 'component_id']) - mock_create_block.return_value = mock_component_version - - result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False - ) - - self.assertEqual(mock_create_block.call_count, 2) - self.assertEqual(len(result), 2) - self.assertEqual(result[0], mock_component_version) - self.assertEqual(result[1], mock_component_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_import_children_with_containers( - self, mock_content_library, mock_create_block, mock_create_container, mock_update_container - ): - xml = """ - - - - - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - mock_component_version = mock.MagicMock(spec=['id', 'component_id']) - mock_create_block.return_value = mock_component_version - - mock_container_version = mock.MagicMock(spec=ContainerVersion) - mock_create_container.return_value = mock_container_version - - result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', str(uuid4()), False - ) - - self.assertEqual(mock_create_container.call_count, 2) - self.assertEqual(mock_create_block.call_count, 1) - self.assertEqual(len(result), 1) - self.assertEqual(result[0], mock_container_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_import_children_empty_block_xblock_level(self, mock_content_library, mock_create_block): - xml = '' - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - mock_component_version = mock.MagicMock() - mock_create_block.return_value = mock_component_version - - result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False - ) - - mock_create_block.assert_called_once() - self.assertEqual(len(result), 1) - self.assertEqual(result[0], mock_component_version) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_import_children_with_override(self, mock_content_library, mock_create_block): - xml = '' - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - import_id = str(uuid4()) - - import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', import_id, True - ) - - mock_create_block.assert_called_once_with( - block_to_import, - self.problem_usage_key, - self.library_key, - self.user_id, - self.staged_content.id, - import_id, - True, - ) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_import_children_library_not_found(self, mock_content_library): - xml = '' - block_to_import = etree.fromstring(xml) - - mock_content_library.objects.filter.return_value.first.return_value = None - - with self.assertRaises(ValueError): - import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False - ) - - def test_import_children_no_matching_children(self): - xml = """ - - - - - """ - block_to_import = etree.fromstring(xml) - - result = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False - ) - - self.assertEqual(result, []) - - @mock.patch('cms.djangoapps.course_to_library_import.helpers._update_container_components') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_container') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.create_block_in_library') - @mock.patch('cms.djangoapps.course_to_library_import.helpers.ContentLibrary') - def test_import_children_filter_by_composition_level( - self, mock_content_library, mock_create_block, mock_create_container, mock_update_container - ): - xml = """ - - - - - - """ - block_to_import = etree.fromstring(xml) - - mock_library = mock.MagicMock() - mock_content_library.objects.filter.return_value.first.return_value = mock_library - - mock_component_version = mock.MagicMock(spec=['id', 'component_id']) - mock_create_block.return_value = mock_component_version - - mock_container_version = mock.MagicMock(spec=ContainerVersion) - mock_create_container.return_value = mock_container_version - - result_xblock = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'xblock', str(uuid4()), False - ) - - result_chapter = import_children( - block_to_import, self.library_key, self.user_id, self.staged_content, 'chapter', str(uuid4()), False - ) - - self.assertTrue(all(not isinstance(item, ContainerVersion) for item in result_xblock)) - - self.assertEqual(len(result_chapter), 1) - self.assertEqual(result_chapter[0], mock_container_version) diff --git a/cms/djangoapps/course_to_library_import/tests/test_tasks.py b/cms/djangoapps/course_to_library_import/tests/test_tasks.py deleted file mode 100644 index 5c9e2f63b510..000000000000 --- a/cms/djangoapps/course_to_library_import/tests/test_tasks.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -Tests for tasks in course_to_library_import app. -""" - -from unittest.mock import Mock, patch - -from django.test import TestCase -from opaque_keys.edx.keys import CourseKey, UsageKey -from opaque_keys.edx.locator import LibraryLocatorV2 - -from cms.djangoapps.course_to_library_import.data import CourseToLibraryImportStatus -from cms.djangoapps.course_to_library_import.tasks import ( - import_library_from_staged_content_task, - save_courses_to_staged_content_task, -) -from .factories import CourseToLibraryImportFactory - - -class TestSaveCourseSectionsToStagedContentTask(TestCase): - """ - Test cases for save_course_sections_to_staged_content_task. - """ - - @patch('cms.djangoapps.course_to_library_import.tasks.modulestore') - @patch('openedx.core.djangoapps.content_staging.api.stage_xblock_temporarily') - def test_save_courses_to_staged_content_task(self, mock_stage_xblock_temporarily, mock_modulestore): - course_to_library_import = CourseToLibraryImportFactory(status=CourseToLibraryImportStatus.PENDING) - course_ids = course_to_library_import.course_ids.split(' ') - user_id = course_to_library_import.user.id - purpose = 'test_purpose' - version_num = 1 - - mock_course_keys = [CourseKey.from_string(course_id) for course_id in course_ids] - - def mock_get_items_side_effect(*args, **kwargs): - if kwargs.get('qualifiers', {}).get('category') == 'chapter': - return ['section1', 'section2'] - - mock_modulestore().get_items.side_effect = mock_get_items_side_effect - sections = ['section1', 'section2'] - - self.assertEqual(course_to_library_import.status, CourseToLibraryImportStatus.PENDING) - - save_courses_to_staged_content_task(course_ids, user_id, course_to_library_import.id, purpose, version_num) - - for mock_course_key in mock_course_keys: - mock_modulestore().get_items.assert_any_call(mock_course_key, qualifiers={"category": "chapter"}) - - self.assertEqual(mock_stage_xblock_temporarily.call_count, len(sections) * len(course_ids)) - for section in sections: - mock_stage_xblock_temporarily.assert_any_call(section, user_id, purpose=purpose, version_num=version_num) - - -class TestImportLibraryFromStagedContentTask(TestCase): - """ - Test cases for import_library_from_staged_content_task. - """ - - @patch('cms.djangoapps.course_to_library_import.tasks.validate_usage_ids') - @patch( - 'cms.djangoapps.course_to_library_import.tasks.content_staging_api.get_ready_staged_content_by_user_and_purpose' - ) - @patch('cms.djangoapps.course_to_library_import.tasks.get_block_to_import') - @patch('cms.djangoapps.course_to_library_import.tasks.import_container') - @patch('cms.djangoapps.course_to_library_import.tasks.etree') - def test_import_library_from_staged_content_task( - self, mock_etree, mock_import_container, mock_get_block_to_import, - mock_get_ready_staged_content, mock_validate_usage_ids - ): - ctli = CourseToLibraryImportFactory(status=CourseToLibraryImportStatus.READY) - library_key = ctli.library_key - user = ctli.user - usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] - usage_key = UsageKey.from_string(usage_ids[0]) - purpose = 'import_from_{course_id}' - course_id = 'course-v1:edX+Demo+2023' - override = True - - mock_staged_content = mock_get_ready_staged_content.return_value - mock_content_item = Mock() - mock_content_item.olx = 'content' - mock_staged_content.filter.return_value.first.return_value = mock_content_item - - mock_node = Mock() - mock_etree.fromstring.return_value = mock_node - mock_etree.XMLParser.return_value = Mock() - - mock_block = Mock() - mock_get_block_to_import.return_value = mock_block - - library_locator = LibraryLocatorV2.from_string(library_key) - - import_library_from_staged_content_task( - user.id, usage_ids, library_key, purpose, course_id, ctli.uuid, 'xblock', override - ) - - mock_get_ready_staged_content.assert_called_once_with( - user.id, purpose.format(course_id=course_id) - ) - mock_validate_usage_ids.assert_called_once_with(usage_ids, mock_staged_content) - mock_etree.XMLParser.assert_called_once_with(strip_cdata=False) - mock_etree.fromstring.assert_called_once_with(mock_content_item.olx, parser=mock_etree.XMLParser()) - - mock_staged_content.filter.assert_called_once_with(tags__icontains=usage_ids[0]) - mock_get_block_to_import.assert_called_once_with(mock_node, usage_key) - mock_import_container.assert_called_once_with( - usage_key, mock_block, library_locator, user.id, mock_content_item, 'xblock', ctli.uuid, override - ) - - ctli.refresh_from_db() - self.assertEqual(ctli.status, CourseToLibraryImportStatus.IMPORTED) - mock_staged_content.delete.assert_called_once() - - @patch( - 'cms.djangoapps.course_to_library_import.tasks.content_staging_api.get_ready_staged_content_by_user_and_purpose' - ) - @patch('cms.djangoapps.course_to_library_import.tasks.get_block_to_import') - @patch('cms.djangoapps.course_to_library_import.tasks.etree') - def test_import_library_block_not_found( - self, mock_etree, mock_get_block_to_import, mock_get_ready_staged_content - ): - ctli = CourseToLibraryImportFactory() - library_key = ctli.library_key - user = ctli.user - usage_ids = ['block-v1:edX+Demo+2023+type@vertical+block@12345'] - purpose = 'import_from_{course_id}' - course_id = 'course-v1:edX+Demo+2023' - override = True - - mock_staged_content = mock_get_ready_staged_content.return_value - mock_content_item = Mock() - mock_content_item.olx = 'content' - mock_staged_content.filter.return_value.first.return_value = mock_content_item - - mock_node = Mock() - mock_etree.fromstring.return_value = mock_node - mock_etree.XMLParser.return_value = Mock() - - mock_get_block_to_import.return_value = None - - library_locator = LibraryLocatorV2.from_string(library_key) - CourseToLibraryImportFactory( - user=user, - library_key=library_locator, - status=CourseToLibraryImportStatus.READY - ) - - with self.assertRaises(ValueError): - import_library_from_staged_content_task( - user.id, usage_ids, library_key, purpose, course_id, ctli.uuid, 'xblock', override - ) diff --git a/cms/djangoapps/course_to_library_import/views/urls.py b/cms/djangoapps/course_to_library_import/views/urls.py deleted file mode 100644 index ef82a37edd2f..000000000000 --- a/cms/djangoapps/course_to_library_import/views/urls.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Course to Library Import API URLs. -""" - -from django.urls import include, path - -app_name = 'course_to_library_import' -urlpatterns = [ - path('v0/', include('cms.djangoapps.course_to_library_import.views.v0.urls', namespace='v0')), -] diff --git a/cms/djangoapps/course_to_library_import/views/v0/serializers.py b/cms/djangoapps/course_to_library_import/views/v0/serializers.py deleted file mode 100644 index 4c107e0be808..000000000000 --- a/cms/djangoapps/course_to_library_import/views/v0/serializers.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Serializers for the Course to Library Import API. -""" - -from rest_framework import serializers - -from cms.djangoapps.course_to_library_import import api -from cms.djangoapps.course_to_library_import.models import CourseToLibraryImport -from cms.djangoapps.course_to_library_import.validators import validate_composition_level - - -class ImportBlocksSerializer(serializers.Serializer): - """ - Serializer for the import blocks API. - """ - - library_key = serializers.CharField(required=True) - usage_ids = serializers.ListField( - child=serializers.CharField(), - required=True, - ) - course_id = serializers.CharField(required=True) - import_id = serializers.CharField(required=True) - composition_level = serializers.CharField( - required=True, - validators=[validate_composition_level], - ) - override = serializers.BooleanField(default=False, required=False) - - def validate(self, data): - if not CourseToLibraryImport.get_ready_by_uuid(data['import_id']): - raise serializers.ValidationError({'import_id': 'Invalid import ID.'}) - return data - - -class CourseToLibraryImportSerializer(serializers.ModelSerializer): - """ - Serializer for CourseToLibraryImport model. - """ - - course_ids = serializers.ListField() - status = serializers.CharField(allow_blank=True, required=False) - library_key = serializers.CharField(allow_blank=True, required=False) - uuid = serializers.CharField(allow_blank=True, required=False) - - class Meta: - model = CourseToLibraryImport - fields = ('course_ids', 'status', 'library_key', 'uuid') - - def create(self, validated_data): - """ - Run the import creation logic. - Creates a new CourseToLibraryImport instance and related data such as StagedContent. - """ - user = getattr(self.context.get('request'), 'user', None) - course_to_library_import = api.create_import( - validated_data['course_ids'], - getattr(user, 'pk', None), - self.context.get('content_library_id'), - ) - return course_to_library_import - - def to_representation(self, instance): - """ - Converts a string with course IDs into a list of strings with course IDs. - """ - representation = super().to_representation(instance) - representation['course_ids'] = ''.join(representation['course_ids']).split() - return representation diff --git a/cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py b/cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py deleted file mode 100644 index 924fdc246422..000000000000 --- a/cms/djangoapps/course_to_library_import/views/v0/tests/test_serializers.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Unit tests for the ImportBlocksSerializer. -""" - -from unittest.mock import patch - -from django.test import TestCase -from rest_framework.exceptions import ValidationError - -from cms.djangoapps.course_to_library_import.views.v0.serializers import ImportBlocksSerializer - - -class TestImportBlocksSerializer(TestCase): - """ - Tests for the ImportBlocksSerializer. - """ - - def setUp(self): - """ - Set up common test data. - """ - self.valid_data = { - 'library_key': 'lib:v1:org+lib+2023', - 'usage_ids': ['block-v1:org+course+2023+type@html+block@123'], - 'course_id': 'course-v1:org+course+2023', - 'import_id': 'valid-import-id', - 'composition_level': 'vertical', - 'override': False, - } - - @patch('cms.djangoapps.course_to_library_import.models.CourseToLibraryImport.get_ready_by_uuid') - def test_validate_with_valid_import_id(self, mock_get_ready_by_uuid): - """ - Test that validation passes when a valid import_id is provided. - """ - mock_get_ready_by_uuid.return_value = {'some': 'object'} - - serializer = ImportBlocksSerializer(data=self.valid_data) - self.assertTrue(serializer.is_valid()) - - mock_get_ready_by_uuid.assert_called_once_with(self.valid_data['import_id']) - - @patch('cms.djangoapps.course_to_library_import.models.CourseToLibraryImport.get_ready_by_uuid') - def test_validate_with_invalid_import_id(self, mock_get_ready_by_uuid): - """ - Test that validation fails when an invalid import_id is provided. - """ - mock_get_ready_by_uuid.return_value = None - - serializer = ImportBlocksSerializer(data=self.valid_data) - - with self.assertRaises(ValidationError) as context: - serializer.is_valid(raise_exception=True) - - self.assertEqual( - context.exception.detail, - {'import_id': ['Invalid import ID.']} - ) - - mock_get_ready_by_uuid.assert_called_once_with(self.valid_data['import_id']) diff --git a/cms/djangoapps/course_to_library_import/README.rst b/cms/djangoapps/import_from_modulestore/README.rst similarity index 94% rename from cms/djangoapps/course_to_library_import/README.rst rename to cms/djangoapps/import_from_modulestore/README.rst index b2be9e67af7e..47e4c77b2f7a 100644 --- a/cms/djangoapps/course_to_library_import/README.rst +++ b/cms/djangoapps/import_from_modulestore/README.rst @@ -2,7 +2,7 @@ Course to Library Import ======================== -The new Django application `course_to_library_import` is designed to +The new Django application `import_from_modulestore` is designed to automate the process of importing course OLX content from Modulestore to Content Libraries. The application allows users to easily and quickly migrate existing course content, minimizing the manual work and potential diff --git a/cms/djangoapps/course_to_library_import/__init__.py b/cms/djangoapps/import_from_modulestore/__init__.py similarity index 100% rename from cms/djangoapps/course_to_library_import/__init__.py rename to cms/djangoapps/import_from_modulestore/__init__.py diff --git a/cms/djangoapps/import_from_modulestore/admin.py b/cms/djangoapps/import_from_modulestore/admin.py new file mode 100644 index 000000000000..3404538d6de7 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/admin.py @@ -0,0 +1,33 @@ +""" +This module contains the admin configuration for the Import model. +""" +from django.contrib import admin + +from .models import Import, PublishableEntityImport, PublishableEntityMapping + + +class ImportAdmin(admin.ModelAdmin): + """ + Admin configuration for the Import model. + """ + + list_display = ( + 'uuid', + 'status', + 'source_key', + 'target', + ) + list_filter = ( + 'status', + ) + search_fields = ( + 'source_key', + 'target', + ) + + raw_id_fields = ('user',) + + +admin.site.register(Import, ImportAdmin) +admin.site.register(PublishableEntityImport) +admin.site.register(PublishableEntityMapping) diff --git a/cms/djangoapps/import_from_modulestore/api.py b/cms/djangoapps/import_from_modulestore/api.py new file mode 100644 index 000000000000..f9166437db6e --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/api.py @@ -0,0 +1,38 @@ +""" +API for course to library import. +""" +from .models import Import as _Import +from .tasks import import_course_staged_content_to_library_task, save_courses_to_staged_content_task + + +def import_course_staged_content_to_library( + usage_ids: list[str], + import_uuid: str, + user_id: int, + composition_level: str, + override: bool +) -> None: + """ + Import staged content to a library. + """ + import_course_staged_content_to_library_task.delay( + usage_ids, + import_uuid, + user_id, + composition_level, + override, + ) + + +def create_import(source_key, user_id: int, learning_package_id: int) -> _Import: + """ + Create a new import task to import a course to a library. + """ + import_from_modulestore = _Import( + source_key=source_key, + target_id=learning_package_id, + user_id=user_id, + ) + import_from_modulestore.save() + save_courses_to_staged_content_task.delay(import_from_modulestore.uuid) + return import_from_modulestore diff --git a/cms/djangoapps/import_from_modulestore/apps.py b/cms/djangoapps/import_from_modulestore/apps.py new file mode 100644 index 000000000000..5fb0fbc67f40 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/apps.py @@ -0,0 +1,19 @@ +""" +App for importing from the modulestore tools. +""" + +from django.apps import AppConfig + + +class ImportFromModulestoreConfig(AppConfig): + """ + App for importing legacy content from the modulestore. + """ + + name = 'cms.djangoapps.import_from_modulestore' + + def ready(self): + """ + Connect handlers to signals. + """ + from . import signals # pylint: disable=unused-import, import-outside-toplevel diff --git a/cms/djangoapps/import_from_modulestore/constants.py b/cms/djangoapps/import_from_modulestore/constants.py new file mode 100644 index 000000000000..099acfbc2b7b --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/constants.py @@ -0,0 +1,5 @@ +""" +Constants for import_from_modulestore app +""" + +IMPORT_FROM_MODULESTORE_PURPOSE = "import_from_modulestore" diff --git a/cms/djangoapps/import_from_modulestore/data.py b/cms/djangoapps/import_from_modulestore/data.py new file mode 100644 index 000000000000..69aa71e1becf --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/data.py @@ -0,0 +1,50 @@ +""" +This module contains the data models for the import_from_modulestore app. +""" +from enum import Enum + +from django.db.models import TextChoices +from django.utils.translation import gettext_lazy as _ + + +class ImportStatus(TextChoices): + """ + The status of this course import. + """ + + # PENDING: The import has been created, but the OLX and related data are not yet in the library. + # It is not ready to be read. + PENDING = 'pending', _('Pending') + # READY: The content is staged and ready to be read. + READY = 'ready', _('Ready') + # IMPORTED: The content has been imported into the library. + IMPORTED = 'imported', _('Imported') + # CANCELED: The import was canceled before it was imported. + CANCELED = 'canceled', _('Canceled') + # ERROR: The content could not be imported. + ERROR = 'error', _('Error') + + +class CompositionLevel(Enum): + """ + Enumeration of composition levels for course content. + + Defines the different levels of composition for course content, + including chapters, sequentials, verticals, and xblocks. + It also categorizes these levels into complicated and flat + levels for easier processing. + """ + + CHAPTER = 'chapter' + SEQUENTIAL = 'sequential' + VERTICAL = 'vertical' + XBLOCK = 'xblock' + COMPLICATED_LEVELS = [CHAPTER, SEQUENTIAL, VERTICAL] + FLAT_LEVELS = [XBLOCK] + + @classmethod + def values(cls): + """ + Returns all levels of composition levels. + """ + return [composition_level.value for composition_level in cls] diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py new file mode 100644 index 000000000000..66ab20671e9d --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/helpers.py @@ -0,0 +1,370 @@ +""" +Helper functions for importing course content into a library. +""" +from datetime import datetime, timezone +import logging +import mimetypes +import os +import secrets + +from django.db import transaction +from django.db.utils import IntegrityError +from lxml import etree + +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import CourseLocator, LibraryUsageLocatorV2 +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import ContainerVersion + +from openedx.core.djangoapps.content_libraries import api +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.django import modulestore + +from .data import CompositionLevel +from .models import PublishableEntityMapping, PublishableEntityImport + + +log = logging.getLogger(__name__) +parser = etree.XMLParser(strip_cdata=False) + + +class ImportClient: + """ + Client for importing course content into a library. + + This class handles the import of course content from staged content into a + content library, creating the appropriate container hierarchy based on the + specified composition level. + """ + + CONTAINER_GETTERS_MAP = { + 'chapter': authoring_api.get_unit_version, # TODO: replace with get_module_and_version + 'sequential': authoring_api.get_unit_version, # TODO: replace with get_section_and_version + 'vertical': authoring_api.get_unit_version, + } + + CONTAINER_CREATORS_MAP = { + 'chapter': authoring_api.create_unit_and_version, # TODO: replace with create_module_and_version + 'sequential': authoring_api.create_unit_and_version, # TODO: replace with create_section_and_version + 'vertical': authoring_api.create_unit_and_version, + } + + def __init__(self, import_event, block_usage_id_to_import, staged_content, composition_level, override=False): + self.import_event = import_event + self.block_usage_id_to_import = block_usage_id_to_import + self.staged_content = staged_content + self.composition_level = composition_level + self.override = override + + self.user_id = import_event.user_id + self.content_library = import_event.target.contentlibrary + self.library_key = self.content_library.library_key + self.parser = etree.XMLParser(strip_cdata=False) + + def import_from_staged_content(self): + """ + Import staged content into a library. + """ + node = etree.fromstring(self.staged_content.olx, parser=parser) + usage_key = UsageKey.from_string(self.block_usage_id_to_import) + block_to_import = get_block_to_import(node, usage_key) + if block_to_import is None: + return + + self._process_import(self.block_usage_id_to_import, block_to_import) + + def create_container(self, container_type, key, display_name): + """ + Create a container of the specified type. + + Creates a container (e.g., chapter, sequential, vertical) in the + content library. + # TODO: add get or create logic and process override + """ + container_creator_func = self.CONTAINER_CREATORS_MAP.get(container_type) + container_getter_func = self.CONTAINER_GETTERS_MAP.get(container_type) + if not container_creator_func or not container_getter_func: + raise ValueError(f"Unknown container type: {container_type}") + + # container_version = container_getter_func(self.content_library.learning_package.id, key) + # if container_version and self.override: + # create_next_unit_version + # elif not container_version: + # container_creator_func + # else: + # return container_version + _, container_version = container_creator_func( + self.import_event.target_id, + key=key or secrets.token_hex(16), + title=display_name or f"New {container_type}", + components=[], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + + return container_version + + def _process_import(self, usage_id, block_to_import): + """ + Process import of a block from staged content into a library. + + Imports a block and its children into the library based on the + composition level. It handles both simple and complicated blocks, creating + the necessary container hierarchy. + """ + usage_key = UsageKey.from_string(usage_id) + result = [] + + if block_to_import.tag not in CompositionLevel.COMPLICATED_LEVELS.value: + return self._import_simple_block(block_to_import, usage_key) + + for child in block_to_import.getchildren(): + child_usage_id = get_usage_id_from_staged_content(self.staged_content, child.get('url_name')) + if not child_usage_id: + continue + result.extend(self._import_child_block(child, child_usage_id)) + + if self.composition_level in CompositionLevel.FLAT_LEVELS.value: + return [component for component in result if not isinstance(component, ContainerVersion)] + return result + + def _import_simple_block(self, block_to_import, usage_key) -> list: + """ + Import a simple block into the library. + + Creates a block in the library from the staged content block. + It returns a list containing the created component version. + """ + component_version = self._create_block_in_library(block_to_import, usage_key) + return [component_version] if component_version else [] + + def _import_child_block(self, child, child_usage_id): + """ + Import a child block into the library. + + Determines whether the child block is simple or complicated and + delegates the import process to the appropriate helper method. + """ + child_usage_key = UsageKey.from_string(child_usage_id) + if child.tag in CompositionLevel.COMPLICATED_LEVELS.value: + return self._import_complicated_child(child, child_usage_id) + else: + return self._import_simple_block(child, child_usage_key) + + def _import_complicated_child(self, child, child_usage_id): + """ + Import a complicated child block into the library + + Handles the import of complicated child blocks, including creating + containers and updating components. + Returns a list containing the created container version. + """ + if self.composition_level in CompositionLevel.FLAT_LEVELS.value: + return self._process_import(child_usage_id, child) + + container_version = self.create_container( + child.tag, + child.get('url_name'), + child.get('display_name', child.tag) + ) + child_component_versions = self._process_import(child_usage_id, child) + self._update_container_components(container_version, child_component_versions) + return [container_version] + + def _update_container_components(self, container_version, component_versions): + """ + Update components of a container. + """ + return authoring_api.create_next_container_version( + container_pk=container_version.container.pk, + title=container_version.title, + publishable_entities_pks=[ + cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk for cv in component_versions + ], + entity_version_pks=[cv.pk for cv in component_versions], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + container_version_cls=container_version.__class__, + ) + + def _create_block_in_library(self, block_to_import, usage_key): + """ + Create a block in a library from a staged content block. + """ + now = datetime.now(tz=timezone.utc) + staged_content_files = content_staging_api.get_staged_content_static_files(self.staged_content.id) + + with transaction.atomic(): + component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type) + does_component_exist = authoring_api.get_components( + self.import_event.target_id + ).filter(local_key=usage_key.block_id).exists() + + if does_component_exist: + if not self.override: + log.info(f"Component {usage_key.block_id} already exists in library {self.library_key}, skipping.") + return + else: + component_version = self._handle_component_override(usage_key, etree.tostring(block_to_import)) + else: + # Create component (regardless of override path) + # FIXME check override logic + _, library_usage_key = api.validate_can_add_block_to_library( + self.library_key, + block_to_import.tag, + usage_key.block_id, + ) + authoring_api.create_component( + self.import_event.target_id, + component_type=component_type, + local_key=usage_key.block_id, + created=now, + created_by=self.import_event.user_id, + ) + component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import)) + + self._process_staged_content_files( + component_version, + staged_content_files, + usage_key, + block_to_import, + now, + ) + _create_publishable_entity_import(self.import_event, usage_key, component_version) + + return component_version + + def _handle_component_override(self, usage_key, new_content): + """ + Create new ComponentVersion for overridden component. + """ + component_version = None + component = self.import_event.target.component_set.filter(local_key=usage_key.block_id).first() + + if component: + library_usage_key = LibraryUsageLocatorV2( # type: ignore[abstract] + lib_key=self.library_key, + block_type=component.component_type.name, + usage_id=component.local_key, + ) + component_version = api.set_library_block_olx(library_usage_key, new_content) + + return component_version + + def _process_staged_content_files( + self, + component_version, + staged_content_files, + usage_key, + block_to_import, + created_at, + ): + """ + Process staged content files for a component. + + Processes the staged content files for a component, creating the + necessary file content and associating it with the component version. + """ + block_olx = etree.tostring(block_to_import).decode('utf-8') + + for staged_content_file_data in staged_content_files: + original_filename = staged_content_file_data.filename + file_basename = os.path.basename(original_filename) + file_basename_no_ext, _ = os.path.splitext(file_basename) + + # Skip files not referenced in the block + if file_basename not in block_olx and file_basename_no_ext not in block_olx: + log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") + continue + + file_data = content_staging_api.get_staged_content_static_file_data( + self.staged_content.id, + original_filename, + ) + if not file_data: + log.error( + f"Staged content {self.staged_content.id} included referenced " + f"file {original_filename}, but no file data was found." + ) + continue + + filename = f"static/{file_basename}" + media_type_str, _ = mimetypes.guess_type(filename) + if not media_type_str: + media_type_str = "application/octet-stream" + + media_type = authoring_api.get_or_create_media_type(media_type_str) + content = authoring_api.get_or_create_file_content( + self.import_event.target_id, + media_type.id, + data=file_data, + created=created_at, + ) + + try: + authoring_api.create_component_version_content(component_version.pk, content.id, key=filename) + except IntegrityError: + pass # Content already exists + + +def _create_publishable_entity_import(import_event, usage_key, component_version) -> PublishableEntityImport: + """ + Creates relations between the imported component and source usage key and import event. + """ + publishable_entity_mapping, _ = _get_or_create_publishable_entity_mapping( + usage_key, + component_version.component + ) + return PublishableEntityImport.objects.create( + import_event=import_event, + result=publishable_entity_mapping, + resulting_draft=component_version.publishable_entity_version, + ) + + +def _get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[PublishableEntityMapping, bool]: + """ + Creates a mapping between the source usage key and the target publishable entity. + """ + return PublishableEntityMapping.objects.get_or_create( + source_usage_key=usage_key, + target_entity=component.publishable_entity, + target_package=component.learning_package + ) + + +def get_usage_id_from_staged_content(staged_content, block_id): + """ + Get the usage ID from a staged content by block ID. + """ + return next((block_usage_id for block_usage_id in staged_content.tags if block_usage_id.endswith(block_id)), None) + + +def get_block_to_import(node, usage_key): + """ + Get the block to import from a node. + """ + + if node.get('url_name') == usage_key.block_id: + return node + + for child in node.getchildren(): + found = get_block_to_import(child, usage_key) + if found is not None: + return found + + +def get_items_to_import(import_event): + """ + Collect items to import from a course. + """ + items_to_import = [] + if isinstance(import_event.source_key, CourseLocator): + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "chapter"}) or [] + ) + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "static_tab"}) or [] + ) + + return items_to_import diff --git a/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py b/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py new file mode 100644 index 000000000000..d759788fcc45 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/migrations/0001_initial.py @@ -0,0 +1,81 @@ +# Generated by Django 4.2.18 on 2025-04-03 09:55 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import opaque_keys.edx.django.models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('content_staging', '0005_stagedcontent_version_num'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('oel_publishing', '0003_containers'), + ] + + operations = [ + migrations.CreateModel( + name='Import', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('ready', 'Ready'), ('imported', 'Imported'), ('canceled', 'Canceled'), ('error', 'Error')], default='pending', max_length=100)), + ('source_key', opaque_keys.edx.django.models.LearningContextKeyField(db_index=True, help_text='The modulestore course', max_length=255)), + ('target', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.learningpackage')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Import from modulestore', + 'verbose_name_plural': 'Imports from modulestore', + }, + ), + migrations.CreateModel( + name='PublishableEntityMapping', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('source_usage_key', opaque_keys.edx.django.models.UsageKeyField(help_text='Original usage key/ID of the thing that has been imported.', max_length=255)), + ('target_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishableentity')), + ('target_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), + ], + options={ + 'unique_together': {('source_usage_key', 'target_package')}, + }, + ), + migrations.CreateModel( + name='StagedContentForImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('import_event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='staged_content_for_import', to='import_from_modulestore.import')), + ('staged_content', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='staged_content_for_import', to='content_staging.stagedcontent')), + ], + options={ + 'unique_together': {('import_event', 'staged_content')}, + }, + ), + migrations.CreateModel( + name='PublishableEntityImport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('import_event', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='import_from_modulestore.import')), + ('result', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='import_from_modulestore.publishableentitymapping')), + ('resulting_draft', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_publishing.publishableentityversion')), + ], + options={ + 'unique_together': {('import_event', 'result')}, + }, + ), + ] diff --git a/cms/djangoapps/course_to_library_import/migrations/__init__.py b/cms/djangoapps/import_from_modulestore/migrations/__init__.py similarity index 100% rename from cms/djangoapps/course_to_library_import/migrations/__init__.py rename to cms/djangoapps/import_from_modulestore/migrations/__init__.py diff --git a/cms/djangoapps/import_from_modulestore/models.py b/cms/djangoapps/import_from_modulestore/models.py new file mode 100644 index 000000000000..9d812f697eb3 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/models.py @@ -0,0 +1,175 @@ +""" +Models for the course to library import app. +""" + +import uuid as uuid_tools +from typing import Self, Optional + +from django.contrib.auth import get_user_model +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from model_utils.models import TimeStampedModel +from opaque_keys.edx.django.models import ( + LearningContextKeyField, + UsageKeyField, +) +from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity + +from .data import ImportStatus + +User = get_user_model() + + +# TODO: Rename app to import_from_modulestore + +class Import(TimeStampedModel): + """ + Represents the action of a user importing a modulestore-based course or legacy + library into a learning-core based learning package (today, that is always a content library). + """ + + uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True, db_index=True) + status = models.CharField(max_length=100, choices=ImportStatus.choices, default=ImportStatus.PENDING) + user = models.ForeignKey(User, on_delete=models.CASCADE) + + source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True) + target = models.ForeignKey(LearningPackage, models.SET_NULL, null=True) + + def __str__(self): + return f'{self.source_key} - {self.target}' + + def ready(self) -> None: + """ + Set import status to ready. + """ + self.status = ImportStatus.READY + self.save() + + def imported(self) -> None: + """ + Set import status to imported and clean related staged content. + """ + self.status = ImportStatus.IMPORTED + self.save() + self.clean_related_staged_content() + + def cancel(self) -> None: + """ + Cancel import action and delete related staged content. + """ + self.status = ImportStatus.CANCELED + self.save() + self.clean_related_staged_content() + + def error(self) -> None: + """ + Set import status to error and clean related staged + """ + self.status = ImportStatus.ERROR + self.save() + self.clean_related_staged_content() + + def clean_related_staged_content(self) -> None: + """ + Clean related staged content. + """ + for staged_content_for_import in self.staged_content_for_import.all(): + staged_content_for_import.staged_content.delete() + + class Meta: + verbose_name = _('Import from modulestore') + verbose_name_plural = _('Imports from modulestore') + + @classmethod + def get_by_uuid(cls, import_uuid: str) -> Self | None: + """ + Get an import task by its ID. + """ + return cls.objects.filter(uuid=import_uuid).first() + + @classmethod + def get_ready_by_uuid(cls, import_uuid: str) -> Self | None: + """ + Get an import task by its UUID. + """ + return cls.objects.filter(uuid=import_uuid, status=ImportStatus.READY).first() + + def get_staged_content_by_block_usage_id(self, block_usage_id: str) -> Optional["StagedContent"]: + """ + Get staged content by block usage ID. + """ + staged_content_for_import = self.staged_content_for_import.filter( + staged_content__tags__icontains=block_usage_id + ).first() + return getattr(staged_content_for_import, 'staged_content', None) + + +class PublishableEntityMapping(TimeStampedModel): + """ + Represents a mapping between a source usage key and a target publishable entity. + """ + + source_usage_key = UsageKeyField( + max_length=255, + help_text=_('Original usage key/ID of the thing that has been imported.'), + ) + target_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + target_entity = models.ForeignKey(PublishableEntity, on_delete=models.CASCADE) + + class Meta: + unique_together = ('source_usage_key', 'target_package') + + def __str__(self): + return f'{self.source_usage_key} - {self.target_entity}' + + +class PublishableEntityImport(TimeStampedModel): + """ + Represents a container version that has been imported into a content library. + + This is a many-to-many relationship between a container version and a course to library import. + """ + + import_event = models.ForeignKey(Import, on_delete=models.SET_NULL, null=True, blank=True) + result = models.ForeignKey(PublishableEntityMapping, on_delete=models.SET_NULL, null=True, blank=True) + resulting_draft = models.OneToOneField( + to='oel_publishing.PublishableEntityVersion', + # a version can be pruned, which would set this to NULL, but not delete the + # entire import record + null=True, + on_delete=models.SET_NULL, + ) + + class Meta: + unique_together = ( + ('import_event', 'result'), + ) + + def __str__(self): + return f'{self.import_event} - {self.result}' + + +class StagedContentForImport(TimeStampedModel): + """ + Represents m2m relationship between an import and staged content created for that import. + """ + + import_event = models.ForeignKey( + Import, + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + staged_content = models.OneToOneField( + to='content_staging.StagedContent', + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + + class Meta: + unique_together = ( + ('import_event', 'staged_content'), + ) + + def __str__(self): + return f'{self.import_event} - {self.staged_content}' diff --git a/cms/djangoapps/import_from_modulestore/permissions.py b/cms/djangoapps/import_from_modulestore/permissions.py new file mode 100644 index 000000000000..0d30f8b6af58 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/permissions.py @@ -0,0 +1,17 @@ +""" +Permission classes for the import_from_modulestore app. +""" +from django.shortcuts import get_object_or_404 +from rest_framework import permissions + +from cms.djangoapps.import_from_modulestore.models import Import + + +class IsImportAuthor(permissions.BasePermission): + """ + Permission class to check if the user is the author of the import. + """ + + def has_permission(self, request, view): + import_event = get_object_or_404(Import, uuid=request.data.get('import_uuid')) + return import_event.user_id == request.user.pk diff --git a/cms/djangoapps/import_from_modulestore/signals.py b/cms/djangoapps/import_from_modulestore/signals.py new file mode 100644 index 000000000000..b0c6b48177ec --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/signals.py @@ -0,0 +1,25 @@ +""" +Signals for Import. +""" +from django.dispatch import receiver +from django.db.models.signals import post_save + +from .models import Import + + +@receiver(post_save, sender=Import) +def cancel_incomplete_imports(sender, instance, created, **kwargs): + """ + Cancel any incomplete imports that have the same target as the current import. + + When a new import is created, we want to cancel any other incomplete user imports that have the same target. + """ + if created: + incomplete_user_imports_with_same_target = Import.objects.filter( + user=instance.user, + target=instance.target, + source_key=instance.source_key, + staged_content_for_import__isnull=False + ).exclude(uuid=instance.uuid) + for incomplete_import in incomplete_user_imports_with_same_target: + incomplete_import.cancel() diff --git a/cms/djangoapps/import_from_modulestore/tasks.py b/cms/djangoapps/import_from_modulestore/tasks.py new file mode 100644 index 000000000000..a5eba2d1b90b --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tasks.py @@ -0,0 +1,84 @@ +""" +Tasks for course to library import. +""" + +from celery import shared_task +from celery.utils.log import get_task_logger +from django.db import transaction +from edx_django_utils.monitoring import set_code_owner_attribute + +from openedx.core.djangoapps.content_staging import api as content_staging_api + +from .constants import IMPORT_FROM_MODULESTORE_PURPOSE +from .helpers import get_items_to_import, ImportClient +from .models import Import, StagedContentForImport +from .validators import validate_composition_level + +log = get_task_logger(__name__) + + +@shared_task +@set_code_owner_attribute +def save_courses_to_staged_content_task(import_uuid: str) -> None: + """ + Save courses to staged content task. + """ + import_event = Import.get_by_uuid(import_uuid) + if not import_event: + return + + import_event.clean_related_staged_content() + try: + with transaction.atomic(): + items_to_import = get_items_to_import(import_event) + for item in items_to_import: + staged_content = content_staging_api.stage_xblock_temporarily( + item, + import_event.user.id, + purpose=IMPORT_FROM_MODULESTORE_PURPOSE, + ) + StagedContentForImport.objects.create( + staged_content=staged_content, + import_event=import_event + ) + + if items_to_import: + import_event.ready() + else: + import_event.error() + except Exception as exc: # pylint: disable=broad-except + import_event.error() + raise exc + + +@shared_task +@set_code_owner_attribute +def import_course_staged_content_to_library_task( + usage_ids: list[str], + import_uuid: str, + user_id: int, + composition_level: str, + override: bool +) -> None: + """ + Import staged content to a library task. + """ + validate_composition_level(composition_level) + import_event = Import.get_ready_by_uuid(import_uuid) + if not import_event or import_event.user_id != user_id: + log.info('Ready import from modulestore not found') + return + + with transaction.atomic(): + for usage_id in usage_ids: + if staged_content_item := import_event.get_staged_content_by_block_usage_id(usage_id): + import_client = ImportClient( + import_event, + usage_id, + staged_content_item, + composition_level, + override, + ) + import_client.import_from_staged_content() + + import_event.imported() diff --git a/cms/djangoapps/course_to_library_import/tests/__init__.py b/cms/djangoapps/import_from_modulestore/tests/__init__.py similarity index 100% rename from cms/djangoapps/course_to_library_import/tests/__init__.py rename to cms/djangoapps/import_from_modulestore/tests/__init__.py diff --git a/cms/djangoapps/import_from_modulestore/tests/factories.py b/cms/djangoapps/import_from_modulestore/tests/factories.py new file mode 100644 index 000000000000..2207a799ef33 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/factories.py @@ -0,0 +1,30 @@ +""" +Factories for Import model. +""" + +import uuid + +import factory +from factory.django import DjangoModelFactory +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries.tests import factories + + +class ImportFactory(DjangoModelFactory): + """ + Factory for Import model. + """ + + class Meta: + model = Import + + @factory.lazy_attribute + def source_key(self): + return CourseKey.from_string(f'course-v1:edX+DemoX+{self.uuid}') + + target = factory.SubFactory(factories.LearningPackageFactory) + uuid = factory.LazyFunction(lambda: str(uuid.uuid4())) + user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/import_from_modulestore/tests/test_api.py b/cms/djangoapps/import_from_modulestore/tests/test_api.py new file mode 100644 index 000000000000..34262a83d777 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_api.py @@ -0,0 +1,77 @@ +""" +Test cases for import_from_modulestore.api module. +""" + +from unittest.mock import patch + +import pytest +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.api import create_import, import_course_staged_content_to_library +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries.tests import factories +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from .factories import ImportFactory + + +@pytest.mark.django_db +class TestCourseToLibraryImportAPI(ModuleStoreTestCase): + """ + Test cases for Import API. + """ + + def setUp(self): + super().setUp() + + self.library = factories.ContentLibraryFactory() + + def test_create_import(self): + """ + Test create_import function. + """ + course_id = "course-v1:edX+DemoX+Demo_Course" + user = UserFactory() + with patch( + "cms.djangoapps.import_from_modulestore.api.save_courses_to_staged_content_task" + ) as save_courses_to_staged_content_task_mock: + create_import(course_id, user.id, self.library.learning_package_id) + + import_event = Import.objects.get() + assert import_event.source_key == CourseKey.from_string(course_id) + assert import_event.target == self.library.learning_package + assert import_event.user_id == user.id + save_courses_to_staged_content_task_mock.delay.assert_called_once_with(import_event.uuid) + + def test_import_course_staged_content_to_library(self): + """ + Test import_course_staged_content_to_library function with different override values. + """ + import_event = ImportFactory( + target=self.library.learning_package, + source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"), + ) + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@html+block@123", + "block-v1:edX+DemoX+Demo_Course+type@html+block@456", + ] + override = False + + with patch( + "cms.djangoapps.import_from_modulestore.api.import_course_staged_content_to_library_task" + ) as import_course_staged_content_to_library_task_mock: + import_course_staged_content_to_library( + usage_ids, + import_event.uuid, + import_event.user.id, + 'xblock', + override + ) + + import_course_staged_content_to_library_task_mock.delay.assert_called_once_with( + usage_ids, + import_event.uuid, + import_event.user.id, + 'xblock', + override + ) diff --git a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py new file mode 100644 index 000000000000..858124c0716f --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py @@ -0,0 +1,168 @@ +""" +Tests for the import_from_modulestore helper functions. +""" +import ddt +from organizations.models import Organization +from unittest.mock import patch + +from lxml import etree + +from cms.djangoapps.import_from_modulestore import api +from cms.djangoapps.import_from_modulestore.helpers import ImportClient +from common.djangoapps.student.tests.factories import UserFactory + +from openedx.core.djangoapps.content_libraries import api as content_libraries_api +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory + + +@ddt.ddt +class TestImportClient(ModuleStoreTestCase): + """ + Functional tests for the ImportClient class. + """ + + def setUp(self): + super().setUp() + self.library = content_libraries_api.create_library( + org=Organization.objects.create(name='Organization 1', short_name='org1'), + slug='lib_1', + title='Library Org 1', + description='This is a library from Org 1', + ) + self.user = UserFactory() + self.course = CourseFactory.create() + self.chapter = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter') + self.sequential = BlockFactory.create(category='sequential', parent=self.chapter, display_name='Sequential') + self.vertical = BlockFactory.create(category='vertical', parent=self.sequential, display_name='Vertical') + self.problem = BlockFactory.create( + category='problem', + parent=self.vertical, + display_name='Problem', + data="""""", + ) + self.video = BlockFactory.create( + category='video', + parent=self.vertical, + display_name='Video', + data="""