Support redis migrations

backend_entrypoint.sh

@@ -20,6 +20,7 @@ cmd_init() {
     ~/manage.py migrate
 
     wait-for-it "${CVAT_REDIS_INMEM_HOST}:${CVAT_REDIS_INMEM_PORT:-6379}" -t 0
+    ~/manage.py migrateredis
     ~/manage.py syncperiodicjobs
 }
 

cvat/apps/engine/redis_migrations/001_cleanup_scheduled_jobs.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import django_rq
+from django.conf import settings
+from rq_scheduler import Scheduler
+
+from cvat.apps.redis_handler.redis_migrations import BaseMigration
+
+
+class Migration(BaseMigration):
+    @staticmethod
+    def run():
+        scheduler: Scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.EXPORT_DATA.value)
+
+        for job in scheduler.get_jobs():
+            if job.func_name == "cvat.apps.dataset_manager.views.clear_export_cache":
+                scheduler.cancel(job)
+                job.delete()

cvat/apps/engine/redis_migrations/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+

cvat/apps/redis_handler/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class RedisHandlerConfig(AppConfig):
+    name = "cvat.apps.redis_handler"

cvat/apps/redis_handler/management/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/management/commands/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/management/commands/migrateredis.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+import traceback
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from cvat.apps.redis_handler.migration_loader import MigrationLoader
+from cvat.apps.redis_handler.models import RedisMigration
+
+
+class Command(BaseCommand):
+    help = "Applies Redis migrations and records them in the database"
+
+    def handle(self, *args, **options) -> None:
+        loader = MigrationLoader()
+
+        if not loader:
+            self.stdout.write("No migrations to apply")
+            sys.exit(0)
+
+        for migration in loader:
+            try:
+                with transaction.atomic():
+                    RedisMigration.objects.create(
+                        name=migration.name, app_label=migration.app_label
+                    )
+                    migration.run()
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"[{migration.app_label}] Successfully applied migration: {migration.name}"
+                    )
+                )
+            except Exception as ex:
+                self.stderr.write(
+                    self.style.ERROR(
+                        f"[{migration.app_label}] Failed to apply migration: {migration.name}"
+                    )
+                )
+                self.stderr.write(self.style.ERROR(f"\n{traceback.format_exc()}"))
+                raise CommandError(str(ex))

cvat/apps/redis_handler/migration_loader.py

@@ -0,0 +1,96 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import importlib
+from pathlib import Path
+from typing import Any, Generator, Mapping
+
+from django.apps import AppConfig, apps
+from django.utils.module_loading import module_has_submodule
+
+from cvat.apps.redis_handler.models import RedisMigration as DBRedisMigration
+from cvat.apps.redis_handler.redis_migrations import BaseMigration as BaseRedisMigration
+
+_MigrationsPerApp = dict[int, list[str]]
+
+
+class LoaderError(Exception):
+    pass
+
+
+class MigrationLoader:
+    REDIS_MIGRATIONS_DIR_NAME = "redis_migrations"
+    REDIS_MIGRATION_CLASS_NAME = "Migration"
+
+    class AppConfigs:
+        def __init__(self):
+            self._app_configs = self._find_app_configs()
+            self._app_configs_mapping: Mapping[str, AppConfig] = self._init_app_configs_mapping()
+
+        def _find_app_configs(self):
+            return [
+                app_config
+                for app_config in apps.get_app_configs()
+                if app_config.name.startswith("cvat")
+                and module_has_submodule(
+                    app_config.module, MigrationLoader.REDIS_MIGRATIONS_DIR_NAME
+                )
+            ]
+
+        def _init_app_configs_mapping(self):
+            return {app_config.label: app_config for app_config in self._app_configs}
+
+        def __getitem__(self, label: str) -> AppConfig:
+            return self._app_configs_mapping[label]
+
+        def __iter__(self) -> Generator[AppConfig, Any, Any]:
+            yield from self._app_configs
+
+    def __init__(self) -> None:
+        self._app_configs = self.AppConfigs()
+        self._disk_migrations_per_app: _MigrationsPerApp = {}
+        self._unapplied_migrations: list[BaseRedisMigration] = []
+
+        self._load_from_disk()
+        self._init_unapplied_migrations()
+
+    @property
+    def app_configs(self) -> "AppConfigs":
+        return self._app_configs
+
+    def _load_from_disk(self):
+        for app_config in self._app_configs:
+            migrations_dir = Path(app_config.path) / self.REDIS_MIGRATIONS_DIR_NAME
+            for migration_file in sorted(migrations_dir.glob("[0-9]*.py")):
+                migration_name = migration_file.stem
+                (self._disk_migrations_per_app.setdefault(app_config.label, [])).append(
+                    migration_name
+                )
+
+    def _init_unapplied_migrations(self):
+        applied_migrations = DBRedisMigration.objects.all().values_list("name", "app_label")
+
+        for app_label, migration_names in self._disk_migrations_per_app.items():
+            app_config = self.app_configs[app_label]
+            app_applied_migrations = {m[0] for m in applied_migrations if m[1] == app_config.label}
+            app_unapplied_migrations = sorted(set(migration_names) - app_applied_migrations)
+            for migration_name in app_unapplied_migrations:
+                MigrationClass = self.get_migration_class(app_config.name, migration_name)
+                self._unapplied_migrations.append(MigrationClass(migration_name, app_config.label))
+
+    def get_migration_class(self, app_name: str, migration_name: str) -> BaseRedisMigration:
+        migration_module_path = ".".join([app_name, self.REDIS_MIGRATIONS_DIR_NAME, migration_name])
+        module = importlib.import_module(migration_module_path)
+        MigrationClass = getattr(module, self.REDIS_MIGRATION_CLASS_NAME, None)
+
+        if not MigrationClass or not issubclass(MigrationClass, BaseRedisMigration):
+            raise LoaderError(f"Invalid migration: {migration_module_path}")
+
+        return MigrationClass
+
+    def __iter__(self):
+        yield from self._unapplied_migrations
+
+    def __len__(self):
+        return len(self._unapplied_migrations)

cvat/apps/redis_handler/migrations/0001_initial.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.17 on 2025-01-15 12:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="RedisMigration",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("app_label", models.CharField(max_length=128)),
+                ("name", models.CharField(max_length=128)),
+                ("applied_date", models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="redismigration",
+            constraint=models.UniqueConstraint(
+                fields=("app_label", "name"), name="migration_name_unique"
+            ),
+        ),
+    ]

cvat/apps/redis_handler/migrations/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/models.py

@@ -0,0 +1,15 @@
+from django.db import models
+
+
+class RedisMigration(models.Model):
+    app_label = models.CharField(max_length=128)
+    name = models.CharField(max_length=128)
+    applied_date = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                name="migration_name_unique",
+                fields=("app_label", "name"),
+            ),
+        ]

cvat/apps/redis_handler/redis_migrations/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from abc import ABCMeta, abstractmethod
+
+from attrs import define, field, validators
+
+
+@define
+class BaseMigration(metaclass=ABCMeta):
+    name: str = field(validator=[validators.instance_of(str)])
+    app_label: str = field(validator=[validators.instance_of(str)])
+
+    @staticmethod
+    @abstractmethod
+    def run() -> None: ...

cvat/settings/base.py

@@ -117,6 +117,7 @@ def generate_secret_key():
     'cvat.apps.events',
     'cvat.apps.quality_control',
     'cvat.apps.analytics_report',
+    'cvat.apps.redis_handler',
 ]
 
 SITE_ID = 1

dev/format_python_code.sh

@@ -37,6 +37,7 @@ for paths in \
     "cvat/apps/events/signals.py" \
     "cvat/apps/engine/management/commands/syncperiodicjobs.py" \
     "cvat/apps/dataset_manager/management/commands/cleanuplegacyexportcache.py" \
+    "cvat/apps/redis_handler/**/*.py" \
     ; do
     ${BLACK} -- ${paths}
     ${ISORT} -- ${paths}