Support redis migrations

backend_entrypoint.sh

@@ -20,6 +20,7 @@ cmd_init() {
     ~/manage.py migrate
 
     wait-for-it "${CVAT_REDIS_INMEM_HOST}:${CVAT_REDIS_INMEM_PORT:-6379}" -t 0
+    ~/manage.py migrateredis
     ~/manage.py syncperiodicjobs
 }
 
@@ -39,6 +40,12 @@ cmd_run() {
         sleep 10
     done
 
+    wait-for-it "${CVAT_REDIS_INMEM_HOST}:${CVAT_REDIS_INMEM_PORT:-6379}" -t 0
+    echo "waiting for Redis migrations to complete..."
+    while ! ~/manage.py migrateredis --check; do
+        sleep 10
+    done
+
     exec supervisord -c "supervisord/$1.conf"
 }
 

changelog.d/20250117_174701_maria_redis_migrations.md

@@ -0,0 +1,4 @@
+### Added
+
+- Support for managing Redis migrations
+  (<https://github.com/cvat-ai/cvat/pull/8898>)

cvat-ui/src/components/export-backup/styles.scss

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: MIT
 
-@import '../../base.scss';
+@import '../../base';
 
 .cvat-modal-export-option-item > .ant-select-item-option-content,
 .cvat-modal-export-select .ant-select-selection-item {

cvat-ui/src/components/layout-grid/styles.scss

@@ -3,7 +3,7 @@
 //
 // SPDX-License-Identifier: MIT
 
-@import './../../base.scss';
+@import './../../base';
 
 .grid {
     display: grid;

cvat-ui/src/components/storage/styles.scss

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: MIT
 
-@import '../../base.scss';
+@import '../../base';
 
 .cvat-question-circle-filled-icon {
     font-size: $grid-unit-size * 14;

cvat/apps/engine/redis_migrations/001_cleanup_scheduled_jobs.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import django_rq
+from django.conf import settings
+from rq_scheduler import Scheduler
+
+from cvat.apps.redis_handler.redis_migrations import BaseMigration
+
+
+class Migration(BaseMigration):
+    @classmethod
+    def run(cls):
+        scheduler: Scheduler = django_rq.get_scheduler(settings.CVAT_QUEUES.EXPORT_DATA.value)
+
+        for job in scheduler.get_jobs():
+            if job.func_name == "cvat.apps.dataset_manager.views.clear_export_cache":
+                scheduler.cancel(job)
+                job.delete()

cvat/apps/engine/redis_migrations/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/apps.py

@@ -0,0 +1,10 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+
+from django.apps import AppConfig
+
+
+class RedisHandlerConfig(AppConfig):
+    name = "cvat.apps.redis_handler"

cvat/apps/redis_handler/management/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/management/commands/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/redis_handler/management/commands/migrateredis.py

@@ -0,0 +1,76 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+import traceback
+from argparse import ArgumentParser
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+from redis import Redis
+
+from cvat.apps.redis_handler.migration_loader import AppliedMigration, MigrationLoader
+
+
+class Command(BaseCommand):
+    help = "Applies Redis migrations and records them in the database"
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "--check",
+            action="store_true",
+            help="Checks whether Redis migrations have been applied; exits with non-zero status if not",
+        )
+
+    def handle(self, *args, **options) -> None:
+        loader = MigrationLoader()
+
+        if options["check"]:
+            if not loader:
+                return
+
+            sys.exit(1)
+
+        if not loader:
+            self.stdout.write("No migrations to apply")
+            return
+
+        conn = Redis(
+            host=settings.REDIS_INMEM_SETTINGS["HOST"],
+            port=settings.REDIS_INMEM_SETTINGS["PORT"],
+            db=settings.REDIS_INMEM_SETTINGS["DB"],
+            password=settings.REDIS_INMEM_SETTINGS["PASSWORD"],
+        )
+
+        for migration in loader:
+            with conn.pipeline() as pipe:
+                try:
+                    migration.run()
+
+                    # add migration to applied ones
+                    applied_migration = AppliedMigration(
+                        name=migration.name,
+                        app_label=migration.app_label,
+                        applied_date=timezone.now(),
+                    )
+                    applied_migration_key = applied_migration.get_key()
+                    pipe.hset(applied_migration_key, mapping=applied_migration.to_dict())
+                    pipe.zadd(applied_migration.SORTED_SET_KEY, {applied_migration_key: 1})
+
+                except Exception as ex:
+                    self.stderr.write(
+                        self.style.ERROR(
+                            f"[{migration.app_label}] Failed to apply migration: {migration.name}"
+                        )
+                    )
+                    self.stderr.write(self.style.ERROR(f"\n{traceback.format_exc()}"))
+                    raise CommandError(str(ex))
+
+                pipe.execute()
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"[{migration.app_label}] Successfully applied migration: {migration.name}"
+                    )
+                )

cvat/apps/redis_handler/migration_loader.py

@@ -0,0 +1,134 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import importlib
+from datetime import datetime
+from pathlib import Path
+from typing import Any, ClassVar
+from uuid import UUID, uuid4
+
+from attrs import asdict, field, frozen, validators
+from django.apps import AppConfig, apps
+from django.conf import settings
+from redis import Redis
+
+from cvat.apps.redis_handler.redis_migrations import BaseMigration
+
+
+def to_datetime(value: float | str | datetime) -> datetime:
+    if isinstance(value, datetime):
+        return value
+    elif isinstance(value, str):
+        value = float(value)
+
+    return datetime.fromtimestamp(value)
+
+
+def to_uuid(value: str | UUID) -> UUID:
+    if isinstance(value, UUID):
+        return value
+
+    return UUID(value)
+
+
+@frozen
+class AppliedMigration:
+    SORTED_SET_KEY: ClassVar[str] = "cvat:applied_migrations:"
+    KEY_PREFIX: ClassVar[str] = "cvat:applied_migration:"
+
+    name: str = field(validator=[validators.instance_of(str), validators.max_len(128)])
+    app_label: str = field(validator=[validators.instance_of(str), validators.max_len(128)])
+    applied_date: datetime = field(
+        validator=[validators.instance_of(datetime)], converter=to_datetime
+    )
+    identifier: UUID = field(factory=uuid4, converter=to_uuid)
+
+    def get_key(self) -> str:
+        return self.KEY_PREFIX + str(self.identifier)
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self, filter=lambda a, _: a.name != "identifier")
+        d["applied_date"] = self.applied_date.timestamp()
+
+        return d
+
+
+class LoaderError(Exception):
+    pass
+
+
+class MigrationLoader:
+    REDIS_MIGRATIONS_DIR_NAME = "redis_migrations"
+    REDIS_MIGRATION_CLASS_NAME = "Migration"
+
+    def __init__(self) -> None:
+        self._app_config_mapping = {
+            app_config.label: app_config for app_config in self._find_app_configs()
+        }
+        self._disk_migrations_per_app: dict[str, list[str]] = {}
+        self._applied_migrations: list[AppliedMigration] = []
+        self._unapplied_migrations: list[BaseMigration] = []
+
+        self._load_from_disk()
+        self._init_applied_migrations()
+        self._init_unapplied_migrations()
+
+    def _find_app_configs(self) -> list[AppConfig]:
+        return [
+            app_config
+            for app_config in apps.get_app_configs()
+            if app_config.name.startswith("cvat")
+            and (Path(app_config.path) / self.REDIS_MIGRATIONS_DIR_NAME).exists()
+        ]
+
+    def _load_from_disk(self):
+        for app_label, app_config in self._app_config_mapping.items():
+            migrations_dir = Path(app_config.path) / self.REDIS_MIGRATIONS_DIR_NAME
+            for migration_file in sorted(migrations_dir.glob("[0-9]*.py")):
+                migration_name = migration_file.stem
+                (self._disk_migrations_per_app.setdefault(app_label, [])).append(migration_name)
+
+    def _init_applied_migrations(self):
+        conn = Redis(
+            host=settings.REDIS_INMEM_SETTINGS["HOST"],
+            port=settings.REDIS_INMEM_SETTINGS["PORT"],
+            db=settings.REDIS_INMEM_SETTINGS["DB"],
+            password=settings.REDIS_INMEM_SETTINGS["PASSWORD"],
+        )
+        applied_migration_keys = [
+            i.decode("utf-8") for i in conn.zrange(AppliedMigration.SORTED_SET_KEY, 0, -1)
+        ]
+        for key in applied_migration_keys:
+            self._applied_migrations.append(
+                AppliedMigration(
+                    **{k.decode("utf-8"): v.decode("utf-8") for k, v in conn.hgetall(key).items()}
+                )
+            )
+
+    def _init_unapplied_migrations(self):
+        for app_label, migration_names in self._disk_migrations_per_app.items():
+            app_config = self._app_config_mapping[app_label]
+            app_applied_migrations = {
+                m.name for m in self._applied_migrations if m.app_label == app_config.label
+            }
+            app_unapplied_migrations = sorted(set(migration_names) - app_applied_migrations)
+            for migration_name in app_unapplied_migrations:
+                MigrationClass = self.get_migration_class(app_config.name, migration_name)
+                self._unapplied_migrations.append(MigrationClass(migration_name, app_config.label))
+
+    def get_migration_class(self, app_name: str, migration_name: str) -> BaseMigration:
+        migration_module_path = ".".join([app_name, self.REDIS_MIGRATIONS_DIR_NAME, migration_name])
+        module = importlib.import_module(migration_module_path)
+        MigrationClass = getattr(module, self.REDIS_MIGRATION_CLASS_NAME, None)
+
+        if not MigrationClass or not issubclass(MigrationClass, BaseMigration):
+            raise LoaderError(f"Invalid migration: {migration_module_path}")
+
+        return MigrationClass
+
+    def __iter__(self):
+        yield from self._unapplied_migrations
+
+    def __len__(self):
+        return len(self._unapplied_migrations)

cvat/apps/redis_handler/redis_migrations/__init__.py

@@ -0,0 +1,26 @@
+# Copyright (C) 2025 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from abc import ABCMeta, abstractmethod
+from typing import ClassVar
+
+from attrs import define, field, validators
+from django.conf import settings
+from redis import Redis
+
+
+@define
+class BaseMigration(metaclass=ABCMeta):
+    CONNECTION: ClassVar[Redis] = Redis(
+        host=settings.REDIS_INMEM_SETTINGS["HOST"],
+        port=settings.REDIS_INMEM_SETTINGS["PORT"],
+        db=settings.REDIS_INMEM_SETTINGS["DB"],
+        password=settings.REDIS_INMEM_SETTINGS["PASSWORD"],
+    )
+    name: str = field(validator=[validators.instance_of(str)])
+    app_label: str = field(validator=[validators.instance_of(str)])
+
+    @classmethod
+    @abstractmethod
+    def run(cls) -> None: ...