Automatic categorization (labelling) of reports based on asking LLM questions.

example.env

@@ -109,6 +109,10 @@ REPORT_LLM_PROVIDER_URL="http://host.docker.internal:11434/v1"
 # 'cli generate-example-reports'.
 REPORT_LLM_PROVIDER_API_KEY="ollama"
 
+# Labels
+# Automatically enqueue a backfill when a new label question is created.
+LABELS_AUTO_BACKFILL_ON_NEW_QUESTION=true
+
 # Docker swarm mode does not respect the Docker Proxy client configuration
 # (see https://docs.docker.com/network/proxy/#configure-the-docker-client),
 # but we can set those environment variables manually.

radis/core/static/core/core.js

@@ -70,10 +70,18 @@ function FormSet(rootEl) {
       console.log(this.formCount);
     },
     addForm() {
+      if (!template || !container || !totalForms) {
+        return;
+      }
       const newForm = template.content.cloneNode(true);
       const idx = totalForms.value;
       container.append(newForm);
-      const lastForm = container.querySelector(".formset-form:last-child");
+      const lastForm =
+        container.querySelector(".formset-form:last-child") ??
+        container.querySelector("c-formset-form:last-child");
+      if (!lastForm) {
+        return;
+      }
       lastForm.innerHTML = lastForm.innerHTML.replace(/__prefix__/g, idx);
       totalForms.value = (parseInt(idx) + 1).toString();
       this.formCount = parseInt(totalForms.value);
@@ -82,7 +90,12 @@ function FormSet(rootEl) {
      * @param {HTMLElement} btnEl - The delete button element that was clicked
      */
     removeForm(btnEl) {
-      btnEl.closest(".formset-form").remove();
+      const formEl =
+        btnEl.closest(".formset-form") ?? btnEl.closest("c-formset-form");
+      if (!formEl) {
+        return;
+      }
+      formEl.remove();
       const idx = totalForms.value;
       totalForms.value = (parseInt(idx) - 1).toString();
       this.formCount = parseInt(totalForms.value);

radis/core/templates/cotton/formset.html

@@ -7,7 +7,7 @@
         <c-formset-form>{% crispy formset.empty_form %}</c-formset-form>
     </template>
     <div class="formset-forms">
-        {% for form in formset %}<c-formset-form>{{ form|crispy }}</c-formset-form>{% endfor %}
+        {% for form in formset %}<c-formset-form>{% crispy form %}</c-formset-form>{% endfor %}
     </div>
     {% if add_form_label %}
         <div class="d-flex flex-row-reverse">

radis/labels/admin.py

@@ -0,0 +1,47 @@
+from django.contrib import admin
+
+from .models import LabelBackfillJob, LabelChoice, LabelGroup, LabelQuestion, ReportLabel
+
+
+class LabelChoiceInline(admin.TabularInline):
+    model = LabelChoice
+    extra = 0
+
+
+@admin.register(LabelGroup)
+class LabelGroupAdmin(admin.ModelAdmin):
+    list_display = ("name", "is_active", "order")
+    list_filter = ("is_active",)
+    search_fields = ("name",)
+    ordering = ("order", "name")
+
+
+@admin.register(LabelQuestion)
+class LabelQuestionAdmin(admin.ModelAdmin):
+    list_display = ("label", "question", "group", "is_active", "order")
+    list_filter = ("group", "is_active")
+    search_fields = ("label", "question")
+    ordering = ("group__order", "order", "label")
+    inlines = (LabelChoiceInline,)
+
+
+@admin.register(ReportLabel)
+class ReportLabelAdmin(admin.ModelAdmin):
+    list_display = ("report", "question", "choice", "confidence", "verified", "created_at")
+    list_filter = ("verified", "question__group")
+    search_fields = ("report__document_id", "question__name", "choice__label")
+    ordering = ("-created_at",)
+
+
+@admin.register(LabelBackfillJob)
+class LabelBackfillJobAdmin(admin.ModelAdmin):
+    list_display = (
+        "id",
+        "label_group",
+        "status",
+        "processed_reports",
+        "total_reports",
+        "created_at",
+    )
+    list_filter = ("status",)
+    ordering = ("-created_at",)

radis/labels/apps.py

@@ -0,0 +1,42 @@
+from django.apps import AppConfig
+
+
+class LabelsConfig(AppConfig):
+    name = "radis.labels"
+
+    def ready(self) -> None:
+        register_app()
+
+        from radis.reports.site import (
+            ReportsCreatedHandler,
+            ReportsUpdatedHandler,
+            register_reports_created_handler,
+            register_reports_updated_handler,
+        )
+
+        from . import signals  # noqa: F401
+        from .site import handle_reports_created, handle_reports_updated
+
+        register_reports_created_handler(
+            ReportsCreatedHandler(
+                name="Labels",
+                handle=handle_reports_created,
+            )
+        )
+        register_reports_updated_handler(
+            ReportsUpdatedHandler(
+                name="Labels",
+                handle=handle_reports_updated,
+            )
+        )
+
+
+def register_app() -> None:
+    from adit_radis_shared.common.site import MainMenuItem, register_main_menu_item
+
+    register_main_menu_item(
+        MainMenuItem(
+            url_name="label_group_list",
+            label="Auto Labels",
+        )
+    )

radis/labels/constants.py

@@ -0,0 +1,5 @@
+DEFAULT_LABEL_CHOICES = [
+    {"value": "yes", "label": "Yes", "is_unknown": False, "order": 1},
+    {"value": "no", "label": "No", "is_unknown": False, "order": 2},
+    {"value": "cannot_decide", "label": "Cannot decide", "is_unknown": True, "order": 3},
+]

radis/labels/forms.py

@@ -0,0 +1,66 @@
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Column, Layout, Row
+from django import forms
+
+from .models import LabelGroup, LabelQuestion
+
+
+class LabelGroupForm(forms.ModelForm):
+    class Meta:
+        model = LabelGroup
+        fields = [
+            "name",
+            "description",
+            "is_active",
+            "order",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.layout = Layout(
+            Row(
+                Column("name", "description"),
+                Column("is_active", "order", css_class="col-3"),
+            )
+        )
+
+
+class LabelQuestionForm(forms.ModelForm):
+    class Meta:
+        model = LabelQuestion
+        fields = [
+            "label",
+            "question",
+            "is_active",
+            "order",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        self.group = kwargs.pop("group", None)
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.layout = Layout(
+            "label",
+            "question",
+            Row(Column("is_active"), Column("order", css_class="col-3")),
+        )
+
+        self.fields["question"].required = False
+        self.fields["question"].help_text = "Optional. If left empty, the label is used."
+
+    def clean_label(self):
+        label = self.cleaned_data.get("label", "")
+        if not label or not self.group:
+            return label
+
+        existing = LabelQuestion.objects.filter(group=self.group, label__iexact=label)
+        if self.instance and self.instance.pk:
+            existing = existing.exclude(pk=self.instance.pk)
+        if existing.exists():
+            raise forms.ValidationError("A question with this label already exists in this group.")
+        return label

radis/labels/management/commands/labels_backfill.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from itertools import batched
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from radis.reports.models import Report
+
+from ...models import LabelBackfillJob, LabelGroup
+from ...tasks import process_label_group
+
+
+class Command(BaseCommand):
+    help = "Enqueue labeling tasks for existing reports."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--group",
+            dest="group",
+            help="Label group name or ID. If omitted, all active groups are used.",
+        )
+        parser.add_argument(
+            "--batch-size",
+            dest="batch_size",
+            type=int,
+            default=None,
+            help="Override the task batch size.",
+        )
+        parser.add_argument(
+            "--limit",
+            dest="limit",
+            type=int,
+            default=None,
+            help="Limit the number of reports to enqueue.",
+        )
+
+    def handle(self, *args, **options):
+        group_value = options.get("group")
+        batch_size = options.get("batch_size")
+        limit = options.get("limit")
+
+        if group_value:
+            group = self._get_group(group_value)
+            groups = [group]
+        else:
+            groups = list(LabelGroup.objects.filter(is_active=True))
+
+        if not groups:
+            self.stdout.write(self.style.WARNING("No active label groups found."))
+            return
+
+        report_ids = Report.objects.order_by("id").values_list("id", flat=True)
+        if limit:
+            report_ids = report_ids[:limit]
+        report_ids = list(report_ids)
+
+        if not report_ids:
+            self.stdout.write(self.style.WARNING("No reports found."))
+            return
+
+        if batch_size is None:
+            from django.conf import settings
+
+            batch_size = settings.LABELING_TASK_BATCH_SIZE
+
+        for group in groups:
+            backfill_job = LabelBackfillJob.objects.create(
+                label_group=group,
+                status=LabelBackfillJob.Status.IN_PROGRESS,
+                started_at=timezone.now(),
+                total_reports=len(report_ids),
+            )
+
+            for report_batch in batched(report_ids, batch_size):
+                process_label_group.defer(
+                    label_group_id=group.id,
+                    report_ids=list(report_batch),
+                    backfill_job_id=backfill_job.id,
+                )
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"Enqueued labeling for {len(report_ids)} reports "
+                    f"in group '{group.name}' (backfill job #{backfill_job.id})."
+                )
+            )
+
+    def _get_group(self, value: str) -> LabelGroup:
+        if value.isdigit():
+            group = LabelGroup.objects.filter(id=int(value)).first()
+        else:
+            matches = LabelGroup.objects.filter(name=value)
+            if matches.count() > 1:
+                raise CommandError(
+                    f"Multiple label groups named '{value}' exist. Use the numeric ID."
+                )
+            group = matches.first()
+
+        if not group:
+            raise CommandError(f"Label group '{value}' not found.")
+
+        return group