diff --git a/.gitignore b/.gitignore index 53538a1..26da588 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ coverage.xml .static_storage/ .media/ media/vector/* +media/scenarios/* local_settings.py # Flask stuff: diff --git a/proenergia/datasets/admin.py b/proenergia/datasets/admin.py index 2710548..4f51916 100644 --- a/proenergia/datasets/admin.py +++ b/proenergia/datasets/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin, messages +from django.forms import ModelForm from unfold.admin import ModelAdmin -from .models import VectorDataset, VectorFile +from .models import DataModel, Scenario, ScenarioFile, VectorDataset, VectorFile class PermissionBasedModelAdmin(ModelAdmin): @@ -107,3 +108,64 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): kwargs["initial"] = request.user.id return super().formfield_for_foreignkey(db_field, request, **kwargs) + + +class DataModelAdminForm(ModelForm): + class Meta: + model = DataModel + fields = ["name", "filter_fields", "popup_fields"] + + def clean(self): + cleaned_data = super().clean() + filter_fields = cleaned_data.get("filter_fields") + popup_fields = cleaned_data.get("popup_fields") + + if filter_fields: + if type(filter_fields) is not list: + self.add_error("filter_fields", "Content should be a list.") + else: + for i in enumerate(filter_fields): + keys = i[1].keys() + if ( + "label" not in keys + or "description" not in keys + or "column" not in keys + ): + self.add_error("filter_fields", "Missing a required key.") + + if popup_fields: + if type(popup_fields) is not list: + self.add_error("popup_fields", "Content should be a list") + else: + for i in enumerate(popup_fields): + keys = i[1].keys() + if ( + "label" not in keys + or "description" not in keys + or "column" not in keys + ): + self.add_error("popup_fields", "Missing a required key.") + + +@admin.register(DataModel) +class DataModelAdmin(ModelAdmin): + form = DataModelAdminForm + + +@admin.register(Scenario) +class ScenarioAdmin(ModelAdmin): + list_display = ["id", "name", "model"] + fields = ["name", "model", "vector_dataset"] + + +@admin.register(ScenarioFile) +class ScenarioFileAdmin(ModelAdmin): + list_display = ["id", "scenario", "created", "status"] + fields = ["scenario", "file"] + + def save_model(self, request, obj, form, change): + if not change: + obj.created_by = request.user + + obj.last_updated_by = request.user + super().save_model(request, obj, form, change) diff --git a/proenergia/datasets/migrations/0003_datamodel_scenario_scenariofile.py b/proenergia/datasets/migrations/0003_datamodel_scenario_scenariofile.py new file mode 100644 index 0000000..9a30414 --- /dev/null +++ b/proenergia/datasets/migrations/0003_datamodel_scenario_scenariofile.py @@ -0,0 +1,140 @@ +# Generated by Django 5.2.9 on 2026-01-09 12:47 + +import django.core.validators +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("datasets", "0002_vectorfile"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="DataModel", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=155, unique=True)), + ( + "filter_fields", + models.JSONField( + default=[], + help_text="A list containing JSON objects following this structure: {'label': 'Field label', 'description': 'Field description', 'column': 'File/Database column name'}", + ), + ), + ( + "popup_fields", + models.JSONField( + default=[], + help_text="A list containing JSON objects following this structure: {'label': 'Field label', 'description': 'Field description', 'column': 'File/Database column name'}", + ), + ), + ], + options={ + "ordering": ["id"], + }, + ), + migrations.CreateModel( + name="Scenario", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=155, unique=True)), + ( + "model", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="datasets.datamodel", + ), + ), + ( + "vector_dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="datasets.vectordataset", + ), + ), + ], + options={ + "ordering": ["id"], + }, + ), + migrations.CreateModel( + name="ScenarioFile", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "status", + models.CharField( + choices=[ + ("created", "Created"), + ("processing", "Processing"), + ("ready", "Ready"), + ("error", "Error"), + ], + default="created", + max_length=155, + ), + ), + ( + "file", + models.FileField( + unique=True, + upload_to="scenarios/", + validators=[ + django.core.validators.FileExtensionValidator( + allowed_extensions=["csv"] + ) + ], + ), + ), + ( + "created_by", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="scenario_files", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "scenario", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="files", + to="datasets.scenario", + ), + ), + ], + options={ + "ordering": ["id"], + }, + ), + ] diff --git a/proenergia/datasets/models.py b/proenergia/datasets/models.py index 73e668b..0c803d7 100644 --- a/proenergia/datasets/models.py +++ b/proenergia/datasets/models.py @@ -67,3 +67,53 @@ def delete_vector_file(sender, instance, **kwargs): # Using default_storage for better compatibility with different storage backends if default_storage.exists(instance.file.name): default_storage.delete(instance.file.name) + + +class DataModel(models.Model): + name = models.CharField(max_length=155, unique=True) + filter_fields = models.JSONField( + default=list(), + help_text="A list containing JSON objects following this structure: {'label': 'Field label', 'description': 'Field description', 'column': 'File/Database column name'}", + ) + popup_fields = models.JSONField( + default=list(), + help_text="A list containing JSON objects following this structure: {'label': 'Field label', 'description': 'Field description', 'column': 'File/Database column name'}", + ) + + def __str__(self): + return f"{self.name}" + + class Meta: + ordering = ["id"] + + +class Scenario(models.Model): + name = models.CharField(max_length=155, unique=True) + model = models.ForeignKey(DataModel, on_delete=models.PROTECT) + vector_dataset = models.ForeignKey(VectorDataset, on_delete=models.PROTECT) + + def __str__(self): + return f"{self.name}" + + class Meta: + ordering = ["id"] + + +class ScenarioFile(models.Model): + scenario = models.ForeignKey(Scenario, models.PROTECT, related_name="files") + created = models.DateTimeField(auto_now_add=True) + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, models.PROTECT, related_name="scenario_files" + ) + status = models.CharField(max_length=155, choices=STATUS, default="created") + file = models.FileField( + upload_to="scenarios/", + unique=True, + validators=[FileExtensionValidator(allowed_extensions=["csv"])], + ) + + def __str__(self): + return f"{self.scenario} ({self.created})" + + class Meta: + ordering = ["id"] diff --git a/proenergia/datasets/serializers.py b/proenergia/datasets/serializers.py index c130083..a0385fd 100644 --- a/proenergia/datasets/serializers.py +++ b/proenergia/datasets/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from .models import VectorDataset, VectorFile +from .models import Scenario, ScenarioFile, VectorDataset, VectorFile class VectorDatasetSerializer(serializers.ModelSerializer): @@ -31,3 +31,29 @@ def get_raw_file(self, obj): return vector_file.file.name except VectorFile.DoesNotExist: return None + + +class ScenarioSerializer(serializers.ModelSerializer): + model_file = serializers.SerializerMethodField() + model = serializers.ReadOnlyField(source="model.name") + filter_fields = serializers.ReadOnlyField(source="model.filter_fields") + popup_fields = serializers.ReadOnlyField(source="model.popup_fields") + + class Meta: + model = Scenario + fields = [ + "id", + "name", + "model", + "model_file", + "filter_fields", + "popup_fields", + ] + + def get_model_file(self, obj): + try: + # update status to ready when we have the file conversion working + model_file = obj.files.latest("created") + return model_file.file.name + except ScenarioFile.DoesNotExist: + return None diff --git a/proenergia/datasets/tests/test_scenario_admin.py b/proenergia/datasets/tests/test_scenario_admin.py new file mode 100644 index 0000000..fdfd381 --- /dev/null +++ b/proenergia/datasets/tests/test_scenario_admin.py @@ -0,0 +1,161 @@ +import json + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse + +from proenergia.datasets.models import DataModel + + +class TestScenarioAdmin(TestCase): + def setUp(self): + self.superadmin_user = get_user_model().objects.create_superuser( + username="superadmin", + email="superadmin@example.com", + password="testpass123", + ) + self.url = reverse("admin:datasets_datamodel_add") + + def test_validation(self): + self.client.login(username="superadmin", password="testpass123") + data = { + "name": "Least Cost Eletrification", + "filter_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + "popup_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + } + self.client.post(self.url, data) + self.assertEqual(DataModel.objects.count(), 1) + + # same name + data = { + "name": "Least Cost Eletrification", + "filter_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + "popup_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + } + self.client.post(self.url, data) + self.assertEqual(DataModel.objects.count(), 1) + # missing column in filter fields + data = { + "name": "PUE", + "filter_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + } + ] + ), + "popup_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + } + self.client.post(self.url, data) + self.assertEqual(DataModel.objects.count(), 1) + # missing label in filter fields + data = { + "name": "Clean Cooking", + "filter_fields": json.dumps( + [ + { + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + "popup_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + } + self.client.post(self.url, data) + # missing column in popup_fields + self.assertEqual(DataModel.objects.count(), 1) + data = { + "name": "Another Model", + "filter_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + "popup_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + } + ] + ), + } + self.client.post(self.url, data) + self.assertEqual(DataModel.objects.count(), 1) + # missing label in popup_fields + self.assertEqual(DataModel.objects.count(), 1) + data = { + "name": "Least Cost Eletrification 2", + "filter_fields": json.dumps( + [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + "popup_fields": json.dumps( + [ + { + "description": "Population in 2025", + "column": "Pop", + } + ] + ), + } + self.client.post(self.url, data) + self.assertEqual(DataModel.objects.count(), 1) diff --git a/proenergia/datasets/tests/test_scenario_views.py b/proenergia/datasets/tests/test_scenario_views.py new file mode 100644 index 0000000..11aed96 --- /dev/null +++ b/proenergia/datasets/tests/test_scenario_views.py @@ -0,0 +1,164 @@ +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from ..models import DataModel, Scenario, ScenarioFile, VectorDataset + + +class TestScenarioListDetailViews(APITestCase): + def setUp(self): + self.superadmin_user = get_user_model().objects.create_superuser( + username="superadmin", + email="superadmin@example.com", + password="testpass123", + ) + self.admin_user = get_user_model().objects.create_user( + username="admin", + email="admin@example.com", + password="testpass123", + is_staff=True, + ) + self.dataset_1 = VectorDataset.objects.create( + name="Boundaries", + description="Administratives Boundaries", + source="OSM", + is_public=True, + is_approved=True, + created_by=self.superadmin_user, + last_updated_by=self.superadmin_user, + ) + self.dataset_2 = VectorDataset.objects.create( + name="Population Clusters", + is_public=True, + is_approved=True, + created_by=self.superadmin_user, + last_updated_by=self.superadmin_user, + ) + self.model_1 = DataModel.objects.create( + name="PUE", + filter_fields=[ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ], + popup_fields=[ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ], + ) + self.model_2 = DataModel.objects.create( + name="Clean Cooking", + filter_fields=[ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + }, + { + "label": "State", + "description": "State name", + "column": "State", + }, + ], + popup_fields=[ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ], + ) + self.scenario_1 = Scenario.objects.create( + name="Least Cost Eletrification", + vector_dataset=self.dataset_1, + model=self.model_1, + ) + self.scenario_2 = Scenario.objects.create( + name="Clean Cooking 1", + vector_dataset=self.dataset_2, + model=self.model_2, + ) + file = SimpleUploadedFile( + "old.csv", b"id,col_b\n1,blah", content_type="text/csv" + ) + self.scenario_file_1 = ScenarioFile.objects.create( + scenario=self.scenario_1, + file=file, + created_by=self.superadmin_user, + ) + file = SimpleUploadedFile( + "new.csv", b"id,col_b\n1,blah", content_type="text/csv" + ) + self.scenario_file_2 = ScenarioFile.objects.create( + scenario=self.scenario_2, + file=file, + created_by=self.superadmin_user, + ) + self.url = reverse("datasets:scenario-list") + + def test_scenario_list_unauthenticated(self): + req = self.client.get(self.url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 2 + assert req.data.get("results")[0]["name"] == "Least Cost Eletrification" + assert req.data.get("results")[1]["name"] == "Clean Cooking 1" + assert req.data.get("results")[0]["model"] == "PUE" + assert req.data.get("results")[1]["model"] == "Clean Cooking" + assert ( + req.data.get("results")[0]["model_file"] == self.scenario_file_1.file.name + ) + assert ( + req.data.get("results")[1]["model_file"] == self.scenario_file_2.file.name + ) + assert req.data.get("results")[0]["filter_fields"] == [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + assert req.data.get("results")[1]["filter_fields"] == [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + }, + {"label": "State", "description": "State name", "column": "State"}, + ] + assert req.data.get("results")[0]["popup_fields"] == [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + assert req.data.get("results")[1]["popup_fields"] == [ + { + "label": "Population", + "description": "Population in 2025", + "column": "Pop", + } + ] + + def test_scenario_detail_unauthenticated(self): + url = reverse("datasets:scenario-detail", args=[self.scenario_1.id]) + req = self.client.get(url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("name") == "Least Cost Eletrification" + assert req.data.get("model_file") == self.scenario_file_1.file.name + + url = reverse("datasets:scenario-detail", args=[self.scenario_2.id]) + req = self.client.get(url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("name") == "Clean Cooking 1" + assert req.data.get("model_file") == self.scenario_file_2.file.name + + def tearDown(self): + ScenarioFile.objects.all().delete() diff --git a/proenergia/datasets/tests/test_admin.py b/proenergia/datasets/tests/test_vector_admin.py similarity index 100% rename from proenergia/datasets/tests/test_admin.py rename to proenergia/datasets/tests/test_vector_admin.py diff --git a/proenergia/datasets/urls.py b/proenergia/datasets/urls.py index ac15992..8748a78 100644 --- a/proenergia/datasets/urls.py +++ b/proenergia/datasets/urls.py @@ -11,4 +11,10 @@ views.VectorDatasetDetailView.as_view(), name="vector-detail", ), + path("scenario/", views.ScenarioListView.as_view(), name="scenario-list"), + path( + "scenario//", + views.ScenarioDetailView.as_view(), + name="scenario-detail", + ), ] diff --git a/proenergia/datasets/views.py b/proenergia/datasets/views.py index ec0c4d8..65c8188 100644 --- a/proenergia/datasets/views.py +++ b/proenergia/datasets/views.py @@ -6,9 +6,9 @@ ) from .filters import VectorDatasetFilter -from .models import VectorDataset +from .models import Scenario, VectorDataset from .pagination import StandardResultsSetPagination -from .serializers import VectorDatasetSerializer +from .serializers import ScenarioSerializer, VectorDatasetSerializer class PublicApprovedDataset(BasePermission): @@ -37,3 +37,15 @@ class VectorDatasetDetailView(RetrieveAPIView): queryset = VectorDataset.objects.all() serializer_class = VectorDatasetSerializer permission_classes = [PublicApprovedDataset] + + +class ScenarioListView(ListAPIView): + queryset = Scenario.objects.all() + serializer_class = ScenarioSerializer + permission_classes = [IsAuthenticatedOrReadOnly] + + +class ScenarioDetailView(RetrieveAPIView): + queryset = Scenario.objects.all() + serializer_class = ScenarioSerializer + permission_classes = [IsAuthenticatedOrReadOnly]