[ch31621] handle Integrity Error on progress report creation

django_api/etools_prp/apps/core/helpers.py

@@ -10,6 +10,8 @@
 from datetime import date, timedelta
 from itertools import combinations, product
 
+from django.db import IntegrityError
+
 from dateutil.relativedelta import relativedelta
 
 from etools_prp.apps.core.common import PD_FREQUENCY_LEVEL
@@ -494,18 +496,20 @@ def create_pr_for_report_type(pd, idx, reporting_period, generate_from_date):
         report_number = 1
         report_type = reporting_period.report_type
         is_final = False
+    try:
+        next_progress_report = ProgressReport.objects.create(
+            start_date=start_date,
+            end_date=end_date,
+            due_date=due_date,
+            programme_document=pd,
+            report_type=report_type,
+            report_number=report_number,
+            is_final=is_final,
+        )
+    except IntegrityError as exc:
+        logger.exception(exc)
 
-    next_progress_report = ProgressReport.objects.create(
-        start_date=start_date,
-        end_date=end_date,
-        due_date=due_date,
-        programme_document=pd,
-        report_type=report_type,
-        report_number=report_number,
-        is_final=is_final,
-    )
-
-    return (next_progress_report, start_date, end_date, due_date)
+    return next_progress_report, start_date, end_date, due_date
 
 
 def create_pr_ir_for_reportable(pd, reportable, pai_ir_for_period, start_date, end_date, due_date):

django_api/etools_prp/apps/unicef/tasks.py

@@ -5,6 +5,7 @@
 from django.contrib.auth.models import Group
 from django.contrib.contenttypes.models import ContentType
 from django.db import transaction
+from django.db.models import F
 
 from celery import shared_task
 from rest_framework.exceptions import ValidationError
@@ -19,6 +20,7 @@
     Disaggregation,
     DisaggregationValue,
     IndicatorBlueprint,
+    IndicatorLocationData,
     Reportable,
     ReportableLocationGoal,
 )
@@ -35,6 +37,7 @@
     PDResultLink,
     Person,
     ProgrammeDocument,
+    ProgressReport,
     ReportingPeriodDates,
     Section,
 )
@@ -100,6 +103,50 @@ def save_person_and_user(person_data, create_user=False):
     return person, user
 
 
+def handle_reporting_dates(business_area_code, pd, reporting_reqs):
+    """
+    Function that handles misaligned start/end dates from etools reporting requirements
+    :param business_area_code: workspace business_area_code
+    :param pd: programme document
+    :param reporting_reqs: the pd reporting requirements from etools API
+    """
+    for report_req in reporting_reqs:
+        try:
+            reporting_period = pd.reporting_periods.get(
+                external_id=report_req['id'],
+                report_type=report_req['report_type'],
+                external_business_area_code=business_area_code)
+        except ReportingPeriodDates.DoesNotExist:
+            continue
+
+        if reporting_period.start_date.strftime('%Y-%m-%d') == report_req['start_date'] and \
+                reporting_period.end_date.strftime('%Y-%m-%d') == report_req['end_date']:
+            continue
+
+        # if start/end dates are not aligned for a ReportingPeriodDates obj, check the corresponding pd ProgressReports
+        try:
+            progress_rep = pd.progress_reports.get(
+                start_date=reporting_period.start_date, end_date=reporting_period.end_date)
+        except ProgressReport.DoesNotExist:
+            # if no progress report found, delete ReportingPeriodDates obj from the db
+            reporting_period.delete()
+            continue
+
+        # if there is any data input from the partner on the progress report
+        # (including indicator reports, indicator location data)
+        if progress_rep.created != progress_rep.modified or \
+                progress_rep.attachments.exists() or \
+                progress_rep.indicator_reports.exclude(created=F('modified')).exists() or \
+                IndicatorLocationData.objects.filter(indicator_report__progress_report=progress_rep).exclude(created=F('modified')).exists():
+            # log exception and skip the report in reporting_requirements
+            logger.exception(f'Misaligned start and end dates for Progress Report id {progress_rep.pk} with user input data. Skipping..')
+            report_req['skip'] = True
+        else:
+            # if there is no user input data, delete the Progress Report and the ReportingPeriodDates obj
+            progress_rep.delete()
+            reporting_period.delete()
+
+
 @shared_task
 def process_programme_documents(fast=False, area=False):
     """
@@ -297,7 +344,10 @@ def process_programme_documents(fast=False, area=False):
 
                         # Create Reporting Date Periods for QPR and HR report type
                         reporting_requirements = item['reporting_requirements']
+                        handle_reporting_dates(workspace.business_area_code, pd, reporting_requirements)
                         for reporting_requirement in reporting_requirements:
+                            if 'skip' in reporting_requirement and reporting_requirements['skip']:
+                                continue
                             reporting_requirement['programme_document'] = pd.id
                             reporting_requirement['external_business_area_code'] = workspace.business_area_code
                             process_model(

django_api/etools_prp/apps/unicef/tests/test_tasks.py

@@ -1,10 +1,14 @@
+from unittest import mock
+
 from rest_framework.exceptions import ValidationError
 
+from etools_prp.apps.core.common import INDICATOR_REPORT_STATUS, OVERALL_STATUS
+from etools_prp.apps.core.helpers import generate_data_combination_entries
 from etools_prp.apps.core.models import Location
 from etools_prp.apps.core.serializers import PMPLocationSerializer
 from etools_prp.apps.core.tests import factories
 from etools_prp.apps.core.tests.base import BaseAPITestCase
-from etools_prp.apps.indicator.models import IndicatorBlueprint, Reportable
+from etools_prp.apps.indicator.models import IndicatorBlueprint, IndicatorLocationData, IndicatorReport, Reportable
 from etools_prp.apps.indicator.serializers import PMPIndicatorBlueprintSerializer, PMPReportableSerializer
 from etools_prp.apps.partner.models import Partner
 from etools_prp.apps.partner.serializers import PMPPartnerSerializer
@@ -14,7 +18,7 @@
     PMPProgrammeDocumentSerializer,
     PMPSectionSerializer,
 )
-from etools_prp.apps.unicef.tasks import process_model
+from etools_prp.apps.unicef.tasks import handle_reporting_dates, process_model
 
 
 class TestProcessModel(BaseAPITestCase):
@@ -248,3 +252,167 @@ def test_location(self):
             filter_dict=filter_dict,
         )
         self.assertTrue(location_qs.exists())
+
+
+class TestHandleReportingDates(BaseAPITestCase):
+    def setUp(self):
+        self.workspace = factories.WorkspaceFactory(business_area_code=1234)
+        self.pd = factories.ProgrammeDocumentFactory(workspace=self.workspace)
+
+        self.location_1 = factories.LocationFactory()
+        self.location_1.workspaces.add(self.workspace)
+
+        self.location_2 = factories.LocationFactory()
+        self.location_2.workspaces.add(self.workspace)
+
+        self.cp_output = factories.PDResultLinkFactory(
+            programme_document=self.pd,
+        )
+        self.llo = factories.LowerLevelOutputFactory(
+            cp_output=self.cp_output,
+        )
+        self.llo_reportable = factories.QuantityReportableToLowerLevelOutputFactory(
+            content_object=self.llo,
+            blueprint=factories.QuantityTypeIndicatorBlueprintFactory(
+                unit=IndicatorBlueprint.NUMBER,
+                calculation_formula_across_locations=IndicatorBlueprint.SUM,
+            )
+        )
+        factories.LocationWithReportableLocationGoalFactory(
+            location=self.location_1,
+            reportable=self.llo_reportable,
+        )
+
+        factories.LocationWithReportableLocationGoalFactory(
+            location=self.location_2,
+            reportable=self.llo_reportable,
+        )
+        self.reporting_requirements = [
+            {
+                "id": 11,
+                "start_date": "2023-11-01",
+                "end_date": "2024-01-15",
+                "due_date": "2024-02-14",
+                "report_type": "QPR"
+            },
+            {
+                "id": 12,
+                "start_date": "2023-08-01",
+                "end_date": "2023-10-31",
+                "due_date": "2023-11-30",
+                "report_type": "QPR"
+            },
+            {
+                "id": 13,
+                "start_date": "2023-05-01",
+                "end_date": "2023-07-31",
+                "due_date": "2023-08-30",
+                "report_type": "QPR"
+            }
+        ]
+        for index, reporting_reqs in enumerate(self.reporting_requirements, start=1):
+            factories.QPRReportingPeriodDatesFactory(
+                programme_document=self.pd, external_id=reporting_reqs['id'],
+                external_business_area_code=self.workspace.business_area_code, **reporting_reqs)
+            progress_report = factories.ProgressReportFactory(
+                programme_document=self.pd, report_number=index, **reporting_reqs)
+            indicator_report = factories.ProgressReportIndicatorReportFactory(
+                progress_report=progress_report,
+                reportable=self.llo_reportable,
+                report_status=INDICATOR_REPORT_STATUS.due,
+                overall_status=OVERALL_STATUS.met,
+            )
+            factories.IndicatorLocationDataFactory(
+                indicator_report=indicator_report,
+                location=self.location_1,
+                num_disaggregation=3,
+                level_reported=3,
+                disaggregation_reported_on=list(
+                    indicator_report.disaggregations.values_list(
+                        'id', flat=True)),
+                disaggregation=generate_data_combination_entries(
+                    indicator_report.disaggregation_values(
+                        id_only=True), indicator_type='quantity', r=3
+                )
+            )
+
+        super().setUp()
+
+    def test_handle_reporting_dates_no_data_input(self):
+
+        self.assertEqual(self.pd.reporting_periods.count(), 3)
+        self.assertEqual(self.pd.progress_reports.count(), 3)
+
+        # no deletion occurs at this step as there are no misaligned dates
+        handle_reporting_dates(self.workspace.business_area_code, self.pd, self.reporting_requirements)
+        self.assertEqual(self.pd.reporting_periods.count(), 3)
+        self.assertEqual(self.pd.progress_reports.count(), 3)
+        self.assertEqual(IndicatorReport.objects.filter(progress_report__programme_document=self.pd).count(), 3)
+        self.assertEqual(
+            IndicatorLocationData.objects.filter(
+                indicator_report__progress_report__programme_document=self.pd).count(), 3)
+
+        # alter end date for last reporting requirement
+        self.reporting_requirements[2]['end_date'] = '2023-07-15'
+        handle_reporting_dates(self.workspace.business_area_code, self.pd, self.reporting_requirements)
+        # the ReportingPeriodDates and progress report are deleted cascaded when no user data input
+        self.assertEqual(self.pd.reporting_periods.count(), 2)
+        self.assertEqual(self.pd.progress_reports.count(), 2)
+        self.assertEqual(IndicatorReport.objects.filter(progress_report__programme_document=self.pd).count(), 2)
+        self.assertEqual(
+            IndicatorLocationData.objects.filter(
+                indicator_report__progress_report__programme_document=self.pd).count(), 2)
+
+    @mock.patch("etools_prp.apps.unicef.tasks.logger.exception")
+    def test_handle_reporting_dates_with_indicator_report_data_input(self, mock_logger_exc):
+        self.assertEqual(self.pd.reporting_periods.count(), 3)
+        self.assertEqual(self.pd.progress_reports.count(), 3)
+
+        # alter end date for last reporting requirement
+        self.reporting_requirements[2]['end_date'] = '2023-07-16'
+        # user input data at indicator report level
+        indicator_report = IndicatorReport.objects.filter(progress_report__programme_document=self.pd).last()
+        indicator_report.narrative_assessment = 'Some narrative_assessment'
+        indicator_report.save()
+
+        handle_reporting_dates(self.workspace.business_area_code, self.pd, self.reporting_requirements)
+        # when there is user data input, an exception is logged, the record is skipped and nothing gets deleted
+        self.assertTrue(self.reporting_requirements[2]['skip'])
+        self.assertTrue(mock_logger_exc.call_count, 1)
+        self.assertEqual(self.pd.reporting_periods.count(), 3)
+        self.assertEqual(self.pd.progress_reports.count(), 3)
+        self.assertEqual(IndicatorReport.objects.filter(progress_report__programme_document=self.pd).count(), 3)
+        self.assertEqual(
+            IndicatorLocationData.objects.filter(
+                indicator_report__progress_report__programme_document=self.pd).count(), 3)
+
+    @mock.patch("etools_prp.apps.unicef.tasks.logger.exception")
+    def test_handle_reporting_dates_with_indicator_location_data_input(self, mock_logger_exc):
+        self.assertEqual(self.pd.reporting_periods.count(), 3)
+        self.assertEqual(self.pd.progress_reports.count(), 3)
+
+        # alter end date for last reporting requirement
+        self.reporting_requirements[2]['end_date'] = '2023-07-16'
+        # user input data at indicator location data level
+        indicator_location = IndicatorLocationData.objects.filter(
+            indicator_report__progress_report__programme_document=self.pd).last()
+        self.assertIsNotNone(indicator_location.disaggregation['()'])
+        indicator_location.disaggregation = {
+            '()': {
+                'v': 1234,
+                'd': 1,
+                'c': 0
+            }
+        }
+        indicator_location.save()
+
+        handle_reporting_dates(self.workspace.business_area_code, self.pd, self.reporting_requirements)
+        # when there is user data input, an exception is logged, the record is skipped and nothing gets deleted
+        self.assertTrue(self.reporting_requirements[2]['skip'])
+        self.assertTrue(mock_logger_exc.call_count, 1)
+        self.assertEqual(self.pd.reporting_periods.count(), 3)
+        self.assertEqual(self.pd.progress_reports.count(), 3)
+        self.assertEqual(IndicatorReport.objects.filter(progress_report__programme_document=self.pd).count(), 3)
+        self.assertEqual(
+            IndicatorLocationData.objects.filter(
+                indicator_report__progress_report__programme_document=self.pd).count(), 3)