[ENG-9856] create a task to periodically clean up notification sent logs

api_tests/notifications/test_notifications_cleanup.py

@@ -0,0 +1,189 @@
+import pytest
+from osf.models import Notification, NotificationType, EmailTask, NotificationSubscription
+from notifications.tasks import (
+    notifications_cleanup_task
+)
+from osf_tests.factories import AuthUserFactory
+from website.settings import NOTIFICATIONS_CLEANUP_AGE
+from django.utils import timezone
+from datetime import timedelta
+
+def create_notification(subscription, sent_date=None):
+    return Notification.objects.create(
+        subscription=subscription,
+        event_context={},
+        sent=sent_date
+    )
+
+def create_email_task(user, created_date):
+    et = EmailTask.objects.create(
+        task_id=f'test-{created_date.timestamp()}',
+        user=user,
+        status='SUCCESS',
+    )
+    et.created_at = created_date
+    et.save()
+    return et
+
+@pytest.mark.django_db
+class TestNotificationCleanUpTask:
+
+    @pytest.fixture()
+    def user(self):
+        return AuthUserFactory()
+
+    @pytest.fixture()
+    def notification_type(self):
+        return NotificationType.objects.get_or_create(
+            name='Test Notification',
+            subject='Hello',
+            template='Sample Template',
+        )[0]
+
+    @pytest.fixture()
+    def subscription(self, user, notification_type):
+        return NotificationSubscription.objects.get_or_create(
+            user=user,
+            notification_type=notification_type,
+            message_frequency='daily',
+        )[0]
+
+    def test_dry_run_does_not_delete_records(self, user, subscription):
+        now = timezone.now()
+
+        old_notification = create_notification(
+            subscription,
+            sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+        old_email_task = create_email_task(
+            user,
+            created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+
+        notifications_cleanup_task(dry_run=True)
+
+        assert Notification.objects.filter(id=old_notification.id).exists()
+        assert EmailTask.objects.filter(id=old_email_task.id).exists()
+
+    def test_deletes_old_notifications_and_email_tasks(self, user, subscription):
+        now = timezone.now()
+
+        old_notification = create_notification(
+            subscription,
+            sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+        new_notification = create_notification(
+            subscription,
+            sent_date=now - timedelta(days=10),
+        )
+
+        old_email_task = create_email_task(
+            user,
+            created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+        new_email_task = create_email_task(
+            user,
+            created_date=now - timedelta(days=10),
+        )
+
+        notifications_cleanup_task()
+
+        assert not Notification.objects.filter(id=old_notification.id).exists()
+        assert Notification.objects.filter(id=new_notification.id).exists()
+
+        assert not EmailTask.objects.filter(id=old_email_task.id).exists()
+        assert EmailTask.objects.filter(id=new_email_task.id).exists()
+
+    def test_records_at_cutoff_are_not_deleted(self, user, subscription):
+        now = timezone.now()
+        cutoff = now - NOTIFICATIONS_CLEANUP_AGE + timedelta(hours=1)
+
+        notification = create_notification(
+            subscription,
+            sent_date=cutoff,
+        )
+        email_task = create_email_task(
+            user,
+            created_date=cutoff,
+        )
+
+        notifications_cleanup_task()
+
+        assert Notification.objects.filter(id=notification.id).exists()
+        assert EmailTask.objects.filter(id=email_task.id).exists()
+
+    def test_cleanup_when_only_notifications_exist(self, user, subscription):
+        now = timezone.now()
+
+        notification = create_notification(
+            subscription,
+            sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+
+        notifications_cleanup_task()
+
+        assert not Notification.objects.filter(id=notification.id).exists()
+
+    def test_cleanup_when_only_email_tasks_exist(self, user, subscription):
+        now = timezone.now()
+
+        email_task = create_email_task(
+            user,
+            created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+
+        notifications_cleanup_task()
+
+        assert not EmailTask.objects.filter(id=email_task.id).exists()
+
+    def test_task_is_idempotent(self, user, subscription):
+        now = timezone.now()
+
+        create_notification(
+            subscription,
+            sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+        create_email_task(
+            user,
+            created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+
+        notifications_cleanup_task()
+        notifications_cleanup_task()
+
+        assert Notification.objects.count() == 0
+        assert EmailTask.objects.count() == 0
+
+    def test_recent_records_are_not_deleted(self, user, subscription):
+        now = timezone.now()
+
+        create_notification(
+            subscription,
+            sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+        create_email_task(
+            user,
+            created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1),
+        )
+        create_notification(
+            subscription,
+            sent_date=now,
+        )
+        create_email_task(
+            user,
+            created_date=now,
+        )
+
+        notifications_cleanup_task()
+
+        assert Notification.objects.count() == 1
+        assert EmailTask.objects.count() == 1
+
+    def test_not_sent_notifications_are_not_deleted(self, user, subscription):
+        create_notification(subscription)
+        create_notification(subscription)
+        create_notification(subscription)
+
+        notifications_cleanup_task()
+
+        assert Notification.objects.count() == 3

notifications/tasks.py

@@ -490,3 +490,22 @@ def send_no_addon_email(self, dry_run=False, **kwargs):
                     pass
             else:
                 notification.mark_sent()
+
+
+@celery_app.task(bind=True, name='notifications.tasks.notifications_cleanup_task')
+def notifications_cleanup_task(self, dry_run=False, **kwargs):
+    """Remove old notifications and email tasks from the database."""
+
+    cutoff_date = timezone.now() - settings.NOTIFICATIONS_CLEANUP_AGE
+    old_notifications = Notification.objects.filter(sent__lt=cutoff_date)
+    old_email_tasks = EmailTask.objects.filter(created_at__lt=cutoff_date)
+
+    if dry_run:
+        notifications_count = old_notifications.count()
+        email_tasks_count = old_email_tasks.count()
+        logger.info(f'[Dry Run] Notifications Cleanup Task: {notifications_count} notifications and {email_tasks_count} email tasks would be deleted.')
+        return
+
+    deleted_notifications_count, _ = old_notifications.delete()
+    deleted_email_tasks_count, _ = old_email_tasks.delete()
+    logger.info(f'Notifications Cleanup Task: Deleted {deleted_notifications_count} notifications and {deleted_email_tasks_count} email tasks older than {cutoff_date}.')

website/settings/defaults.py

@@ -186,6 +186,7 @@ def parent_dir(path):
 NO_ADDON_WAIT_TIME = timedelta(weeks=8)  # 2 months for "Link an add-on to your OSF project" email
 NO_LOGIN_WAIT_TIME = timedelta(weeks=52)   # 1 year for "We miss you at OSF" email
 NO_LOGIN_OSF4M_WAIT_TIME = timedelta(weeks=52)  # 1 year for "We miss you at OSF" email to users created from OSF4M
+NOTIFICATIONS_CLEANUP_AGE = timedelta(weeks=4)  # 1 month to clean up old notifications and email tasks
 
 # Configuration for "We miss you at OSF" email (`NotificationType.Type.USER_NO_LOGIN`)
 # Note: 1) we can gradually increase `MAX_DAILY_NO_LOGIN_EMAILS` to 10000, 100000, etc. or set it to `None` after we
@@ -648,6 +649,11 @@ class CeleryConfig:
             'schedule': crontab(minute=0, hour=5),  # Daily 12 a.m
             'kwargs': {'dry_run': False},
         },
+        'notifications_cleanup_task': {
+            'task': 'notifications.tasks.notifications_cleanup_task',
+            'schedule': crontab(minute=0, hour=17),  # Daily 12 p.m
+            'kwargs': {'dry_run': False},
+        },
         'clear_expired_sessions': {
             'task': 'osf.management.commands.clear_expired_sessions',
             'schedule': crontab(minute=0, hour=5),  # Daily 12 a.m

website/settings/local-ci.py

@@ -85,6 +85,8 @@ class CeleryConfig(defaults.CeleryConfig):
 MAX_DAILY_NO_LOGIN_EMAILS = None
 NO_LOGIN_EMAIL_CUTOFF = None
 
+NOTIFICATIONS_CLEANUP_AGE = timedelta(weeks=4)  # 1 month to clean up old notifications and email tasks
+
 USE_CDN_FOR_CLIENT_LIBS = False
 
 SENTRY_DSN = None