Refactor sponsorship email to send asynchronously via Celery

Dockerfile

@@ -19,11 +19,11 @@ RUN apt-get update && apt-get install -y gettext
 ###############################################################################
 FROM base AS dev
 
-ARG USER_ID
-ARG GROUP_ID
+ARG USER_ID=1000
+ARG GROUP_ID=1000
 
-RUN groupadd -o -g $GROUP_ID -r usergrp
-RUN useradd -o -m -u $USER_ID -g $GROUP_ID user
+RUN groupadd -r usergrp -g $GROUP_ID && \
+    useradd -r -u $USER_ID -g usergrp user
 RUN chown user /code
 
 COPY requirements-dev.txt /code/

compose.yml

@@ -8,7 +8,7 @@ services:
       POSTGRES_USER: pyladiescon
       POSTGRES_PASSWORD: pyladiescon
       POSTGRES_DB: pyladiescon
-      POSTGRES_HOST_AUTH_METHOD: trust  # never do this in production!
+      POSTGRES_HOST_AUTH_METHOD: trust
       POSTGRES_FSYNC: null
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "pyladiescon", "-d", "pyladiescon"]
@@ -22,22 +22,25 @@ services:
     ports:
       - "6379:6379"
     healthcheck:
-     test: ["CMD", "redis-cli","ping"]
-     interval: 1s
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 1s
+    volumes:
+      - redisdata:/data
 
   web:
     build:
       target: dev
     image: pyladiescon-portal-web:docker-compose
-    command: python manage.py runserver 0.0.0.0:8000
+    command: ["./wait-for-postgres.sh", "postgres", "python", "manage.py", "runserver", "0.0.0.0:8000"]
     volumes:
       - .:/code
       - ./media:/code/media
+      - ./wait-for-postgres.sh:/wait-for-postgres.sh
     ports:
       - "8000:8000"
     environment:
-      DEBUG: True
-      DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,[::1]
+      DEBUG: "1"
+      DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]"
       SECRET_KEY: verysecure
       DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon
       DJANGO_DEFAULT_FROM_EMAIL: PyLadiesCon <[email protected]>
@@ -49,6 +52,45 @@ services:
       postgres:
         condition: service_healthy
 
+  celery:
+    image: pyladiescon-portal-web:docker-compose
+    command: celery -A portal worker --loglevel=info
+    volumes:
+      - .:/code
+    environment:
+      DEBUG: "1"
+      SECRET_KEY: verysecure
+      DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+      DJANGO_DEFAULT_FROM_EMAIL: PyLadiesCon <[email protected]>
+      DJANGO_EMAIL_HOST: maildev
+      DJANGO_EMAIL_PORT: 1025
+      DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]"
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+
+  celery-beat:
+    image: pyladiescon-portal-web:docker-compose
+    command: celery -A portal beat --loglevel=info
+    volumes:
+      - .:/code
+    environment:
+      DEBUG: "1"
+      SECRET_KEY: verysecure
+      DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+      DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]"
+    depends_on:
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+
   maildev:
     image: maildev/maildev:2.2.1
     ports:
@@ -57,3 +99,4 @@ services:
 
 volumes:
   pgdata:
+  redisdata:

portal/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

portal/celery.py

@@ -0,0 +1,11 @@
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings')
+app = Celery('portal')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
+
+@app.task(bind=True, ignore_result=True)
+def debug_task(self):
+    print(f'Request: {self.request!r}')

portal/settings.py

@@ -52,9 +52,9 @@
             ),
         ],
     )
-ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS")
-if ALLOWED_HOSTS:
-    ALLOWED_HOSTS = ALLOWED_HOSTS.split(",")
+ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "localhost")
+ALLOWED_HOSTS = [host.strip() for host in ALLOWED_HOSTS.split(",")]
+
 
 # Application definition
 
@@ -335,3 +335,24 @@
 
 PRETIX_API_TOKEN = os.getenv("PRETIX_API_TOKEN")
 PRETIX_WEBHOOK_SECRET = os.getenv("PRETIX_WEBHOOK_SECRET")
+
+
+CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
+CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379/0')
+
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TIMEZONE = 'UTC'
+
+CELERY_TASK_ROUTES = {
+    'sponsorship.tasks.*': {'queue': 'emails'},
+    'volunteer.tasks.*': {'queue': 'emails'},
+}
+
+CELERY_TASK_TRACK_STARTED = True
+CELERY_TASK_TIME_LIMIT = 30 * 60 
+CELERY_TASK_SOFT_TIME_LIMIT = 25 * 60  
+
+CELERY_WORKER_LOG_FORMAT = '[%(asctime)s: %(levelname)s/%(processName)s] %(message)s'
+CELERY_WORKER_TASK_LOG_FORMAT = '[%(asctime)s: %(levelname)s/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s'

requirements-app.txt

@@ -18,4 +18,6 @@ django-import-export[all]==4.3.12
 markdown==3.7
 bleach==6.3.0
 sentry-sdk[django]==2.43.0
-requests==2.32.5
+requests==2.32.5
+celery==5.4.0
+redis==5.2.1

sponsorship/signals.py

@@ -5,92 +5,24 @@
 from django.dispatch import receiver
 
 from common.send_emails import send_email
-from volunteer.models import RoleTypes, VolunteerProfile
+from volunteer.constants import RoleTypes
+from volunteer.models import VolunteerProfile
 
 from .models import SponsorshipProfile
 
-
-def _send_internal_email(
-    subject,
-    *,
-    markdown_template,
-    context=None,
-):
-    """Helper function to send an internal email.
-
-    Lookup who the internal team members who should receive the email and then send the emails individually.
-    Send the email to staff, admin, and sponsorship team members
-
-    Only supports Markdown templates going forward.
-    """
-
-    recipients = User.objects.filter(
-        Q(
-            id__in=VolunteerProfile.objects.prefetch_related("roles")
-            .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF])
-            .values_list("id", flat=True)
-        )
-        | Q(is_superuser=True)
-        | Q(is_staff=True)
-    ).distinct()
-
-    if not recipients.exists():
-        return
-
-    # send each email individually to each recipient, for privacy reasons
-    for recipient in recipients:
-        context["recipient_name"] = recipient.get_full_name() or recipient.username
-
-        send_email(
-            subject,
-            [recipient.email],
-            markdown_template=markdown_template,
-            context=context,
-        )
-
-
-def send_internal_sponsor_onboarding_email(instance):
-    """Send email to team whenever a new sponsor is created.
-
-    Emails will be sent to team members with the role type Staff or Admin, and to sponsorship team.
-    Emails will also be sent to users with is_superuser or is_staff set to True.
-    """
-    context = {"profile": instance}
-    subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {instance.organization_name}"
-
-    markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md"
-
-    _send_internal_email(
-        subject,
-        markdown_template=markdown_template,
-        context=context,
-    )
-
-
-def send_internal_sponsor_progress_update_email(instance):
-    """Send email to team whenever there is a change in sponsorship progress."""
-
-    context = {"profile": instance}
-    subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {instance.organization_name}"
-
-    markdown_template = "emails/sponsorship/internal_sponsor_updated.md"
-
-    _send_internal_email(
-        subject,
-        markdown_template=markdown_template,
-        context=context,
-    )
-
-
 @receiver(post_save, sender=SponsorshipProfile)
 def sponsorship_profile_signal(sender, instance, created, **kwargs):
     """Send emails when sponsorship profile is created or updated.
     Do not send emails if the instance was created via import/export. (too noisy).
     """
     if hasattr(instance, "from_import_export"):
         return
+    from sponsorship.tasks import (
+        send_internal_sponsor_onboarding_email_task,
+        send_internal_sponsor_progress_update_email_task
+    )
+    if created:
+        send_internal_sponsor_onboarding_email_task.delay(instance.id)
     else:
-        if created:
-            send_internal_sponsor_onboarding_email(instance)
-        else:
-            send_internal_sponsor_progress_update_email(instance)
+        send_internal_sponsor_progress_update_email_task.delay(instance.id)
+