WIP Significantly optimize submissions pages with UX trade-offs

judge/views/organization.py

@@ -4,8 +4,9 @@
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
+from django.core.cache import cache
 from django.core.exceptions import ImproperlyConfigured, PermissionDenied
-from django.db.models import Count, FilteredRelation, Q, Sum
+from django.db.models import Count, FilteredRelation, Q, Sum, Prefetch
 from django.db.models.expressions import F, Value
 from django.db.models.functions import Coalesce
 from django.forms import Form, modelformset_factory
@@ -22,19 +23,22 @@
 from judge.forms import OrganizationForm
 from judge.models import BlogPost, Comment, Contest, Language, Organization, OrganizationRequest, \
     Problem, ProblemData, Profile
+from judge.models.problem import ProblemTranslation
 from judge.models.profile import OrganizationMonthlyUsage
+from judge.models.submission import Submission
 from judge.tasks import on_new_problem
 from judge.utils import cache_helper
 from judge.utils.infinite_paginator import InfinitePaginationMixin
 from judge.utils.organization import add_admin_to_group
 from judge.utils.ranker import ranker
+from judge.utils.raw_sql import use_straight_join
 from judge.utils.stats import get_lines_chart
 from judge.utils.views import DiggPaginatorMixin, QueryStringSortMixin, TitleMixin, generic_message, \
     paginate_query_context
 from judge.views.blog import BlogPostCreate, PostListBase
 from judge.views.contests import ContestList, CreateContest
 from judge.views.problem import ProblemCreate, ProblemList
-from judge.views.submission import SubmissionsListBase
+from judge.views.submission import SubmissionsListBase, submission_related
 
 __all__ = ['OrganizationList', 'OrganizationHome', 'OrganizationUsers', 'OrganizationMembershipChange',
            'JoinOrganization', 'LeaveOrganization', 'EditOrganization', 'RequestJoinOrganization',
@@ -652,14 +656,52 @@ def get_context_data(self, **kwargs):
             context['title'] = self.organization.name
         return context
 
+def get_last_submission():
+    key = 'last_submission_pk_cache'
+    last_submission = cache.get(key)
+    if last_submission is None:
+        last_submission = Submission.objects.all().order_by('-id').first()
+        if last_submission:
+            last_submission = last_submission.id
+        cache.set(key, last_submission, 600)
+    return last_submission
 
 class SubmissionListOrganization(InfinitePaginationMixin, PrivateOrganizationMixin, SubmissionsListBase):
     template_name = 'organization/submission-list.html'
     permission_bypass = ['judge.view_all_submission']
 
     def _get_queryset(self):
-        query_set = super(SubmissionListOrganization, self)._get_queryset()
-        query_set = query_set.filter(problem__organization=self.organization)
+        queryset = Submission.objects.all()
+        use_straight_join(queryset)
+        queryset = submission_related(queryset.order_by('-id'))
+        if self.show_problem:
+            queryset = queryset.prefetch_related(Prefetch('problem__translations',
+                                                          queryset=ProblemTranslation.objects.filter(
+                                                              language=self.request.LANGUAGE_CODE), to_attr='_trans'))
+        # if not org admin -> only view submissions to public problems
+        if not self.object.is_admin(self.request.profile) and not self.request.user.is_superuser:
+            queryset = queryset.filter(
+                contest_object__isnull=True,
+                problem__is_organization_private=True,
+                problem__organization=self.organization,
+                problem__is_public=True,
+            )
+        else:
+            # orgs admin can view submissions of others admin
+            # not gud but for the sake of time being ...
+            queryset = queryset.filter(
+                problem__organization=self.organization,
+            )
+
+        queryset = self._do_filter_queryset(queryset)
+
+        return queryset
+
+    def get_queryset(self):
+        query_set = self._get_queryset()
+        last_submission = get_last_submission()
+        if last_submission is not None:
+            query_set = query_set.filter(id__gte=last_submission - 20_000)
         return query_set
 
     def get_context_data(self, **kwargs):

judge/views/submission.py

@@ -7,6 +7,7 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import LoginRequiredMixin
+from django.contrib.auth.models import AnonymousUser
 from django.core.cache import cache
 from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist, PermissionDenied
 from django.core.files.storage import default_storage
@@ -382,6 +383,24 @@ def is_contest_scoped(self):
     def contest(self):
         return self.request.profile.current_contest.contest
 
+    def _do_filter_queryset(self, queryset):
+        if self.selected_languages:
+            # MariaDB can't optimize this subquery for some insane, unknown reason,
+            # so we are forcing an eager evaluation to get the IDs right here.
+            # Otherwise, with multiple language filters, MariaDB refuses to use an index
+            # (or runs the subquery for every submission, which is even more horrifying to think about).
+            queryset = queryset.filter(language__in=list(
+                Language.objects.filter(key__in=self.selected_languages).values_list('id', flat=True)))
+        if self.selected_statuses:
+            status_filter = Q(result__in=self.selected_statuses)
+            if self.could_filter_by_status():
+                status_filter |= Q(status__in=self.selected_statuses)
+            queryset = queryset.filter(status_filter)
+        if self.selected_organization:
+            organization_object = get_object_or_404(Organization, pk=self.selected_organization)
+            queryset = queryset.filter(user__organizations=organization_object)
+        return queryset
+
     def _get_queryset(self):
         queryset = Submission.objects.all()
         use_straight_join(queryset)
@@ -408,21 +427,7 @@ def _get_queryset(self):
                                            Q(contest_object__in=contest_queryset) |
                                            Q(contest_object__isnull=True))
 
-        if self.selected_languages:
-            # MariaDB can't optimize this subquery for some insane, unknown reason,
-            # so we are forcing an eager evaluation to get the IDs right here.
-            # Otherwise, with multiple language filters, MariaDB refuses to use an index
-            # (or runs the subquery for every submission, which is even more horrifying to think about).
-            queryset = queryset.filter(language__in=list(
-                Language.objects.filter(key__in=self.selected_languages).values_list('id', flat=True)))
-        if self.selected_statuses:
-            status_filter = Q(result__in=self.selected_statuses)
-            if self.could_filter_by_status():
-                status_filter |= Q(status__in=self.selected_statuses)
-            queryset = queryset.filter(status_filter)
-        if self.selected_organization:
-            organization_object = get_object_or_404(Organization, pk=self.selected_organization)
-            queryset = queryset.filter(user__organizations=organization_object)
+        queryset = self._do_filter_queryset(queryset)
 
         return queryset
 
@@ -650,7 +655,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class ProblemSubmissions(ProblemSubmissionsBase):
+class ProblemSubmissions(InfinitePaginationMixin, ProblemSubmissionsBase):
     def get_my_submissions_page(self):
         if self.request.user.is_authenticated:
             if hasattr(self, 'contest'):
@@ -730,6 +735,27 @@ def single_submission(request):
 class AllSubmissions(InfinitePaginationMixin, SubmissionsListBase):
     stats_update_interval = 3600
 
+    def public_problem_queryset(self):
+        queryset = Submission.objects.all()
+        use_straight_join(queryset)
+        queryset = submission_related(queryset.order_by('-id'))
+        if self.show_problem:
+            queryset = queryset.prefetch_related(Prefetch('problem__translations',
+                                                          queryset=ProblemTranslation.objects.filter(
+                                                              language=self.request.LANGUAGE_CODE), to_attr='_trans'))
+        queryset = queryset.filter(contest_object__isnull=True)
+
+        return self._do_filter_queryset(queryset)
+
+    def get_queryset(self):
+        if not self.is_contest_scoped:
+            queryset = self.public_problem_queryset()
+            filter_submissions_by_visible_problems(queryset, AnonymousUser())
+        else:
+            queryset = self._get_queryset()
+
+        return queryset
+
     @property
     def use_infinite_pagination(self):
         return not self.is_contest_scoped
@@ -742,20 +768,14 @@ def get_context_data(self, **kwargs):
         context = super(AllSubmissions, self).get_context_data(**kwargs)
         context['dynamic_update'] = context['page_obj'].number == 1
         context['stats_update_interval'] = self.stats_update_interval
+        # in the all submissions page -> no need to show view submsission source link ...
+        context['completed_problem_ids'] = []
+        context['editable_problem_ids'] = []
+        context['tester_problem_ids'] = []
         return context
 
     def _get_result_data(self, queryset=None):
-        if queryset is not None or self.is_contest_scoped or self.selected_languages or \
-           self.selected_statuses or self.selected_organization:
-            return super(AllSubmissions, self)._get_result_data(queryset)
-
-        key = 'global_submission_result_data'
-        result = cache.get(key)
-        if result:
-            return result
-        result = super(AllSubmissions, self)._get_result_data(Submission.objects.all())
-        cache.set(key, result, self.stats_update_interval)
-        return result
+        return {'categories': [], 'total': 0}
 
 
 class ForceContestMixin(object):