[CAT-126] ADD: pagination to ListSubmissions (#124)

Author: Astha Patni

examples/submission.py

@@ -105,3 +105,13 @@
 job = client.call(SubmitReview(submission.id, changes=changes, rejected=rejected))
 job = client.call(JobStatus(job.id))
 print("Review", job.id, "has result", job.result)
+
+"""
+Example 5
+Use the client paginator to retrieve all PROCESSING submissions
+Without the paginator, the hard limit is 1000
+"""
+sub_filter = SubmissionFilter(status="PROCESSING")
+for submission in client.paginate(ListSubmissions(filters=sub_filter)):
+    print(f"Submission {submission.id}")
+    # do other cool things

indico/queries/document_report.py

@@ -11,46 +11,49 @@ class _DocumentReportList(BaseType):
     submissions: List[DocumentReport]
     pass
 
+
 class GetDocumentReport(PagedRequest):
-  """
-  Query to generate a Document Report.
-  Generates a paged request and paged response.
-  See examples for a sample query.
-  """
-  query = """
-           query SubmissionsLog($filters: SubmissionLogFilter, $limit: Int){
-  submissionsLog(filters: $filters, limit: $limit){
-    submissions{
-      datasetId
-      workflowId
-      status
-      createdAt
-      updatedAt
-      updatedBy
-      completedAt
-      errors
-      retrieved
-      inputFiles{
-        filename
-        submissionId
-      }
-    }
-    pageInfo{
-      startCursor
-      endCursor
-      hasNextPage
-      aggregateCount
-    }
-  }
-}
-        """
+    """
+    Query to generate a Document Report.
+    Generates a paged request and paged response.
+    See examples for a sample query.
+    """
 
-  def __init__(self, filters: Union[dict, DocumentReportFilter] = None, limit: int = None):
-        variables = {
-            "filters": filters,
-            "limit": limit
+    query = """
+       query SubmissionsLog($filters: SubmissionLogFilter, $limit: Int, $after: Int){
+          submissionsLog(filters: $filters, limit: $limit, after: $after){
+            submissions{
+              datasetId
+              workflowId
+              status
+              createdAt
+              updatedAt
+              updatedBy
+              completedAt
+              errors
+              retrieved
+              inputFiles{
+                filename
+                submissionId
+              }
+            }
+            pageInfo{
+              startCursor
+              endCursor
+              hasNextPage
+              aggregateCount
+            }
+          }
         }
+        """
+
+    def __init__(
+        self, filters: Union[dict, DocumentReportFilter] = None, limit: int = None
+    ):
+        variables = {"filters": filters, "limit": limit}
         super().__init__(self.query, variables=variables)
 
-  def process_response(self, response):
-        return _DocumentReportList(**super().process_response(response)["submissionsLog"]).submissions
+    def process_response(self, response):
+        return _DocumentReportList(
+            **super().process_response(response)["submissionsLog"]
+        ).submissions

indico/queries/submission.py

@@ -4,17 +4,18 @@
 from operator import eq, ne
 from typing import Dict, List, Union
 
-from indico.client.request import GraphQLRequest, RequestChain
+from indico.client.request import GraphQLRequest, RequestChain, PagedRequest
 from indico.errors import IndicoInputError, IndicoTimeoutError
 from indico.filters import SubmissionFilter
 from indico.queries import JobStatus
 from indico.types import Job, Submission
 from indico.types.submission import VALID_SUBMISSION_STATUSES
 
 
-class ListSubmissions(GraphQLRequest):
+class ListSubmissions(PagedRequest):
     """
     List all Submissions visible to the authenticated user by most recent.
+    Supports pagination (limit becomes page_size)
 
     Options:
         submission_ids (List[int]): Submission ids to filter by
@@ -26,6 +27,7 @@ class ListSubmissions(GraphQLRequest):
 
     Returns:
         List[Submission]: All the found Submission objects
+        If paginated, yields results one at a time
     """
 
     query = """
@@ -35,7 +37,8 @@ class ListSubmissions(GraphQLRequest):
             $filters: SubmissionFilter,
             $limit: Int,
             $orderBy: SUBMISSION_COLUMN_ENUM,
-            $desc: Boolean
+            $desc: Boolean,
+            $after: Int
 
         ){
             submissions(
@@ -44,7 +47,8 @@ class ListSubmissions(GraphQLRequest):
                 filters: $filters,
                 limit: $limit
                 orderBy: $orderBy,
-                desc: $desc
+                desc: $desc,
+                after: $after
 
             ){
                 submissions {
@@ -59,6 +63,10 @@ class ListSubmissions(GraphQLRequest):
                     retrieved
                     errors
                 }
+                pageInfo {
+                    endCursor
+                    hasNextPage
+                }
             }
         }
     """

tests/integration/queries/test_workflow.py

@@ -1,5 +1,3 @@
-import io
-
 from indico.queries.workflow import GetWorkflow
 import pytest
 from pathlib import Path
@@ -248,6 +246,23 @@ def test_list_workflow_submission_retrieved(
     assert all([not s.retrieved for s in submissions])
     assert submission_id not in [s.id for s in submissions]
 
+def test_list_workflow_submission_paginate(
+    indico, airlines_dataset, airlines_model_group: ModelGroup
+):
+    client = IndicoClient()
+    wfs = client.call(ListWorkflows(dataset_ids=[airlines_dataset.id]))
+    wf = max(wfs, key=lambda w: w.id)
+
+    dataset_filepath = str(Path(__file__).parents[1]) + "/data/mock.pdf"
+
+    submission_ids = client.call(
+        WorkflowSubmission(workflow_id=wf.id, files=[dataset_filepath]*5)
+    )
+    for sub in client.paginate(ListSubmissions(workflow_ids=[wf.id], limit=3)):
+
+        if not submission_ids:
+            break
+        assert sub.id == submission_ids.pop()  # list is desc by default
 
 def test_workflow_submission_missing_workflow(indico):
     client = IndicoClient()