API integration for user-submitted differential expression calculations (SCP-5767)

app/controllers/api/v1/site_controller.rb

@@ -2,8 +2,8 @@ module Api
   module V1
     class SiteController < ApiBaseController
       before_action :set_current_api_user!
-      before_action :authenticate_api_user!, only: [:download_data, :stream_data, :get_study_analysis_config,
-                                                    :submit_study_analysis, :get_study_submissions,
+      before_action :authenticate_api_user!, only: [:download_data, :stream_data, :submit_differential_expression,
+                                                    :get_study_analysis_config, :submit_study_analysis, :get_study_submissions,
                                                     :get_study_submission, :sync_submission_outputs]
       before_action :set_study, except: [:studies, :check_terra_tos_acceptance, :analyses, :get_analysis, :renew_user_access_token]
       before_action :set_analysis_configuration, only: [:get_analysis, :get_study_analysis_config]
@@ -403,6 +403,135 @@ def stream_data
         end
       end
 
+      swagger_path '/site/studies/{accession}/differential_expression' do
+        operation :post do
+          key :tags, [
+            'Site'
+          ]
+          key :summary, 'Submit a differential expression calculation request'
+          key :description, 'Request differential expression calculations for a given cluster/annotation in a study'
+          key :operationId, 'site_study_submit_differential_expression_path'
+          parameter do
+            key :name, :accession
+            key :in, :path
+            key :description, 'Accession of Study to use'
+            key :required, true
+            key :type, :string
+          end
+          parameter do
+            key :name, :de_job
+            key :type, :object
+            key :in, :body
+            schema do
+              property :cluster_name do
+                key :description, 'Name of cluster group.  Use "_default" to use the default cluster'
+                key :required, true
+                key :type, :string
+              end
+              property :annotation_name do
+                key :description, 'Name of annotation'
+                key :required, true
+                key :type, :string
+              end
+              property :annotation_scope do
+                key :description, 'Scope of annotation.  One of "study" or "cluster".'
+                key :type, :string
+                key :required, true
+                key :enum, Api::V1::Visualization::AnnotationsController::VALID_SCOPE_VALUES
+              end
+              property :de_type do
+                key :description, 'Type of differential expression analysis. Either "rest" (one-vs-rest) or "pairwise"'
+                key :type, :string
+                key :required, true
+                key :enum, %w[rest pairwise]
+              end
+              property :group1 do
+                key :description, 'First group for pairwise analysis (optional)'
+                key :type, :string
+              end
+              property :group2 do
+                key :description, 'Second group for pairwise analysis (optional)'
+                key :type, :string
+              end
+            end
+          end
+          response 204 do
+            key :description, 'Job successfully submitted'
+          end
+          response 401 do
+            key :description, ApiBaseController.unauthorized
+          end
+          response 403 do
+            key :description, ApiBaseController.forbidden('view study')
+          end
+          response 404 do
+            key :description, ApiBaseController.not_found(Study, 'Cluster', 'Annotation')
+          end
+          response 406 do
+            key :description, ApiBaseController.not_acceptable
+          end
+          response 409 do
+            key :description, "Results are processing or already exist"
+          end
+          response 410 do
+            key :description, ApiBaseController.resource_gone
+          end
+          response 422 do
+            key :description, "Job parameters failed validation"
+          end
+        end
+      end
+
+      def submit_differential_expression
+        # disallow DE calculation requests for studies with author DE
+        if @study.differential_expression_results.where(is_author_de: true).any?
+          render json: {
+            error: 'User requests are disabled for this study as it contains author-supplied differential expression results'
+          }, status: 403 and return
+        end
+
+        cluster_name = params[:cluster_name]
+        cluster = cluster_name == '_default' ? @study.default_cluster : @study.cluster_groups.by_name(cluster_name)
+        render json: { error: "Requested cluster #{cluster_name} not found" }, status: 404 and return if cluster.nil?
+
+        annotation_name = params[:annotation_name]
+        annotation_scope = params[:annotation_scope]
+        de_type = params[:de_type]
+        pairwise = de_type == 'pairwise'
+        group1 = params[:group1]
+        group2 = params[:group2]
+        annotation = AnnotationVizService.get_selected_annotation(
+          @study, cluster:, annot_name: annotation_name, annot_type: 'group', annot_scope: annotation_scope
+        )
+        render json: { error: 'No matching annotation found' }, status: 404 and return if annotation.nil?
+
+        de_params = { annotation_name:, annotation_scope:, de_type:, group1:, group2: }
+
+        # check if these results already exist
+        # for pairwise, also check if requested comparisons exist
+        result = DifferentialExpressionResult.find_by(
+          study: @study, cluster_group: cluster, annotation_name:, annotation_scope:, is_author_de: false
+        )
+        if result && (!pairwise || (pairwise && result.has_pairwise_comparison?(group1, group2)))
+          render json: { error: "Requested results already exist" }, status: 409 and return
+        end
+
+        begin
+          submitted = DifferentialExpressionService.run_differential_expression_job(
+            cluster, @study, current_api_user, **de_params
+          )
+          if submitted
+            head 204
+          else
+            # submitted: false here means that there is a matching running DE job
+            render json: { error: "Requested results are processing - please check back later" }, status: 409
+          end
+        rescue ArgumentError => e
+          # job parameters failed to validate
+          render json: { error: e.message}, status: 422 and return
+        end
+      end
+
       swagger_path '/site/analyses' do
         operation :get do
           key :tags, [
@@ -1136,6 +1265,10 @@ def get_download_quota
         end
       end
 
+      def de_job_params
+        params.require(:de_job).permit(:cluster_name, :annotation_name, :annotation_scope, :de_type, :group1, :group2)
+      end
+
       # update AnalysisSubmissions when loading study analysis tab
       # will not backfill existing workflows to keep our submission history clean
       def update_analysis_submission(submission)

app/lib/differential_expression_service.rb

@@ -163,25 +163,27 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_
     params_object.machine_type = machine_type if machine_type.present? # override :machine_type if specified
     return true if dry_run # exit before submission if specified as annotation was already validated
 
-    # check if there's already a job running using these parameters and exit if so
-    job_params = ApplicationController.batch_api_client.format_command_line(
-      study_file:, action: :differential_expression, user_metrics_uuid: user.metrics_uuid, params_object:
-    )
-    running = ApplicationController.batch_api_client.find_matching_jobs(
-      params: job_params, job_states: BatchApiClient::RUNNING_STATES
-    )
-    if running.any?
-      log_message "Found #{running.count} running DE jobs: #{running.map(&:name).join(', ')}"
-      log_message "Matching these parameters: #{job_params.join(' ')}"
-      log_message "Exiting without queuing new job"
-      false
-    elsif params_object.valid?
-      # launch DE job
-      job = IngestJob.new(study:, study_file:, user:, action: :differential_expression, params_object:)
-      job.delay.push_remote_and_launch_ingest
-      true
+    if params_object.valid?
+      # check if there's already a job running using these parameters and exit if so
+      job_params = ApplicationController.batch_api_client.format_command_line(
+        study_file:, action: :differential_expression, user_metrics_uuid: user.metrics_uuid, params_object:
+      )
+      running = ApplicationController.batch_api_client.find_matching_jobs(
+        params: job_params, job_states: BatchApiClient::RUNNING_STATES
+      )
+      if running.any?
+        log_message "Found #{running.count} running DE jobs: #{running.map(&:name).join(', ')}"
+        log_message "Matching these parameters: #{job_params.join(' ')}"
+        log_message "Exiting without queuing new job"
+        false
+      else
+        # launch DE job
+        job = IngestJob.new(study:, study_file:, user:, action: :differential_expression, params_object:)
+        job.delay.push_remote_and_launch_ingest
+        true
+      end
     else
-      raise ArgumentError, "job parameters failed to validate: #{params_object.errors.full_messages}"
+      raise ArgumentError, "job parameters failed to validate: #{params_object.errors.full_messages.join(', ')}"
     end
   end
 
@@ -377,12 +379,14 @@ def self.validate_annotation(cluster_group, study, annotation_name, annotation_s
     if !pairwise && cells_by_label.keys.count < 2
       raise ArgumentError, "#{identifier} does not have enough labels represented in #{cluster_group.name}"
     elsif pairwise
-      missing = {
+      missing = [group1, group2] - annotation[:values]
+      raise ArgumentError, "#{annotation_name} does not contain '#{missing.join(', ')}'" if missing.any?
+      cell_count = {
         "#{group1}" => cells_by_label[group1].count,
         "#{group2}" => cells_by_label[group2].count
       }.keep_if { |_, c| c < 2 }
       raise ArgumentError,
-            "#{missing.keys.join(', ')} does not have enough cells represented in #{identifier}" if missing.any?
+            "#{cell_count.keys.join(', ')} does not have enough cells represented in #{identifier}" if cell_count.any?
     end
   end
 

app/models/ingest_job.rb

@@ -1011,9 +1011,13 @@ def get_job_analytics
       job_props.merge!(
         {
           numCells: cluster&.points,
-          numAnnotationValues: annotation[:values]&.size
+          numAnnotationValues: annotation[:values]&.size,
+          deType: params_object.de_type
         }
       )
+      if params_object.de_type == 'pairwise'
+        job_props.merge!( { pairwiseGroups: [params_object.group1, params_object.group2]})
+      end
     when :image_pipeline
       data_cache_perftime =  params_object.data_cache_perftime
       job_props.merge!(

config/routes.rb

@@ -87,6 +87,7 @@
             get 'studies/:accession', to: 'site#view_study', as: :site_study_view
             get 'studies/:accession/download', to: 'site#download_data', as: :site_study_download_data
             get 'studies/:accession/stream', to: 'site#stream_data', as: :site_study_stream_data
+            post 'studies/:accession/differential_expression', to: 'site#submit_differential_expression', as: :site_study_submit_differential_expression
             get 'studies/:accession/renew_read_only_access_token', to: 'site#renew_read_only_access_token', as: :site_renew_read_only_access_token
             get 'renew_user_access_token', to: 'site#renew_user_access_token', as: :site_renew_user_access_token
 

test/api/api_site_controller_test.rb

@@ -38,6 +38,7 @@ class ApiSiteControllerTest < ActionDispatch::IntegrationTest
   end
 
   teardown do
+    DifferentialExpressionResult.delete_all
     OmniAuth.config.mock_auth[:google_oauth2] = nil
     reset_user_tokens
     @study.update(public: true)
@@ -156,4 +157,133 @@ class ApiSiteControllerTest < ActionDispatch::IntegrationTest
                    "Did not return correct download url for external fastq; #{external_entry['download_url']} != #{external_sequence_file.human_fastq_url}"
     end
   end
+
+  test 'should submit differential expression request' do
+    user = FactoryBot.create(:api_user, test_array: @@users_to_clean)
+    study = FactoryBot.create(:detached_study, name_prefix: 'DiffExp Submit Test', user:, test_array: @@studies_to_clean)
+    cells = %w[A B C D E F G]
+    coordinates = 1.upto(7).to_a
+    species = %w[dog cat dog dog cat cat cat]
+    cell_types = ['B cell', 'T cell', 'B cell', 'T cell', 'T cell', 'B cell', 'B cell']
+    custom_cell_types = ['Custom 1', 'Custom 2', 'Custom 1', 'Custom 2', 'Custom 1', 'Custom 2', 'Custom 2']
+    raw_matrix = FactoryBot.create(
+      :expression_file, name: 'raw.txt', study:, expression_file_info: {
+      is_raw_counts: true, units: 'raw counts', library_preparation_protocol: 'Drop-seq',
+      biosample_input_type: 'Whole cell', modality: 'Proteomic' }
+    )
+    cluster_file = FactoryBot.create(:cluster_file,
+                                     name: 'umap',
+                                     study:,
+                                     cell_input: { x: coordinates, y: coordinates, cells: })
+    cluster_group = ClusterGroup.find_by(study:, study_file: cluster_file)
+
+    FactoryBot.create(
+      :metadata_file, name: 'metadata.txt', study:, cell_input: cells, annotation_input: [
+        { name: 'species', type: 'group', values: species },
+        { name: 'cell_type__ontology_label', type: 'group', values: cell_types },
+        { name: 'cell_type__custom', type: 'group', values: custom_cell_types }]
+    )
+
+    DifferentialExpressionResult.create(
+      study:, cluster_group:, annotation_name: 'species', annotation_scope: 'study', matrix_file_id: raw_matrix.id,
+      pairwise_comparisons: { dog: %w[cat]}
+    )
+    mock_not_detached study, :find_by do
+      sign_in_and_update user
+
+      # stub :raw_matrix_for_cluster_cells to avoid having to create cell arrays manually
+      ClusterVizService.stub :raw_matrix_for_cluster_cells, raw_matrix do
+        valid_params = [
+          {
+            cluster_name: 'umap', annotation_name: 'cell_type__ontology_label',
+            annotation_scope: 'study', de_type: 'rest'
+          },
+          {
+            cluster_name: 'umap', annotation_name: 'cell_type__ontology_label',
+            annotation_scope: 'study', de_type: 'pairwise', group1: 'B cell', group2: 'T cell'
+          }
+        ]
+        # test normal submission
+        valid_params.each do |job_params|
+          job_mock = Minitest::Mock.new
+          job_mock.expect :push_remote_and_launch_ingest, nil
+          delay_mock = Minitest::Mock.new
+          delay_mock.expect :delay, job_mock
+          IngestJob.stub :new, delay_mock do
+            execute_http_request(:post,
+                                 api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                                 request_payload: job_params,
+                                 user:)
+            assert_response 204
+            delay_mock.verify
+          end
+        end
+        # check for existing results
+        existing_params = {
+          cluster_name: 'umap', annotation_name: 'species',
+          annotation_scope: 'study', de_type: 'pairwise', group1: 'dog', group2: 'cat'
+        }
+        execute_http_request(:post,
+                             api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                             request_payload: existing_params,
+                             user:)
+        assert_response 409
+        # request parameter validations
+        execute_http_request(:post,
+                             api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                             request_payload: { cluster_name: 'foo'},
+                             user:)
+        assert_response :not_found
+        execute_http_request(:post,
+                             api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                             request_payload: {
+                               cluster_name: 'umap', annotation_name: 'foo', annotation_scope: 'study'
+                             },
+                             user:)
+        assert_response :not_found
+
+        execute_http_request(:post,
+                             api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                             request_payload: {
+                               cluster_name: 'umap', annotation_name: 'cell_type__ontology_label',
+                               annotation_scope: 'study', de_type: 'foo'
+                             },
+                             user:)
+        assert_response 422
+        execute_http_request(:post,
+                             api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                             request_payload: {
+                               cluster_name: 'umap', annotation_name: 'cell_type__ontology_label',
+                               annotation_scope: 'study', de_type: 'pairwise'
+                             },
+                             user:)
+        assert_response 422
+        # check for author results
+        study.differential_expression_results.delete_all
+        de_file = FactoryBot.create(:differential_expression_file,
+                                    study:,
+                                    name: 'user_de.txt',
+                                    annotation_name: 'cell_type__custom',
+                                    annotation_scope: 'study',
+                                    cluster_group:,
+                                    computational_method: 'custom'
+        )
+        author_result = DifferentialExpressionResult.create(
+          study:, cluster_group:, annotation_name: 'cell_type__custom', annotation_scope: 'study',
+          study_file: de_file, is_author_de: true, one_vs_rest_comparisons: ['Custom 1', 'Custom 2']
+        )
+        assert author_result.persisted?
+        params = {
+          cluster_name: 'umap', annotation_name: 'cell_type__ontology_label',
+          annotation_scope: 'study', de_type: 'rest'
+        }
+        execute_http_request(:post,
+                             api_v1_site_study_submit_differential_expression_path(accession: study.accession),
+                             request_payload: params,
+                             user:)
+        assert_response :forbidden
+        assert json['error'].starts_with? 'User requests are disabled'
+      end
+    end
+  end
 end