diff --git a/Gemfile.lock b/Gemfile.lock index 73a8d85db7..2f0d1db051 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -368,7 +368,7 @@ GEM puma (5.6.9) nio4r (~> 2.0) racc (1.8.1) - rack (2.2.11) + rack (2.2.13) rack-brotli (1.1.0) brotli (>= 0.1.7) rack (>= 1.4) diff --git a/app/controllers/analysis_configurations_controller.rb b/app/controllers/analysis_configurations_controller.rb index a070929853..e37ca2b986 100644 --- a/app/controllers/analysis_configurations_controller.rb +++ b/app/controllers/analysis_configurations_controller.rb @@ -3,7 +3,6 @@ class AnalysisConfigurationsController < ApplicationController :submission_preview, :load_study_for_submission_preview, :update_analysis_parameter] before_action :set_analysis_parameter, only: [:update_analysis_parameter] - before_action :check_firecloud_status, only: [:new, :create] before_action do authenticate_user! authenticate_admin @@ -177,16 +176,4 @@ def analysis_parameter_params :association_method, :association_data_type, :_destroy]) end - - def check_firecloud_status - unless ApplicationController.firecloud_client.services_available?(FireCloudClient::RAWLS_SERVICE) - alert = 'The Methods Repository is temporarily unavailable, so we cannot complete your request. Please try again later.' - respond_to do |format| - format.js {render js: "$('.modal').modal('hide'); alert('#{alert}')" and return} - format.html {redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]), - alert: alert and return} - format.json {head 503} - end - end - end end diff --git a/app/controllers/api/v1/site_controller.rb b/app/controllers/api/v1/site_controller.rb index ea231bc80b..24bb971c04 100644 --- a/app/controllers/api/v1/site_controller.rb +++ b/app/controllers/api/v1/site_controller.rb @@ -2,8 +2,8 @@ module Api module V1 class SiteController < ApiBaseController before_action :set_current_api_user! - before_action :authenticate_api_user!, only: [:download_data, :stream_data, :get_study_analysis_config, - :submit_study_analysis, :get_study_submissions, + before_action :authenticate_api_user!, only: [:download_data, :stream_data, :submit_differential_expression, + :get_study_analysis_config, :submit_study_analysis, :get_study_submissions, :get_study_submission, :sync_submission_outputs] before_action :set_study, except: [:studies, :check_terra_tos_acceptance, :analyses, :get_analysis, :renew_user_access_token] before_action :set_analysis_configuration, only: [:get_analysis, :get_study_analysis_config] @@ -403,6 +403,151 @@ def stream_data end end + swagger_path '/site/studies/{accession}/differential_expression' do + operation :post do + key :tags, [ + 'Site' + ] + key :summary, 'Submit a differential expression calculation request' + key :description, 'Request differential expression calculations for a given cluster/annotation in a study' + key :operationId, 'site_study_submit_differential_expression_path' + parameter do + key :name, :accession + key :in, :path + key :description, 'Accession of Study to use' + key :required, true + key :type, :string + end + parameter do + key :name, :de_job + key :type, :object + key :in, :body + schema do + property :cluster_name do + key :description, 'Name of cluster group. Use "_default" to use the default cluster' + key :required, true + key :type, :string + end + property :annotation_name do + key :description, 'Name of annotation' + key :required, true + key :type, :string + end + property :annotation_scope do + key :description, 'Scope of annotation. One of "study" or "cluster".' + key :type, :string + key :required, true + key :enum, Api::V1::Visualization::AnnotationsController::VALID_SCOPE_VALUES + end + property :de_type do + key :description, 'Type of differential expression analysis. Either "rest" (one-vs-rest) or "pairwise"' + key :type, :string + key :required, true + key :enum, %w[rest pairwise] + end + property :group1 do + key :description, 'First group for pairwise analysis (optional)' + key :type, :string + end + property :group2 do + key :description, 'Second group for pairwise analysis (optional)' + key :type, :string + end + end + end + response 204 do + key :description, 'Job successfully submitted' + end + response 401 do + key :description, ApiBaseController.unauthorized + end + response 403 do + key :description, ApiBaseController.forbidden('view study, study has author DE') + end + response 404 do + key :description, ApiBaseController.not_found(Study, 'Cluster', 'Annotation') + end + response 406 do + key :description, ApiBaseController.not_acceptable + end + response 409 do + key :description, "Results are processing or already exist" + end + response 410 do + key :description, ApiBaseController.resource_gone + end + response 422 do + key :description, "Job parameters failed validation" + end + response 429 do + key :description, 'Weekly user quota exceeded' + end + end + end + + def submit_differential_expression + # disallow DE calculation requests for studies with author DE + if @study.differential_expression_results.where(is_author_de: true).any? + render json: { + error: 'User requests are disabled for this study as it contains author-supplied differential expression results' + }, status: 403 and return + end + + # check user quota before proceeding + if DifferentialExpressionService.job_exceeds_quota?(current_api_user) + # minimal log props to help gauge overall user interest, as well as annotation/de types + log_props = { + studyAccession: @study.accession, annotationName: params[:annotation_name], de_type: params[:de_type] + } + MetricsService.log('quota-exceeded-de', log_props, current_api_user, request:) + current_quota = DifferentialExpressionService.get_weekly_user_quota + render json: { error: "You have exceeded your weekly quota of #{current_quota} requests" }, + status: 429 and return + end + + cluster_name = params[:cluster_name] + cluster = cluster_name == '_default' ? @study.default_cluster : @study.cluster_groups.by_name(cluster_name) + render json: { error: "Requested cluster #{cluster_name} not found" }, status: 404 and return if cluster.nil? + + annotation_name = params[:annotation_name] + annotation_scope = params[:annotation_scope] + de_type = params[:de_type] + pairwise = de_type == 'pairwise' + group1 = params[:group1] + group2 = params[:group2] + annotation = AnnotationVizService.get_selected_annotation( + @study, cluster:, annot_name: annotation_name, annot_type: 'group', annot_scope: annotation_scope + ) + render json: { error: 'No matching annotation found' }, status: 404 and return if annotation.nil? + + de_params = { annotation_name:, annotation_scope:, de_type:, group1:, group2: } + + # check if these results already exist + # for pairwise, also check if requested comparisons exist + result = DifferentialExpressionResult.find_by( + study: @study, cluster_group: cluster, annotation_name:, annotation_scope:, is_author_de: false + ) + if result && (!pairwise || (pairwise && result.has_pairwise_comparison?(group1, group2))) + render json: { error: "Requested results already exist" }, status: 409 and return + end + + begin + submitted = DifferentialExpressionService.run_differential_expression_job( + cluster, @study, current_api_user, **de_params + ) + if submitted + DifferentialExpressionService.increment_user_quota(current_api_user) + head 204 + else + # submitted: false here means that there is a matching running DE job + render json: { error: "Requested results are processing - please check back later" }, status: 409 + end + rescue ArgumentError => e + # job parameters failed to validate + render json: { error: e.message}, status: 422 and return + end + end + swagger_path '/site/analyses' do operation :get do key :tags, [ @@ -1136,6 +1281,10 @@ def get_download_quota end end + def de_job_params + params.require(:de_job).permit(:cluster_name, :annotation_name, :annotation_scope, :de_type, :group1, :group2) + end + # update AnalysisSubmissions when loading study analysis tab # will not backfill existing workflows to keep our submission history clean def update_analysis_submission(submission) diff --git a/app/controllers/api/v1/studies_controller.rb b/app/controllers/api/v1/studies_controller.rb index bb3cdea7e9..04bc39ca63 100644 --- a/app/controllers/api/v1/studies_controller.rb +++ b/app/controllers/api/v1/studies_controller.rb @@ -1,8 +1,6 @@ module Api module V1 class StudiesController < ApiBaseController - include Concerns::FireCloudStatus - def firecloud_independent_methods # add file_info is essentially a more extensive 'show' method [:index, :show, :file_info] diff --git a/app/controllers/api/v1/study_files_controller.rb b/app/controllers/api/v1/study_files_controller.rb index 027567b089..3035c91f6c 100644 --- a/app/controllers/api/v1/study_files_controller.rb +++ b/app/controllers/api/v1/study_files_controller.rb @@ -1,8 +1,6 @@ module Api module V1 class StudyFilesController < ApiBaseController - include Concerns::FireCloudStatus - before_action :authenticate_api_user! before_action :set_study before_action :check_study_edit_permission diff --git a/app/controllers/api/v1/study_shares_controller.rb b/app/controllers/api/v1/study_shares_controller.rb index f4ea294dce..7d5c5cd00e 100644 --- a/app/controllers/api/v1/study_shares_controller.rb +++ b/app/controllers/api/v1/study_shares_controller.rb @@ -1,8 +1,6 @@ module Api module V1 class StudySharesController < ApiBaseController - include Concerns::FireCloudStatus - before_action :authenticate_api_user! before_action :set_study before_action :check_study_permission diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 3c5e2faefb..d6f2fa8f65 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -264,7 +264,7 @@ def verify_file_download_permissions(study) # next check if downloads have been disabled by administrator, this will abort the download # download links shouldn't be rendered in any case, this just catches someone doing a straight GET on a file # also check if workspace google buckets are available - if !AdminConfiguration.firecloud_access_enabled? || !ApplicationController.firecloud_client.services_available?(FireCloudClient::BUCKETS_SERVICE) + if !AdminConfiguration.firecloud_access_enabled? head 503 and return end end diff --git a/app/controllers/billing_projects_controller.rb b/app/controllers/billing_projects_controller.rb index ee372c3175..44c3d53ae2 100644 --- a/app/controllers/billing_projects_controller.rb +++ b/app/controllers/billing_projects_controller.rb @@ -18,7 +18,6 @@ class BillingProjectsController < ApplicationController respond_to :html, :js, :json before_action :authenticate_user! before_action :check_firecloud_registration, except: :access_request - before_action :check_firecloud_status, except: :access_request before_action :create_firecloud_client, except: :access_request before_action :check_project_permissions, except: [:index, :create, :access_request] before_action :load_service_account, except: [:new_user, :create_user, :delete_user, :storage_estimate, :access_request] @@ -214,18 +213,5 @@ def check_project_permissions alert: "You do not have permission to perform that action. #{SCP_SUPPORT_EMAIL}" and return end end - - # check on FireCloud API status and respond accordingly - def check_firecloud_status - unless ApplicationController.firecloud_client.services_available?(FireCloudClient::THURLOE_SERVICE) - alert = "Billing projects are temporarily unavailable, so we cannot complete your request. Please try again later. #{SCP_SUPPORT_EMAIL}" - respond_to do |format| - format.js {render js: "$('.modal').modal('hide'); alert('#{alert}')" and return} - format.html {redirect_to merge_default_redirect_params(site_path, scpbr: params[:scpbr]), - alert: alert and return} - format.json {head 503} - end - end - end end diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb index 31cdaee42d..b36666a0a8 100644 --- a/app/controllers/profiles_controller.rb +++ b/app/controllers/profiles_controller.rb @@ -19,22 +19,18 @@ def show @study_shares = StudyShare.where(email: @user.email) @studies = Study.where(user_id: @user.id) @fire_cloud_profile = FireCloudProfile.new - # check if profile services are available - @profiles_available = ApplicationController.firecloud_client.services_available?(FireCloudClient::THURLOE_SERVICE) - if @profiles_available - begin - user_client = FireCloudClient.new(user: current_user, project: FireCloudClient::PORTAL_NAMESPACE) - profile = user_client.get_profile - profile['keyValuePairs'].each do |attribute| - if @fire_cloud_profile.respond_to?("#{attribute['key']}=") - @fire_cloud_profile.send("#{attribute['key']}=", attribute['value']) - end + begin + user_client = FireCloudClient.new(user: current_user, project: FireCloudClient::PORTAL_NAMESPACE) + profile = user_client.get_profile + profile['keyValuePairs'].each do |attribute| + if @fire_cloud_profile.respond_to?("#{attribute['key']}=") + @fire_cloud_profile.send("#{attribute['key']}=", attribute['value']) end - rescue => e - ErrorTracker.report_exception(e, current_user, params) - MetricsService.report_error(e, request, current_user) - logger.info "#{Time.zone.now}: unable to retrieve FireCloud profile for #{current_user.email}: #{e.message}" end + rescue => e + ErrorTracker.report_exception(e, current_user, params) + MetricsService.report_error(e, request, current_user) + logger.info "#{Time.zone.now}: unable to retrieve FireCloud profile for #{current_user.email}: #{e.message}" end end diff --git a/app/controllers/site_controller.rb b/app/controllers/site_controller.rb index 450d92554f..bad75ccde2 100644 --- a/app/controllers/site_controller.rb +++ b/app/controllers/site_controller.rb @@ -118,12 +118,7 @@ def update_study_settings @cluster_annotations = ClusterVizService.load_cluster_group_annotations(@study, @cluster, current_user) set_selected_annotation end - - # double check on download availability: first, check if administrator has disabled downloads - # then check if FireCloud is available and disable download links if either is true - @allow_downloads = ApplicationController.firecloud_client.services_available?(FireCloudClient::BUCKETS_SERVICE) end - set_firecloud_permissions(@study.detached?) set_study_permissions(@study.detached?) set_study_default_options set_study_download_options @@ -156,7 +151,6 @@ def study # double check on download availability: first, check if administrator has disabled downloads # then check individual statuses to see what to enable/disable # if the study is 'detached', then everything is set to false by default - set_firecloud_permissions(@study.detached?) set_study_permissions(@study.detached?) set_study_default_options set_study_download_options @@ -652,46 +646,19 @@ def set_workspace_samples end end - # check various firecloud statuses/permissions, but only if a study is not 'detached' - def set_firecloud_permissions(study_detached) - @allow_firecloud_access = false - @allow_downloads = false - @allow_edits = false - return if study_detached - begin - @allow_firecloud_access = AdminConfiguration.firecloud_access_enabled? - api_status = ApplicationController.firecloud_client.api_status - # reuse status object because firecloud_client.services_available? each makes a separate status call - # calling Hash#dig will gracefully handle any key lookup errors in case of a larger outage - if api_status.is_a?(Hash) - system_status = api_status['systems'] - sam_ok = system_status.dig(FireCloudClient::SAM_SERVICE, 'ok') == true # do equality check in case 'ok' node isn't present - rawls_ok = system_status.dig(FireCloudClient::RAWLS_SERVICE, 'ok') == true - buckets_ok = system_status.dig(FireCloudClient::BUCKETS_SERVICE, 'ok') == true - @allow_downloads = buckets_ok - @allow_edits = sam_ok && rawls_ok - end - rescue => e - logger.error "Error checking FireCloud API status: #{e.class.name} -- #{e.message}" - ErrorTracker.report_exception(e, current_user, @study, { firecloud_status: api_status}) - MetricsService.report_error(e, request, current_user, @study) - end - end - # set various study permissions based on the results of the above FC permissions def set_study_permissions(study_detached) @user_can_edit = false @user_can_compute = false @user_can_download = false @user_embargoed = false + @allow_firecloud_access = AdminConfiguration.firecloud_access_enabled? return if study_detached || !@allow_firecloud_access begin @user_can_edit = @study.can_edit?(current_user) - if @allow_downloads - @user_can_download = @user_can_edit ? true : @study.can_download?(current_user) - @user_embargoed = @user_can_edit ? false : @study.embargoed?(current_user) - end + @user_can_download = @user_can_edit ? true : @study.can_download?(current_user) + @user_embargoed = @user_can_edit ? false : @study.embargoed?(current_user) rescue => e logger.error "Error setting study permissions: #{e.class.name} -- #{e.message}" ErrorTracker.report_exception(e, current_user, @study) @@ -781,21 +748,12 @@ def check_view_permissions # check compute permissions for study def check_compute_permissions - if ApplicationController.firecloud_client.services_available?(FireCloudClient::SAM_SERVICE, FireCloudClient::RAWLS_SERVICE) - if !user_signed_in? || !@study.can_compute?(current_user) - @alert = "You do not have permission to perform that action. #{SCP_SUPPORT_EMAIL}" - respond_to do |format| - format.js {render action: :notice} - format.html {redirect_to merge_default_redirect_params(site_path, scpbr: params[:scpbr]), alert: @alert and return} - format.json {head 403} - end - end - else - @alert = "Compute services are currently unavailable - please check back later. #{SCP_SUPPORT_EMAIL}" + if !user_signed_in? || !@study.can_compute?(current_user) + @alert = "You do not have permission to perform that action. #{SCP_SUPPORT_EMAIL}" respond_to do |format| format.js {render action: :notice} format.html {redirect_to merge_default_redirect_params(site_path, scpbr: params[:scpbr]), alert: @alert and return} - format.json {head 503} + format.json {head 403} end end end diff --git a/app/controllers/studies_controller.rb b/app/controllers/studies_controller.rb index 89399f7ddc..cc1c69cc02 100644 --- a/app/controllers/studies_controller.rb +++ b/app/controllers/studies_controller.rb @@ -22,9 +22,6 @@ class StudiesController < ApplicationController authenticate_user! check_access_settings end - # special before_action to make sure FireCloud is available and pre-empt any calls when down - before_action :check_firecloud_status, except: [:index, :do_upload, :resume_upload, :update_status, - :retrieve_wizard_upload, :parse] before_action :check_study_detached, only: [:edit, :update, :initialize_study, :sync_study, :sync_submission_outputs] helper_method :visible_unsynced_files, :hidden_unsynced_files ### @@ -46,7 +43,7 @@ def show @directories = @study.directory_listings.are_synced @primary_data = @study.directory_listings.primary_data @other_data = @study.directory_listings.non_primary_data - @allow_downloads = ApplicationController.firecloud_client.services_available?(FireCloudClient::BUCKETS_SERVICE) && !@study.detached + @allow_downloads = !@study.detached @analysis_metadata = @study.analysis_metadata.to_a # load study default options set_study_default_options @@ -1146,19 +1143,6 @@ def check_edit_permissions end end - # check on FireCloud API status and respond accordingly - def check_firecloud_status - unless ApplicationController.firecloud_client.services_available?(FireCloudClient::SAM_SERVICE, FireCloudClient::RAWLS_SERVICE) - alert = "Study workspaces are temporarily unavailable, so we cannot complete your request. Please try again later. #{SCP_SUPPORT_EMAIL}" - respond_to do |format| - format.js {render js: "$('.modal').modal('hide'); alert('#{alert}')" and return} - format.html {redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]), - alert: alert and return} - format.json {head 503} - end - end - end - # check if a study is 'detached' and handle accordingly def check_study_detached if @study.detached? diff --git a/app/lib/differential_expression_service.rb b/app/lib/differential_expression_service.rb index d8003d7697..2296c7a8c8 100644 --- a/app/lib/differential_expression_service.rb +++ b/app/lib/differential_expression_service.rb @@ -9,6 +9,8 @@ class DifferentialExpressionService ALLOWED_ANNOTS = Regexp.union(CELL_TYPE_MATCHER, CLUSTERING_MATCHER) # specific annotations to exclude from automation EXCLUDED_ANNOTS = /(enrichment__cell_type)/i + # default quota for user-requested DE jobs + DEFAULT_USER_QUOTA = 5 # run a differential expression job for a given study on the default cluster/annotation # @@ -163,25 +165,27 @@ def self.run_differential_expression_job(cluster_group, study, user, annotation_ params_object.machine_type = machine_type if machine_type.present? # override :machine_type if specified return true if dry_run # exit before submission if specified as annotation was already validated - # check if there's already a job running using these parameters and exit if so - job_params = ApplicationController.batch_api_client.format_command_line( - study_file:, action: :differential_expression, user_metrics_uuid: user.metrics_uuid, params_object: - ) - running = ApplicationController.batch_api_client.find_matching_jobs( - params: job_params, job_states: BatchApiClient::RUNNING_STATES - ) - if running.any? - log_message "Found #{running.count} running DE jobs: #{running.map(&:name).join(', ')}" - log_message "Matching these parameters: #{job_params.join(' ')}" - log_message "Exiting without queuing new job" - false - elsif params_object.valid? - # launch DE job - job = IngestJob.new(study:, study_file:, user:, action: :differential_expression, params_object:) - job.delay.push_remote_and_launch_ingest - true + if params_object.valid? + # check if there's already a job running using these parameters and exit if so + job_params = ApplicationController.batch_api_client.format_command_line( + study_file:, action: :differential_expression, user_metrics_uuid: user.metrics_uuid, params_object: + ) + running = ApplicationController.batch_api_client.find_matching_jobs( + params: job_params, job_states: BatchApiClient::RUNNING_STATES + ) + if running.any? + log_message "Found #{running.count} running DE jobs: #{running.map(&:name).join(', ')}" + log_message "Matching these parameters: #{job_params.join(' ')}" + log_message "Exiting without queuing new job" + false + else + # launch DE job + job = IngestJob.new(study:, study_file:, user:, action: :differential_expression, params_object:) + job.delay.push_remote_and_launch_ingest + true + end else - raise ArgumentError, "job parameters failed to validate: #{params_object.errors.full_messages}" + raise ArgumentError, "job parameters failed to validate: #{params_object.errors.full_messages.join(', ')}" end end @@ -377,12 +381,14 @@ def self.validate_annotation(cluster_group, study, annotation_name, annotation_s if !pairwise && cells_by_label.keys.count < 2 raise ArgumentError, "#{identifier} does not have enough labels represented in #{cluster_group.name}" elsif pairwise - missing = { + missing = [group1, group2] - annotation[:values] + raise ArgumentError, "#{annotation_name} does not contain '#{missing.join(', ')}'" if missing.any? + cell_count = { "#{group1}" => cells_by_label[group1].count, "#{group2}" => cells_by_label[group2].count }.keep_if { |_, c| c < 2 } raise ArgumentError, - "#{missing.keys.join(', ')} does not have enough cells represented in #{identifier}" if missing.any? + "#{cell_count.keys.join(', ')} does not have enough cells represented in #{identifier}" if cell_count.any? end end @@ -451,4 +457,39 @@ def self.cluster_file_url(cluster_group) study_file.gs_url end end + + # retrieve the weekly user quota value + # + # * *returns* + # - (Integer) + def self.get_weekly_user_quota + config = AdminConfiguration.find_by(config_type: 'Weekly User DE Quota') + config ? config.value.to_i : DEFAULT_USER_QUOTA + end + + # check if a user has exceeded their weekly quota + # + # * *params* + # - +user+ (User) => user requesting DE job + # + # * *returns* + # - (Boolean) + def self.job_exceeds_quota?(user) + user.weekly_de_quota.to_i >= get_weekly_user_quota + end + + # increment a given user's weekly quota + # + # * *params* + # - +user+ (User) => user requesting DE job + def self.increment_user_quota(user) + current_quota = user.weekly_de_quota.to_i + current_quota += 1 + user.update(weekly_de_quota: current_quota) + end + + # reset all user-requested DE quotas on a weekly basis + def self.reset_all_user_quotas + User.update_all(weekly_de_quota: 0) + end end diff --git a/app/models/admin_configuration.rb b/app/models/admin_configuration.rb index bdb399d2d6..56d1f47432 100644 --- a/app/models/admin_configuration.rb +++ b/app/models/admin_configuration.rb @@ -20,8 +20,9 @@ class AdminConfiguration API_NOTIFIER_NAME = 'API Health Check Notifier'.freeze INGEST_DOCKER_NAME = 'Ingest Pipeline Docker Image'.freeze NUMERIC_VALS = %w[byte kilobyte megabyte terabyte petabyte exabyte].freeze - CONFIG_TYPES = [INGEST_DOCKER_NAME, 'Daily User Download Quota', 'Portal FireCloud User Group', 'TDR Snapshot IDs', - 'Reference Data Workspace', 'Read-Only Access Control', 'QA Dev Email', API_NOTIFIER_NAME].freeze + CONFIG_TYPES = [INGEST_DOCKER_NAME, 'Daily User Download Quota', 'Weekly User DE Quota', 'Portal FireCloud User Group', + 'TDR Snapshot IDs', 'Reference Data Workspace', 'Read-Only Access Control', 'QA Dev Email', + API_NOTIFIER_NAME].freeze ALL_CONFIG_TYPES = (CONFIG_TYPES.dup << FIRECLOUD_ACCESS_NAME).freeze VALUE_TYPES = %w[Numeric Boolean String].freeze diff --git a/app/models/import_service_config/nemo.rb b/app/models/import_service_config/nemo.rb index 97d4e7a4ba..b32274169f 100644 --- a/app/models/import_service_config/nemo.rb +++ b/app/models/import_service_config/nemo.rb @@ -58,7 +58,7 @@ def association_order # return hash of file access info, like url, sizes, etc def file_access_info(protocol: nil) - urls = load_file&.[]('urls') + urls = load_file&.[]('manifest_file_urls') protocol ? urls.detect { |url| url.with_indifferent_access[:protocol] == protocol.to_s } : urls end @@ -90,7 +90,7 @@ def study_file_mappings upload_file_size: :size, name: :file_name, expression_file_info: { - library_preparation_protocol: :technique + library_preparation_protocol: :techniques } }.with_indifferent_access end @@ -108,9 +108,10 @@ def populate_study_file(scp_study_id) study_file = to_study_file(scp_study_id, preferred_name) library = study_file.expression_file_info.library_preparation_protocol if library.blank? - library = load_study&.[]('technique') + libs = load_study&.[]('techniques') + libraries = libs.map { |lib| find_library_prep(lib) }.compact + study_file.expression_file_info.library_preparation_protocol = libraries.first end - study_file.expression_file_info.library_preparation_protocol = find_library_prep(library) exp_fragment = expression_data_fragment(study_file) study_file.ann_data_file_info.data_fragments << exp_fragment http_url = file_access_info(protocol: :http)&.[]('url') diff --git a/app/models/ingest_job.rb b/app/models/ingest_job.rb index 49bc9ecfee..c1a44e79ed 100644 --- a/app/models/ingest_job.rb +++ b/app/models/ingest_job.rb @@ -382,10 +382,8 @@ def poll_for_completion(run_at: 1.minute.from_now) if done? && !failed? Rails.logger.info "IngestJob poller: #{pipeline_name} is done!" Rails.logger.info "IngestJob poller: #{pipeline_name} status: #{current_status}" - unless special_action? || (action == :ingest_anndata && study_file.is_viz_anndata?) - study_file.update(parse_status: 'parsed') - study_file.bundled_files.each { |sf| sf.update(parse_status: 'parsed') } - end + study_file.update(parse_status: 'parsed') + study_file.bundled_files.each { |sf| sf.update(parse_status: 'parsed') } study.reload # refresh cached instance of study study_file.reload # refresh cached instance of study_file # check if another process marked file for deletion, can happen if this is an AnnData file @@ -412,6 +410,7 @@ def poll_for_completion(run_at: 1.minute.from_now) end elsif done? && failed? Rails.logger.error "IngestJob poller: #{pipeline_name} has failed." + study_file.update(parse_status: 'parsed') # log errors to application log for inspection log_error_messages log_to_mixpanel # log before queuing file for deletion to preserve properties @@ -515,9 +514,9 @@ def set_study_state_after_ingest when :image_pipeline set_has_image_cache when :ingest_anndata + set_anndata_file_info launch_anndata_subparse_jobs if study_file.is_viz_anndata? launch_differential_expression_jobs if study_file.is_viz_anndata? - set_anndata_file_info end set_study_initialized end @@ -708,6 +707,7 @@ def launch_subsample_jobs submission = ApplicationController.batch_api_client.run_job( study_file:, user:, action: :ingest_subsample, params_object: subsample_params ) + study_file.update(parse_status: 'parsing') IngestJob.new( pipeline_name: submission.name, study:, study_file:, user:, action: :ingest_subsample, params_object: subsample_params, reparse:, persist_on_fail: @@ -741,10 +741,10 @@ def create_differential_expression_results Rails.logger.info "Creating differential expression result object for annotation: #{annotation_identifier}" cluster = ClusterGroup.find_by(study_id: study.id, study_file_id: study_file.id, name: params_object.cluster_name) matrix_url = params_object.matrix_file_path - matrix_filename = matrix_url.split("gs://#{study.bucket_id}/").last - matrix_file = StudyFile.where(study: study, 'expression_file_info.is_raw_counts' => true).any_of( - { upload_file_name: matrix_filename }, { remote_location: matrix_filename } - ).first + bucket_path = matrix_url.split("gs://#{study.bucket_id}/").last + matrix_file = StudyFile.where( + study:, :upload_file_name.in => [bucket_path, bucket_path.split('/').last] + ).detect(&:is_raw_counts_file?) de_result = DifferentialExpressionResult.find_or_initialize_by( study: study, cluster_group: cluster, cluster_name: cluster.name, annotation_name: params_object.annotation_name, annotation_scope: params_object.annotation_scope, @@ -819,6 +819,7 @@ def launch_anndata_subparse_jobs # reference AnnData uploads don't have extract parameter so exit immediately return if params_object.extract.blank? + study_file.update(parse_status: 'parsing') params_object.extract.each do |extract| case extract when 'cluster' @@ -862,14 +863,8 @@ def launch_anndata_subparse_jobs job.delay.push_remote_and_launch_ingest end - # if this was only a raw counts extraction, update parse status - if params_object.extract == %w[raw_counts] - study_file.update(parse_status: 'parsed') - launch_differential_expression_jobs - else - # unset anndata_summary flag to allow reporting summary later unless this is only a raw counts extraction - study_file.unset_anndata_summary! - end + # unset anndata_summary flag to allow reporting summary later unless this is only a raw counts extraction + study_file.unset_anndata_summary! unless params_object.extract == %w[raw_counts] end end @@ -1016,9 +1011,13 @@ def get_job_analytics job_props.merge!( { numCells: cluster&.points, - numAnnotationValues: annotation[:values]&.size + numAnnotationValues: annotation[:values]&.size, + deType: params_object.de_type } ) + if params_object.de_type == 'pairwise' + job_props.merge!( { pairwiseGroups: [params_object.group1, params_object.group2]}) + end when :image_pipeline data_cache_perftime = params_object.data_cache_perftime job_props.merge!( diff --git a/app/models/user.rb b/app/models/user.rb index c8d07ea5aa..3a0d42831b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -87,6 +87,7 @@ def owned_by(user) field :admin, type: Boolean field :reporter, type: Boolean field :daily_download_quota, type: Integer, default: 0 + field :weekly_de_quota, type: Integer, default: 0 # limits number of user-requested DE jobs field :admin_email_delivery, type: Boolean, default: true field :registered_for_firecloud, type: Boolean, default: false # { diff --git a/app/views/profiles/show.html.erb b/app/views/profiles/show.html.erb index fcbc701a0f..56a4d79129 100644 --- a/app/views/profiles/show.html.erb +++ b/app/views/profiles/show.html.erb @@ -7,16 +7,9 @@ - <% if @profiles_available %> - - <% else %> - - <% end %> +
@@ -144,28 +137,26 @@
- <% if @profiles_available %> -
- <% if current_user.registered_for_firecloud %> - <%= render partial: 'user_firecloud_profile' %> - <% else %> -
-
-

Please complete your Terra registration

-

- You may not update your Terra profile until you have registered with Terra and accepted the terms of service. - Please <%= link_to 'visit Terra', 'https://app.terra.bio', target: :_blank, rel: 'noopener noreferrer' %>, - select 'Sign in with Google' from the top-lefthand nav menu, and complete the sign in and registration process. -

-

- <%= link_to "Complete Registration Now ".html_safe, 'https://app.terra.bio', - target: :_blank, rel: 'noopener noreferrer', class: 'btn btn-lg btn-default' %> -

-
+
+ <% if current_user.registered_for_firecloud %> + <%= render partial: 'user_firecloud_profile' %> + <% else %> +
+
+

Please complete your Terra registration

+

+ You may not update your Terra profile until you have registered with Terra and accepted the terms of service. + Please <%= link_to 'visit Terra', 'https://app.terra.bio', target: :_blank, rel: 'noopener noreferrer' %>, + select 'Sign in with Google' from the top-lefthand nav menu, and complete the sign in and registration process. +

+

+ <%= link_to "Complete Registration Now ".html_safe, 'https://app.terra.bio', + target: :_blank, rel: 'noopener noreferrer', class: 'btn btn-lg btn-default' %> +

- <% end %> -
- <% end %> +
+ <% end %> +
diff --git a/app/views/site/_study_download_data.html.erb b/app/views/site/_study_download_data.html.erb index 02bc17346f..7d072daf19 100644 --- a/app/views/site/_study_download_data.html.erb +++ b/app/views/site/_study_download_data.html.erb @@ -4,10 +4,9 @@

Study Files <%= @study_files.count %> - <%= link_to " Bulk download".html_safe, '#/', class: "btn btn-default pull-right #{!@allow_downloads ? 'disabled' : nil}", id: 'download-help' %> + <%= link_to " Bulk download".html_safe, '#/', class: "btn btn-default pull-right", id: 'download-help' %>

- <% if @allow_downloads %>