refactor sheerid webhook code, do not trigger on confirmed

app/models/security_log.rb

@@ -107,6 +107,10 @@ class SecurityLog < ApplicationRecord
     attempted_to_add_school_not_cached_yet
     school_added_to_user_from_sheerid_webhook
     user_lead_id_updated_from_salesforce
+    sheerid_webhook_ignored
+    sheerid_api_call_failed
+    sheerid_webhook_duplicate_ignored
+    sheerid_webhook_doc_upload_required
   ]
 
   json_serialize :event_data, Hash

app/routines/update_user_contact_info.rb

@@ -66,6 +66,8 @@ def call
                                 :rejected_faculty
                               when "rejected_by_sheerid"
                                 :rejected_by_sheerid
+                              when "pending_sheerid"
+                                :pending_sheerid
                               when "incomplete_signup"
                                 :incomplete_signup
                               when "no_faculty_info"

db/migrate/20250814215045_add_enhanced_fields_to_sheerid_verifications.rb

@@ -0,0 +1,26 @@
+class AddEnhancedFieldsToSheeridVerifications < ActiveRecord::Migration[6.1]
+  def change
+    add_column :sheerid_verifications, :program_id, :string, if_not_exists: true
+    add_column :sheerid_verifications, :segment, :string, if_not_exists: true
+    add_column :sheerid_verifications, :sub_segment, :string, if_not_exists: true
+    add_column :sheerid_verifications, :locale, :string, if_not_exists: true
+    add_column :sheerid_verifications, :reward_code, :string, if_not_exists: true
+    add_column :sheerid_verifications, :organization_id, :string, if_not_exists: true
+    add_column :sheerid_verifications, :postal_code, :string, if_not_exists: true
+    add_column :sheerid_verifications, :country, :string, if_not_exists: true
+    add_column :sheerid_verifications, :phone_number, :string, if_not_exists: true
+    add_column :sheerid_verifications, :birth_date, :string, if_not_exists: true
+    add_column :sheerid_verifications, :ip_address, :string, if_not_exists: true
+    add_column :sheerid_verifications, :device_fingerprint_hash, :string, if_not_exists: true
+    add_column :sheerid_verifications, :doc_upload_rejection_count, :integer, default: 0, if_not_exists: true
+    add_column :sheerid_verifications, :doc_upload_rejection_reasons, :text, array: true, default: [], if_not_exists: true
+    add_column :sheerid_verifications, :error_ids, :text, array: true, default: [], if_not_exists: true
+    add_column :sheerid_verifications, :metadata, :jsonb, default: {}, if_not_exists: true
+
+    add_index :sheerid_verifications, :program_id, if_not_exists: true
+    add_index :sheerid_verifications, :segment, if_not_exists: true
+    add_index :sheerid_verifications, :organization_id, if_not_exists: true
+    add_index :sheerid_verifications, :error_ids, using: :gin, if_not_exists: true
+    add_index :sheerid_verifications, :metadata, using: :gin, if_not_exists: true
+  end
+end

db/migrate/20250814215101_add_sheerid_metadata_to_users.rb

@@ -0,0 +1,23 @@
+class AddSheeridMetadataToUsers < ActiveRecord::Migration[6.1]
+  def change
+    add_column :users, :sheerid_program_id, :string, if_not_exists: true
+    add_column :users, :sheerid_segment, :string, if_not_exists: true
+    add_column :users, :sheerid_organization_id, :string, if_not_exists: true
+    add_column :users, :sheerid_postal_code, :string, if_not_exists: true
+    add_column :users, :sheerid_country, :string, if_not_exists: true
+    add_column :users, :sheerid_phone_number, :string, if_not_exists: true
+    add_column :users, :sheerid_birth_date, :string, if_not_exists: true
+    add_column :users, :sheerid_ip_address, :string, if_not_exists: true
+    add_column :users, :sheerid_device_fingerprint_hash, :string, if_not_exists: true
+    add_column :users, :sheerid_doc_upload_rejection_count, :integer, default: 0, if_not_exists: true
+    add_column :users, :sheerid_doc_upload_rejection_reasons, :text, array: true, default: [], if_not_exists: true
+    add_column :users, :sheerid_error_ids, :text, array: true, default: [], if_not_exists: true
+    add_column :users, :sheerid_metadata, :jsonb, default: {}, if_not_exists: true
+
+    add_index :users, :sheerid_program_id, if_not_exists: true
+    add_index :users, :sheerid_segment, if_not_exists: true
+    add_index :users, :sheerid_organization_id, if_not_exists: true
+    add_index :users, :sheerid_error_ids, using: :gin, if_not_exists: true
+    add_index :users, :sheerid_metadata, using: :gin, if_not_exists: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2024_11_07_193019) do
+ActiveRecord::Schema.define(version: 2025_08_14_215101) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "citext"
@@ -409,6 +409,27 @@
     t.string "organization_name"
     t.datetime "created_at", default: -> { "CURRENT_TIMESTAMP" }, null: false
     t.datetime "updated_at", default: -> { "CURRENT_TIMESTAMP" }, null: false
+    t.string "program_id"
+    t.string "segment"
+    t.string "sub_segment"
+    t.string "locale"
+    t.string "reward_code"
+    t.string "organization_id"
+    t.string "postal_code"
+    t.string "country"
+    t.string "phone_number"
+    t.string "birth_date"
+    t.string "ip_address"
+    t.string "device_fingerprint_hash"
+    t.integer "doc_upload_rejection_count", default: 0
+    t.text "doc_upload_rejection_reasons", default: [], array: true
+    t.text "error_ids", default: [], array: true
+    t.jsonb "metadata", default: {}
+    t.index ["error_ids"], name: "index_sheerid_verifications_on_error_ids", using: :gin
+    t.index ["metadata"], name: "index_sheerid_verifications_on_metadata", using: :gin
+    t.index ["organization_id"], name: "index_sheerid_verifications_on_organization_id"
+    t.index ["program_id"], name: "index_sheerid_verifications_on_program_id"
+    t.index ["segment"], name: "index_sheerid_verifications_on_segment"
   end
 
   create_table "user_external_uuids", id: :serial, force: :cascade do |t|
@@ -473,6 +494,19 @@
     t.string "adopter_status"
     t.jsonb "consent_preferences"
     t.boolean "is_deleted"
+    t.string "sheerid_program_id"
+    t.string "sheerid_segment"
+    t.string "sheerid_organization_id"
+    t.string "sheerid_postal_code"
+    t.string "sheerid_country"
+    t.string "sheerid_phone_number"
+    t.string "sheerid_birth_date"
+    t.string "sheerid_ip_address"
+    t.string "sheerid_device_fingerprint_hash"
+    t.integer "sheerid_doc_upload_rejection_count", default: 0
+    t.text "sheerid_doc_upload_rejection_reasons", default: [], array: true
+    t.text "sheerid_error_ids", default: [], array: true
+    t.jsonb "sheerid_metadata", default: {}
     t.index "lower((first_name)::text)", name: "index_users_on_first_name"
     t.index "lower((last_name)::text)", name: "index_users_on_last_name"
     t.index "lower((username)::text)", name: "index_users_on_username_case_insensitive"
@@ -482,6 +516,11 @@
     t.index ["salesforce_contact_id"], name: "index_users_on_salesforce_contact_id"
     t.index ["school_id"], name: "index_users_on_school_id"
     t.index ["school_type"], name: "index_users_on_school_type"
+    t.index ["sheerid_error_ids"], name: "index_users_on_sheerid_error_ids", using: :gin
+    t.index ["sheerid_metadata"], name: "index_users_on_sheerid_metadata", using: :gin
+    t.index ["sheerid_organization_id"], name: "index_users_on_sheerid_organization_id"
+    t.index ["sheerid_program_id"], name: "index_users_on_sheerid_program_id"
+    t.index ["sheerid_segment"], name: "index_users_on_sheerid_segment"
     t.index ["source_application_id"], name: "index_users_on_source_application_id"
     t.index ["username"], name: "index_users_on_username", unique: true
     t.index ["uuid"], name: "index_users_on_uuid", unique: true

lib/sheerid_api/constants.rb

@@ -0,0 +1,10 @@
+module SheeridAPI
+  module Constants
+    AUTHORIZATION_HEADER = "Bearer #{Rails.application.secrets.sheerid_api_secret}"
+    HEADERS = {
+      'Authorization': AUTHORIZATION_HEADER,
+      'Accept': 'application/json',
+      'Content-Type': 'application/json'
+    }.freeze
+  end
+end

lib/sheerid_api/request.rb

@@ -1,14 +1,8 @@
+require_relative 'constants'
+
 module SheeridAPI
   class Request
-
-    AUTHORIZATION_HEADER = "Bearer #{Rails.application.secrets.sheerid_api_secret}"
-    HEADERS = {
-      'Authorization': AUTHORIZATION_HEADER,
-      'Accept': 'application/json',
-      'Content-Type': 'application/json'
-    }.freeze
-
-    private_constant(:AUTHORIZATION_HEADER, :HEADERS)
+    include Constants
 
     def initialize(http_method, url, request_body = nil)
       @http_method = http_method
@@ -20,28 +14,43 @@ def response
       @response ||= call_api
     end
 
-    private #################
+    private
 
     def call_api
-      http_response = Faraday.send(@http_method, @url, @request_body, HEADERS)
-      return Response.new(parse_body(http_response.body))
-    rescue Net::ReadTimeout => ee
-      message = 'SheeridAPI: timeout'
-      Sentry.capture_message(message)
-      Rails.logger.warn(message)
-      return NullResponse.instance
+      http_response = send_request
+      Response.new(parse_body(http_response.body))
+    rescue Net::ReadTimeout
+      handle_timeout
     rescue => ee
-      # We don't want explosions here to trickle out and impact callers
-      Sentry.capture_exception(ee)
-      Rails.logger.warn(ee)
-      return NullResponse.instance
+      handle_exception(ee)
     end
 
-    private
+    def send_request
+      case @http_method
+      when :get
+        Faraday.get(@url, @request_body, HEADERS)
+      when :post
+        Faraday.post(@url, @request_body, HEADERS)
+      else
+        raise ArgumentError, "Unsupported HTTP method: #{@http_method}"
+      end
+    end
 
     def parse_body(response)
       JSON.parse(response).to_h
     end
 
+    def handle_timeout
+      message = 'SheeridAPI: timeout'
+      Sentry.capture_message(message)
+      Rails.logger.warn(message)
+      NullResponse.instance
+    end
+
+    def handle_exception(exception)
+      Sentry.capture_exception(exception)
+      Rails.logger.warn(exception)
+      NullResponse.instance
+    end
   end
 end

lib/sheerid_api/response.rb

@@ -21,22 +21,61 @@ module SheeridAPI
   class Response < SheeridAPI::Base
 
     def initialize(body_as_hash)
-      @current_step = body_as_hash.fetch('lastResponse', {}).fetch('currentStep', {})
+      last_response = body_as_hash.fetch('lastResponse', {})
       person_info = body_as_hash.fetch('personInfo', {})
+      organization = person_info&.fetch('organization', {})
+
+      # Basic verification info
+      @current_step = last_response.fetch('currentStep', '')
+      @verification_id = last_response.fetch('verificationId', '')
+      @segment = last_response.fetch('segment', '')
+      @sub_segment = last_response.fetch('subSegment', '')
+      @locale = last_response.fetch('locale', '')
+      @reward_code = last_response.fetch('rewardCode', '')
+      @error_ids = last_response.fetch('errorIds', [])
+
+      # Program and tracking info
+      @program_id = body_as_hash.fetch('programId', '')
+      @tracking_id = body_as_hash.fetch('trackingId', '')
+      @created = body_as_hash.fetch('created', '')
+      @updated = body_as_hash.fetch('updated', '')
+
+      # Person info
       @first_name = person_info&.fetch('firstName', '')
       @last_name = person_info&.fetch('lastName', '')
       @email = person_info&.fetch('email', '')
-      @organization_name = person_info&.fetch('organization', {})&.fetch('name', '')
+      @birth_date = person_info&.fetch('birthDate', '')
+      @device_fingerprint_hash = person_info&.fetch('deviceFingerprintHash', '')
+      @phone_number = person_info&.fetch('phoneNumber', '')
+      @country = person_info&.fetch('country', '')
+      @person_locale = person_info&.fetch('locale', '')
+      @postal_code = person_info&.fetch('postalCode', '')
+      @ip_address = person_info&.fetch('ipAddress', '')
+      @metadata = person_info&.fetch('metadata', {})
+
+      # Organization info
+      @organization_name = organization&.fetch('name', '')
+      @organization_id = organization&.fetch('id', '')
+
+      # Document upload info
+      @doc_upload_rejection_count = body_as_hash.fetch('docUploadRejectionCount', 0)
+      @doc_upload_rejection_reasons = body_as_hash.fetch('docUploadRejectionReasons', [])
     end
 
     def success?
       true
     end
 
     def relevant?
-      # TODO: is this really a good test of relevance?
-      @email.present? && @organization_name.present?
+      # A response is relevant if it has basic verification info
+      @email.present? && @current_step.present?
     end
 
+    # Expose additional fields for enhanced data tracking
+    attr_reader :verification_id, :segment, :sub_segment, :locale, :reward_code, :error_ids,
+                :program_id, :tracking_id, :created, :updated, :birth_date, :device_fingerprint_hash,
+                :phone_number, :country, :person_locale, :postal_code, :ip_address, :metadata,
+                :organization_id, :doc_upload_rejection_count, :doc_upload_rejection_reasons
+
   end
 end

spec/factories/sheerid_verification.rb

@@ -8,5 +8,21 @@
     first_name { Faker::Name.first_name }
     last_name { Faker::Name.last_name }
     organization_name { Faker::Company.name }
+    program_id { "5e150b86ce2a5a1d94874660" }
+    segment { "teacher" }
+    sub_segment { nil }
+    locale { "en-US" }
+    reward_code { "EXAMPLE-CODE" }
+    organization_id { rand(1000..9999).to_s }
+    postal_code { Faker::Address.zip_code }
+    country { "United States" }
+    phone_number { Faker::PhoneNumber.phone_number }
+    birth_date { nil }
+    ip_address { Faker::Internet.ip_v4_address }
+    device_fingerprint_hash { nil }
+    doc_upload_rejection_count { 0 }
+    doc_upload_rejection_reasons { [] }
+    error_ids { [] }
+    metadata { { "marketConsentValue" => "false" } }
   end
 end