diff --git a/docs/Running-Suites-Against-Each-Other.md b/docs/Running-Suites-Against-Each-Other.md index fbb33f82..66a80160 100644 --- a/docs/Running-Suites-Against-Each-Other.md +++ b/docs/Running-Suites-Against-Each-Other.md @@ -79,4 +79,5 @@ Some tests will fail, including: - Client test 2.1.2.01 will fail because the CRD client simulation in the Server suite does not automatically update the Bundle with resource updates in `systemActions`. - The server tests will fail as expected because no responses were sent by the client suite. +- - Client tests 1.2.x.3.01 validating that the hook requests structure and content will fail with errors on the 4th and 5th requests only because the server suite purposefully sends non-conformant requests with unexpected fields to verify that the server ignores them. diff --git a/lib/davinci_crd_test_kit/cross_suite/tags.rb b/lib/davinci_crd_test_kit/cross_suite/tags.rb index 0cadf40f..d37ffea1 100644 --- a/lib/davinci_crd_test_kit/cross_suite/tags.rb +++ b/lib/davinci_crd_test_kit/cross_suite/tags.rb @@ -15,6 +15,9 @@ module DaVinciCRDTestKit LONG_RUNNING_GROUP_TAG = 'long_running_request'.freeze DUPLICATED_HOOK_INSTANCE_TAG = 'duplicate_hook_instance'.freeze COVERAGE_INFO_DISABLED_TAG = 'coverage-info-disabled'.freeze + UNKNOWN_CONFIGURATION_TAG = 'unknown-configuration'.freeze + UNKNOWN_CONTEXT_TAG = 'unknown-context'.freeze + UNKNOWN_ELEMENT_TAG = 'unknown-element'.freeze module TagMethods def hook_instance_tag(hook_instance) diff --git a/lib/davinci_crd_test_kit/requirements/generated/crd_server_v221_requirements_coverage.csv b/lib/davinci_crd_test_kit/requirements/generated/crd_server_v221_requirements_coverage.csv index 0ff60314..d2bf6d5f 100644 --- a/lib/davinci_crd_test_kit/requirements/generated/crd_server_v221_requirements_coverage.csv +++ b/lib/davinci_crd_test_kit/requirements/generated/crd_server_v221_requirements_coverage.csv @@ -24,7 +24,7 @@ hl7.fhir.us.davinci-crd_2.2.1,dev-9,https://hl7.org/fhir/us/davinci-crd/2.2.1/en hl7.fhir.us.davinci-crd_2.2.1,dev-10,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-10,CRD servers providing more than one type of coverage requirement information/guidance **SHOULD** expose configuration options allowing clients to dynamically control what information is returned by the service.,SHOULD,CRD Server,false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,dev-14,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-14,CRD Servers **SHALL** behave in the manner prescribed by any supported configuration information received from the CRD Client.,SHALL,CRD Server,false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,dev-15,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-15,CRD Servers **SHALL NOT** require the inclusion of configuration information in a hook call (i.e. no hook invocation is permitted to fail because configuration information was not included).,SHALL NOT,CRD Server,false,,,"","" -hl7.fhir.us.davinci-crd_2.2.1,dev-17,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-17,CRD Servers **SHALL** ignore unsupported configuration information.,SHALL,CRD Server,false,,,"","" +hl7.fhir.us.davinci-crd_2.2.1,dev-17,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-17,CRD Servers **SHALL** ignore unsupported configuration information.,SHALL,CRD Server,false,,,"3.1.3.12, 3.2.3.11, 3.3.3.11, 3.4.3.13, 3.5.3.12, 3.6.3.14","crd_server_v221-crd_v221_server_hooks-crd_v221_server_appointment_book-Group03-crd_v221_unknown_configuration, crd_server_v221-crd_v221_server_hooks-crd_v221_server_encounter_start-Group03-crd_v221_unknown_configuration, crd_server_v221-crd_v221_server_hooks-crd_v221_server_encounter_discharge-Group03-crd_v221_unknown_configuration, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_select-Group03-crd_v221_unknown_configuration, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_dispatch-Group03-crd_v221_unknown_configuration, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_sign-Group03-crd_v221_unknown_configuration" hl7.fhir.us.davinci-crd_2.2.1,dev-19,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-19,"When included with a Task, the creation of the Questionnaire needs to be conditional - it **SHOULD** only occur if that specific Questionnaire version does not already exist",SHOULD,CRD Server,false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,dev-20,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-20,the CRD server **SHALL** query to determine if the client has a copy of the Questionnaire before sending the request.,SHALL,CRD Server,false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,dev-24,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/deviations.html#ci-c-dev-24,"If a hook service is invoked on a collection of resources, all cards returned that are specific to only a subset of the resources passed as context **SHALL** disambiguate in the `detail` element which resources they are associated with in a human-friendly way.",SHALL,CRD Server,true,,,"","" @@ -45,7 +45,7 @@ hl7.fhir.us.davinci-crd_2.2.1,found-27,https://hl7.org/fhir/us/davinci-crd/2.2.1 hl7.fhir.us.davinci-crd_2.2.1,found-28,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-28,"When information is only needed for certain invocations of the hook (e.g., for specific types of medications or services), that information **SHOULD** only be retrieved by query using the provided token, not requested universally via prefetch.",SHOULD,CRD Server,true,,,"","" hl7.fhir.us.davinci-crd_2.2.1,found-29,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-29,Servers **SHALL** use prefetch expressions in the manner described below if those data elements are relevant to their coverage determination or other decision support.,SHALL,CRD Server,true,,,"","" hl7.fhir.us.davinci-crd_2.2.1,found-32,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-32,HTTP 412 responses **SHALL NOT** be used in situations where the prefetch was provided or the query was successfully performed but the record in question did not have all the data the payer might have needed/desired.,SHALL NOT,CRD Server,false,,,"","" -hl7.fhir.us.davinci-crd_2.2.1,found-33,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-33,CRD clients and servers **SHALL** ignore unexpected elements when processing instances.,SHALL,"CRD Client,CRD Server",false,,,"","" +hl7.fhir.us.davinci-crd_2.2.1,found-33,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-33,CRD clients and servers **SHALL** ignore unexpected elements when processing instances.,SHALL,"CRD Client,CRD Server",false,,,"3.1.3.14, 3.2.3.13, 3.3.3.13, 3.4.3.15, 3.5.3.14, 3.6.3.16","crd_server_v221-crd_v221_server_hooks-crd_v221_server_appointment_book-Group03-crd_v221_unknown_cds_hooks_elements, crd_server_v221-crd_v221_server_hooks-crd_v221_server_encounter_start-Group03-crd_v221_unknown_cds_hooks_elements, crd_server_v221-crd_v221_server_hooks-crd_v221_server_encounter_discharge-Group03-crd_v221_unknown_cds_hooks_elements, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_select-Group03-crd_v221_unknown_cds_hooks_elements, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_dispatch-Group03-crd_v221_unknown_cds_hooks_elements, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_sign-Group03-crd_v221_unknown_cds_hooks_elements" hl7.fhir.us.davinci-crd_2.2.1,found-34,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-34,CRD servers **SHALL** provide what coverage requirements they can based on the information available.,SHALL,CRD Server,false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,found-37,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-37,"Therefore, in addition to any logging performed for security purposes, both CRD clients and CRD servers **SHALL** retain logs of all CRD-related hook invocations and their responses for access in the event of a dispute.",SHALL,"CRD Client,CRD Server",false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,found-40,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/foundation.html#ci-c-found-40,Conformant systems **SHOULD** omit non-significant whitespace in transmitted instances for performance reasons.,SHOULD,"CRD Client,CRD Server",false,,,"","" @@ -59,7 +59,7 @@ hl7.fhir.us.davinci-crd_2.2.1,hook-14,https://hl7.org/fhir/us/davinci-crd/2.2.1/ hl7.fhir.us.davinci-crd_2.2.1,hook-16,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-16,"Unless reporting an error or coverage information already exists and there is no new information, CRD Servers **SHALL** include a Coverage Information system action in the response for 'primary' hooks, even if the response indicates that further information is needed or that the level of detail provided is insufficient to determine coverage.",SHALL,CRD Server,false,,,"3.1.3.08, 3.5.3.08, 3.6.3.08","crd_server_v221-crd_v221_server_hooks-crd_v221_server_appointment_book-Group03-crd_v221_all_responses_include_coverage_information, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_dispatch-Group03-crd_v221_order_dispatch_coverage_information, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_sign-Group03-crd_v221_all_responses_include_coverage_information" hl7.fhir.us.davinci-crd_2.2.1,hook-19,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-19,"If Coverage Information is returned for these hooks, it **SHALL NOT** include messages indicating a need for [clinical](https://hl7.org/fhir/us/davinci-crd/2.2.1/en/ValueSet-AdditionalDocumentation.html) or [administrative](https://hl7.org/fhir/us/davinci-crd/2.2.1/en/ValueSet-AdditionalDocumentation.html) information, as such information is expected to be made available later in the process and therefore such guidance is not useful.",SHALL NOT,CRD Server,true,,,"","" hl7.fhir.us.davinci-crd_2.2.1,hook-22,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-22,CRD servers **SHALL NOT** depend on data not covered by the identified profiles in order to return valid coverage-information responses.,SHALL NOT,CRD Server,false,,,"","" -hl7.fhir.us.davinci-crd_2.2.1,hook-23,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-23,CRD Servers **SHALL** handle unrecognized context elements by ignoring them.,SHALL,CRD Server,false,,,"","" +hl7.fhir.us.davinci-crd_2.2.1,hook-23,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-23,CRD Servers **SHALL** handle unrecognized context elements by ignoring them.,SHALL,CRD Server,false,,,"3.1.3.13, 3.2.3.12, 3.3.3.12, 3.4.3.14, 3.5.3.13, 3.6.3.15","crd_server_v221-crd_v221_server_hooks-crd_v221_server_appointment_book-Group03-crd_v221_unknown_context, crd_server_v221-crd_v221_server_hooks-crd_v221_server_encounter_start-Group03-crd_v221_unknown_context, crd_server_v221-crd_v221_server_hooks-crd_v221_server_encounter_discharge-Group03-crd_v221_unknown_context, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_select-Group03-crd_v221_unknown_context, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_dispatch-Group03-crd_v221_unknown_context, crd_server_v221-crd_v221_server_hooks-crd_v221_server_order_sign-Group03-crd_v221_unknown_context" hl7.fhir.us.davinci-crd_2.2.1,hook-25,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-25,CRD Servers **MAY** use the appointment-book hook as a basis for associating a patient with a particular practitioner from a payer attribution perspective.,MAY,CRD Server,false,,,"","" hl7.fhir.us.davinci-crd_2.2.1,hook-26,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-26,"CRD clients and servers **SHALL**, at minimum, support returning and processing the [Coverage Information](https://hl7.org/fhir/us/davinci-crd/2.2.1/en/StructureDefinition-ext-coverage-information.html) system action for all invocations of the appointment-book hook.",SHALL,"CRD Client,CRD Server",false,,,3.1.3.08,crd_server_v221-crd_v221_server_hooks-crd_v221_server_appointment_book-Group03-crd_v221_all_responses_include_coverage_information hl7.fhir.us.davinci-crd_2.2.1,hook-28,https://hl7.org/fhir/us/davinci-crd/2.2.1/en/hooks.html#ci-c-hook-28,CRD Servers **MAY** use the encounter-start hook as a basis for associating a patient with a particular practitioner from a payer attribution perspective.,MAY,CRD Server,false,,,"","" diff --git a/lib/davinci_crd_test_kit/server/jobs/invoke_hook.rb b/lib/davinci_crd_test_kit/server/jobs/invoke_hook.rb index 410ce490..3158f5a2 100644 --- a/lib/davinci_crd_test_kit/server/jobs/invoke_hook.rb +++ b/lib/davinci_crd_test_kit/server/jobs/invoke_hook.rb @@ -41,7 +41,11 @@ def perform_hook_invocations(request_bodies) request_body = prepare_hook_request(request) response = send_hook_invocation(request_body.to_json) send_coverage_info_configuration_invocation(request_body, response) + send_unknown_configuration_invocation(request_body, response) + send_unknown_context_invocation(request_body, response) + send_unknown_cds_hooks_element_invocation(request_body, response) end + return unless test_waiting? # end the wait to continue the tests @@ -128,6 +132,67 @@ def send_coverage_info_configuration_invocation(request_body, response) @coverage_info_configuration_invoked = true end + def send_unknown_configuration_invocation(request_body, response) + return if @unknown_configuration_invoked + return unless response.status == 200 + return unless coverage_info_response?(parsed_response_body(response)) + return unless test_waiting? + + request_body = JSON.parse(request_body.to_json) + prepare_hook_request(request_body) + add_unknown_configuration(request_body) + send_hook_invocation(request_body.to_json, [UNKNOWN_CONFIGURATION_TAG]) + + @unknown_configuration_invoked = true + end + + def send_unknown_context_invocation(request_body, response) + return if @unknown_context_invoked + return unless response.status == 200 + return unless coverage_info_response?(parsed_response_body(response)) + return unless test_waiting? + + request_body = JSON.parse(request_body.to_json) + prepare_hook_request(request_body) + add_unknown_context(request_body) + send_hook_invocation(request_body.to_json, [UNKNOWN_CONTEXT_TAG]) + + @unknown_context_invoked = true + end + + def send_unknown_cds_hooks_element_invocation(request_body, response) + return if @unknown_cds_hooks_element_invoked + return unless response.status == 200 + return unless coverage_info_response?(parsed_response_body(response)) + return unless test_waiting? + + request_body = JSON.parse(request_body.to_json) + prepare_hook_request(request_body) + add_unknown_element(request_body) + send_hook_invocation(request_body.to_json, [UNKNOWN_ELEMENT_TAG]) + + @unknown_cds_hooks_element_invoked = true + end + + def random_key + ('a'..'z').to_a.sample(16).join + end + + def add_unknown_configuration(request_body) + request_body['extension'] ||= {} + request_body['extension']['davinci-crd.configuration'] ||= {} + request_body['extension']['davinci-crd.configuration'][random_key] = true + end + + def add_unknown_context(request_body) + request_body['context'] ||= {} + request_body['context'][random_key] ||= random_key + end + + def add_unknown_element(request_body) + request_body[random_key] ||= random_key + end + def parsed_response_body(response) JSON.parse(response.env.response_body.to_s) rescue JSON::ParserError, TypeError diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/server_appointment_book_group.rb b/lib/davinci_crd_test_kit/server/v2.2.1/server_appointment_book_group.rb index 1a28ae3f..287f0c14 100644 --- a/lib/davinci_crd_test_kit/server/v2.2.1/server_appointment_book_group.rb +++ b/lib/davinci_crd_test_kit/server/v2.2.1/server_appointment_book_group.rb @@ -14,6 +14,9 @@ require_relative 'verify_response/launch_smart_app_card_validation_test' require_relative 'verify_response/create_or_update_coverage_info_response_validation_test' require_relative 'verify_response/all_responses_include_coverage_information_test' +require_relative 'verify_response/unknown_configuration_test' +require_relative 'verify_response/unknown_context_test' +require_relative 'verify_response/unknown_cds_hooks_elements_test' module DaVinciCRDTestKit module V221 @@ -200,6 +203,9 @@ class ServerAppointmentBookGroup < Inferno::TestGroup } } test from: :crd_v221_coverage_info_configuration + test from: :crd_v221_unknown_configuration + test from: :crd_v221_unknown_context + test from: :crd_v221_unknown_cds_hooks_elements end end end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_discharge_group.rb b/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_discharge_group.rb index eeb569a4..0f42f785 100644 --- a/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_discharge_group.rb +++ b/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_discharge_group.rb @@ -11,6 +11,9 @@ require_relative 'verify_response/form_completion_response_validation_test' require_relative 'verify_response/create_or_update_coverage_info_response_validation_test' require_relative 'verify_response/coverage_info_configuration_test' +require_relative 'verify_response/unknown_configuration_test' +require_relative 'verify_response/unknown_context_test' +require_relative 'verify_response/unknown_cds_hooks_elements_test' module DaVinciCRDTestKit module V221 @@ -182,6 +185,9 @@ class ServerEncounterDischargeGroup < Inferno::TestGroup } } test from: :crd_v221_coverage_info_configuration + test from: :crd_v221_unknown_configuration + test from: :crd_v221_unknown_context + test from: :crd_v221_unknown_cds_hooks_elements end end end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_start_group.rb b/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_start_group.rb index 40473e2c..05a009a2 100644 --- a/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_start_group.rb +++ b/lib/davinci_crd_test_kit/server/v2.2.1/server_encounter_start_group.rb @@ -11,6 +11,9 @@ require_relative 'verify_response/form_completion_response_validation_test' require_relative 'verify_response/create_or_update_coverage_info_response_validation_test' require_relative 'verify_response/coverage_info_configuration_test' +require_relative 'verify_response/unknown_configuration_test' +require_relative 'verify_response/unknown_context_test' +require_relative 'verify_response/unknown_cds_hooks_elements_test' module DaVinciCRDTestKit module V221 @@ -182,6 +185,9 @@ class ServerEncounterStartGroup < Inferno::TestGroup } } test from: :crd_v221_coverage_info_configuration + test from: :crd_v221_unknown_configuration + test from: :crd_v221_unknown_context + test from: :crd_v221_unknown_cds_hooks_elements end end end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/server_order_dispatch_group.rb b/lib/davinci_crd_test_kit/server/v2.2.1/server_order_dispatch_group.rb index 50a14faa..30d00fff 100644 --- a/lib/davinci_crd_test_kit/server/v2.2.1/server_order_dispatch_group.rb +++ b/lib/davinci_crd_test_kit/server/v2.2.1/server_order_dispatch_group.rb @@ -14,6 +14,9 @@ require_relative 'verify_response/form_completion_response_validation_test' require_relative 'verify_response/create_or_update_coverage_info_response_validation_test' require_relative 'verify_response/order_dispatch_coverage_information_test' +require_relative 'verify_response/unknown_configuration_test' +require_relative 'verify_response/unknown_context_test' +require_relative 'verify_response/unknown_cds_hooks_elements_test' module DaVinciCRDTestKit module V221 @@ -201,6 +204,9 @@ class ServerOrderDispatchGroup < Inferno::TestGroup } } test from: :crd_v221_coverage_info_configuration + test from: :crd_v221_unknown_configuration + test from: :crd_v221_unknown_context + test from: :crd_v221_unknown_cds_hooks_elements end end end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/server_order_select_group.rb b/lib/davinci_crd_test_kit/server/v2.2.1/server_order_select_group.rb index 64575d30..f6cde8ee 100644 --- a/lib/davinci_crd_test_kit/server/v2.2.1/server_order_select_group.rb +++ b/lib/davinci_crd_test_kit/server/v2.2.1/server_order_select_group.rb @@ -13,6 +13,9 @@ require_relative 'verify_response/form_completion_response_validation_test' require_relative 'verify_response/create_or_update_coverage_info_response_validation_test' require_relative 'verify_response/coverage_info_configuration_test' +require_relative 'verify_response/unknown_configuration_test' +require_relative 'verify_response/unknown_context_test' +require_relative 'verify_response/unknown_cds_hooks_elements_test' module DaVinciCRDTestKit module V221 @@ -207,6 +210,9 @@ class ServerOrderSelectGroup < Inferno::TestGroup } } test from: :crd_v221_coverage_info_configuration + test from: :crd_v221_unknown_configuration + test from: :crd_v221_unknown_context + test from: :crd_v221_unknown_cds_hooks_elements end end end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/server_order_sign_group.rb b/lib/davinci_crd_test_kit/server/v2.2.1/server_order_sign_group.rb index f305907e..4fdf6780 100644 --- a/lib/davinci_crd_test_kit/server/v2.2.1/server_order_sign_group.rb +++ b/lib/davinci_crd_test_kit/server/v2.2.1/server_order_sign_group.rb @@ -16,6 +16,9 @@ require_relative 'verify_response/form_completion_response_validation_test' require_relative 'verify_response/create_or_update_coverage_info_response_validation_test' require_relative 'verify_response/all_responses_include_coverage_information_test' +require_relative 'verify_response/unknown_configuration_test' +require_relative 'verify_response/unknown_context_test' +require_relative 'verify_response/unknown_cds_hooks_elements_test' module DaVinciCRDTestKit module V221 @@ -228,6 +231,9 @@ class ServerOrderSignGroup < Inferno::TestGroup } } test from: :crd_v221_coverage_info_configuration + test from: :crd_v221_unknown_configuration + test from: :crd_v221_unknown_context + test from: :crd_v221_unknown_cds_hooks_elements end end end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_cds_hooks_elements_test.rb b/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_cds_hooks_elements_test.rb new file mode 100644 index 00000000..2db2888c --- /dev/null +++ b/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_cds_hooks_elements_test.rb @@ -0,0 +1,78 @@ +require_relative '../../../cross_suite/cards_identification' +require_relative '../../server_hook_helper' +require_relative '../server_urls' + +module DaVinciCRDTestKit + module V221 + class UnknownCDSHooksElementsTest < Inferno::Test + include DaVinciCRDTestKit::CardsIdentification + include DaVinciCRDTestKit::ServerHookHelper + include DaVinciCRDTestKit::V221::ServerURLs + + title 'Server ignores unknown CDS Hooks elements' + id :crd_v221_unknown_cds_hooks_elements + description %( + If a request resulted in a successful response with a coverage + information system action, a follow-up request is made with an element + with a random key and value added to the CDS hooks request. This test + verifes that this follow-up request also resulted in a successful + response with a coverage information system action to verify that + unknown CDS Hooks elements are ignored. + ) + + verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@found-33' + + def primary_hook? + ['appointment-book', 'order-sign', 'order-dispatch'].include? tested_hook_name + end + + def parsed_body(json) + JSON.parse(json) + rescue JSON::ParserError + nil + end + + run do + load_tagged_requests(tested_hook_name, UNKNOWN_ELEMENT_TAG) + + if requests.empty? + message = "No successful #{tested_hook_name} response contained a coverage-info action." + if primary_hook? + skip message + else + omit message + end + end + + requests.each do |request| + unless request.status == 200 + add_message( + 'error', + "Server request returned HTTP #{request.status} for a request with an unknown CDS Hooks element; " \ + 'expected HTTP 200.' + ) + next + end + + response_body = parsed_body(request.response_body) + unless response_body.is_a?(Hash) + add_message('error', 'Server response to a request with an unknown CDS Hooks element was not valid JSON.') + next + end + + next if coverage_info_response?(response_body) + + add_message( + 'error', + 'Server response to a request with an unknown CDS Hooks element did not contain ' \ + 'a coverage information system action.' + ) + end + + assert_no_error_messages( + 'Responses to requests with an unknown CDS Hooks element were not valid. Check messages for details.' + ) + end + end + end +end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_configuration_test.rb b/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_configuration_test.rb new file mode 100644 index 00000000..bb95a806 --- /dev/null +++ b/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_configuration_test.rb @@ -0,0 +1,78 @@ +require_relative '../../../cross_suite/cards_identification' +require_relative '../../server_hook_helper' +require_relative '../server_urls' + +module DaVinciCRDTestKit + module V221 + class UnknownConfigurationTest < Inferno::Test + include DaVinciCRDTestKit::CardsIdentification + include DaVinciCRDTestKit::ServerHookHelper + include DaVinciCRDTestKit::V221::ServerURLs + + title 'Server ignores unknown configuration information' + id :crd_v221_unknown_configuration + description %( + If a request resulted in a successful response with a coverage + information system action, a follow-up request is made with a random + configuration key set to `true`. This test verifes that this follow-up + request also resulted in a successful response with a coverage + information system action to verify that unknown configuration values + are ignored. + ) + + verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@dev-17' + + def primary_hook? + ['appointment-book', 'order-sign', 'order-dispatch'].include? tested_hook_name + end + + def parsed_body(json) + JSON.parse(json) + rescue JSON::ParserError + nil + end + + run do + load_tagged_requests(tested_hook_name, UNKNOWN_CONFIGURATION_TAG) + + if requests.empty? + message = "No successful #{tested_hook_name} response contained a coverage-info action." + if primary_hook? + skip message + else + omit message + end + end + + requests.each do |request| + unless request.status == 200 + add_message( + 'error', + "Server request returned HTTP #{request.status} for a request with an unknown configuration option; " \ + 'expected HTTP 200.' + ) + next + end + + response_body = parsed_body(request.response_body) + unless response_body.is_a?(Hash) + add_message('error', 'Server response to a request with an unknown configuration value was not valid JSON.') + next + end + + next if coverage_info_response?(response_body) + + add_message( + 'error', + 'Server response to a request with an unknown configuration value did not contain ' \ + 'a coverage information system action.' + ) + end + + assert_no_error_messages( + 'Responses to requests with unknown configuration were not valid. Check messages for details.' + ) + end + end + end +end diff --git a/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_context_test.rb b/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_context_test.rb new file mode 100644 index 00000000..7e25ddca --- /dev/null +++ b/lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_context_test.rb @@ -0,0 +1,78 @@ +require_relative '../../../cross_suite/cards_identification' +require_relative '../../server_hook_helper' +require_relative '../server_urls' + +module DaVinciCRDTestKit + module V221 + class UnknownContextTest < Inferno::Test + include DaVinciCRDTestKit::CardsIdentification + include DaVinciCRDTestKit::ServerHookHelper + include DaVinciCRDTestKit::V221::ServerURLs + + title 'Server ignores unknown context information' + id :crd_v221_unknown_context + description %( + If a request resulted in a successful response with a coverage + information system action, a follow-up request is made with a random + context key set to a random value. This test verifes that this follow-up + request also resulted in a successful response with a coverage + information system action to verify that unknown context values are + ignored. + ) + + verifies_requirements 'hl7.fhir.us.davinci-crd_2.2.1@hook-23' + + def primary_hook? + ['appointment-book', 'order-sign', 'order-dispatch'].include? tested_hook_name + end + + def parsed_body(json) + JSON.parse(json) + rescue JSON::ParserError + nil + end + + run do + load_tagged_requests(tested_hook_name, UNKNOWN_CONTEXT_TAG) + + if requests.empty? + message = "No successful #{tested_hook_name} response contained a coverage-info action." + if primary_hook? + skip message + else + omit message + end + end + + requests.each do |request| + unless request.status == 200 + add_message( + 'error', + "Server request returned HTTP #{request.status} for a request with an unknown context option; " \ + 'expected HTTP 200.' + ) + next + end + + response_body = parsed_body(request.response_body) + unless response_body.is_a?(Hash) + add_message('error', 'Server response to a request with an unknown context element was not valid JSON.') + next + end + + next if coverage_info_response?(response_body) + + add_message( + 'error', + 'Server response to a request with an unknown context element did not contain ' \ + 'a coverage information system action.' + ) + end + + assert_no_error_messages( + 'Responses to requests with unknown context were not valid. Check messages for details.' + ) + end + end + end +end diff --git a/spec/davinci_crd_test_kit/invoke_hook_spec.rb b/spec/davinci_crd_test_kit/invoke_hook_spec.rb index 0619f63e..5aeeed4e 100644 --- a/spec/davinci_crd_test_kit/invoke_hook_spec.rb +++ b/spec/davinci_crd_test_kit/invoke_hook_spec.rb @@ -95,26 +95,62 @@ expect(hook_request).to have_been_made.once end - it 'sends one follow-up request with coverage-info disabled when coverage-info content is returned' do - original_request = stub_request(:post, service_endpoint) - .with do |request| - JSON.parse(request.body).dig('extension', 'davinci-crd.configuration', 'coverage-info').nil? - end - .to_return(status: 200, body: coverage_info_response.to_json) - coverage_info_disabled_request = stub_request(:post, service_endpoint) - .with do |request| - JSON.parse(request.body).dig('extension', 'davinci-crd.configuration', 'coverage-info') == false - end - .to_return(status: 200, body: filtered_response.to_json) + it 'sends follow-up requests when coverage-info content is returned' do + job = described_class.new + + random_key = 'RANDOM_KEY' + allow(job).to receive(:random_key).and_return(random_key) + + original_request = + stub_request(:post, service_endpoint) + .with do |request| + JSON.parse(request.body).dig('extension', 'davinci-crd.configuration', 'coverage-info').nil? && + JSON.parse(request.body).dig('extension', 'davinci-crd.configuration', random_key).nil? && + JSON.parse(request.body).dig('context', random_key).nil? && + JSON.parse(request.body)[random_key].nil? + end + .to_return(status: 200, body: coverage_info_response.to_json) + + coverage_info_disabled_request = + stub_request(:post, service_endpoint) + .with do |request| + JSON.parse(request.body).dig('extension', 'davinci-crd.configuration', 'coverage-info') == false + end + .to_return(status: 200, body: filtered_response.to_json) + + unknown_configuration_request = + stub_request(:post, service_endpoint) + .with do |request| + JSON.parse(request.body).dig('extension', 'davinci-crd.configuration', random_key) == true + end + .to_return(status: 200, body: filtered_response.to_json) + + unknown_context_request = + stub_request(:post, service_endpoint) + .with do |request| + JSON.parse(request.body).dig('context', random_key).present? + end + .to_return(status: 200, body: filtered_response.to_json) + + unknown_element_request = + stub_request(:post, service_endpoint) + .with do |request| + JSON.parse(request.body)[random_key].present? + end + .to_return(status: 200, body: filtered_response.to_json) + stub_request(:get, continuation_url).to_return(status: 200) - described_class.new.perform( + job.perform( test_session_id, service_request_bodies, service_endpoint, inferno_base_url, nil, encryption_method, invoked_hook, continuation_url, failure_url, false, true ) expect(original_request).to have_been_made.once expect(coverage_info_disabled_request).to have_been_made.once + expect(unknown_configuration_request).to have_been_made.once + expect(unknown_context_request).to have_been_made.once + expect(unknown_element_request).to have_been_made.once end it 'sends only one coverage-info disabled follow-up request per job' do @@ -131,7 +167,12 @@ .to_return(status: 200, body: filtered_response.to_json) stub_request(:get, continuation_url).to_return(status: 200) - described_class.new.perform( + job = described_class.new + job.instance_variable_set(:@unknown_configuration_invoked, true) + job.instance_variable_set(:@unknown_context_invoked, true) + job.instance_variable_set(:@unknown_cds_hooks_element_invoked, true) + + job.perform( test_session_id, request_bodies, service_endpoint, inferno_base_url, nil, encryption_method, invoked_hook, continuation_url, failure_url, false, true ) diff --git a/spec/davinci_crd_test_kit/v2.2.1/unknown_cds_hooks_elements_test_spec.rb b/spec/davinci_crd_test_kit/v2.2.1/unknown_cds_hooks_elements_test_spec.rb new file mode 100644 index 00000000..a6634584 --- /dev/null +++ b/spec/davinci_crd_test_kit/v2.2.1/unknown_cds_hooks_elements_test_spec.rb @@ -0,0 +1,134 @@ +require_relative '../../../lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_cds_hooks_elements_test' + +RSpec.describe DaVinciCRDTestKit::V221::UnknownCDSHooksElementsTest do + let(:suite_id) { 'crd_server_v221' } + let(:runnable) { described_class } + let(:results_repo) { Inferno::Repositories::Results.new } + let(:result) { repo_create(:result, test_session_id: test_session.id) } + + let(:service_endpoint) { 'http://example.com/cds-services/order-sign-service' } + let(:hook_request) do + JSON.parse(File.read(File.join( + __dir__, '..', '..', 'fixtures', 'order_sign_hook_request.json' + ))) + end + let(:coverage_info_action) do + { + 'type' => 'update', + 'description' => 'Added coverage information', + 'resource' => { + 'resourceType' => 'ServiceRequest', + 'id' => 'service-request-1', + 'status' => 'draft', + 'intent' => 'order', + 'extension' => [ + { + 'url' => DaVinciCRDTestKit::CardsIdentification::COVERAGE_INFO_EXT_URL + } + ] + } + } + end + let(:guideline_card) do + { + 'summary' => 'Guideline', + 'indicator' => 'info', + 'source' => { 'topic' => { 'code' => 'guideline' } } + } + end + let(:original_response) do + { + 'cards' => [guideline_card], + 'systemActions' => [coverage_info_action] + } + end + let(:filtered_response) do + { + 'cards' => [guideline_card], + 'systemActions' => [] + } + end + + def create_service_request( + body: original_response, + status: 200, + tags: [DaVinciCRDTestKit::ORDER_SIGN_TAG] + ) + repo_create( + :request, + direction: 'outgoing', + url: service_endpoint, + test_session_id: test_session.id, + request_body: unknown_cds_hooks_element_request_body.to_json, + response_body: body.to_json, + result:, + status:, + tags:, + headers: nil + ) + end + + def create_unknown_cds_hooks_element_request(**args) + create_service_request( + **args, + tags: [DaVinciCRDTestKit::ORDER_SIGN_TAG, DaVinciCRDTestKit::UNKNOWN_ELEMENT_TAG] + ) + end + + def unknown_cds_hooks_element_request_body + hook_request.deep_dup.tap do |request_body| + request_body['hookInstance'] = SecureRandom.uuid + request_body['RANDOM_KEY'] = 'RANDOM_KEY' + end + end + + def entity_result_messages + results_repo.current_results_for_test_session_and_runnables(test_session.id, [runnable]) + .first + .messages + end + + before do + allow_any_instance_of(runnable).to receive(:tested_hook_name).and_return('order-sign') + end + + it 'passes when unknown cds hooks element responses include coverage-info' do + create_unknown_cds_hooks_element_request(body: original_response) + + result = run(runnable) + + expect(result.result).to eq('pass'), result.result_message + end + + it 'fails if the unknown response does not contain coverage-info' do + create_unknown_cds_hooks_element_request(body: filtered_response) + + result = run(runnable) + + expect(result.result).to eq('fail') + expect(result.result_message).to match(/unknown CDS Hooks element were not valid/) + expect( + entity_result_messages.map(&:message).join(' ') + ).to match(/did not contain a coverage information system action/) + end + + it 'skips if no unknown cds hooks element follow-up request was made for primary hooks' do + create_service_request + + result = run(runnable) + + expect(result.result).to eq('skip') + expect(result.result_message).to match(/response contained a coverage-info action/) + end + + it 'omits if no unknown cds hooks element follow-up request was made for secondary hooks' do + allow_any_instance_of(runnable).to receive(:tested_hook_name).and_return('order-select') + + create_service_request + + result = run(runnable) + + expect(result.result).to eq('omit') + expect(result.result_message).to match(/response contained a coverage-info action/) + end +end diff --git a/spec/davinci_crd_test_kit/v2.2.1/unknown_configuration_test_spec.rb b/spec/davinci_crd_test_kit/v2.2.1/unknown_configuration_test_spec.rb new file mode 100644 index 00000000..32ef1ef4 --- /dev/null +++ b/spec/davinci_crd_test_kit/v2.2.1/unknown_configuration_test_spec.rb @@ -0,0 +1,138 @@ +require_relative '../../../lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_configuration_test' + +RSpec.describe DaVinciCRDTestKit::V221::UnknownConfigurationTest do + let(:suite_id) { 'crd_server_v221' } + let(:runnable) { described_class } + let(:results_repo) { Inferno::Repositories::Results.new } + let(:result) { repo_create(:result, test_session_id: test_session.id) } + + let(:service_endpoint) { 'http://example.com/cds-services/order-sign-service' } + let(:hook_request) do + JSON.parse(File.read(File.join( + __dir__, '..', '..', 'fixtures', 'order_sign_hook_request.json' + ))) + end + let(:coverage_info_action) do + { + 'type' => 'update', + 'description' => 'Added coverage information', + 'resource' => { + 'resourceType' => 'ServiceRequest', + 'id' => 'service-request-1', + 'status' => 'draft', + 'intent' => 'order', + 'extension' => [ + { + 'url' => DaVinciCRDTestKit::CardsIdentification::COVERAGE_INFO_EXT_URL + } + ] + } + } + end + let(:guideline_card) do + { + 'summary' => 'Guideline', + 'indicator' => 'info', + 'source' => { 'topic' => { 'code' => 'guideline' } } + } + end + let(:original_response) do + { + 'cards' => [guideline_card], + 'systemActions' => [coverage_info_action] + } + end + let(:filtered_response) do + { + 'cards' => [guideline_card], + 'systemActions' => [] + } + end + + def create_service_request( + body: original_response, + status: 200, + tags: [DaVinciCRDTestKit::ORDER_SIGN_TAG] + ) + repo_create( + :request, + direction: 'outgoing', + url: service_endpoint, + test_session_id: test_session.id, + request_body: unknown_configuration_request_body.to_json, + response_body: body.to_json, + result:, + status:, + tags:, + headers: nil + ) + end + + def create_unknown_configuration_request(**args) + create_service_request( + **args, + tags: [DaVinciCRDTestKit::ORDER_SIGN_TAG, DaVinciCRDTestKit::UNKNOWN_CONFIGURATION_TAG] + ) + end + + def unknown_configuration_request_body + hook_request.deep_dup.tap do |request_body| + request_body['hookInstance'] = SecureRandom.uuid + request_body['extension'] = { + 'davinci-crd.configuration' => { + 'RANDOM_KEY' => true + } + } + end + end + + def entity_result_messages + results_repo.current_results_for_test_session_and_runnables(test_session.id, [runnable]) + .first + .messages + end + + before do + allow_any_instance_of(runnable).to receive(:tested_hook_name).and_return('order-sign') + end + + it 'passes when unknown configuration responses include coverage-info' do + create_unknown_configuration_request(body: original_response) + + result = run(runnable) + + expect(result.result).to eq('pass'), result.result_message + end + + it 'fails if the unknown response does not contain coverage-info' do + create_unknown_configuration_request(body: filtered_response) + + result = run(runnable) + + expect(result.result).to eq('fail') + expect(result.result_message).to match(/unknown configuration were not valid/) + expect( + entity_result_messages.map(&:message).join(' ') + ).to match(/did not contain a coverage information system action/) + end + + it 'skips if no unknown configuration follow-up request was made for primary hooks' do + create_service_request + + result = run(runnable) + + expect(result.result).to eq('skip') + expect(result.result_message).to match(/response contained a coverage-info action/) + end + + it 'omits if no unknown configuration follow-up request was made for secondary hooks' do + allow_any_instance_of(runnable).to receive(:tested_hook_name).and_return('order-select') + + create_service_request + + result = run(runnable) + + expect(result.result).to eq('omit') + expect(result.result_message).to match(/response contained a coverage-info action/) + end +end diff --git a/spec/davinci_crd_test_kit/v2.2.1/unknown_context_test_spec.rb b/spec/davinci_crd_test_kit/v2.2.1/unknown_context_test_spec.rb new file mode 100644 index 00000000..3d2a6a72 --- /dev/null +++ b/spec/davinci_crd_test_kit/v2.2.1/unknown_context_test_spec.rb @@ -0,0 +1,134 @@ +require_relative '../../../lib/davinci_crd_test_kit/server/v2.2.1/verify_response/unknown_context_test' + +RSpec.describe DaVinciCRDTestKit::V221::UnknownContextTest do + let(:suite_id) { 'crd_server_v221' } + let(:runnable) { described_class } + let(:results_repo) { Inferno::Repositories::Results.new } + let(:result) { repo_create(:result, test_session_id: test_session.id) } + + let(:service_endpoint) { 'http://example.com/cds-services/order-sign-service' } + let(:hook_request) do + JSON.parse(File.read(File.join( + __dir__, '..', '..', 'fixtures', 'order_sign_hook_request.json' + ))) + end + let(:coverage_info_action) do + { + 'type' => 'update', + 'description' => 'Added coverage information', + 'resource' => { + 'resourceType' => 'ServiceRequest', + 'id' => 'service-request-1', + 'status' => 'draft', + 'intent' => 'order', + 'extension' => [ + { + 'url' => DaVinciCRDTestKit::CardsIdentification::COVERAGE_INFO_EXT_URL + } + ] + } + } + end + let(:guideline_card) do + { + 'summary' => 'Guideline', + 'indicator' => 'info', + 'source' => { 'topic' => { 'code' => 'guideline' } } + } + end + let(:original_response) do + { + 'cards' => [guideline_card], + 'systemActions' => [coverage_info_action] + } + end + let(:filtered_response) do + { + 'cards' => [guideline_card], + 'systemActions' => [] + } + end + + def create_service_request( + body: original_response, + status: 200, + tags: [DaVinciCRDTestKit::ORDER_SIGN_TAG] + ) + repo_create( + :request, + direction: 'outgoing', + url: service_endpoint, + test_session_id: test_session.id, + request_body: unknown_context_request_body.to_json, + response_body: body.to_json, + result:, + status:, + tags:, + headers: nil + ) + end + + def create_unknown_context_request(**args) + create_service_request( + **args, + tags: [DaVinciCRDTestKit::ORDER_SIGN_TAG, DaVinciCRDTestKit::UNKNOWN_CONTEXT_TAG] + ) + end + + def unknown_context_request_body + hook_request.deep_dup.tap do |request_body| + request_body['hookInstance'] = SecureRandom.uuid + request_body['context'] = { 'RANDOM_KEY' => 'RANDOM_KEY' } + end + end + + def entity_result_messages + results_repo.current_results_for_test_session_and_runnables(test_session.id, [runnable]) + .first + .messages + end + + before do + allow_any_instance_of(runnable).to receive(:tested_hook_name).and_return('order-sign') + end + + it 'passes when unknown context responses include coverage-info' do + create_unknown_context_request(body: original_response) + + result = run(runnable) + + expect(result.result).to eq('pass'), result.result_message + end + + it 'fails if the unknown response does not contain coverage-info' do + create_unknown_context_request(body: filtered_response) + + result = run(runnable) + + expect(result.result).to eq('fail') + expect(result.result_message).to match(/unknown context were not valid/) + expect( + entity_result_messages.map(&:message).join(' ') + ).to match(/did not contain a coverage information system action/) + end + + it 'skips if no unknown context follow-up request was made for primary hooks' do + create_service_request + + result = run(runnable) + + expect(result.result).to eq('skip') + expect(result.result_message).to match(/response contained a coverage-info action/) + end + + it 'omits if no unknown context follow-up request was made for secondary hooks' do + allow_any_instance_of(runnable).to receive(:tested_hook_name).and_return('order-select') + + create_service_request + + result = run(runnable) + + expect(result.result).to eq('omit') + expect(result.result_message).to match(/response contained a coverage-info action/) + end +end