output urls for interacting with waits for scripting

.github/workflows/run_inferno_execution_scripts.yml

@@ -0,0 +1,105 @@
+name: Run Inferno Execution Scripts
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+
+  find-execution-scripts:
+    runs-on: ubuntu-latest
+    outputs:
+      scripts: ${{ steps.list.outputs.scripts }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: list
+        run: |
+          scripts=$(find execution_scripts -name '*.yaml' | sort | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "scripts=$scripts" >> "$GITHUB_OUTPUT"
+
+  test:
+    needs: find-execution-scripts
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false          # let all files run even if one fails
+      matrix:
+        config: ${{ fromJson(needs.find-execution-scripts.outputs.scripts) }}
+
+    steps:
+      - name: Checkout Inferno
+        uses: actions/checkout@v4
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Start Inferno
+        run: |
+          gem install foreman
+          mkdir -p data/redis && chmod 777 data/redis
+          bundle exec inferno services start
+          timeout 30 bash -c \
+            'until docker compose -f docker-compose.background.yml exec -T redis redis-cli ping 2>/dev/null; do
+              echo "Waiting for Redis..."; sleep 2
+            done'
+          bundle exec inferno migrate
+          bundle exec inferno start &> /tmp/inferno.log &
+          echo $! > /tmp/inferno.pid
+          echo "Inferno PID: $(cat /tmp/inferno.pid)"
+
+      - name: Wait for Inferno to be ready
+        run: |
+          timeout 60 bash -c \
+            'until curl -sf http://localhost:4567 > /dev/null; do
+              echo "Waiting for Inferno..."; sleep 3
+            done'
+          echo "Inferno is ready"
+
+      - name: Run execution script
+        run: |
+          flags=""
+          [[ "${{ matrix.config }}" == *_with_commands.yaml ]] && flags="--allow-commands"
+          bundle exec inferno execute_script $flags "${{ matrix.config }}"
+
+      - name: Compute debug artifact paths
+        if: failure()
+        id: debug-paths
+        run: |
+          dir=$(dirname "${{ matrix.config }}")
+          prefix=$(basename "${{ matrix.config }}" .yaml)
+          echo "name=debug-$prefix" >> "$GITHUB_OUTPUT"
+          mkdir -p /tmp/docker_logs /tmp/debug_results
+          for service in $(docker compose -f docker-compose.background.yml config --services 2>/dev/null || true); do
+            docker compose -f docker-compose.background.yml logs "$service" > "/tmp/docker_logs/${service}.log" 2>&1 || true
+          done
+          find "$dir" -maxdepth 1 \( -name "${prefix}*_actual_results_*.json" -o -name "${prefix}*_compared_results_*.csv" \) \
+            -exec cp {} /tmp/debug_results/ \; 2>/dev/null || true
+
+      - name: Upload debug artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.debug-paths.outputs.name }}
+          path: |
+            /tmp/debug_results/*
+            /tmp/inferno.log
+            /tmp/docker_logs/*.log
+          if-no-files-found: ignore
+
+      - name: Stop Inferno
+        if: ${{ always() }}
+        run: |
+          if [[ -f /tmp/inferno.pid ]]; then
+            kill "$(cat /tmp/inferno.pid)" 2>/dev/null || true
+            rm /tmp/inferno.pid
+          fi
+
+      - name: Stop Inferno services
+        if: ${{ always() }}
+        run: bundle exec inferno services stop

.gitignore

@@ -13,10 +13,15 @@
 .settings/org.eclipse.wst.jsdt.ui.superType.container
 .settings/org.eclipse.wst.jsdt.ui.superType.name
 .idea
-*.gem
 
 /coverage
 /spec/examples.txt
 /docs/yard
 .yardoc
 node_modules
+
+.byebug_history
+
+# execution script failed run artifacts
+execution_scripts/**/*_actual_results*.json
+execution_scripts/**/*_compared_results*.csv

Gemfile.lock

@@ -2,7 +2,8 @@ PATH
   remote: .
   specs:
     subscriptions_test_kit (0.12.0)
-      inferno_core (~> 1.0, >= 1.1.2)
+      faraday (~> 1.10.5)
+      inferno_core (~> 1.2.1)
 
 GEM
   remote: https://rubygems.org/
@@ -142,7 +143,7 @@ GEM
       domain_name (~> 0.5)
     i18n (1.14.8)
       concurrent-ruby (~> 1.0)
-    inferno_core (1.1.2)
+    inferno_core (1.2.1)
       activesupport (~> 6.1.7.5)
       base62-rb (= 0.3.1)
       blueprinter (= 0.25.2)

Rakefile

@@ -5,6 +5,19 @@ begin
 rescue LoadError # rubocop:disable Lint/SuppressedException
 end
 
+namespace :execute_scripts do
+  desc 'Run all execution script YAML files against a local Inferno instance (already running). ' \
+       'Optional FILTER env var restricts by File.fnmatch pattern, e.g. FILTER="execution_scripts/demo/*". ' \
+       'Optional INFERNO_BASE_URL env var sets the target Inferno URL, e.g. INFERNO_BASE_URL="http://localhost:4567/"'
+  task :run_all do
+    require 'inferno/utils/execution_script_runner'
+    Inferno::Utils::ExecutionScriptRunner.run_all(
+      pattern: ENV.fetch('FILTER', 'execution_scripts/**/*.yaml'),
+      inferno_base_url: ENV.fetch('INFERNO_BASE_URL', nil)
+    )
+  end
+end
+
 namespace :db do
   desc 'Apply changes to the database'
   task :migrate do

execution_scripts/EXECUTION_SCRIPTS.md

@@ -0,0 +1,50 @@
+# Execution Scripts
+
+Automated end-to-end scripts that drive the Inferno Subscriptions Test Kit
+using the `inferno execute_script` CLI. They simulate a human tester running
+the suite without any manual browser interaction.
+
+For general background on the `execute_script` framework, see the
+[Scripting Suite Execution](https://inferno-framework.github.io/docs/advanced-test-features/scripting-execution.html)
+documentation. For CLI usage and how to start Inferno and run scripts, see the
+[Inferno CLI](https://inferno-framework.github.io/docs/getting-started/inferno-cli#complex-scripted-execution)
+documentation.
+
+---
+
+## Files
+
+| File | Payload |
+|------|---------|
+| `subscriptions_r4_empty_with_commands.yaml` | `empty` |
+| `subscriptions_r4_id_only_with_commands.yaml` | `id-only` |
+| `subscriptions_r4_full_resource_with_commands.yaml` | `full-resource` |
+| `advance_wait.rb` | Shared helper — advances an Inferno wait state via GET |
+
+Run a script from the repository root:
+
+```bash
+bundle exec inferno execute_script execution_scripts/subscriptions_r4_empty_with_commands.yaml --allow-commands
+```
+```bash
+bundle exec inferno execute_script execution_scripts/subscriptions_r4_id_only_with_commands.yaml --allow-commands
+```
+```bash
+bundle exec inferno execute_script execution_scripts/subscriptions_r4_full_resource_with_commands.yaml --allow-commands
+```
+
+The `_with_commands` suffix signals to the `execute_scripts:run_all` Rake task
+and the GitHub Actions workflow that these scripts require the `--allow-commands`
+flag, which is passed automatically.
+
+---
+
+## Result Comparison
+
+For general documentation on how result comparison works, see
+[Check Results](https://inferno-framework.github.io/docs/advanced-test-features/scripting-execution#check-results).
+
+Each script normalises the following values before comparison so that results
+are portable across runs and Inferno instances:
+- The Inferno host URL (replaced with `<INFERNO_HOST>`)
+- UUIDs (replaced with `<UUID>`)

execution_scripts/advance_wait.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Advances an Inferno wait state by sending a GET request to the given URL.
+# An optional delay can be specified to allow background jobs to complete
+# before advancing (e.g. waiting for notifications to be delivered).
+#
+# Args:
+#   ARGV[0] - url          (the wait URL to GET)
+#   ARGV[1] - delay        (seconds to sleep before advancing, default: 0)
+
+require 'faraday'
+
+url   = ARGV[0]
+delay = ARGV[1].to_i
+
+raise "Usage: #{$PROGRAM_NAME} <url> [delay_seconds]" if url.nil?
+
+if delay.positive?
+  puts "Waiting #{delay}s for notifications to be delivered..."
+  sleep delay
+end
+
+puts "Advancing wait: #{url}"
+response = Faraday.get(url)
+puts "Response: #{response.status}"

execution_scripts/subscriptions_r4_empty_with_commands.yaml

@@ -0,0 +1,81 @@
+sessions:
+  - suite: subscriptions_r5_backport_r4_client
+    name: client
+    preset: inferno-subscriptions_r5_backport_r4_client_empty
+  - suite: subscriptions_r5_backport_r4_server
+    name: server
+    preset: inferno-subscriptions_r5_backport_r4_server_preset_empty
+
+comparison_config:
+  normalized_strings:
+    - replacement: <INFERNO_HOST>
+      patterns:
+        - http://localhost:4567/inferno # local inferno core ruby
+        - http://localhost:4567 # local ruby
+        - http://localhost # local docker
+        - https://inferno.healthit.gov/suites # prod
+        - https://inferno-qa.healthit.gov/suites # qa
+    - replacement: <UUID>
+      pattern: /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
+
+steps:
+  - session: client
+    status: created
+    state_description: >
+      Step 1: Start client suite — it runs until it reaches the InteractionTest wait,
+      where it is ready to receive a Subscription from the server under test.
+    start_run:
+      session: client
+      runnable: suite
+    next_poll_session: client
+    action_description: Start client suite
+
+  - session: client
+    status: waiting
+    last_completed: 1.1.01
+    state_description: >
+      Step 2: Client is waiting at InteractionTest (1.1.01). Start the server suite,
+      which will POST a Subscription to the client's FHIR endpoint. The client
+      then sends handshake and event notifications to Inferno's own notification
+      endpoint inside Inferno.
+    start_run:
+      session: server
+      runnable: suite
+    next_poll_session: server
+    action_description: Start server suite once client is waiting at InteractionTest
+
+  - session: server
+    status: waiting
+    last_completed: 1.1.02
+    state_description: >
+      Step 3: Server is waiting at notification delivery (1.1.02). Sleep to allow the
+      client to finish delivering notifications, then advance the server wait.
+    command: bundle exec ruby execution_scripts/advance_wait.rb '{server.wait_outputs.confirmation_url}' 15
+    next_poll_session: server
+    action_description: Wait for client to deliver notifications then advance server wait
+
+  - session: server
+    status: done
+    last_completed: suite
+    state_description: >
+      Step 4: Server suite is done. Advance the client interaction wait so the client
+      suite can continue with conformance verification.
+    command: bundle exec ruby execution_scripts/advance_wait.rb '{client.wait_outputs.confirmation_url}'
+    next_poll_session: client
+    action_description: Advance client interaction wait after server suite completes
+
+  - session: client
+    status: waiting
+    last_completed: 1.3.04
+    state_description: >
+      Step 5: Client is waiting at the processing attestation (1.3.04). Attest that
+      the event notification was processed correctly.
+    command: curl -s '{client.wait_outputs.attest_true_url}'
+    next_poll_session: client
+    action_description: Attest that event notification was processed correctly
+
+  - session: client
+    status: done
+    last_completed: suite
+    action: END_SCRIPT
+    action_description: Script complete