Add conformance testing for Explorer search backends

.github/workflows/search-backend-conformance.yml

@@ -0,0 +1,218 @@
+name: Search backend conformance
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 3 * * 0'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: search-backend-conformance
+  cancel-in-progress: false
+
+env:
+  SYNBIOHUB_DOCKER_REF: marpaia/sbol-db-and-explorer
+  SBOL_TEST_SUITE_REVISION: 0044284331b2f915a6e4b9d50e1cbf3ea2f62dcd
+  SYNBIOHUB_CONFORMANCE_IMAGE: synbiohub/synbiohub:search-conformance
+
+jobs:
+  build:
+    name: Build SynBioHub image
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub
+      - name: Build Docker image
+        run: |
+          docker build . \
+            --file docker/Dockerfile \
+            --tag "$SYNBIOHUB_CONFORMANCE_IMAGE"
+      - name: Package Docker image
+        run: |
+          docker save "$SYNBIOHUB_CONFORMANCE_IMAGE" | gzip > sbh.tar.gz
+      - uses: actions/upload-artifact@v4
+        name: Upload Docker image
+        with:
+          name: search-conformance-sbh-image
+          path: sbh.tar.gz
+          retention-days: 1
+
+  sbolexplorer:
+    name: Full corpus (SBOLExplorer)
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      COMPOSE_PROJECT_NAME: search-conformance-explorer
+      SYNBIOHUB_IMAGE: synbiohub/synbiohub:search-conformance
+      SYNBIOHUB_PORT: '37777'
+      TRIPLESTORE_PORT: '38890'
+      EXPLORER_PORT: '33162'
+      ELASTICSEARCH_PORT: '39200'
+    steps:
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub Docker topologies
+        with:
+          repository: SynBioHub/synbiohub-docker
+          ref: ${{ env.SYNBIOHUB_DOCKER_REF }}
+          path: synbiohub-docker
+      - uses: actions/checkout@v5
+        name: Checkout pinned SBOLTestSuite corpus
+        with:
+          repository: SynBioDex/SBOLTestSuite
+          ref: ${{ env.SBOL_TEST_SUITE_REVISION }}
+          path: SBOLTestSuite
+      - uses: actions/download-artifact@v4
+        name: Download SynBioHub image
+        with:
+          name: search-conformance-sbh-image
+      - name: Import SynBioHub image
+        run: |
+          gzip --decompress --stdout sbh.tar.gz | docker load
+      - name: Create artifact directory
+        run: mkdir -p artifacts/search-backends
+      - name: Start sbol-db with SBOLExplorer
+        run: |
+          docker compose \
+            -f synbiohub-docker/docker-compose.sboldb.yml \
+            -f synbiohub-docker/docker-compose.sboldb-store-only.yml \
+            -f synbiohub-docker/docker-compose.explorer.yml \
+            up -d --wait --wait-timeout 600
+      - name: Run full-corpus SBOLExplorer baseline
+        run: |
+          python3 tests/search-backends/run-conformance.py \
+            --topology sboldb-explorer-fixed \
+            --base-url http://127.0.0.1:37777/ \
+            --corpus-root SBOLTestSuite \
+            --report artifacts/search-backends/sboldb-explorer-fixed.json
+      - name: Capture SBOLExplorer topology logs
+        if: always()
+        run: |
+          docker compose \
+            -f synbiohub-docker/docker-compose.sboldb.yml \
+            -f synbiohub-docker/docker-compose.sboldb-store-only.yml \
+            -f synbiohub-docker/docker-compose.explorer.yml \
+            logs --no-color > artifacts/search-backends/sboldb-explorer-fixed.compose.log 2>&1 || true
+      - name: Stop SBOLExplorer topology
+        if: always()
+        run: |
+          docker compose \
+            -f synbiohub-docker/docker-compose.sboldb.yml \
+            -f synbiohub-docker/docker-compose.sboldb-store-only.yml \
+            -f synbiohub-docker/docker-compose.explorer.yml \
+            down --volumes --remove-orphans
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: search-conformance-sbolexplorer
+          path: artifacts/search-backends/
+          if-no-files-found: error
+          retention-days: 30
+
+  sboldb:
+    name: Full corpus (sbol-db)
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      COMPOSE_PROJECT_NAME: search-conformance-sboldb
+      SYNBIOHUB_IMAGE: synbiohub/synbiohub:search-conformance
+      SYNBIOHUB_PORT: '27777'
+      TRIPLESTORE_PORT: '28890'
+    steps:
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub Docker topologies
+        with:
+          repository: SynBioHub/synbiohub-docker
+          ref: ${{ env.SYNBIOHUB_DOCKER_REF }}
+          path: synbiohub-docker
+      - uses: actions/checkout@v5
+        name: Checkout pinned SBOLTestSuite corpus
+        with:
+          repository: SynBioDex/SBOLTestSuite
+          ref: ${{ env.SBOL_TEST_SUITE_REVISION }}
+          path: SBOLTestSuite
+      - uses: actions/download-artifact@v4
+        name: Download SynBioHub image
+        with:
+          name: search-conformance-sbh-image
+      - name: Import SynBioHub image
+        run: |
+          gzip --decompress --stdout sbh.tar.gz | docker load
+      - name: Create artifact directory
+        run: mkdir -p artifacts/search-backends
+      - name: Start sbol-db native search topology
+        run: |
+          docker compose \
+            -f synbiohub-docker/docker-compose.sboldb.yml \
+            -f synbiohub-docker/docker-compose.sboldb-search.yml \
+            up -d --wait --wait-timeout 600
+      - name: Run full-corpus sbol-db candidate
+        run: |
+          python3 tests/search-backends/run-conformance.py \
+            --topology sboldb-native \
+            --base-url http://127.0.0.1:27777/ \
+            --metrics-url http://127.0.0.1:28890/metrics \
+            --corpus-root SBOLTestSuite \
+            --report artifacts/search-backends/sboldb-native.json
+      - name: Capture sbol-db topology logs
+        if: always()
+        run: |
+          docker compose \
+            -f synbiohub-docker/docker-compose.sboldb.yml \
+            -f synbiohub-docker/docker-compose.sboldb-search.yml \
+            logs --no-color > artifacts/search-backends/sboldb-native.compose.log 2>&1 || true
+      - name: Stop sbol-db topology
+        if: always()
+        run: |
+          docker compose \
+            -f synbiohub-docker/docker-compose.sboldb.yml \
+            -f synbiohub-docker/docker-compose.sboldb-search.yml \
+            down --volumes --remove-orphans
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: search-conformance-sboldb
+          path: artifacts/search-backends/
+          if-no-files-found: error
+          retention-days: 30
+
+  compare:
+    name: Compare full-corpus reports
+    needs: [sbolexplorer, sboldb]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub
+      - uses: actions/download-artifact@v4
+        name: Download SBOLExplorer report
+        with:
+          name: search-conformance-sbolexplorer
+          path: artifacts/search-backends/baseline
+      - uses: actions/download-artifact@v4
+        name: Download sbol-db report
+        with:
+          name: search-conformance-sboldb
+          path: artifacts/search-backends/candidate
+      - name: Compare backend reports
+        run: |
+          python3 tests/search-backends/compare-reports.py \
+            --baseline artifacts/search-backends/baseline/sboldb-explorer-fixed.json \
+            --candidate artifacts/search-backends/candidate/sboldb-native.json \
+            --output artifacts/search-backends/comparison.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: search-conformance-comparison
+          path: artifacts/search-backends/comparison.json
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/test.yml

@@ -2,6 +2,9 @@ name: Integration testing
 
 on: push
 
+env:
+  SYNBIOHUB_DOCKER_REF: marpaia/sbol-db-and-explorer
+
 jobs:
   build:
     name: Build Docker image
@@ -23,6 +26,21 @@ jobs:
         with:
           name: sbh-image
           path: sbh.tar.gz
+  search-backend-harness:
+    name: Search backend harness unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        name: Checkout source tree
+      - uses: actions/setup-python@v6
+        name: Install Python
+        with:
+          python-version: '3.9' # To match the integration-test jobs
+      - name: Run search backend harness unit tests
+        run: |
+          python -m unittest \
+            tests/search-backends/test_run_conformance.py \
+            tests/search-backends/test_compare_reports.py
   sboltests:
     name: SBOL Test Suite
     needs: build
@@ -129,9 +147,58 @@ jobs:
       - name: Run tests
         run: |
           tests/test.sh --stopaftertestsuite
+  explorer-compatibility:
+    name: Explorer compatibility (${{ matrix.label }})
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: Virtuoso + SBOLExplorer
+            store: virtuoso
+            search: sbol-explorer
+          - label: sbol-db + SBOLExplorer
+            store: sboldb
+            search: sbol-explorer
+          - label: sbol-db + sbol-db
+            store: sboldb
+            search: sbol-db
+    env:
+      SBH_TRIPLESTORE: ${{ matrix.store }}
+      SBH_SEARCH_BACKEND: ${{ matrix.search }}
+      SBH_DOCKER_DIR: ${{ github.workspace }}/synbiohub-docker
+      SBH_EXPLORER_INDEX_TIMEOUT: "600"
+    steps:
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub
+      - uses: actions/checkout@v5
+        name: Checkout SynBioHub Docker topologies
+        with:
+          repository: SynBioHub/synbiohub-docker
+          ref: ${{ env.SYNBIOHUB_DOCKER_REF }}
+          path: synbiohub-docker
+      - uses: actions/download-artifact@v4
+        name: Download Docker image
+        with:
+          name: sbh-image
+      - name: Import saved Docker image
+        run: |
+          gzip --decompress --stdout sbh.tar.gz | docker load
+      - uses: actions/setup-python@v6
+        name: Install Python
+        with:
+          python-version: '3.9'
+      - name: Install test dependencies
+        run: |
+          pip install -r tests/test_requirements.txt
+      - name: Run indexed Explorer HTML contract
+        run: |
+          tests/test.sh --stopaftertestsuite
   publish:
     name: Publish snapshot image
-    needs: [sboltests, sbhtests, sboltests-sboldb, sbhtests-sboldb]
+    needs: [search-backend-harness, sboltests, sbhtests, sboltests-sboldb, sbhtests-sboldb, explorer-compatibility]
     runs-on: ubuntu-latest
     if: endsWith(github.ref, 'master')
     steps:

config.json

@@ -7,6 +7,7 @@
   "javaPath": "java",
   "SBOLExplorerEndpoint": "http://localhost:13162/",
   "useSBOLExplorer": true,
+  "SBOLExplorerFallback": true,
   "theme": "default",
   "themeParameters": {
       "default": {

lib/config.js

@@ -77,6 +77,45 @@ function mergeConfigs () {
   mergedConfig = deepmerge(config, localConfig, {
     arrayMerge: (dest, src) => src
   })
+  applyEnvironmentOverrides()
+}
+
+// Deployment-only overrides for the external search topology. These values
+// deliberately sit above config.local.json without being persisted back into
+// it, so the same SynBioHub image can be exercised against stock SBOLExplorer
+// and sbol-db's compatibility listener in isolated Compose projects.
+function applyEnvironmentOverrides () {
+  if (process.env.SBH_EXPLORER_ENDPOINT !== undefined) {
+    if (process.env.SBH_EXPLORER_ENDPOINT === '') {
+      throw new Error('SBH_EXPLORER_ENDPOINT cannot be empty')
+    }
+    mergedConfig.SBOLExplorerEndpoint = process.env.SBH_EXPLORER_ENDPOINT
+  }
+  if (process.env.SBH_USE_EXPLORER !== undefined) {
+    mergedConfig.useSBOLExplorer = parseEnvironmentBoolean(
+      'SBH_USE_EXPLORER', process.env.SBH_USE_EXPLORER)
+  }
+  if (process.env.SBH_EXPLORER_FALLBACK !== undefined) {
+    mergedConfig.SBOLExplorerFallback = parseEnvironmentBoolean(
+      'SBH_EXPLORER_FALLBACK', process.env.SBH_EXPLORER_FALLBACK)
+  }
+}
+
+function parseEnvironmentBoolean (name, value) {
+  switch (String(value).trim().toLowerCase()) {
+    case '1':
+    case 'true':
+    case 'yes':
+    case 'on':
+      return true
+    case '0':
+    case 'false':
+    case 'no':
+    case 'off':
+      return false
+    default:
+      throw new Error(name + ' must be a boolean (true/false, 1/0, yes/no, on/off)')
+  }
 }
 
 function saveLocalConfig () {

lib/search.js

@@ -103,6 +103,9 @@ function search (graphUri, criteria, offset, limit, user, explorer = false) {
   }).catch((err) => {
     console.error('Virtuoso not responding search')
     console.error(err.stack)
+    if (explorer && config.get('SBOLExplorerFallback') === false) {
+      return Promise.reject(err)
+    }
     return Promise.resolve({ count: 0, results: [] })
   })
 }

lib/sparql/sparql.js

@@ -140,6 +140,10 @@ function queryJson (sparql, graphUri, explorer = false) {
 
   function handleError (error) {
     if (explorer) {
+      if (config.get('SBOLExplorerFallback') === false) {
+        console.error(error + ' -> SBOLExplorer fallback disabled')
+        return Promise.reject(error)
+      }
       console.log(error + ' -> retrying query with SBOLExplorer bypassed')
       return query(sparql, graphUri, 'application/sparql-results+json', false).then(parseResult).catch((err) => {
         console.error('Virtuoso not responding search')