NCIOCPL · blairlearn · Mar 15, 2024
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -27,10 +27,10 @@ jobs:
 
     steps:
       - name: Get the code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up node
-        uses:  actions/setup-node@v3
+        uses:  actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -48,9 +48,9 @@ jobs:
           CI: true
 
       - name: Archive test artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
-          name: test-results
+          name: test-results-${{ matrix.operating-system }}
           path: coverage
 
 
@@ -73,10 +73,10 @@ jobs:
     steps:
 
       - name: Get the code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up node
-        uses:  actions/setup-node@v3
+        uses:  actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -98,14 +98,9 @@ jobs:
             {
               \"pipeline\": {
                   \"source\": {
-                      \"module\": \"./lib/sources/github-bestbets-source\",
+                      \"module\": \"./lib/sources/filesystem-bestbets-source\",
                       \"config\": {
-                          \"repoUrl\": \"https://github.com/nciocpl/bestbets-content\",
-                          \"branchName\": \"integration-testing\",
-                          \"resourcesPath\": \"/content\",
-                          \"authentication\": {
-                              \"token\": \"${GITHUB_TOKEN}\"
-                          }
+                          \"contentPath\": \"integration-tests/data/content\"
                       }
                   },
                   \"transformers\": [
@@ -143,24 +138,10 @@ jobs:
               ## Variable is picked up by karate-config.js
               export KARATE_ESHOST="http://localhost:${{ job.services.elasticsearch.ports[9200] }}"
               cd integration-tests && ./bin/karate ./features
-              ## Store the exit code off so we can pass this step and
-              ## capture the test output in the next step, but still
-              ## fail the entire job
-              echo "TEST_EXIT_CODE=$?" >> $GITHUB_ENV
-              exit 0
 
       - name: Upload Integration test results
-        uses: actions/upload-artifact@v1
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
           name: integration-test-results
           path: integration-tests/target
-
-      - name: Fail build on bad tests
-        run: |
-              ## Check if we had errors on the test step, and if so, fail the job
-              if [ $TEST_EXIT_CODE -ne 0 ]; then
-                echo "Tests Failed -- See Run Integration Test step or integration-test-results artifact for more information"
-                exit $TEST_EXIT_CODE
-              else
-                echo "Tests passed"
-              fi
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-lts/gallium
+lts/iron
diff --git a/config/default.json b/config/default.json
@@ -4,14 +4,9 @@
     },
     "pipeline": {
         "source": {
-          "module": "./lib/sources/github-bestbets-source",
+          "module": "./lib/sources/filesystem-bestbets-source",
           "config": {
-            "repoUrl": "https://github.com/nciocpl/bestbets-content",
-            "resourcesPath": "/content",
-            //"authentication": {
-            //    "type": "token",
-            //    "token": "SECRET"
-            //}
+            "contentPath": "<PATH_TO_FILES>"
           }
         },
         "transformers": [
@@ -21,10 +16,8 @@
                     "eshosts": [ "http://localhost:9200" ],
                     "settingsPath": "es-mappings/settings.json",
                     "analyzer": "nostem",
-                    // A good value for this might be 14 times the number of ES nodes.
                     "concurrencyLimit": 14
                 }
-                //"socketLimit": 80
             }
         ],
         "loader": {
@@ -36,7 +29,6 @@
                "mappingPath": "es-mappings/mappings.json",
                "settingsPath": "es-mappings/settings.json"
             }
-            //"socketLimit": 80
         }
     }
 }
diff --git a/integration-tests/bin/load-integration-data.sh b/integration-tests/bin/load-integration-data.sh
@@ -46,10 +46,8 @@ export NODE_CONFIG="
   {                                                                                 \
     \"pipeline\": {                                                                 \
         \"source\": {                                                               \
-            \"module\": \"./lib/sources/github-bestbets-source\",                   \
+            \"module\": \"./lib/sources/filesystem-bestbets-source\",               \
             \"config\": {                                                           \
-                \"repoUrl\": \"https://github.com/nciocpl/bestbets-content\",       \
-                \"ref\": \"integration-testing\",                                   \
                 \"resourcesPath\": \"/content\"                                     \
             }                                                                       \
         },                                                                          \

diff --git a/integration-tests/data/README.md b/integration-tests/data/README.md
@@ -0,0 +1,2 @@
+# bestbets-content
+Content source for the best bets API
diff --git a/integration-tests/data/content/1045389.xml b/integration-tests/data/content/1045389.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1045389
+</CategoryId>
+<CategoryName><![CDATA[Cancer Research Ideas]]></CategoryName>
+<CategoryWeight>100</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>en</Language>
+<Display>true</Display>
+<IncludeSynonyms><synonym IsExactMatch="false"><![CDATA[Clinical Trial Ideas]]></synonym>
+</IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="https://cancerresearchideas.cancer.gov">Cancer Research Ideas</a><!-- start description -->
+<div class="description"><p class="body">An online platform for cancer researchers to submit their best scientific ideas for bringing about a decade’s worth of advances in 5 years, making more therapies available to more patients, and spurring progress in cancer prevention, treatment, and care. </p></div><!-- end description --></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>	
diff --git a/integration-tests/data/content/1055466.xml b/integration-tests/data/content/1055466.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1055466
+</CategoryId>
+<CategoryName><![CDATA[Lynch Syndrome]]></CategoryName>
+<CategoryWeight>50</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>en</Language>
+<Display>true</Display>
+<IncludeSynonyms></IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="/about-cancer/causes-prevention/genetics/genetic-testing-fact-sheet">Genetic Testing for Hereditary Cancer Syndromes</a><!-- start description -->
+<div class="description"><p class="body">A fact sheet about genetic testing for inherited cancer risk, including types of tests, who should consider testing, how to understand test results, and who has access to a person’s test results. Also contains information about at-home, or direct-to-consumer, genetic tests. </p></div><!-- end description --></div><!-- end title & desc container -->
+</li><li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="/types/breast/hp/breast-ovarian-genetics-pdq">Genetics of Breast and Gynecologic Cancers (PDQ®)–Health Professional Version</a><!-- start description -->
+<div class="description"><p class="body">Genetics of Breast and Gynecologic Cancers includes information on BRCA1 and BRCA2 variants (breast and ovarian cancer) and Lynch syndrome (endometrial cancer). Get more information about hereditary breast and gynecologic cancer syndromes in this clinician summary.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li><li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="/types/colorectal/hp/colorectal-genetics-pdq">Genetics of Colorectal Cancer (PDQ®)–Health Professional Version</a><!-- start description -->
+<div class="description"><p class="body">Hereditary colorectal cancer syndromes include Lynch syndrome and several polyposis syndromes (familial adenomatous polyposis, MUTYH-associated polyposis, juvenile polyposis syndrome, Peutz-Jeghers syndrome, and serrated polyposis syndrome). Learn about the genetics, clinical manifestations, management, and psychosocial aspects of these and other hereditary colon cancer syndromes in this expert-reviewed summary.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>	
diff --git a/integration-tests/data/content/1056403.xml b/integration-tests/data/content/1056403.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1056403
+</CategoryId>
+<CategoryName><![CDATA[Cáncer de faringe]]></CategoryName>
+<CategoryWeight>25</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>es-us</Language>
+<Display>true</Display>
+<IncludeSynonyms><synonym IsExactMatch="false"><![CDATA[faringe]]></synonym>
+</IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="/espanol/tipos/cabeza-cuello">Cáncer de cabeza y cuello—Versión para pacientes</a><!-- start description -->
+<div class="description"><p class="body">Información del Instituto Nacional del Cáncer sobre el tratamiento, la prevención, las causas y los exámenes de detección de los cánceres de cabeza y cuello, así como referencias a estudios clínicos, investigaciones, estadísticas y temas relacionados.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>	
diff --git a/integration-tests/data/content/1061550.xml b/integration-tests/data/content/1061550.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1061550
+</CategoryId>
+<CategoryName><![CDATA[Gynecologic Cancers]]></CategoryName>
+<CategoryWeight>1</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>en</Language>
+<Display>true</Display>
+<IncludeSynonyms><synonym IsExactMatch="false"><![CDATA[gynecologic cancer]]></synonym>
+</IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="/types/by-body-location">Cancers by Body Location/System</a><!-- start description -->
+<div class="description"><p class="body">A list of all cancers, organized by location and/or function in the body. Cancers on the list are linked to information about treatment, supportive care, screening, prevention, clinical trials, and other topics.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>	
diff --git a/integration-tests/data/content/1071562.xml b/integration-tests/data/content/1071562.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1071562
+</CategoryId>
+<CategoryName><![CDATA[Cancer Risk Assessment Tools]]></CategoryName>
+<CategoryWeight>200</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>en</Language>
+<Display>true</Display>
+<IncludeSynonyms><synonym IsExactMatch="false"><![CDATA[risk assessemt]]></synonym>
+<synonym IsExactMatch="false"><![CDATA[risk assessment tool]]></synonym>
+<synonym IsExactMatch="false"><![CDATA[assessment tool]]></synonym>
+</IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="https://bcrisktool.cancer.gov">Breast Cancer Risk Assessment Tool</a><!-- start description -->
+<div class="description"><p class="body">A web-based tool for health professionals to estimate a woman’s risk of developing invasive breast cancer.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li><li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="https://ccrisktool.cancer.gov/">Colorectal Cancer Risk Assessment Tool</a><!-- start description -->
+<div class="description"><p class="body">An interactive tool designed by scientists at the National Cancer Institute to estimate a person's risk of developing colorectal cancer.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li><li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="http://www.cancer.gov/melanomarisktool/">Melanoma Risk Assessment Tool</a><!-- start description -->
+<div class="description"><p class="body">An interactive tool designed by scientists to estimate a person's absolute risk of developing invasive melanoma. </p></div><!-- end description --></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>	
diff --git a/integration-tests/data/content/1075147.xml b/integration-tests/data/content/1075147.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1075147
+</CategoryId>
+<CategoryName><![CDATA[Common Data Elements]]></CategoryName>
+<CategoryWeight>150</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>en</Language>
+<Display>true</Display>
+<IncludeSynonyms></IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="https://cbiit.nci.nih.gov/ncip/biomedical-informatics-resources/interoperability-and-semantics/metadata-and-models">Cancer Data Standards Registry and Repository</a></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>	
diff --git a/integration-tests/data/content/1091074.xml b/integration-tests/data/content/1091074.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<cde:BestBetsCategory xsi:schemaLocation="http://www.example.org/CDESchema CDESchema.xsd" xmlns:cde="http://www.example.org/CDESchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<CategoryId>
+1091074
+</CategoryId>
+<CategoryName><![CDATA[Health Information National Trends Survey (HINTS)]]></CategoryName>
+<CategoryWeight>300</CategoryWeight>
+<IsExactMatch>false</IsExactMatch>
+<Language>en</Language>
+<Display>true</Display>
+<IncludeSynonyms></IncludeSynonyms>
+<ExcludeSynonyms></ExcludeSynonyms>
+<CategoryDisplay>
+<![CDATA[
+<div class="managed list">
+<ul>
+<li class="general-list-item general list-item">
+<!-- cgvSnListItemGeneral -->
+<!-- Image -->
+<!-- End Image -->
+<div class="title-and-desc title desc container"><a class="title" href="https://hints.cancer.gov">Health Information National Trends Survey</a><!-- start description -->
+<div class="description"><p class="body">HINTS collects data about the use of cancer-related information by the American public. These data provide opportunities to understand and improve health communication.</p></div><!-- end description --></div><!-- end title & desc container -->
+</li></ul>
+</div>
+
+]]>
+</CategoryDisplay>
+</cde:BestBetsCategory>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# bestbets-content
		Content source for the best bets API