(#93) Update to Elasticsearch 8

.editorconfig

@@ -5,7 +5,7 @@ root = true
 
 [*]
 trim_trailing_whitespace = true
-indent_size = 4
+indent_size = 2
 indent_style = space
 
 end_of_line = lf

.github/copilot-instructions.md

@@ -0,0 +1,17 @@
+When working with files in this repository, please follow these guidelines:
+
+- If you don't know how do something, say you don't know. Do not make assumptions.
+- If an instruction is unclear, ask for clarification.
+- Write clear and concise code comments where necessary.
+- Follow the existing coding style and conventions used in the repository.
+- Ensure proper error handling and input validation.
+
+- Use the modern Elastic.Clients.Elasticsearch.
+- Use System.Text.Json instead of Newtonsoft.Json.
+- Remove any unneeded Using statements.
+- Using statements should be split into groups in this order:
+  1. "Out of the box" packages.
+  2. Third-party packages (e.g. the elasticsearch packages)
+  3. Packages which are part of this solution.
+- Within a group, Using statements should be in alphabetical order.
+- When committing changes, include a brief summary of what was changed and why.

.github/workflows/workflow.yml

@@ -39,10 +39,10 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v4
+        uses: actions/setup-dotnet@v5
         with:
           global-json-file: global.json
 
@@ -58,10 +58,10 @@ jobs:
                 dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=../../${{matrix.package}}.lcov   test/${{matrix.package}}.Tests
               fi
 
-      - name: Save Code Coverage Output
+      - name: Save code coverage output
         # Coverage output isn't created on Windows, so don't upload it.
         if: matrix.operating-system != 'windows-latest'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: coverage-${{matrix.package}}-${{ matrix.operating-system }}
           path: ${{matrix.package}}.lcov
@@ -86,7 +86,7 @@ jobs:
             output-name: "${{ github.workspace }}/out/wwwroot/build-info.json"
 
       - name: Upload Published Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: ${{matrix.package}}-${{ matrix.operating-system }}
           path: out
@@ -109,13 +109,29 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v4
+        uses: actions/setup-dotnet@v5
         with:
           global-json-file: global.json
 
+      - name: Set up reporting tools
+        shell: bash
+        run: |
+              dotnet tool install --global coverlet.console --version 6.*
+              dotnet tool install --global dotnet-reportgenerator-globaltool --version 5.*
+              mkdir -p "${{github.workspace}}/coverage-report"
+
+      - name: Retrieve unit test coverage reports
+        uses: actions/download-artifact@v7
+        with:
+          name: coverage-NCI.OCPL.Api.Common-ubuntu-latest
+
+      - name: Rename unit test coverage file
+        run: |
+              mv "${{github.workspace}}/NCI.OCPL.Api.Common.lcov" "${{github.workspace}}/integration-coverage.lcov"
+
       - name: Setup test output directory
         shell: bash
         run: |
@@ -141,12 +157,16 @@ jobs:
           APP_ASSEMBLY: integration-test-harness.dll
         shell: bash
         run: |
-              ## TODO: This should become a GitHub Action.
-
-              ## Start the app and log output
+              ## Start the app and log output while recording code coverage.
               ## NOTE: We must change directory because while you can call `dotnet "${APP_PATH}/${APP_ASSEMBLY}"`
               ## it will not find the appsettings.json, so we must cd into the APP_PATH first
-              cd $APP_PATH && dotnet $APP_ASSEMBLY > ../integration-tests/target/api_log.txt 2>&1 &
+              cd $APP_PATH && \
+                coverlet .  --target "dotnet" \
+                            --targetargs "$APP_ASSEMBLY"  \
+                            --format lcov \
+                            --include-directory "." \
+                            --include "[NCI.OCPL.Api.Common]*" \
+                            --merge-with "${{github.workspace}}/integration-coverage.lcov" > ../integration-tests/target/api_log.txt 2>&1 &
 
               time_waited=1
               echo "Checking status of ${API_URL}."
@@ -164,6 +184,13 @@ jobs:
 
               echo "API is up"
 
+      - name: Save the API startup Log
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: integration-tests-api-startup-log
+          path: integration-tests/target/api_log.txt
+
       - name: Run tests requiring no data be present.
         ## Normally bash runs with -e which exits the shell upon hitting
         ## an error which breaks our ability to capture those errors.
@@ -190,9 +217,6 @@ jobs:
               ./integration-tests/bin/load-integration-data.sh
 
       - name: Run Integration Test
-        ## Normally bash runs with -e which exits the shell upon hitting
-        ## an error which breaks our ability to capture those errors.
-        shell: bash --noprofile --norc -o pipefail {0}
         run: |
               ## Run Karate
               cd integration-tests && ./bin/karate ./features
@@ -203,22 +227,40 @@ jobs:
               echo TEST_EXIT_CODE_2=$? >> $GITHUB_ENV
               exit 0
 
+      - name: Stop API
+        shell: bash
+        run: |
+              ## Cause a **controlled** shutdown the API so coverlet can get the coverage statistics.
+              curl -X POST http://localhost:5000/shutdown
+
       - name: Upload Integration test results
-        uses: actions/upload-artifact@v4
+        if: always()
+        uses: actions/upload-artifact@v6
         with:
           name: integration-test-results
           path: integration-tests/target
 
-      - name: Fail build on bad tests
-        shell: bash
+      - name: Upload code coverage data
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: coverage-integration
+          path: ${{github.workspace}}/integration-coverage.lcov
+
+      - name: Generate coverage report
         run: |
-              ## Check if we had errors on the test step, and if so, fail the job
-              if [ $TEST_EXIT_CODE_1 -ne 0 ] || [ $TEST_EXIT_CODE_2 -ne 0 ]; then
-                echo "Tests Failed -- See Run Integration Test step or integration-test-results artifact for more information"
-                exit $TEST_EXIT_CODE
-              else
-                echo "Tests passed"
-              fi
+              reportgenerator \
+                -reports:${{github.workspace}}/integration-coverage.lcov \
+                -targetdir:${{github.workspace}}/coverage-report \
+                -reporttypes:Html \
+                -sourcedirs:${{github.workspace}} \
+                -verbosity:Verbose
+
+      - name: Upload Coverage Report
+        uses: actions/upload-artifact@v6
+        with:
+          name: Code Coverage Report
+          path: ${{github.workspace}}/coverage-report
 
   determine_whether_to_publish:
 
@@ -235,8 +277,6 @@ jobs:
     runs-on: windows-latest
     outputs:
       should_publish: ${{ steps.set_outputs.outputs.should_publish }}
-    needs:
-      - integration_tests
 
     steps:
       - name: Set outputs
@@ -286,12 +326,12 @@ jobs:
           path: ${{ github.workspace }}
 
       - name: Download global.json
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           path: source
 
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v4
+        uses: actions/setup-dotnet@v5
         with:
           global-json-file: source/global.json
           source-url: "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json"

integration-tests/docker-nciocpl-api-common/docker-compose.yml

@@ -1,11 +1,20 @@
 services:
     elasticsearch:
-        image: elasticsearch:7.17.28
+        build:
+            # Point to the root of the project.
+            context: ../../
+            # This path is relative to the context.
+            dockerfile: integration-tests/docker-nciocpl-api-common/elasticsearch/Dockerfile
         ## All of the ES settings can be set via the environment
         ## vars.
         environment:
             - discovery.type=single-node
             - ES_JAVA_OPTS=-Xms750m -Xmx750m
+            # Turn security settings off for testing. (Allows http instead of https.)
+            - xpack.security.enabled=false
+            - xpack.security.transport.ssl.enabled=false
+            - xpack.security.http.ssl.enabled=false
+            - http.host=0.0.0.0
         ## These exposed ports are for debugging only. .NET +
         ## Docker + MacOS == bad scene. (.NET always wants to
         ## use the hosts name, and on a mac that is actually

integration-tests/docker-nciocpl-api-common/elasticsearch/Dockerfile

@@ -0,0 +1,12 @@
+FROM elasticsearch:8.19.8
+
+USER root
+
+# Get the NIH Root certificates and install them so we work on VPN
+# Download from https://ocio.nih.gov/Smartcard/Pages/PKI_chain.aspx
+RUN mkdir -p /usr/local/share/ca-certificates/nih \
+    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-ROOT-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-ROOT-1A-base64.cer \
+    && curl -o /usr/local/share/ca-certificates/nih/NIH-DPKI-CA-1A-base64.crt https://ocio.nih.gov/Smartcard/Documents/Certificates/NIH-DPKI-CA-1A-base64.cer \
+    && update-ca-certificates
+
+USER elasticsearch

integration-tests/features/default-route/custom-serialization.feature

@@ -8,7 +8,7 @@ Feature: The API library allows for custom deserialization of data retrieved fro
     Given path 'test/custom-serialization/', identifier
     When method get
     Then status 200
-    And match each $.[*].default == 'Default serialization'
+    And match $.default == 'Default serialization'
 
     Examples:
       | identifier    |
@@ -23,6 +23,6 @@ Feature: The API library allows for custom deserialization of data retrieved fro
     And match $.custom == expected
 
     Examples:
-      | identifier    | expected                 |
-      | simple-string | Took the string path     |
-      | has-array     | Took the not string path |
+      | identifier    | expected            |
+      | simple-string | simple-string-value |
+      | has-array     | first-string-value  |

src/NCI.OCPL.Api.Common.Testing/ElastcsearchTestingTools.cs

@@ -1,6 +1,5 @@
 using System.IO;
 using System.Text;
-using Newtonsoft.Json.Linq;
 
 namespace NCI.OCPL.Api.Common.Testing
 {

src/NCI.OCPL.Api.Common.Testing/ElasticTools.cs

@@ -1,8 +1,8 @@
 using System;
+using System.Collections.Generic;
 
-using Elasticsearch.Net;
-using Nest;
-using Nest.JsonNetSerializer;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Transport;
 
 namespace NCI.OCPL.Api.Common.Testing
 {
@@ -13,50 +13,62 @@ namespace NCI.OCPL.Api.Common.Testing
     public static class ElasticTools {
 
         /// <summary>
-        /// Gets an ElasticClient backed by an InMemoryConnection.  This is used to mock the
-        /// JSON returned by the elastic search so that we test the Nest mappings to our models.
+        /// Gets an ElasticsearchClient backed by an InMemoryConnection.  This is used to mock the
+        /// JSON returned by the elastic search so that we test the Elasticsearch mappings to our models.
         /// </summary>
         /// <param name="testFile"></param>
         /// <returns></returns>
-        public static IElasticClient GetInMemoryElasticClient(string testFile) {
+        public static ElasticsearchClient GetInMemoryElasticClient(string testFile) {
 
             //Get Response JSON
             byte[] responseBody = TestingTools.GetTestFileAsBytes(testFile);
 
             //While this has a URI, it does not matter, an InMemoryConnection never requests
             //from the server.
-            var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
+            var pool = new SingleNodePool(new Uri("http://localhost:9200"));
+
+            // Headers required to identify the server as a genuine Elasticsearch product.
+            var headers = new Dictionary<string, IEnumerable<string>>
+            {
+                { "x-elastic-product", new[] { "Elasticsearch" } }
+            };
 
             // Setup ElasticSearch stuff using the contents of the JSON file as the client response.
-            InMemoryConnection conn = new InMemoryConnection(responseBody);
+            InMemoryRequestInvoker conn = new InMemoryRequestInvoker(responseBody, headers: headers);
 
-            var connectionSettings = new ConnectionSettings(pool, conn, sourceSerializer: JsonNetSerializer.Default);
+            var connectionSettings = new ElasticsearchClientSettings(pool, conn);
 
-            return new ElasticClient(connectionSettings);
+            return new ElasticsearchClient(connectionSettings);
         }
 
         /// <summary>
-        /// Gets an ElasticClient which simulates a failed request.  Success is defined by
+        /// Gets an ElasticsearchClient which simulates a failed request.  Success is defined by
         /// statuses with a 200-series response, so anything from the 400 or 503 series
         /// should be treated as an error.
         /// </summary>
         /// <param name="statusCode"></param>
         /// <returns></returns>
-        public static IElasticClient GetErrorElasticClient(int statusCode)
+        public static ElasticsearchClient GetErrorElasticClient(int statusCode)
         {
           //While this has a URI, it does not matter, an InMemoryConnection never requests
           //from the server.
-          var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
+          var pool = new SingleNodePool(new Uri("http://localhost:9200"));
 
           //Get Response JSON
-          byte[] responseBody = new byte[0];
+          byte[] responseBody = Array.Empty<byte>();
+
+          // Headers required to identify the server as a genuine Elasticsearch product.
+          var headers = new Dictionary<string, IEnumerable<string>>
+          {
+              { "x-elastic-product", new[] { "Elasticsearch" } }
+          };
 
           // Setup ElasticSearch stuff using the contents of the JSON file as the client response.
-          InMemoryConnection conn = new InMemoryConnection(responseBody, statusCode);
+          InMemoryRequestInvoker conn = new InMemoryRequestInvoker(responseBody, statusCode: statusCode, headers: headers);
 
-          var connectionSettings = new ConnectionSettings(pool, conn, sourceSerializer: JsonNetSerializer.Default);
+          var connectionSettings = new ElasticsearchClientSettings(pool, conn);
 
-          return new ElasticClient(connectionSettings);
+          return new ElasticsearchClient(connectionSettings);
         }
 
   }