[AUTO] Incremented version to 2.16.1.

.github/workflows/ci.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches:
       - "*"
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
   Get-CI-Image-Tag:
@@ -17,7 +19,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [21]
         os: [ ubuntu-latest ]
     name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -26,11 +28,13 @@ jobs:
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
       # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
@@ -40,38 +44,40 @@ jobs:
       - name: Build and Test
         run: |
           chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "whoami && java -version && ./gradlew build"
+          su `id -un 1000` -c "whoami && java -version && ./gradlew build --refresh-dependencies"
 
       - name: Create Artifact Path
         run: |
           mkdir -p security-analytics-artifacts
           cp ./build/distributions/*.zip security-analytics-artifacts
 
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: logs-ubuntu
           path: build/testclusters/integTest-*/logs/*
+          overwrite: true
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: security-analytics-plugin-${{ matrix.os }}
           path: security-analytics-artifacts
+          overwrite: true
 
   build-windows-macos:
     env:
       BUILD_ARGS: ${{ matrix.os_build_args }}
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [21]
         os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
@@ -86,7 +92,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # This is a hack, but this step creates a link to the X: mounted drive, which makes the path
       # short enough to work on Windows
@@ -101,7 +107,7 @@ jobs:
 
       - name: Build and Test
         working-directory: ${{ env.WORKING_DIR }}
-        run: ./gradlew build ${{ env.BUILD_ARGS }}
+        run: ./gradlew build --refresh-dependencies ${{ env.BUILD_ARGS }}
         env:
           _JAVA_OPTIONS: ${{ matrix.os_java_options }}
 
@@ -111,21 +117,24 @@ jobs:
           cp ./build/distributions/*.zip security-analytics-artifacts
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ failure() && matrix.os == 'macos-latest' }}
         with:
           name: logs-mac
           path: build/testclusters/integTest-*/logs/*
+          overwrite: true
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ failure() && matrix.os == 'windows-latest' }}
         with:
           name: logs-windows
           path: build\testclusters\integTest-*\logs\*
+          overwrite: true
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: security-analytics-plugin-${{ matrix.os }}
           path: security-analytics-artifacts
+          overwrite: true

.github/workflows/maven-publish.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: aws-actions/configure-aws-credentials@v1
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}

.github/workflows/multi-node-test-workflow.yml

@@ -7,7 +7,6 @@ on:
   push:
     branches:
       - "*"
-
 jobs:
   Get-CI-Image-Tag:
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
@@ -18,7 +17,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     # Job name
     name: Build and test Security Analytics on linux
     # This job runs on Linux
@@ -28,23 +27,25 @@ jobs:
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
       # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Run integration tests with multi node config
         run: |
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c "./gradlew integTest -PnumNodes=3"
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: logs

.github/workflows/security-test-workflow.yml

@@ -9,12 +9,13 @@ on:
       - "*"
 env:
   OPENSEARCH_INITIAL_ADMIN_PASSWORD: myStrongPassword123!
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
   build:
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     # Job name
     name: Build and test SecurityAnalytics
     # This job runs on Linux
@@ -27,7 +28,7 @@ jobs:
           java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1

build.gradle

@@ -7,21 +7,31 @@ import org.opensearch.gradle.test.RestIntegTestTask
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.15.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.19.1-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
         plugin_no_snapshot = opensearch_build
         opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
+        sa_commons_version = '1.0.0'
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
         }
+
+        alerting_spi_build = opensearch_build
+        alerting_spi_build += "-SNAPSHOT"
         if (isSnapshot) {
             opensearch_build += "-SNAPSHOT"
+
+            // TODO consider enabling snapshot options once SA commons is published to maven central
+//            sa_commons_version += "-SNAPSHOT"
         }
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-        kotlin_version = '1.6.10'
+        kotlin_version = '1.8.21'
+
+        sa_commons_file_name = "security-analytics-commons-${sa_commons_version}.jar"
+        sa_commons_file_path = "${project.rootDir}/${sa_commons_file_name}"
     }
 
     repositories {
@@ -54,7 +64,7 @@ ext {
     noticeFile = rootProject.file('NOTICE')
 }
 
-licenseHeaders.enabled = true
+licenseHeaders.enabled = false
 testingConventions.enabled = false
 forbiddenApis.ignoreFailures = true
 
@@ -68,7 +78,7 @@ opensearchplugin {
     name 'opensearch-security-analytics'
     description 'OpenSearch Security Analytics plugin'
     classname 'org.opensearch.securityanalytics.SecurityAnalyticsPlugin'
-    extendedPlugins = ['opensearch-job-scheduler']
+    extendedPlugins = ['opensearch-job-scheduler', 'opensearch-alerting']
 }
 
 javaRestTest {
@@ -150,27 +160,36 @@ configurations {
         resolutionStrategy {
             // for spotless transitive dependency CVE
             force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0"
-            force "com.google.guava:guava:32.1.2-jre"
+            force "com.google.guava:guava:32.1.3-jre"
         }
     }
 }
 
 dependencies {
     javaRestTestImplementation project.sourceSets.main.runtimeClasspath
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}"
-    implementation "org.antlr:antlr4-runtime:4.10.1"
-    implementation "com.cronutils:cron-utils:9.1.6"
-    api "org.opensearch:common-utils:${common_utils_version}@jar"
-    api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
-    implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
+    compileOnly "org.antlr:antlr4-runtime:4.10.1"
+    compileOnly "com.cronutils:cron-utils:9.1.7"
+    compileOnly "org.opensearch:common-utils:${common_utils_version}@jar"
+    compileOnly "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
+    compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
+    compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}"
     implementation "org.apache.commons:commons-csv:1.10.0"
+    compileOnly "com.google.guava:guava:32.1.3-jre"
+
+    // TODO uncomment once SA commons is published to maven central
+//    api "org.opensearch:security-analytics-commons:${sa_commons_version}@jar"
+
+    // TODO remove once SA commons is published to maven central
+    api files(sa_commons_file_path)
 
     // Needed for integ tests
     zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
 }
 
 // RPM & Debian build
@@ -364,6 +383,12 @@ afterEvaluate {
             into opensearchplugin.name
         }
 
+        // TODO remove once SA commons is published to maven central
+        from(project.rootDir) {
+            include sa_commons_file_name
+            into opensearchplugin.name
+        }
+
         user 'root'
         permissionGroup 'root'
         fileMode 0644

release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md

@@ -2,6 +2,10 @@
 
 Compatible with OpenSearch 2.15.0
 
+### Features
+* Alerts in correlations [Experminental] ([#1040](https://github.com/opensearch-project/security-analytics/pull/1040))
+* Alerts in Correlations Part 2 ([#1062](https://github.com/opensearch-project/security-analytics/pull/1062))
+
 ### Maintenance
 * Increment version to 2.15.0-SNAPSHOT. ([#1055](https://github.com/opensearch-project/security-analytics/pull/1055))
 * Fix codecov calculation ([#1021](https://github.com/opensearch-project/security-analytics/pull/1021))
@@ -12,4 +16,4 @@ Compatible with OpenSearch 2.15.0
 * Change default filter to time based fields ([#1030](https://github.com/opensearch-project/security-analytics/pull/1030))
 
 ### Documentation
-* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061))
+* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061))

release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md

@@ -0,0 +1,35 @@
+## Version 2.16.0.0 2024-07-23
+
+Compatible with OpenSearch 2.16.0
+
+### Features
+* Threat Intel Analytics ([#1098](https://github.com/opensearch-project/security-analytics/pull/1098))
+
+### Maintenance
+* Incremented version to 2.16.0. ([#1197](https://github.com/opensearch-project/security-analytics/pull/1197))
+* Fix build CI error due to action runner env upgrade node 20 ([#1143](https://github.com/opensearch-project/security-analytics/pull/1143))
+
+### Enhancement
+* added correlationAlert integ tests ([#1099](https://github.com/opensearch-project/security-analytics/pull/1099))
+* add filter to list ioc api to fetch only from available and refreshing apis. null check for alias of ioc indices ([#1131](https://github.com/opensearch-project/security-analytics/pull/1131))
+* Changes threat intel default store config model ([#1133](https://github.com/opensearch-project/security-analytics/pull/1133))
+* adds new tif source config type - url download ([#1142](https://github.com/opensearch-project/security-analytics/pull/1142))
+
+### Bug Fixes
+* pass integ tests ([#1082](https://github.com/opensearch-project/security-analytics/pull/1082))
+* set blank response when indexNotFound exception ([#1125](https://github.com/opensearch-project/security-analytics/pull/1125))
+* throw error when no iocs are stored due to incompatible ioc types from S3 downloaded iocs file ([#1129](https://github.com/opensearch-project/security-analytics/pull/1129))
+* fix findingIds filter on ioc findings search api ([#1130](https://github.com/opensearch-project/security-analytics/pull/1130))
+* Adjusted IOCTypes usage ([#1156](https://github.com/opensearch-project/security-analytics/pull/1156))
+* Fix the job scheduler parser, action listeners, and multi-node test ([#1157](https://github.com/opensearch-project/security-analytics/pull/1157))
+* ListIOCs API to return number of findings per IOC ([#1163](https://github.com/opensearch-project/security-analytics/pull/1163))
+* Ioc upload integ tests and fix update ([#1162](https://github.com/opensearch-project/security-analytics/pull/1162))
+* [BUG] Resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs ([#1173](https://github.com/opensearch-project/security-analytics/pull/1173))
+* Enum fix ([#1178](https://github.com/opensearch-project/security-analytics/pull/1178))
+* fix bug: threat intel monitor finding doesnt contain all doc_ids containing malicious IOC ([#1184](https://github.com/opensearch-project/security-analytics/pull/1184))
+* Fixed bulk indexing for IOCs ([#1187](https://github.com/opensearch-project/security-analytics/pull/1187))
+* Fix ioc upload update behavior and change error response ([#1192](https://github.com/opensearch-project/security-analytics/pull/1192))
+* Catch and wrap exceptions. ([#1198](https://github.com/opensearch-project/security-analytics/pull/1198))
+
+### Documentation
+* Added 2.16.0 release notes. ([#1196](https://github.com/opensearch-project/security-analytics/pull/1196))

release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md

@@ -0,0 +1,27 @@
+## Version 2.17.0.0 2024-09-05
+
+Compatible with OpenSearch 2.17.0
+
+### Maintenance
+* update build.gradle to use alerting-spi snapshot version ([#1217](https://github.com/opensearch-project/security-analytics/pull/1217))
+
+### Enhancement
+* added triggers in getDetectors API response ([#1226](https://github.com/opensearch-project/security-analytics/pull/1226))
+* secure rest tests for threat intel monitor apis ([#1212](https://github.com/opensearch-project/security-analytics/pull/1212))
+
+### Bug Fixes
+* Adds user validation for threat intel transport layer classes and stashes the thread context for all system index interactions ([#1207](https://github.com/opensearch-project/security-analytics/pull/1207))
+* fix mappings integ tests ([#1213](https://github.com/opensearch-project/security-analytics/pull/1213))
+* Bug fixes for threat intel ([#1223](https://github.com/opensearch-project/security-analytics/pull/1223))
+* make threat intel run with standard detectors ([#1234](https://github.com/opensearch-project/security-analytics/pull/1234))
+* Fixed searchString bug. Removed nested IOC mapping structure. ([#1239](https://github.com/opensearch-project/security-analytics/pull/1239))
+* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs ([#1240](https://github.com/opensearch-project/security-analytics/pull/1240))
+* Make threat intel source config release lock event driven ([#1254](https://github.com/opensearch-project/security-analytics/pull/1254))
+* Fix S3 validation errors not caught by action listener ([#1257](https://github.com/opensearch-project/security-analytics/pull/1257))
+* Clean up empty IOC indices created by failed source configs ([#1267](https://github.com/opensearch-project/security-analytics/pull/1267))
+* Fix threat intel multinode tests ([#1274](https://github.com/opensearch-project/security-analytics/pull/1274))
+* Update threat intel job mapping to new version ([#1272](https://github.com/opensearch-project/security-analytics/pull/1272))
+* Stash context for List IOCs Api ([#1278](https://github.com/opensearch-project/security-analytics/pull/1278))
+
+### Documentation
+* Added 2.17.0 release notes. ([#1290](https://github.com/opensearch-project/security-analytics/pull/1290))